├── .gitignore
├── Makefile
├── conf.py
├── content
    ├── All Buttons Pressed - CMS & blog for Django-nonrel.rst
    ├── Django-nonrel - NoSQL support for Django.rst
    ├── HTML5 offline manifests with django-mediagenerator.rst
    ├── Managing per-field indexes on App Engine.rst
    ├── Writing a non-relational Django backend.rst
    ├── django-autoload.rst
    ├── django-dbindexer - Expressive NoSQL.rst
    ├── django-filetransfers - File upload-download abstraction.rst
    ├── djangoappengine - Django App Engine backends (DB, email, etc.).rst
    └── djangotoolbox.rst
├── index.rst
└── readme.rst


/.gitignore:
--------------------------------------------------------------------------------
1 | _build
2 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
  1 | # Makefile for Sphinx documentation
  2 | #
  3 | 
  4 | # You can set these variables from the command line.
  5 | SPHINXOPTS    =
  6 | SPHINXBUILD   = sphinx-build
  7 | PAPER         =
  8 | BUILDDIR      = _build
  9 | 
 10 | # Internal variables.
 11 | PAPEROPT_a4     = -D latex_paper_size=a4
 12 | PAPEROPT_letter = -D latex_paper_size=letter
 13 | ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 14 | 
 15 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest
 16 | 
 17 | help:
 18 | 	@echo "Please use \`make <target>' where <target> is one of"
 19 | 	@echo "  html       to make standalone HTML files"
 20 | 	@echo "  dirhtml    to make HTML files named index.html in directories"
 21 | 	@echo "  singlehtml to make a single large HTML file"
 22 | 	@echo "  pickle     to make pickle files"
 23 | 	@echo "  json       to make JSON files"
 24 | 	@echo "  htmlhelp   to make HTML files and a HTML help project"
 25 | 	@echo "  qthelp     to make HTML files and a qthelp project"
 26 | 	@echo "  devhelp    to make HTML files and a Devhelp project"
 27 | 	@echo "  epub       to make an epub"
 28 | 	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
 29 | 	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
 30 | 	@echo "  text       to make text files"
 31 | 	@echo "  man        to make manual pages"
 32 | 	@echo "  changes    to make an overview of all changed/added/deprecated items"
 33 | 	@echo "  linkcheck  to check all external links for integrity"
 34 | 	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
 35 | 
 36 | clean:
 37 | 	-rm -rf $(BUILDDIR)/*
 38 | 
 39 | html:
 40 | 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 41 | 	@echo
 42 | 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 43 | 
 44 | dirhtml:
 45 | 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
 46 | 	@echo
 47 | 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
 48 | 
 49 | singlehtml:
 50 | 	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
 51 | 	@echo
 52 | 	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
 53 | 
 54 | pickle:
 55 | 	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
 56 | 	@echo
 57 | 	@echo "Build finished; now you can process the pickle files."
 58 | 
 59 | json:
 60 | 	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
 61 | 	@echo
 62 | 	@echo "Build finished; now you can process the JSON files."
 63 | 
 64 | htmlhelp:
 65 | 	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
 66 | 	@echo
 67 | 	@echo "Build finished; now you can run HTML Help Workshop with the" \
 68 | 	      ".hhp project file in $(BUILDDIR)/htmlhelp."
 69 | 
 70 | qthelp:
 71 | 	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
 72 | 	@echo
 73 | 	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
 74 | 	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
 75 | 	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Django-nonrel.qhcp"
 76 | 	@echo "To view the help file:"
 77 | 	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Django-nonrel.qhc"
 78 | 
 79 | devhelp:
 80 | 	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
 81 | 	@echo
 82 | 	@echo "Build finished."
 83 | 	@echo "To view the help file:"
 84 | 	@echo "# mkdir -p $$HOME/.local/share/devhelp/Django-nonrel"
 85 | 	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Django-nonrel"
 86 | 	@echo "# devhelp"
 87 | 
 88 | epub:
 89 | 	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
 90 | 	@echo
 91 | 	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
 92 | 
 93 | latex:
 94 | 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
 95 | 	@echo
 96 | 	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
 97 | 	@echo "Run \`make' in that directory to run these through (pdf)latex" \
 98 | 	      "(use \`make latexpdf' here to do that automatically)."
 99 | 
100 | latexpdf:
101 | 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
102 | 	@echo "Running LaTeX files through pdflatex..."
103 | 	make -C $(BUILDDIR)/latex all-pdf
104 | 	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
105 | 
106 | text:
107 | 	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
108 | 	@echo
109 | 	@echo "Build finished. The text files are in $(BUILDDIR)/text."
110 | 
111 | man:
112 | 	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
113 | 	@echo
114 | 	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
115 | 
116 | changes:
117 | 	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
118 | 	@echo
119 | 	@echo "The overview file is in $(BUILDDIR)/changes."
120 | 
121 | linkcheck:
122 | 	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
123 | 	@echo
124 | 	@echo "Link check complete; look for any errors in the above output " \
125 | 	      "or in $(BUILDDIR)/linkcheck/output.txt."
126 | 
127 | doctest:
128 | 	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
129 | 	@echo "Testing of doctests in the sources finished, look at the " \
130 | 	      "results in $(BUILDDIR)/doctest/output.txt."
131 | 


--------------------------------------------------------------------------------
/conf.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | #
  3 | # Django-nonrel documentation build configuration file, created by
  4 | # sphinx-quickstart on Tue Nov 29 11:54:41 2011.
  5 | #
  6 | # This file is execfile()d with the current directory set to its containing dir.
  7 | #
  8 | # Note that not all possible configuration values are present in this
  9 | # autogenerated file.
 10 | #
 11 | # All configuration values have a default; values that are commented out
 12 | # serve to show the default.
 13 | 
 14 | import sys, os
 15 | 
 16 | # If extensions (or modules to document with autodoc) are in another directory,
 17 | # add these directories to sys.path here. If the directory is relative to the
 18 | # documentation root, use os.path.abspath to make it absolute, like shown here.
 19 | #sys.path.insert(0, os.path.abspath('.'))
 20 | 
 21 | # -- General configuration -----------------------------------------------------
 22 | 
 23 | # If your documentation needs a minimal Sphinx version, state it here.
 24 | #needs_sphinx = '1.0'
 25 | 
 26 | # Add any Sphinx extension module names here, as strings. They can be extensions
 27 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 28 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo']
 29 | 
 30 | # Add any paths that contain templates here, relative to this directory.
 31 | templates_path = ['_templates']
 32 | 
 33 | # The suffix of source filenames.
 34 | source_suffix = '.rst'
 35 | 
 36 | # The encoding of source files.
 37 | #source_encoding = 'utf-8-sig'
 38 | 
 39 | # The master toctree document.
 40 | master_doc = 'index'
 41 | 
 42 | # General information about the project.
 43 | project = u'Django-nonrel'
 44 | copyright = u'2011, Tom Brander'
 45 | 
 46 | # The version info for the project you're documenting, acts as replacement for
 47 | # |version| and |release|, also used in various other places throughout the
 48 | # built documents.
 49 | #
 50 | # The short X.Y version.
 51 | version = '0'
 52 | # The full version, including alpha/beta/rc tags.
 53 | release = '0'
 54 | 
 55 | # The language for content autogenerated by Sphinx. Refer to documentation
 56 | # for a list of supported languages.
 57 | #language = None
 58 | 
 59 | # There are two options for replacing |today|: either, you set today to some
 60 | # non-false value, then it is used:
 61 | #today = ''
 62 | # Else, today_fmt is used as the format for a strftime call.
 63 | #today_fmt = '%B %d, %Y'
 64 | 
 65 | # List of patterns, relative to source directory, that match files and
 66 | # directories to ignore when looking for source files.
 67 | exclude_patterns = ['_build']
 68 | 
 69 | # The reST default role (used for this markup: `text`) to use for all documents.
 70 | #default_role = None
 71 | 
 72 | # If true, '()' will be appended to :func: etc. cross-reference text.
 73 | #add_function_parentheses = True
 74 | 
 75 | # If true, the current module name will be prepended to all description
 76 | # unit titles (such as .. function::).
 77 | #add_module_names = True
 78 | 
 79 | # If true, sectionauthor and moduleauthor directives will be shown in the
 80 | # output. They are ignored by default.
 81 | #show_authors = False
 82 | 
 83 | # The name of the Pygments (syntax highlighting) style to use.
 84 | pygments_style = 'sphinx'
 85 | 
 86 | # A list of ignored prefixes for module index sorting.
 87 | #modindex_common_prefix = []
 88 | 
 89 | 
 90 | # -- Options for HTML output ---------------------------------------------------
 91 | 
 92 | # The theme to use for HTML and HTML Help pages.  See the documentation for
 93 | # a list of builtin themes.
 94 | html_theme = 'default'
 95 | 
 96 | # Theme options are theme-specific and customize the look and feel of a theme
 97 | # further.  For a list of options available for each theme, see the
 98 | # documentation.
 99 | #html_theme_options = {}
100 | 
101 | # Add any paths that contain custom themes here, relative to this directory.
102 | #html_theme_path = []
103 | 
104 | # The name for this set of Sphinx documents.  If None, it defaults to
105 | # "<project> v<release> documentation".
106 | #html_title = None
107 | 
108 | # A shorter title for the navigation bar.  Default is the same as html_title.
109 | #html_short_title = None
110 | 
111 | # The name of an image file (relative to this directory) to place at the top
112 | # of the sidebar.
113 | #html_logo = None
114 | 
115 | # The name of an image file (within the static path) to use as favicon of the
116 | # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
117 | # pixels large.
118 | #html_favicon = None
119 | 
120 | # Add any paths that contain custom static files (such as style sheets) here,
121 | # relative to this directory. They are copied after the builtin static files,
122 | # so a file named "default.css" will overwrite the builtin "default.css".
123 | html_static_path = ['_static']
124 | 
125 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
126 | # using the given strftime format.
127 | #html_last_updated_fmt = '%b %d, %Y'
128 | 
129 | # If true, SmartyPants will be used to convert quotes and dashes to
130 | # typographically correct entities.
131 | #html_use_smartypants = True
132 | 
133 | # Custom sidebar templates, maps document names to template names.
134 | #html_sidebars = {}
135 | 
136 | # Additional templates that should be rendered to pages, maps page names to
137 | # template names.
138 | #html_additional_pages = {}
139 | 
140 | # If false, no module index is generated.
141 | #html_domain_indices = True
142 | 
143 | # If false, no index is generated.
144 | #html_use_index = True
145 | 
146 | # If true, the index is split into individual pages for each letter.
147 | #html_split_index = False
148 | 
149 | # If true, links to the reST sources are added to the pages.
150 | #html_show_sourcelink = True
151 | 
152 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
153 | #html_show_sphinx = True
154 | 
155 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
156 | #html_show_copyright = True
157 | 
158 | # If true, an OpenSearch description file will be output, and all pages will
159 | # contain a <link> tag referring to it.  The value of this option must be the
160 | # base URL from which the finished HTML is served.
161 | #html_use_opensearch = ''
162 | 
163 | # This is the file name suffix for HTML files (e.g. ".xhtml").
164 | #html_file_suffix = None
165 | 
166 | # Output file base name for HTML help builder.
167 | htmlhelp_basename = 'Django-nonreldoc'
168 | 
169 | 
170 | # -- Options for LaTeX output --------------------------------------------------
171 | 
172 | # The paper size ('letter' or 'a4').
173 | #latex_paper_size = 'letter'
174 | 
175 | # The font size ('10pt', '11pt' or '12pt').
176 | #latex_font_size = '10pt'
177 | 
178 | # Grouping the document tree into LaTeX files. List of tuples
179 | # (source start file, target name, title, author, documentclass [howto/manual]).
180 | latex_documents = [
181 |   ('index', 'Django-nonrel.tex', u'Django-nonrel Documentation',
182 |    u'Tom Brander', 'manual'),
183 | ]
184 | 
185 | # The name of an image file (relative to this directory) to place at the top of
186 | # the title page.
187 | #latex_logo = None
188 | 
189 | # For "manual" documents, if this is true, then toplevel headings are parts,
190 | # not chapters.
191 | #latex_use_parts = False
192 | 
193 | # If true, show page references after internal links.
194 | #latex_show_pagerefs = False
195 | 
196 | # If true, show URL addresses after external links.
197 | #latex_show_urls = False
198 | 
199 | # Additional stuff for the LaTeX preamble.
200 | #latex_preamble = ''
201 | 
202 | # Documents to append as an appendix to all manuals.
203 | #latex_appendices = []
204 | 
205 | # If false, no module index is generated.
206 | #latex_domain_indices = True
207 | 
208 | 
209 | # -- Options for manual page output --------------------------------------------
210 | 
211 | # One entry per manual page. List of tuples
212 | # (source start file, name, description, authors, manual section).
213 | man_pages = [
214 |     ('index', 'django-nonrel', u'Django-nonrel Documentation',
215 |      [u'Tom Brander'], 1)
216 | ]
217 | 
218 | 
219 | # Example configuration for intersphinx: refer to the Python standard library.
220 | intersphinx_mapping = {'http://docs.python.org/': None}
221 | 


--------------------------------------------------------------------------------
/content/All Buttons Pressed - CMS & blog for Django-nonrel.rst:
--------------------------------------------------------------------------------
 1 | .. toctree::
 2 |    :maxdepth: 2
 3 | 
 4 | All Buttons Pressed Blog Sample Project
 5 | ================================================
 6 | 
 7 | This is the the website you're currently looking at. All Buttons Pressed provides a simple "CMS" with support for multiple independent blogs. It's compatible with Django-nonrel_ and the normal SQL-based Django, so you can use it with App Engine, MongoDB, MySQL, PostgreSQL, sqlite, and all other backends that work with either Django or Django-nonrel. Actually the project was started to demonstrate what's possible with Django-nonrel and to give you an impression of how to write code for non-relational databases.
 8 | 
 9 | .. raw:: html
10 | 
11 |    <ul class="project-links">
12 |       <li><a href="#documentation">Documentation</a></li>
13 |       <li><a href="http://bitbucket.org/wkornewald/allbuttonspressed/src">Source</a></li>
14 |       <li><a href="http://bitbucket.org/wkornewald/allbuttonspressed/downloads/allbuttonspressed.zip">Download</a></li>
15 |    </ul>
16 | 
17 | Documentation
18 | ------------------------------------
19 | 
20 | Note: Some parts of the code are explained in the `4 things to know for NoSQL Django coders`_ blog post.
21 | 
22 | Before you start you'll need to install Sass_ and Compass_. Then you can just download_ the zip file. Make sure that your App Engine SDK is at least version 1.5.0 prerelease.
23 | 
24 | Alternatively, if you want to clone the latest code you'll have to also install django-mediagenerator_. By default, the settings file is configured to use App Engine, so you'll also need djangoappengine_ and all of its dependencies (Django-nonrel_, djangotoolbox_, django-dbindexer_, and django-autoload_). In contrast, on SQL databases you won't need any of those dependencies except for django-mediagenerator_.
25 | 
26 | First, create an admin user with ``manage.py createsupeuser``. Then, run ``manage.py runserver`` and go to http://localhost:8000/admin/ and create a few pages and posts. Otherwise you'll only see a 404 error page.
27 | 
28 | All Buttons Pressed has a concept called "Block". Blocks can be created and edited via the admin UI. The sidebar's content is defined via a block called "sidebar".
29 | 
30 | The menu is defined via a "menu" block where each menu item is in the format ``<label> <url>``:
31 | 
32 | .. sourcecode:: text
33 | 
34 |     Some label     /url
35 |     Other label    http://www.other.url/
36 | 
37 | .. _Sass: http://sass-lang.com/
38 | .. _Compass: http://compass-style.org/
39 | .. _4 things to know for NoSQL Django coders: /blog/django/2010/02/4-things-to-know-for-NoSQL-Django-coders
40 | .. _Django-nonrel: /projects/django-nonrel
41 | .. _django-mediagenerator: /projects/django-mediagenerator
42 | .. _download: http://bitbucket.org/wkornewald/allbuttonspressed/downloads/allbuttonspressed.zip
43 | .. _djangoappengine: /projects/djangoappengine
44 | .. _djangotoolbox: /projects/djangotoolbox
45 | .. _django-dbindexer: /projects/django-dbindexer
46 | .. _django-autoload: /projects/django-autoload
47 | 


--------------------------------------------------------------------------------
/content/Django-nonrel - NoSQL support for Django.rst:
--------------------------------------------------------------------------------
  1 | Django-nonrel - NoSQL support for Django
  2 | =================================================
  3 | 
  4 | Django-nonrel is an independent branch of Django that adds NoSQL database support to the ORM. The long-term goal is to add NoSQL support to the official Django release. Take a look at the documentation below for more information.
  5 | 
  6 | .. raw:: html
  7 | 
  8 |    <ul class="project-links">
  9 |       <li><a href="#documentation">Documentation</a></li>
 10 |       <li><a href="http://bitbucket.org/wkornewald/django-nonrel/src">Source</a></li>
 11 |       <li><a href="http://bitbucket.org/wkornewald/django-nonrel/get/tip.zip">Download</a></li>
 12 |       <li><a href="http://groups.google.com/group/django-non-relational">Discussion group</a></li>
 13 |       <li><a href="/">Blog</a></li>
 14 |    </ul>
 15 | 
 16 | Why Django on NoSQL
 17 | --------------------------------------
 18 | We believe that development on non-relational databases is unnecessarily unproductive:
 19 | 
 20 | * you can't reuse code written for other non-relational DBs even if they're very similar
 21 | * you can't reuse code written for SQL DBs even if it could actually run unmodified on a non-relational DB
 22 | * you have to manually maintain indexes with hand-written code (denormalization, etc.)
 23 | * you have to manually write code for merging the results of multiple queries (JOINs, ``select_related()``, etc.)
 24 | * some databases are coupled to a specific provider (App Engine, SimpleDB), so you're locked into their hosting solution and can't easily move away if at some point you need something that's impossible on their platform
 25 | 
 26 | How can we fix the situation? Basically, we need a powerful abstraction that automates all the manual indexing work and provides an advanced query API which simplifies common nonrel development patterns. The Django ORM is such an abstraction and it fits our goals pretty well. It even works with SQL and thus allows to reuse existing code written for SQL DBs.
 27 | 
 28 | Django-nonrel is in fact only a small patch to Django that fixes a few assumptions (e.g.: ``AutoField`` might not be an integer). The real work is done by django-dbindexer_. The dbindexer will automatically take care of denormalization, JOINs, and other important features in the future too. Currently, it extends the NoSQL DB's native query language with support for filters like ``__month`` and ``__icontains`` and adds support for simple JOINs. If you want to stay up-to-date with our progress you should subscribe_ to the `Django-nonrel blog`_ for the latest news and tutorials.
 29 | 
 30 | This page hosts general information that is independent of the backend. You should also take a look at the `4 things to know for NoSQL Django coders`_ blog post for practical examples.
 31 | 
 32 | The App Engine documentation is hosted on the djangoappengine_ project site. We've also published some information in the `Django on App Engine`_ blog post.
 33 | 
 34 | The MongoDB backend is introduced in the `MongoDB backend for Django-nonrel`_ blog post.
 35 | 
 36 | Backends for ElasticSearch_ and Cassandra_ are also in development.
 37 | 
 38 | Documentation
 39 | ----------------------------------
 40 | Differences to Django
 41 | ---------------------------------------------------
 42 | Django should mostly behave as described in the `Django documentation`_. You can run Django-nonrel in combined SQL and NoSQL multi-database setups without any problems. Whenever possible, we'll try to reuse existing Django APIs or at least provide our own APIs that abstract away the platform-specific API, so you can write portable, database-independent code. There might be a few exceptions and you should consult the respective backend documentation (e.g., djangoappengine_) for more details. Here are a few general rules which can help you write better code:
 43 | 
 44 | Some backends (e.g., MongoDB) use a string instead of an integer for ``AutoField``. If you want to be on the safe side always write code and urlpatterns that would work with both strings and integers. Note that ``QuerySet`` automatically converts strings to integers where necessary, so you don't need to deal with that in your code.
 45 | 
 46 | There is no integrity guarantee. When deleting a model instance the related objects will **not** be deleted. This had to be done because such a deletion process can take too much time. We might solve this in a later release.
 47 | 
 48 | JOINs don't work unless you use django-dbindexer_, but even then they're very limited at the moment. Without dbindexer, queries can only be executed on one single model (filters can't span relationships like ``user__username=...``).
 49 | 
 50 | You can't use Django's transactions API. If your particular DB supports a special kind of transaction (e.g., ``run_in_transaction()`` on App Engine) you have to use the platform-specific functions.
 51 | 
 52 | Not all DBs provide strong consistency. If you want your code to be fully portable you should assume an eventual consistency model. For example, recently App Engine introduced an eventually-consistent high-replication datastore which is now the preferred DB type because it provides very high availability. Strongly consistent operations should be implemented via ``QuerySet.update()`` instead of ``Model.save()`` (unless you use ``run_in_transaction()`` on App Engine, of course).
 53 | 
 54 | Workarounds
 55 | -------------------------------------------------
 56 | You can only edit users in the admin interface if you add "djangotoolbox" to your ``INSTALLED_APPS``. Otherwise you'll get an exception about an unsupported query which requires JOINs.
 57 | 
 58 | Florian Hahn has also written an `authentication backend`_ which provides permission support on non-relational backends. You should use that backend if you want to use Django's permission system.
 59 | 
 60 | Writing a backend
 61 | -------------------------------------------------
 62 | The `backend template`_ provides a simple starting point with simple code samples, so you understand what each function does.
 63 | 
 64 | Sample projects
 65 | -------------------------------------------------
 66 | You can use allbuttonspressed_ or django-testapp_ as starting points to see what a nonrel-compatible project looks like.
 67 | 
 68 | Internals
 69 | -------------------------------------------------
 70 | Django-nonrel is based on Django 1.3. The modifications to Django are minimal (maybe less than 100 lines). Backends can override the ``save()`` process, so no distinction between ``INSERT`` and ``UPDATE`` is made and so there is no unnecessary ``exists()`` check in ``save()``. Also, deletion of related objects is disabled on NoSQL because it conflicts with transaction support at least on App Engine. These are the most important changes we've made. The actual DB abstraction is handled by django-dbindexer_ and it will stay a separate component even after NoSQL support is added to the official Django release.
 71 | 
 72 | Contributors
 73 | -----------------------------------
 74 | * Alberto Paro
 75 | * Andi Albrecht
 76 | * Flavio Percoco Premoli
 77 | * Florian Hahn
 78 | * Jonas Haag
 79 | * Kyle Finley
 80 | * George Karpenkov
 81 | * Mariusz Kryński
 82 | * Matt Bierner
 83 | * Thomas Wanschik
 84 | * Waldemar Kornewald
 85 | 
 86 | .. _Django documentation: http://docs.djangoproject.com/
 87 | .. _4 things to know for NoSQL Django coders: /blog/django/2010/02/4-things-to-know-for-NoSQL-Django-coders
 88 | .. _Django on App Engine: /blog/django/2010/01/Native-Django-on-App-Engine
 89 | .. _MongoDB backend for Django-nonrel: /blog/django/2010/05/MongoDB-backend-for-Django-nonrel-released
 90 | .. _Django-nonrel blog: /blog/django
 91 | .. _djangoappengine: /projects/djangoappengine
 92 | .. _subscribe: http://feeds.feedburner.com/AllButtonsPressed
 93 | .. _backend template: http://www.allbuttonspressed.com/blog/django/2010/04/Writing-a-non-relational-Django-backend
 94 | .. _django-testapp: http://bitbucket.org/wkornewald/django-testapp
 95 | .. _allbuttonspressed: /projects/allbuttonspressed
 96 | .. _django-dbindexer: http://www.allbuttonspressed.com/projects/django-dbindexer
 97 | .. _authentication backend: http://bitbucket.org/fhahn/django-permission-backend-nonrel
 98 | .. _Cassandra: http://github.com/vaterlaus/django_cassandra_backend
 99 | .. _ElasticSearch: http://github.com/aparo/django-elasticsearch
100 | 


--------------------------------------------------------------------------------
/content/HTML5 offline manifests with django-mediagenerator.rst:
--------------------------------------------------------------------------------
  1 | HTML5 offline manifests with django-mediagenerator
  2 | ========================================================
  3 | 
  4 | This is actually part 3 of our django-mediagenerator_ Python canvas app series (see `part 1`_ and `part 2`_), but since it has nothing to do with client-side Python we name it differently. In this part you'll see how to make your web app load without an Internet connection. HTML5 supports offline web apps through manifest_ files.
  5 | 
  6 | Manifest files
  7 | ---------------------------------
  8 | First here's some background, so you know what a manifest file is. A manifest file is really simple. In its most basic form it lists the URLs of the files that should be cached. Here's an ``example.manifest``:
  9 | 
 10 | .. sourcecode:: text
 11 | 
 12 |     CACHE MANIFEST
 13 |     /media/main.css
 14 |     /media/main.js
 15 | 
 16 | The first line is always ``CACHE MANIFEST``. The next lines can list the files that should be cached. In this case we've added the ``main.css`` and ``main.js`` bundles. Additionally, the main HTML file which includes the manifest is cached, automatically. You can include the manifest in the ``<html>`` tag:
 17 | 
 18 | .. sourcecode:: html
 19 | 
 20 |     <html manifest="example.manifest">
 21 | 
 22 | When the browser sees this it loads the manifest and adds the current HTML and manifest file and all files listed in the manifest to the cache. The next time you visit the page the browser will try to load the manifest file from your server and compare it to the cached version. If the content of the manifest file hasn't changed the browser just loads all files from the cache. If the content of the manifest file has changed the browser refreshes its cache.
 23 | 
 24 | This is important, so I repeat it: The browser updates its cache only when the **content** of the **manifest** file is modified. Changes to your JavaScript, CSS, and image files will go unnoticed if the manifest file is not changed! That's exactly where things become annoying. Imagine you've changed the ``main.js`` file. Now you have to change your manifest file, too. One possibility is to add a comment to their manifest file which represents the current version number:
 25 | 
 26 | .. sourcecode:: text
 27 | 
 28 |     CACHE MANIFEST
 29 |     # version 2
 30 |     /media/main.css
 31 |     /media/main.js
 32 | 
 33 | Whenever you change something in your JS or CSS or image files you have to increment the version number, manually. That's not really nice.
 34 | 
 35 | django-mediagenerator to the rescue
 36 | -------------------------------------------
 37 | This is where the media generator comes in. It automatically modifies the manifest file whenever your media files are changed. Since media files are versioned automatically by django-mediagenerator the version hash in the file name serves as a natural and automatic solution to our problem. With the media generator a manifest file could look like this:
 38 | 
 39 | .. sourcecode:: text
 40 | 
 41 |     CACHE MANIFEST
 42 |     /media/main-bf1e7dfbd511baf660e57a1f36048750f1ee660f.css
 43 |     /media/main-fb16702a27fc6c8073aa4df0b0b5b3dd8057cc12.js
 44 | 
 45 | Whenever you change your media files the version hash of the affected files becomes different and thus the manifest file changes automatically, too.
 46 | 
 47 | Now how do we tell django-mediagenerator_ to create such a manifest file? Just add this to your ``settings.py``:
 48 | 
 49 | .. sourcecode:: python
 50 | 
 51 |     OFFLINE_MANIFEST = 'webapp.manifest'
 52 | 
 53 | With this simple snippet the media generator will create a manifest file called ``webapp.manifest``. However, the manifest file will contain **all** of the assets in your project. In other words, the whole ``_generated_media`` folder will be listed in the manifest file.
 54 | 
 55 | Often you only want specific files to be cached. You can do that by specifying a list of regular expressions matching path names (relative to your media directories, exactly like in ``MEDIA_BUNDLES``):
 56 | 
 57 | .. sourcecode:: python
 58 | 
 59 |     OFFLINE_MANIFEST = {
 60 |         'webapp.manifest': {
 61 |             'cache': (
 62 |                 r'main\.css',
 63 |                 r'main\.js',
 64 |                 r'webapp/img/.*',
 65 |             ),
 66 |             'exclude': (
 67 |                 r'webapp/img/online-only/.*',
 68 |             )
 69 |         },
 70 |     }
 71 | 
 72 | Here we've added the ``main.css`` and ``main.js`` bundles and all files under the ``webapp/img/`` folder, except for files under ``webapp/img/online-only/``. Also, you might have guessed it already: You can create multiple manifest files this way. Just add more entries to the ``OFFLINE_MANIFEST`` dict.
 73 | 
 74 | Finally, we also have to include the manifest file in our template:
 75 | 
 76 | .. sourcecode:: django
 77 | 
 78 |     {% load media %}
 79 |     <html manifest="{% media_url 'webapp.manifest' %}">
 80 | 
 81 | Manifest files actually provide more features than this. For example, you can also specify ``FALLBACK`` handlers in case there is no Internet connection. In the following example the "/offline.html" page will be displayed for resources which can't be reached while offline:
 82 | 
 83 | .. sourcecode:: python
 84 | 
 85 |     OFFLINE_MANIFEST = {
 86 |         'webapp.manifest': {
 87 |             'cache': (...),
 88 |             'fallback': {
 89 |                 '/': '/offline.html',
 90 |             },
 91 |         },
 92 |     }
 93 | 
 94 | Here ``/`` is a pattern that matches all pages. You can also define ``NETWORK`` entries which specify allowed URLs that can be accessed even though they're not cached:
 95 | 
 96 | .. sourcecode:: python
 97 | 
 98 |     OFFLINE_MANIFEST = {
 99 |         'webapp.manifest': {
100 |             'cache': (...),
101 |             'network': (
102 |                 '*',
103 |             ),
104 |         },
105 |     }
106 | 
107 | Here ``*`` is a wildcard that allows to access any URL. If you just had an empty ``NETWORK`` section you wouldn't be able to load uncached files, even when you're online (however, not all browsers are so strict).
108 | 
109 | Serving manifest files
110 | ---------------------------------------------------
111 | Manifest files should be served with the MIME type ``text/cache-manifest``. Also it's **critical** that you disable HTTP caching for manifest files! Otherwise the browser will **never** load a new version of your app because it always loads the cached manifest! Make sure that you've configured your web server correctly.
112 | 
113 | As an example, on App Engine you'd configure your ``app.yaml`` like this:
114 | 
115 | .. sourcecode:: yaml
116 | 
117 |     handlers:
118 |     - url: /media/(.*\.manifest)
119 |       static_files: _generated_media/\1
120 |       mime_type: text/cache-manifest
121 |       upload: _generated_media/(.*\.manifest)
122 |       expiration: '0'
123 | 
124 |     - url: /media
125 |       static_dir: _generated_media/
126 |       expiration: '365d'
127 | 
128 | Here we first catch all manifest files and serve them with an expiration of "0" and the correct MIME type. The normal ``/media`` handler must be installed **after** the manifest handler.
129 | 
130 | Like a native iPad/iPhone app
131 | ----------------------------------------
132 | Offline-capable web apps have a nice extra advantage: We can put them on the iPad's/iPhone's home screen, so they appear exactly like native apps! All browser bars will disappear and your whole web app will be full-screen (except for the top-most status bar which shows the current time and battery and network status). Just add the following to your template:
133 | 
134 | .. sourcecode:: django
135 | 
136 |     <head>
137 |     <meta name="apple-mobile-web-app-capable" content="yes" />
138 |     ...
139 | 
140 | Now when you're in the browser you can tap on the "+" icon in the middle of the bottom toolbar (**update:** I just updated to iOS 4.2.1 and the "+" icon got replaced with some other icon, but it's still in the middle of the bottom toolbar :) and select "Add to Home Screen":
141 | 
142 | .. image:: http://lh3.ggpht.com/_03uxRzJMadw/TOfkL5YEULI/AAAAAAAAAIo/sCOT_u4ymdQ/add-to-home-screen.png
143 | 
144 | Then you can enter the name of the home screen icon:
145 | 
146 | .. image:: http://lh4.ggpht.com/_03uxRzJMadw/TOfkLpUSIeI/AAAAAAAAAIk/n3IZTgfZyIo/add-to-home.png
147 | 
148 | Tapping "Add" will add an icon for your web app to the home screen:
149 | 
150 | .. image:: http://lh3.ggpht.com/_03uxRzJMadw/TOfkMLPDyQI/AAAAAAAAAIw/qducGXp4DzE/app-on-home-screen.png
151 | 
152 | When you tap that icon the canvas demo app starts in full-screen:
153 | 
154 | .. image:: http://lh5.ggpht.com/_03uxRzJMadw/TOfkLyiW0SI/AAAAAAAAAIs/lOIzhyI6BMQ/app.png
155 | 
156 | We can also specify an icon for your web app. For example, if your icon is in ``img/app-icon.png`` you can add it like this:
157 | 
158 | .. sourcecode:: django
159 | 
160 |     {% load media %}
161 |     <head>
162 |     <link rel="apple-touch-icon" href="{% media_url 'img/app-icon.png' %}" />
163 |     ...
164 | 
165 | The image should measure 57x57 pixels.
166 | 
167 | Finally, you can also add a startup image which is displayed while your app loads. The following snippet assumes that the startup image is in ``img/startup.png``:
168 | 
169 | .. sourcecode:: django
170 | 
171 |     {% load media %}
172 |     <head>
173 |     <link rel="apple-touch-startup-image" href="{% media_url 'img/startup.png' %}" />
174 |     ...
175 | 
176 | The image dimensions should be 320x460 pixels and it should be in portrait orientation.
177 | 
178 | Summary
179 | --------------------------------
180 | * The manifest file just lists the files that should be cached
181 | * Files are only reloaded if the manifest file's content has changed
182 | * The manifest file must not be cached (!) or the browser will never reload anything
183 | * django-mediagenerator_ automatically maintains the manifest file for you
184 | * Offline web apps can appear like native apps on the iPad and iPhone
185 | * Download_ the latest canvas drawing app source which is now offline-capable
186 | 
187 | As you've seen in this post, it's very easy to make your web app offline-capable with django-mediagenerator_. This is also the foundation for making your app look like a native app on the iPhone and iPad. Offline web apps open up exciting possibilities and allow you to become independent of Apple's slow approval processes for the app store and the iOS platform in general because web apps can run on Android, webOS, and many other mobile platforms. It's also possible to write a little wrapper for the App Store which just opens Safari with your website. That way users can still find your app in the App Store (in addition to the web).
188 | 
189 | The next time you want to write a native app for the iOS platform, consider making a web app, instead (unless you're writing e.g. a real-time game, of course).
190 | 
191 | .. _part 1: /blog/django/2010/11/Offline-HTML5-canvas-app-in-Python-with-django-mediagenerator-Part-1-pyjs
192 | .. _part 2: /blog/django/2010/11/Offline-HTML5-canvas-app-in-Python-with-django-mediagenerator-Part-2-Drawing
193 | .. _manifest: http://www.w3.org/TR/html5/offline.html
194 | .. _django-mediagenerator: /projects/django-mediagenerator
195 | .. _download: http://bitbucket.org/wkornewald/offline-canvas-python-web-app/get/tip.zip
196 | 


--------------------------------------------------------------------------------
/content/Managing per-field indexes on App Engine.rst:
--------------------------------------------------------------------------------
 1 | Managing per-field indexes on App Engine
 2 | ================================================
 3 | 
 4 | An annoying problem when trying to reuse an existing Django app is that some apps use ``TextField`` instead of ``CharField`` and still want to filter on that field. On App Engine ``TextField`` is not indexed and thus can't be filtered against. One app which has this problem is django-openid-auth_. Previously, you had to modify the model source code directly and replace ``TextField`` with ``CharField`` where necessary. However, this is not a good solution because whenever you update the code you have to apply the patch, again. Now, djangoappengine_ provides a solution which allows you to configure indexes for individual fields without changing the models. By decoupling DB-specific indexes from the model definition we simplify maintenance and increase code portability.
 5 | 
 6 | Example
 7 | -------------------------------------
 8 | Let's see how we can get django-openid-auth to work correctly without modifying the app's source code. First, you need to create a module which defines the indexing settings. Let's call it "gae_openid_settings.py":
 9 | 
10 | .. sourcecode:: python
11 | 
12 |     from django_openid_auth.models import Association, UserOpenID
13 |     
14 |     FIELD_INDEXES = {
15 |         Association: {'indexed': ['server_url', 'assoc_type']},
16 |         UserOpenID: {'indexed': ['claimed_id']},
17 |     }
18 | 
19 | Then, in your settings.py you have to specify the list of gae settings modules:
20 | 
21 | .. sourcecode:: python
22 | 
23 |     GAE_SETTINGS_MODULES = (
24 |         'gae_openid_settings',
25 |     )
26 | 
27 | That's it. Now the ``server_url``, ``assoc_type``, and ``claimed_id`` ``TextField``\s will behave like ``CharField`` and get indexed by the datastore.
28 | 
29 | Note that we didn't place the index definition in the ``django_openid_auth`` package. It's better to keep them separate because that way upgrades are easier: Just update the ``django_openid_auth`` folder. No need to re-add the index definition (and you can't mistakenly delete the index definition during updates).
30 | 
31 | Optimization
32 | ----------------------------
33 | You can also use this to optimize your models. For example, you might have fields which don't need to be indexed. The more indexes you have the slower ``Model.save()`` will be. Fields that shouldn't be indexed can be specified via ``'unindexed'``:
34 | 
35 | .. sourcecode:: python
36 | 
37 |     from myapp.models import MyContact
38 |     
39 |     FIELD_INDEXES = {
40 |         MyContact: {
41 |             'indexed': [...],
42 |             'unindexed': ['creation_date', 'last_modified', ...],
43 |         },
44 |     }
45 | 
46 | This also has a nice extra advantage: If you specify a ``CharField`` as "unindexed" it will behave like a ``TextField`` and allow for storing strings that are longer than 500 bytes. This can also be useful when trying to integrate 3rd-party apps.
47 | 
48 | We hope you'll find this feature helpful. Feel free to post sample settings for other reusable Django apps in the comments. See you next time with some very exciting releases!
49 | 
50 | .. _django-openid-auth: https://launchpad.net/django-openid-auth
51 | .. _djangoappengine: /projects/djangoappengine
52 | 


--------------------------------------------------------------------------------
/content/Writing a non-relational Django backend.rst:
--------------------------------------------------------------------------------
  1 | Writing a non-relational Django backend
  2 | =============================================
  3 | 
  4 | In our `April 1st post`_ we claimed to have a simplified backend API. Well, this wasn't true, of course, but yesterday it has become true. The Django ORM is pretty complicated and it takes too much time for contributors to understand all the necessary details. In order to make the process as easy as possible we've implemented a `backend template`_ which provides a simple starting point for a new backend based on our simplified API. It also contains sample code, so you can better understand what each function does. All places where you have to make changes are marked with "``# TODO:``" comments. Note, you'll need djangotoolbox_ which provides the base classes for nonrel backends.
  5 | 
  6 | Let's start with ``base.py``. You can use the ``DatabaseCreation`` class to define a custom ``data_types`` mapping from Django's fields to your database types. The types will later be passed to functions which you'll have to implement to convert values from and to the DB (``convert_value_from_db()`` and ``convert_value_to_db()``). If the `default values <http://bitbucket.org/wkornewald/djangotoolbox/src/tip/db/creation.py>`__ work for you just leave the class untouched.
  7 | 
  8 | Also, if you want to maintain a DB connection we'd recommend storing it in ``DatabaseWrapper``:
  9 | 
 10 | .. sourcecode:: python
 11 | 
 12 |     class DatabaseWrapper(NonrelDatabaseWrapper):
 13 |         def __init__(self, *args, **kwds):
 14 |             super(DatabaseWrapper, self).__init__(*args, **kwds)
 15 |             ...
 16 |             self.db_connection = connect(
 17 |                 self.settings_dict['HOST'], self.settings_dict['PORT'],
 18 |                 self.settings_dict['USER'], self.settings_dict['PASSWORD'])
 19 | 
 20 | The real meat is in ``compiler.py``. Here, you have to define a BackendQuery class which handles query creation and execution. In the constructor you should create a low-level query instance for your connection. Depending on your DB API this might be nothing more than a dict, but let's say your DB provides a ``LowLevelQuery`` class:
 21 | 
 22 | .. sourcecode:: python
 23 | 
 24 |     class BackendQuery(NonrelQuery):
 25 |         def __init__(self, compiler, fields):
 26 |             super(BackendQuery, self).__init__(compiler, fields)
 27 |             self.db_query = LowLevelQuery(self.connection.db_connection)
 28 | 
 29 | Note, ``self.connection`` is the ``DatabaseWrapper`` instance which is the high-level DB connection object in Django.
 30 | 
 31 | Then, you need to define a function that converts Django's filters from Django's internal query object (``SQLQuery``, accessible via ``self.query``) to their counterparts for your DB. This should be done in the ``add_filters()`` function. Since quite a few nonrel DBs seem to only support AND queries we provide a default implementation which makes sure that there is no OR filter (well, it has some logic for converting certain OR filters to AND filters). It expects an ``add_filter()`` function (without the trailing "s"):
 32 | 
 33 | .. sourcecode:: python
 34 | 
 35 |         @safe_call
 36 |         def add_filter(self, column, lookup_type, negated, db_type, value):
 37 |             # Emulated/converted lookups
 38 |             if column == self.query.get_meta().pk.column:
 39 |                 column = '_id'
 40 | 
 41 |             if negated:
 42 |                 try:
 43 |                     op = NEGATION_MAP[lookup_type]
 44 |                 except KeyError:
 45 |                     raise DatabaseError("Lookup type %r can't be negated" % lookup_type)
 46 |             else:
 47 |                 try:
 48 |                     op = OPERATORS_MAP[lookup_type]
 49 |                 except KeyError:
 50 |                     raise DatabaseError("Lookup type %r isn't supported" % lookup_type)
 51 | 
 52 |             # Handle special-case lookup types
 53 |             if callable(op):
 54 |                 op, value = op(lookup_type, value)
 55 | 
 56 |             db_value = self.convert_value_for_db(db_type, value)
 57 |             self.db_query.filter(column, op, db_value)
 58 | 
 59 | This is just an example implementation. You don't have to use the same code. At first, we convert the primary key column to the DB's internal reserved column for the primary key. Then, we check if the filter should be negated or not and retrieve the respective DB comparison operator from a mapping like this:
 60 | 
 61 | .. sourcecode:: python
 62 | 
 63 |     OPERATORS_MAP = {
 64 |         'exact': '=',
 65 |         'gt': '>',
 66 |         'gte': '>=',
 67 |         # ...
 68 |         'isnull': lambda lookup_type, value: ('=' if value else '!=', None),
 69 |     }
 70 | 
 71 |     NEGATION_MAP = {
 72 |         'exact': '!=',
 73 |         'gt': '<=',
 74 |         # ...
 75 |         'isnull': lambda lookup_type, value: ('!=' if value else '=', None),
 76 |     }
 77 | 
 78 | In our example implementation the operator can be a string or a callable that returns the comparison operator and a modified value. Finally, in the last two lines of ``add_filter()`` we convert the value to its low-level DB type and then add a filter to the low-level query object.
 79 | 
 80 | You might have noticed the ``@save_call`` decorator. This is important. It catches database exceptions and converts them to Django's ``DatabaseError``. That decorator should be used for all your public API methods. Just modify the sample implementation in ``compiler.py`` to match your DB's needs.
 81 | 
 82 | Next, you have to define a ``fetch()`` function for retrieving the results from the configured query:
 83 | 
 84 | .. sourcecode:: python
 85 | 
 86 |         @safe_call
 87 |         def fetch(self, low_mark, high_mark):
 88 |             if high_mark is None:
 89 |                 # Infinite fetching
 90 |                 results = self.db_query.fetch_infinite(offset=low_mark)
 91 |             elif high_mark > low_mark:
 92 |                 # Range fetching
 93 |                 results = self.db_query.fetch_range(high_mark - low_mark, low_mark)
 94 |             else:
 95 |                 results = ()
 96 | 
 97 |             for entity in results:
 98 |                 entity[self.query.get_meta().pk.column] = entity['_id']
 99 |                 del entity['_id']
100 |                 yield entity
101 | 
102 | Here, ``low_mark`` and ``high_mark`` define the query range. If ``high_mark`` is not defined you should allow for iterating through the whole result set. At the end, we convert the internal primary key column, again, and return a dict representing the entity. If your DB also supports only fetching specific columns you should get the requested fields from ``self.fields`` (``field.column`` contains the column name).
103 | 
104 | All values in the resulting dict are automatically converted via ``SQLCompiler.convert_value_from_db()``. You have to implement that function (the backend template contains a sample implementation). That function gets a ``db_type`` parameter which is the type string as defined in your field type mapping in ``DatabaseCreation.data_types``.
105 | 
106 | We won't look at the whole API in this post. There are additional functions for ordering, counting, and deleting the query results. It's pretty simple. The API might later get extended with support for aggregates, but currently you'll have to handle them at a lower level in your ``SQLCompiler`` implementation if your DB supports those features.
107 | 
108 | Another important function is called on ``Model.save()``:
109 | 
110 | .. sourcecode:: python
111 | 
112 |     class SQLInsertCompiler(NonrelInsertCompiler, SQLCompiler):
113 |         @safe_call
114 |         def insert(self, data, return_id=False):
115 |             pk_column = self.query.get_meta().pk.column
116 |             if pk_column in data:
117 |                 data['_id'] = data[pk_column]
118 |                 del data[pk_column]
119 | 
120 |             pk = save_entity(self.connection.db_connection,
121 |                 self.query.get_meta().db_table, data)
122 |             return pk
123 | 
124 | Again, ``data`` is a dict because that maps naturally to nonrel DBs. Note, before insert() is called, all values are automatically converted via ``SQLCompiler.convert_value_for_db()`` (which you have to implement, too), so you don't have to deal with value conversions in that function.
125 | 
126 | I hope this gives you enough information to get started with a new backend. Please spread the word, so we can find backend contributors for all non-relational DBs. Django 1.3 development is getting closer and in order to get officially integrated into Django we have to prove that it's possible to use Django-nonrel with a wide variety of NoSQL DBs.
127 | 
128 | Please comment on the API. Should we improve anything? Is it flexible and easy enough?
129 | 
130 | .. _April 1st post: /blog/django/2010/04/SimpleDB-backend-and-JOIN-support
131 | .. _backend template: http://bitbucket.org/wkornewald/backend-template/
132 | .. _djangotoolbox: /projects/djangotoolbox
133 | 


--------------------------------------------------------------------------------
/content/django-autoload.rst:
--------------------------------------------------------------------------------
 1 | django-autoload
 2 | ========================================
 3 | 
 4 | django-autoload is a reuseable django app which allows for correct initialization of your django project by ensuring the loading of signal handlers or indexes before any request is being processed. 
 5 | 
 6 | .. raw:: html
 7 | 
 8 |    <ul class="project-links">
 9 |      <li><a href="https://bitbucket.org/twanschik/django-autoload/src">Source</a></li>
10 |      <li><a href="/blog/django">Blog</a></li>
11 |    </ul>
12 | 
13 | 
14 | 
15 | 
16 | Installation
17 | ----------------------------------------------------
18 | 
19 | #. Add django-autoload/autoload to settings.INSTALLED_APPS
20 |     .. sourcecode:: python
21 | 
22 |        INSTALLED_APPS = (
23 |            'autoload',
24 |        )
25 | 
26 | #. Add the ``autoload.middleware.AutoloadMiddleware`` before any other middelware
27 |     .. sourcecode:: python
28 |    
29 |        MIDDLEWARE_CLASSES = ('autoload.middleware.AutoloadMiddleware', ) + \
30 |                              MIDDLEWARE_CLASSES
31 | 
32 | 
33 | How does django-autoload ensures correct initialization of my project?
34 | --------------------------------------------------------------------------------------------------------------------------------------------------------------------
35 | 
36 | django-autoload provides two mechanisms to allow for correct initializations:
37 | 
38 | #. The ``autoload.middleware.AutoloadMiddleware`` middleware will load all ``models.py`` from settings.INSTALLED_APPS as soon as the first request is being processed. This ensures the registration of signal handlers for example.
39 | 
40 | #. django-autoload provides a site configuration module loading mechanism. Therefore, you have to create a site configuration module. The module name has to be specified in your settings:
41 | 
42 | .. sourcecode:: python
43 | 
44 |    # settings.py:
45 |    AUTOLOAD_SITECONF = 'indexes'
46 | 
47 | Now, django-autoload will load the module ``indexes.py`` before any request is being processed. An example module could look like this: 
48 | 
49 | .. sourcecode:: python
50 | 
51 |    # indexes.py:
52 |    from dbindexer import autodiscover
53 |    autodiscover()
54 | 
55 | This will ensure the registration of all database indexes specified in your project.
56 | 
57 | 
58 | autoload.autodiscover(module_name)
59 | __________________________________________
60 | 
61 | Some apps like django-dbindexer_ or nonrel-search_ provide an autodiscover-mechanism which tries to import index definitions from all apps in settings.INSTALLED_APPS. Since the autodiscover-mechanism is so common django-autoload provides an ``autodiscover`` function for these apps.
62 | 
63 | ``autodiscover`` will search for ``[module_name].py`` in all ``settings.INSTALLED_APPS`` and load them. django-dbindexer's autodiscover function just looks like this for example:
64 | 
65 | .. sourcecode:: python
66 | 
67 |     def autodiscover():
68 |         from autoload import autodiscover as auto_discover
69 |         auto_discover('dbindexes')
70 | 
71 | .. _Get SQL features on NoSQL with django-dbindexer: /blog/django/2010/09/Get-SQL-features-on-NoSQL-with-django-dbindexer
72 | .. _`JOINs for NoSQL databases via django-dbindexer - First steps`: http://www.allbuttonspressed.com/blog/django/joins-for-nosql-databases-via-django-dbindexer-first-steps
73 | .. _`JOINs via denormalization for NoSQL coders, Part 1`: http://www.allbuttonspressed.com/blog/django/2010/09/JOINs-via-denormalization-for-NoSQL-coders-Part-1-Intro
74 | .. _djangoappengine: http://www.allbuttonspressed.com/projects/djangoappengine
75 | .. _django-mongodb-engine: http://github.com/aparo/django-mongodb-engine
76 | .. _djangotoolbox: http://www.allbuttonspressed.com/projects/djangotoolbox
77 | .. _django-dbindexer: http://www.allbuttonspressed.com/projects/django-dbindexer
78 | .. _nonrel-search: http://www.allbuttonspressed.com/projects/nonrel-search
79 | 


--------------------------------------------------------------------------------
/content/django-dbindexer - Expressive NoSQL.rst:
--------------------------------------------------------------------------------
  1 | Django-dbindexer
  2 | =============================
  3 | 
  4 | With django-dbindexer you can use SQL features on NoSQL databases and abstract the differences between NoSQL databases. For example, if your database doesn't support case-insensitive queries (``iexact``, ``istartswith``, etc.) you can just tell the dbindexer which models and fields should support these queries and it'll take care of maintaining the required indexes for you. It's similar for JOINs. Tell the dbindexer that you would like to use in-memory JOINs for a specific query for example and the dbindexer will make it possible. Magically, previously unsupported queries will just work. Currently, this project is in an early development stage. The long-term plan is to support more complex JOINs and at least some simple aggregates, possibly even much more.
  5 | 
  6 | .. raw:: html
  7 | 
  8 |    <ul class="project-links">
  9 |      <li><a href="http://bitbucket.org/wkornewald/django-dbindexer/src">Source</a></li>
 10 |      <li><a href="http://bitbucket.org/wkornewald/django-dbindexer/get/tip.zip">Download</a></li>
 11 |      <li><a href="http://bitbucket.org/twanschik/django-dbindexer-testapp/src">Sample project</a></li>
 12 |       <li><a href="/blog/django">Blog</a></li>
 13 |    </ul>
 14 | 
 15 | Tutorials
 16 | ---------------------------------
 17 | * Getting started: `Get SQL features on NoSQL with django-dbindexer`_
 18 | * `JOINs for NoSQL databases via django-dbindexer - First steps`_
 19 | 
 20 | 
 21 | Documentation
 22 | ---------------------------------
 23 | 
 24 | **Dependencies**: djangotoolbox_, django-autoload_
 25 | 
 26 | Installation
 27 | ----------------------------------------------------
 28 | 
 29 | For installation see `Get SQL features on NoSQL with django-dbindexer`_
 30 | 
 31 | 
 32 | 
 33 | How does django-dbindexer make unsupported field lookup types work?
 34 | --------------------------------------------------------------------------------------------------------------------------------------------------------------------
 35 | 
 36 | For each filter you want to use on a field for a given model, django-dbindexer adds an additional field to that model. For example, if you want to use the ``contains`` filter on a ``CharField`` you have to add the following index definition:
 37 | 
 38 | .. sourcecode:: python
 39 | 
 40 |    register_index(MyModel, {'name': 'contains'})
 41 | 
 42 | django-dbindexer will then store an additional ``ListField`` called 'idxf_<char_field_name>_l_contains' on ``MyModel``. When saving an entity, django-dbindexer will fill the ``ListField`` with all substrings of the ``CharField``'s reversed content i.e. if ``CharField`` stores ``'Jiraiya'`` then the ``ListField`` stores ``['J', 'iJ', 'riJ', 'ariJ' ..., 'ayiariJ']``. When querying on that ``CharField`` using ``contains``,  django-dbindexer delegates this filter using ``startswith`` on the ``ListField`` with the reversed query string i.e. ``filter(<char_field_name>__contains='ira')`` => ``filter('idxf_<char_field_name>_l_contains'__startswith='ari')`` which matches the content of the list and gives back the correct result set. On App Engine ``startswith`` gets converted to ">=" and "<" filters for example.
 43 | 
 44 | In the following is listed which fields will be added for a specific filter/lookup type:
 45 | 
 46 | * ``__iexact`` using an additional ``CharField`` and a ``__exact`` query
 47 | * ``__istartswith`` creates an additional ``CharField``. Uses a ``__startswith`` query
 48 | * ``__endswith`` using an additional ``CharField`` and a ``__startswith`` query
 49 | * ``__iendswith`` using an additional ``CharField`` and a ``__startswith`` query
 50 | * ``__year`` using an additional ``IntegerField``and a ``__exact`` query
 51 | * ``__month`` using an additional ``IntegerField`` and a ``__exact`` query
 52 | * ``__day`` using an additional ``IntegerField`` and a ``__exact`` query
 53 | * ``__week_day`` using an additional ``IntegerField`` and a ``__exact`` query
 54 | * ``__contains`` using an additional ``ListField`` and a ``__startswith`` query
 55 | * ``__icontains`` using an additional ``ListField`` and a ``__startswith`` query
 56 | * ``__regex`` using an additional ``ListField`` and a ``__exact`` query
 57 | * ``__iregex`` using an additional ``ListField`` and a ``__exact`` query
 58 | 
 59 | For App Engine users using djangoappengine_ this means that you can use all django field lookup types for example.
 60 | 
 61 | MongoDB users using django-mongodb-engine_ can benefit from this because case-insensitive filters can be handled as efficient case-sensitive filters for example.
 62 | 
 63 | For regex filters you have to specify which regex filter you would like to execute:
 64 | 
 65 | .. sourcecode:: python
 66 | 
 67 |    register_index(MyModel, {'name': ('iexact', re.compile('\/\*.*?\*\/', re.I)})
 68 | 
 69 | This will allow you to use the following filter:
 70 | 
 71 | .. sourcecode:: python
 72 | 
 73 |    MyModel.objects.all().filter(name__iregex='\/\*.*?\*\/')
 74 | 
 75 | Backend system
 76 | -----------------------------------------------------
 77 | 
 78 | django-dbindexer uses backends to resolve lookups.  You can specify which backends to use via ``DBINDEXER_BACKENDS``
 79 | 
 80 | .. sourcecode:: python
 81 | 
 82 |     # settings.py:
 83 |     
 84 |     DBINDEXER_BACKENDS = (
 85 |         'dbindexer.backends.BaseResolver',
 86 |         'dbindexer.backends.InMemoryJOINResolver',
 87 |     )
 88 | 
 89 | The ``BaseResolver`` is responsible for resolving lookups like ``__iexact`` or ``__regex`` for example.
 90 | The ``InMemoryJOINResolver`` is used to resolve JOINs in-memory.
 91 | The ``ConstantFieldJOINResolver`` uses denormalization in order to resolve JOINs. For more information see `JOINs via denormalization for NoSQL coders, Part 1`_ is then done automatically by the ``ConstantFieldJOINResolver`` for you. :)
 92 | 
 93 | Loading indexes
 94 | ---------------------------------------
 95 | 
 96 | First of all, you need to install django-autoload_. Then you have to create a site configuration module which loads the index definitions. The module name has to be specified in the settings:
 97 | 
 98 | .. sourcecode:: python
 99 | 
100 |    # settings.py:
101 |    AUTOLOAD_SITECONF = 'dbindexes'
102 | 
103 | Now, there are two ways to load database index definitions in the ``AUTOLOAD_SITECONF`` module: auto-detection or manual listing of modules.
104 | 
105 | Note: by default ``AUTOLOAD_SITECONF`` is set to your ``ROOT_URLCONF``.
106 | 
107 | dbindexer.autodiscover
108 | __________________________________________
109 | 
110 | ``autodiscover`` will search for ``dbindexes.py`` in all ``INSTALLED_APPS`` and load them. It's like in django's admin interface. Your ``AUTOLOAD_SITECONF`` module would look like this:
111 | 
112 | .. sourcecode:: python
113 | 
114 |    # dbindexes.py:
115 |    import dbindexer
116 |    dbindexer.autodiscover()
117 | 
118 | Manual imports
119 | ______________________
120 | 
121 | Alternatively, you can import the desired index definition modules directly:
122 | 
123 | .. sourcecode:: python
124 | 
125 |    # dbindexes.py:
126 |    import myapp.dbindexes
127 |    import otherapp.dbindexes
128 | 
129 | .. _Get SQL features on NoSQL with django-dbindexer: /blog/django/2010/09/Get-SQL-features-on-NoSQL-with-django-dbindexer
130 | .. _`JOINs for NoSQL databases via django-dbindexer - First steps`: http://www.allbuttonspressed.com/blog/django/joins-for-nosql-databases-via-django-dbindexer-first-steps
131 | .. _`JOINs via denormalization for NoSQL coders, Part 1`: http://www.allbuttonspressed.com/blog/django/2010/09/JOINs-via-denormalization-for-NoSQL-coders-Part-1-Intro
132 | .. _djangoappengine: http://www.allbuttonspressed.com/projects/djangoappengine
133 | .. _django-mongodb-engine: http://github.com/aparo/django-mongodb-engine
134 | .. _djangotoolbox: http://www.allbuttonspressed.com/projects/djangotoolbox
135 | .. _django-autoload: http://www.allbuttonspressed.com/projects/django-autoload
136 | 


--------------------------------------------------------------------------------
/content/django-filetransfers - File upload-download abstraction.rst:
--------------------------------------------------------------------------------
  1 | django-filetransfers
  2 | ====================================
  3 | 
  4 | With django-filetransfers you can write reusable Django apps that handle uploads and downloads in an abstract way. Django's own file upload and storage API alone is too limited because (1) it doesn't provide a mechanism for file downloads and (2) it can only handle direct uploads which eat a lot of resources and aren't compatible with cloud services like the App Engine Blobstore or asynchronous Amazon S3 uploads (where the file isn't piped through Django, but sent directly to S3). This is where django-filetransfers comes in. You can continue to use Django's ``FileField`` and ``ModelForm`` in your apps. You just need to add a few very simple API calls to your file handling views and templates and select a django-filetransfers backend via your settings.py. With this you can transparently support cloud services for file hosting or even the X-Sendfile mechanism.
  5 | 
  6 | .. raw:: html
  7 | 
  8 |    <ul class="project-links">
  9 |      <li><a href="#documentation">Documentation</a></li>
 10 |      <li><a href="http://bitbucket.org/wkornewald/django-filetransfers/src">Source</a></li>
 11 |      <li><a href="http://bitbucket.org/wkornewald/django-filetransfers/get/tip.zip">Download</a></li>
 12 |      <li><a href="http://bitbucket.org/wkornewald/upload-sample">Sample code</a></li>
 13 |      <li><a href="http://django-filetransfers.appspot.com/">Demo</a></li>
 14 |    </ul>
 15 | 
 16 | Installation
 17 | ----------------------------------------------
 18 | You can install the package via ``setup.py install`` or by copying or linking the "filetransfers" folder to your project (App Engine developers have to use the copy/link method). Then, add "filetransfers" to your ``INSTALLED_APPS``.
 19 | 
 20 | Note for App Engine users: All nrequired backends are already enabled in the default settings. You don't need any special configuration. In order to use the Blobstore on the App Engine production server you have to enable billing. Otherwise, the Blobstore API is disabled.
 21 | 
 22 | Model and form
 23 | ----------------------------------------------
 24 | In the following we'll use this model and form:
 25 | 
 26 | .. sourcecode:: python
 27 | 
 28 |     class UploadModel(models.Model):
 29 |         file = models.FileField(upload_to='uploads/%Y/%m/%d/%H/%M/%S/')
 30 | 
 31 |     class UploadForm(forms.ModelForm):
 32 |         class Meta:
 33 |             model = UploadModel
 34 | 
 35 | The ``upload_to`` parameter for ``FileField`` defines the target folder for file uploads (here, we add the date).
 36 | 
 37 | **Note for App Engine users:** When accessing a file object from ``UploadedModel`` you can get the file's ``BlobInfo`` object via ``uploadedmodel.file.blobstore_info``. Use this to e.g. convert uploaded images via the Image API.
 38 | 
 39 | Handling uploads
 40 | ----------------------------------------------
 41 | File uploads are handled with the ``prepare_upload()`` function which takes the request and the URL of the upload view and returns a tuple with a generated upload URL and extra POST data for the upload. The extra POST data is just a ``dict``, so you can pass it to your JavaScript code if needed. This is an example upload view:
 42 | 
 43 | .. sourcecode:: python
 44 | 
 45 |     from filetransfers.api import prepare_upload
 46 | 
 47 |     def upload_handler(request):
 48 |         view_url = reverse('upload.views.upload_handler')
 49 |         if request.method == 'POST':
 50 |             form = UploadForm(request.POST, request.FILES)
 51 |             form.save()
 52 |             return HttpResponseRedirect(view_url)
 53 | 
 54 |         upload_url, upload_data = prepare_upload(request, view_url)
 55 |         form = UploadForm()
 56 |         return direct_to_template(request, 'upload/upload.html',
 57 |             {'form': form, 'upload_url': upload_url, 'upload_data': upload_data})
 58 | 
 59 | Note that it's important that you send a redirect after an upload. Otherwise, some file hosting services won't work correctly.
 60 | 
 61 | Now, you have to use the generated upload URL and the upload's extra POST data in the template:
 62 | 
 63 | .. sourcecode:: html
 64 | 
 65 |     {% load filetransfers %}
 66 |     <form action="{{ upload_url }}" method="POST" enctype="multipart/form-data">
 67 |       {% csrf_token %}
 68 |       {% render_upload_data upload_data %}
 69 |       <table>{{ form }}</table>
 70 |       <input type="submit" value="Upload" />
 71 |     </form>
 72 | 
 73 | Here we use the ``{% render_upload_data %}`` tag which generates ``<input type="hidden" />`` fields for the extra POST data.
 74 | 
 75 | Security and permissions
 76 | ----------------------------------------------
 77 | By default, uploads are assumed to have a publicly accessible URL if that's supported by the backend. You can tell the backend to mark the upload as private via ``prepare_upload(..., private=True)``. If the backend has no control over the permissions (e.g., because it's your task to configure the web server correctly and not make private files publicly accessible) the ``private=True`` argument might just be ignored.
 78 | 
 79 | Asynchronous backends (like async S3 or even Blobstore) have to take special care of preventing faked uploads. After a successful upload to the actual server these backends have to generate a separate request which contains the POST data and a file ID identifying the uploaded file (the Blobstore automatically sends the blob key and async S3 would send the file and bucket name). The problem here is that a user can manually generate a request which matches the ID of some other user's private file, thus getting access to that file because it's now fake-uploaded to his private files, too. In order to prevent this asynchronous backends have to guarantee that no file ID is used twice for an upload.
 80 | 
 81 | Handling downloads
 82 | ----------------------------------------------
 83 | Since the actual download permissions can be out of the backend's control the download solution consists of two layers.
 84 | 
 85 | The ``serve_file()`` function primarily takes care of private file downloads, but in some configurations it might also have to take care of public downloads because the file hosting solution doesn't provide publicly accessible URLs (e.g., App Engine Blobstore). This means that you should also use that function as a fallback even if you only have public downloads. The function takes two required arguments: the request and the Django ``File`` object that should be served (e.g. from ``FileField``):
 86 | 
 87 | .. sourcecode:: python
 88 | 
 89 |     from filetransfers.api import serve_file
 90 | 
 91 |     def download_handler(request, pk):
 92 |         upload = get_object_or_404(UploadModel, pk=pk)
 93 |         return serve_file(request, upload.file)
 94 | 
 95 | The ``public_download_url`` function, which is also available as a template filter, returns a file's publicly accessible URL if that's supported by the backend. Otherwise it returns ``None``.
 96 | 
 97 | **Important**: Use ``public_download_url`` only for files that should be publicly accessible. Otherwise you should only use ``serve_file()``, so you can check permissions before approving the download.
 98 | 
 99 | A complete solution for public downloads which falls back to ``serve_file()`` would look like this in a template for an instance of ``UploadModel`` called ``upload``:
100 | 
101 | .. sourcecode:: html
102 | 
103 |     {% load filetransfers %}
104 |     {% url upload.views.download_handler pk=upload.pk as fallback_url %}
105 |     <a href="{% firstof upload.file|public_download_url fallback_url %}">Download</a>
106 | 
107 | The second line stores the ``serve_file()`` fallback URL in a variable. In the third line we then use the ``public_download_url`` template filter in order to get the file's publicly accessible URL. If that returns ``None`` the ``{% firstof %}`` template tag returns the second argument which is our fallback URL. Otherwise the public download URL is used.
108 | 
109 | Configuration
110 | ----------------------------------------------
111 | There are three backend types which are supported by django-filetransfers: one for uploads, one for downloads via ``serve_file()``, and one for public downloads. You can specify the backends in your settings.py:
112 | 
113 | .. sourcecode:: python
114 | 
115 |     PREPARE_UPLOAD_BACKEND = 'filetransfers.backends.default.prepare_upload'
116 |     SERVE_FILE_BACKEND = 'filetransfers.backends.default.serve_file'
117 |     PUBLIC_DOWNLOAD_URL_BACKEND = 'filetransfers.backends.default.public_download_url'
118 | 
119 | The default upload backend simply returns the URL unmodified. The default download backend transfers the file in chunks via Django, so it's definitely not the most efficient mechanism, but it uses only a small amount of memory (important for large files) and requires less resources than passing a file object directly to the response. The default public downloads backend simply returns ``None``. This default configuration should work with practically all servers, but it's not the most efficient solution. Please take a look at the backends which are shipped with django-filetransfers to see if something fits your solution better.
120 | 
121 | Private download backends
122 | ----------------------------------------------------------------------
123 | xsendfile.serve_file
124 | __________________________________________________
125 | Many web servers (at least Apache, Lighttpd, and nginx) provide an "X-Sendfile" module which allows for handing off the actual file transfer to the web server. This is much more efficient than the default download backend, so you should install the required module for your web server and then configure the xsendfile download backend in your settings.py:
126 | 
127 | .. sourcecode:: python
128 | 
129 |     SERVE_FILE_BACKEND = 'filetransfers.backends.xsendfile.serve_file'
130 | 
131 | url.serve_file
132 | __________________________________________________
133 | We also provide a backend which simply redirects to ``file.url``. You have to make sure that ``file.url`` actually generates a private download URL, though. This backend should work with the Amazon S3 and similar storage backends from the django-storages_ project. Just add the following to your settings.py:
134 | 
135 | .. sourcecode:: python
136 | 
137 |     SERVE_FILE_BACKEND = 'filetransfers.backends.url.serve_file'
138 | 
139 | Public download backends
140 | -----------------------------------------------------------------------
141 | url.public_download_url
142 | __________________________________________________
143 | If ``file.url`` points to a public download URL you can use this backend:
144 | 
145 | .. sourcecode:: python
146 | 
147 |     PUBLIC_DOWNLOAD_URL_BACKEND = 'filetransfers.backends.url.public_download_url'
148 | 
149 | base_url.public_download_url
150 | __________________________________________________
151 | Alternatively, there's also a simple backend that merely points to a different URL. You just need to specify a base URL and the backend appends ``file.name`` to that base URL.
152 | 
153 | .. sourcecode:: python
154 | 
155 |     PUBLIC_DOWNLOAD_URL_BACKEND = 'filetransfers.backends.base_url.public_download_url'
156 |     PUBLIC_DOWNLOADS_URL_BASE = '/downloads/'
157 | 
158 | Upload backends
159 | ----------------------------------------------------------------------
160 | delegate.prepare_upload
161 | __________________________________________________
162 | This backend delegates the upload to some other backend based on ``private=True`` or ``private=False``. This way you can, for instance, use the App Engine Blobstore for private files and Amazon S3 for public files:
163 | 
164 | .. sourcecode:: python
165 | 
166 |     # Configure "delegate" backend
167 |     PREPARE_UPLOAD_BACKEND = 'filetransfers.backends.delegate.prepare_upload'
168 |     PRIVATE_PREPARE_UPLOAD_BACKEND = 'djangoappengine.storage.prepare_upload'
169 |     PUBLIC_PREPARE_UPLOAD_BACKEND = 's3backend.prepare_upload'
170 | 
171 |     # Use S3 for public_download_url and Blobstore for serve_file
172 |     SERVE_FILE_BACKEND = 'djangoappengine.storage.serve_file'
173 |     PUBLIC_DOWNLOAD_URL_BACKEND = 'filetransfers.backends.base_url.public_download_url'
174 |     PUBLIC_DOWNLOADS_URL_BASE = 'http://s3.amazonaws.com/my-public-bucket/'
175 | 
176 | Reference: ``filetransfers.api`` module
177 | -----------------------------------------------------------------------
178 | 
179 | prepare_upload(request, url, private=False, backend=None)
180 | ______________________________________________________________________________
181 | Returns a tuple with a target URL for the upload form and a ``dict`` with additional POST data for the upload request.
182 | 
183 | Required arguments:
184 | 
185 | * ``request``: The view's request.
186 | * ``url``: The target URL where the files should be sent to.
187 | 
188 | Optional arguments:
189 | 
190 | * ``private``: If ``False`` the backend will try to make the upload publicly accessible, so it can be served via the ``public_download_url`` template filter. If ``True`` the backend will try to make the upload non-accessible to the public, so it can only be served via ``serve_file()``.
191 | * ``backend``: If defined, you can override the backend specified in settings.py.
192 | 
193 | serve_file(request, file, backend=None, save_as=False, content_type=None)
194 | ______________________________________________________________________________
195 | Serves a file to the browser. This is used either for checking permissions before approving a downoad or as a fallback if the backend doesn't support publicly accessible URLs. So, you always have to provide a view that uses this function.
196 | 
197 | Required arguments:
198 | 
199 | * ``request``: The view's request.
200 | * ``file``: The ``File`` object (e.g. from ``FileField``) that should be served.
201 | 
202 | Optional arguments:
203 | 
204 | * ``save_as``: Forces the browser to save the file instead of displaying it (useful for PDF documents, for example). If this is True the file object's ``name`` attribute will be used as the file name in the download dialog. Alternatively, you can pass a string to override the file name. The default is to let the browser decide how to handle the download.
205 | * ``content_type``: Overrides the file's content type in the response. By default the content type will be detected via ``mimetypes.guess_type()`` using ``file.name``.
206 | * ``backend``: If defined, you can override the backend specified in settings.py.
207 | 
208 | public_download_url(file, backend=None)
209 | ______________________________________________________________________________
210 | Tries to generate a publicly accessible URL for the given file. Returns ``None`` if no URL could be generated. The same function is available as a template filter.
211 | 
212 | Required arguments:
213 | 
214 | * ``file``: The ``File`` object (e.g. from ``FileField``) that should be served.
215 | 
216 | Optional arguments:
217 | 
218 | * ``backend``: If defined, you can override the backend specified in settings.py.
219 | 
220 | Reference: ``filetransfers`` template library
221 | -----------------------------------------------------------------------
222 | {% render_upload_data upload_data %}
223 | ______________________________________________________________________________
224 | Renders ``<input type="hidden" ... />`` fields for the extra POST data (``upload_data``) as returned by ``prepare_upload()``.
225 | 
226 | public_download_url
227 | ______________________________________________________________________________
228 | This template filter does the same as the ``public_upload_url()`` function in the ``filetransfers.api`` module: It returns a publicly accessible URL for the given file or ``None`` if it no such URL exists.
229 | 
230 | It takes the ``File`` object (e.g. from ``FileField``) that should be served and optionally a second parameter to override the backend specified in settings.py.
231 | 
232 | .. _sample project: http://bitbucket.org/wkornewald/upload-sample
233 | .. _django-storages: http://bitbucket.org/david/django-storages
234 | .. _issue 3328: http://code.google.com/p/googleappengine/issues/detail?id=3328
235 | 


--------------------------------------------------------------------------------
/content/djangoappengine - Django App Engine backends (DB, email, etc.).rst:
--------------------------------------------------------------------------------
  1 | djangoappengine - Django App Engine backends (DB, email, etc.)
  2 | ===============================================================
  3 | 
  4 | Djangoappengine contains all App Engine backends for Django-nonrel, e.g. the database and email backends. In addition we provide a testapp_ which contains minimal settings for running Django-nonrel on App Engine. Use it as a starting point if you want to use App Engine as your database for Django-nonrel. We've also published some details in the `Django on App Engine`_ blog post.
  5 | 
  6 | Take a look at the documentation below and subscribe_ to our `Django-nonrel blog`_ for the latest updates.
  7 | 
  8 | .. raw:: html
  9 | 
 10 |    <ul class="project-links">
 11 |       <li><a href="#documentation">Documentation</a></li>
 12 |       <li><a href="http://bitbucket.org/wkornewald/djangoappengine/src">Source</a></li>
 13 |       <li><a href="http://bitbucket.org/wkornewald/djangoappengine/get/tip.zip">Download</a></li>
 14 |       <li><a href="http://groups.google.com/group/django-non-relational">Discussion group</a></li>
 15 |       <li><a href="/">Blog</a></li>
 16 |    </ul>
 17 | 
 18 | 
 19 | 
 20 | 
 21 | 
 22 | 
 23 | Installation
 24 | ---------------------------------------
 25 | Make sure you've installed the `App Engine SDK`_. On Windows simply use the default installation path. On Linux you can put it in /usr/local/google_appengine. On MacOS it should work if you put it in your Applications folder. Alternatively, on all systems you can add the google_appengine folder to your PATH (not PYTHONPATH) environment variable.
 26 | 
 27 | Download the following zip files:
 28 | 
 29 | * `django-nonrel <http://bitbucket.org/wkornewald/django-nonrel/get/tip.zip>`__ (or `clone it <https://bitbucket.org/wkornewald/django-nonrel>`__)
 30 | * `djangoappengine <http://bitbucket.org/wkornewald/djangoappengine/get/tip.zip>`__ (or `clone it <https://bitbucket.org/wkornewald/djangoappengine>`__)
 31 | * `djangotoolbox <http://bitbucket.org/wkornewald/djangotoolbox/get/tip.zip>`__ (or `clone it <https://bitbucket.org/wkornewald/djangotoolbox>`__)
 32 | * `django-autoload <http://bitbucket.org/twanschik/django-autoload/get/tip.zip>`__ (or `clone it <https://bitbucket.org/twanschik/django-autoload>`__)
 33 | * `django-dbindexer <http://bitbucket.org/wkornewald/django-dbindexer/get/tip.zip>`__ (or `clone it <https://bitbucket.org/wkornewald/django-dbindexer>`__)
 34 | * `django-testapp <http://bitbucket.org/wkornewald/django-testapp/get/tip.zip>`__ (or `clone it <https://bitbucket.org/wkornewald/django-testapp>`__)
 35 | 
 36 | Unzip everything.
 37 | 
 38 | The django-testapp folder contains a sample project to get you started. If you want to start a new project or port an existing Django project you can just copy all ".py" and ".yaml" files from the root folder into your project and adapt settings.py and app.yaml to your needs.
 39 | 
 40 | Copy the following folders into your project (e.g., django-testapp):
 41 | 
 42 | * django-nonrel/django => ``<project>``/django
 43 | * djangotoolbox/djangotoolbox => ``<project>``/djangotoolbox
 44 | * django-autoload/autoload => ``<project>``/autoload
 45 | * django-dbindexer/dbindexer => ``<project>``/dbindexer
 46 | * djangoappengine => ``<project>``/djangoappengine
 47 | 
 48 | That's it. Your project structure should look like this:
 49 | 
 50 | * <project>/django
 51 | * <project>/djangotoolbox
 52 | * <project>/autoload
 53 | * <project>/dbindexer
 54 | * <project>/djangoappengine
 55 | 
 56 | Alternatively, you can of course clone the respective repositories and create symbolic links instead of copying the folders to your project. That might be easier if you have a lot of projects and don't want to update each one manually.
 57 | 
 58 | Management commands
 59 | ---------------------------------------------
 60 | You can directly use Django's manage.py commands. For example, run ``manage.py createsuperuser`` to create a new admin user and ``manage.py runserver`` to start the development server.
 61 | 
 62 | **Important:**  Don't use dev_appserver.py directly. This won't work as expected because ``manage.py runserver`` uses a customized dev_appserver.py configuration. Also, never run ``manage.py runserver`` together with other management commands at the same time. The changes won't take effect. That's an App Engine SDK limitation which might get fixed in a later release.
 63 | 
 64 | With djangoappengine you get a few extra manage.py commands:
 65 | 
 66 | * ``manage.py remote`` allows you to execute a command on the production database (e.g., ``manage.py remote shell`` or ``manage.py remote createsuperuser``)
 67 | * ``manage.py deploy`` uploads your project to App Engine (use this instead of ``appcfg.py update``)
 68 | 
 69 | Note that you can only use ``manage.py remote`` if your app is deployed and if you have enabled authentication via the Google Accounts API in your app settings in the App Engine Dashboard. Also, if you use a custom app.yaml you have to make sure that it contains the remote_api handler.
 70 | 
 71 | Supported and unsupported  features
 72 | -----------------------------------------------------------
 73 | Field types
 74 | __________________________
 75 | All Django field types are fully supported except for the following:
 76 | 
 77 | * ``ImageField``
 78 | * ``ManyToManyField``
 79 | 
 80 | The following Django field options have no effect on App Engine:
 81 | 
 82 | * ``unique``
 83 | * ``unique_for_date``
 84 | * ``unique_for_month``
 85 | * ``unique_for_year``
 86 | 
 87 | Additionally djangotoolbox_ provides non-Django field types in ``djangotoolbox.fields`` which you can use on App Engine or other non-relational databases. These are
 88 | 
 89 | * ``ListField``
 90 | * ``BlobField``
 91 | 
 92 | The following App Engine properties can be emulated by using a ``CharField`` in Django-nonrel:
 93 | 
 94 | * ``CategoryProperty``
 95 | * ``LinkProperty``
 96 | * ``EmailProperty``
 97 | * ``IMProperty``
 98 | * ``PhoneNumberProperty``
 99 | * ``PostalAddressProperty``
100 | 
101 | QuerySet methods
102 | ______________________________
103 | You can use the following field lookup types on all Fields except on ``TextField`` (unless you use indexes_) and ``BlobField``
104 | 
105 | * ``__exact`` equal to (the default)
106 | * ``__lt`` less than
107 | * ``__lte`` less than or equal to
108 | * ``__gt`` greater than
109 | * ``__gte`` greater than or equal to
110 | * ``__in`` (up to 500 values on primary keys and 30 on other fields)
111 | * ``__range`` inclusive on both boundaries
112 | * ``__startswith`` needs a composite index if combined with other filters 
113 | * ``__year``
114 | * ``__isnull`` requires django-dbindexer_ to work correctly on ``ForeignKey`` (you don't have to define any indexes for this to work)
115 | 
116 | Using django-dbindexer_ all remaining lookup types will automatically work too!
117 | 
118 | Additionally, you can use
119 | 
120 | * ``QuerySet.exclude()``
121 | * ``Queryset.values()`` (only efficient on primary keys)
122 | * ``Q``-objects
123 | * ``QuerySet.count()``
124 | * ``QuerySet.reverse()``
125 | * ...
126 | 
127 | In all cases you have to keep general App Engine restrictions in mind.
128 | 
129 | Model inheritance only works with `abstract base classes`_:
130 | 
131 | .. sourcecode:: python
132 | 
133 |     class MyModel(models.Model):
134 |         # ... fields ...
135 |         class Meta:
136 |             abstract = True # important!
137 | 
138 |     class ChildModel(MyModel):
139 |         # works
140 | 
141 | In contrast, `multi-table inheritance`_ (i.e. inheritance from non-abstract models) will result in query errors. That's because multi-table inheritance, as the name implies, creates separate tables for each model in the inheritance hierarchy, so it requires JOINs to merge the results. This is not the same as `multiple inheritance`_ which is supported as long as you use abstract parent models.
142 | 
143 | Many advanced Django features are not supported at the moment. A few of them are:
144 | 
145 | * JOINs (with django-dbindexer simple JOINs will work)
146 | * many-to-many relations
147 | * aggregates
148 | * transactions (but you can use ``run_in_transaction()`` from App Engine's SDK)
149 | * ``QuerySet.select_related()``
150 | 
151 | Other
152 | __________________________
153 | Additionally, the following features from App Engine are not supported:
154 | 
155 | * entity groups (we don't yet have a ``GAEPKField``, but it should be trivial to add)
156 | * batch puts (it's technically possible, but nobody found the time/need to implement it, yet)
157 | 
158 | Indexes
159 | --------------------------------------------
160 | It's possible to specify which fields should be indexed and which not. This also includes the possibility to convert a ``TextField`` into an indexed field like ``CharField``. You can read more about this feature in our blog post `Managing per-field indexes on App Engine`_.
161 | 
162 | Email handling
163 | ---------------------------------------------
164 | You can (and should) use Django's mail API instead of App Engine's mail API. The App Engine email backend is already enabled in the default settings (``from djangoappengine.settings_base import *``). By default, emails will be deferred to a background task on the production server.
165 | 
166 | Cache API
167 | ---------------------------------------------
168 | You can (and should) use Django's cache API instead of App Engine's memcache module. The memcache backend is already enabled in the default settings.
169 | 
170 | Sessions
171 | ---------------------------------------------
172 | You can use Django's session API in your code. The ``cached_db`` session backend is already enabled in the default settings.
173 | 
174 | Authentication
175 | ---------------------------------------------
176 | You can (and probably should) use ``django.contrib.auth`` directly in your code. We don't recommend to use App Engine's Google Accounts API. This will lock you into App Engine unnecessarily. Use Django's auth API, instead. If you want to support Google Accounts you can do so via OpenID. Django has several apps which provide OpenID support via Django's auth API. This also allows you to support Yahoo and other login options in the future and you're independent of App Engine. Take a look at `Google OpenID Sample Store`_ to see an example of what OpenID login for Google Accounts looks like.
177 | 
178 | Note that username uniqueness is only checked at the form level (and by Django's model validation API if you explicitly use that). Since App Engine doesn't support uniqueness constraints at the DB level it's possible, though very unlikely, that two users register the same username at exactly the same time. Your registration confirmation/activation mechanism (i.e., user receives mail to activate his account) must handle such cases correctly. For example, the activation model could store the username as its primary key, so you can be sure that only one of the created users is activated.
179 | 
180 | File uploads/downloads
181 | ---------------------------------------------
182 | See django-filetransfers_ for an abstract file upload/download API for ``FileField`` which works with the Blobstore_ and X-Sendfile and other solutions. The required backends for the App Engine Blobstore are already enabled in the default settings.
183 | 
184 | Background tasks
185 | ---------------------------------------------
186 | **Contributors:** We've started an experimental API for abstracting background tasks, so the same code can work with App Engine and Celery and others. Please help us finish and improve the API here: https://bitbucket.org/wkornewald/django-defer
187 | 
188 | Make sure that your ``app.yaml`` specifies the correct ``deferred`` handler. It should be:
189 | 
190 | .. sourcecode:: yaml
191 | 
192 |     - url: /_ah/queue/deferred
193 |       script: djangoappengine/deferred/handler.py
194 |       login: admin
195 | 
196 | This custom handler initializes ``djangoappengine`` before it passes the request to App Engine's internal ``deferred`` handler.
197 | 
198 | dbindexer index definitions
199 | -------------------------------------------------------------
200 | By default, djangoappengine installs ``__iexact`` indexes on ``User.username`` and ``User.email``.
201 | 
202 | High-replication datastore settings
203 | -------------------------------------------------------------
204 | In order to use ``manage.py remote`` with the high-replication datastore you need to add the following to the top of your ``settings.py``:
205 | 
206 | .. sourcecode:: python
207 | 
208 |     from djangoappengine.settings_base import *
209 |     DATABASES['default']['HIGH_REPLICATION'] = True
210 | 
211 | App Engine for Business
212 | -------------------------------------------------------------
213 | In order to use ``manage.py remote`` with the ``googleplex.com`` domain you need to add the following to the top of your ``settings.py``:
214 | 
215 | .. sourcecode:: python
216 | 
217 |     from djangoappengine.settings_base import *
218 |     DATABASES['default']['DOMAIN'] = 'googleplex.com'
219 | 
220 | Checking whether you're on the production server
221 | ------------------------------------------------------------------------------------------
222 | 
223 | .. sourcecode:: python
224 | 
225 |     from djangoappengine.utils import on_production_server, have_appserver
226 | 
227 | When you're running on the production server ``on_production_server`` is ``True``. When you're running either the development or production server ``have_appserver`` is ``True`` and for any other ``manage.py`` command it's ``False``.
228 | 
229 | Zip packages
230 | ---------------------------------------------
231 | **Important:** Your instances will load slower when using zip packages because zipped Python files are not precompiled. Also, i18n doesn't work with zip packages. Zipping should only be a **last resort**! If you hit the 3000 files limit you should better try to reduce the number of files by, e.g., deleting unused packages from Django's "contrib" folder. Only when **nothing** (!) else works you should consider zip packages.
232 | 
233 | Since you can't upload more than 3000 files on App Engine you sometimes have to create zipped packages. Luckily, djangoappengine can help you with integrating those zip packages. Simply create a "zip-packages" directory in your project folder and move your zip packages there. They'll automatically get added to ``sys.path``.
234 | 
235 | In order to create a zip package simply select a Python package (e.g., a Django app) and zip it. However, keep in mind that only Python modules can be loaded transparently from such a zip file. You can't easily access templates and JavaScript files from a zip package, for example. In order to be able to access the templates you should move the templates into your global "templates" folder within your project before zipping the Python package.
236 | 
237 | Contribute
238 | ------------------------------------------------------
239 | If you want to help with implementing a missing feature or improving something please fork the source_ and send a pull request via BitBucket or a patch to the `discussion group`_.
240 | 
241 | .. _djangotoolbox: http://www.allbuttonspressed.com/projects/djangotoolbox
242 | .. _`Django-nonrel blog`: /blog/django
243 | .. _testapp: http://bitbucket.org/wkornewald/django-testapp
244 | .. _django-nonrel: http://bitbucket.org/wkornewald/django-nonrel/wiki/Home
245 | .. _djangoappengine: http://bitbucket.org/wkornewald/djangoappengine
246 | .. _source: djangoappengine_
247 | .. _django-testapp: http://bitbucket.org/wkornewald/django-testapp
248 | .. _subscribe: http://feeds.feedburner.com/AllButtonsPressed
249 | .. _Django on App Engine: /blog/django/2010/01/Native-Django-on-App-Engine
250 | .. _Link Shell Extension: http://schinagl.priv.at/nt/hardlinkshellext/hardlinkshellext.html
251 | .. _Mercurial: http://mercurial.selenic.com/
252 | .. _App Engine SDK: http://code.google.com/appengine/downloads.html
253 | .. _abstract base classes: http://docs.djangoproject.com/en/dev/topics/db/models/#abstract-base-classes
254 | .. _multi-table inheritance: http://docs.djangoproject.com/en/dev/topics/db/models/#multi-table-inheritance
255 | .. _multiple inheritance: http://docs.djangoproject.com/en/dev/topics/db/models/#multiple-inheritance
256 | .. _Managing per-field indexes on App Engine: http://www.allbuttonspressed.com/blog/django/2010/07/Managing-per-field-indexes-on-App-Engine
257 | .. _django-dbindexer: http://www.allbuttonspressed.com/projects/django-dbindexer
258 | .. _Google OpenID Sample Store: https://sites.google.com/site/oauthgoog/Home/openidsamplesite
259 | .. _django-filetransfers: http://www.allbuttonspressed.com/projects/django-filetransfers
260 | .. _Blobstore: http://code.google.com/appengine/docs/python/blobstore/
261 | .. _discussion group: http://groups.google.com/group/django-non-relational
262 | 


--------------------------------------------------------------------------------
/content/djangotoolbox.rst:
--------------------------------------------------------------------------------
 1 | Django Toolbox
 2 | ===============================
 3 | 
 4 | Small set of useful Django tools. Goals: 1) be usable with non-relational Django backends 2) load no unnecessary code (faster instance startups) 3) provide good coding conventions and tools which have a real impact on your code (no "matter of taste" utilities).
 5 | 
 6 | We'll add some documentation, soon.
 7 | 
 8 | .. raw:: html
 9 | 
10 |    <ul class="project-links">
11 |      <li><a href="http://bitbucket.org/wkornewald/djangotoolbox/src">Source</a></li>
12 |      <li><a href="http://bitbucket.org/wkornewald/djangotoolbox/get/tip.zip">Download</a></li>
13 |    </ul>
14 | 


--------------------------------------------------------------------------------
/index.rst:
--------------------------------------------------------------------------------
 1 | .. Django-nonrel documentation master file, created by
 2 |    sphinx-quickstart on Tue Nov 29 11:54:41 2011.
 3 |    You can adapt this file completely to your liking, but it should at least
 4 |    contain the root `toctree` directive.
 5 | 
 6 | Welcome to Django-nonrel
 7 | =========================================
 8 | 
 9 | This is is a compilation of the original documentation provided by Waldemar Kornewald. 
10 | 
11 | Contents:
12 | 
13 | .. toctree::
14 |    :maxdepth: 1
15 | 
16 |    /content/Django-nonrel - NoSQL support for Django
17 |    /content/All Buttons Pressed - CMS & blog for Django-nonrel
18 |    /content/djangoappengine - Django App Engine backends (DB, email, etc.)
19 |    /content/django-autoload
20 |    /content/django-dbindexer - Expressive NoSQL
21 |    /content/django-filetransfers - File upload-download abstraction
22 |    /content/django-mediagenerator asset manager
23 |    /content/djangotoolbox
24 |    /content/HTML5 offline manifests with django-mediagenerator
25 |    /content/Managing per-field indexes on App Engine
26 |    /content/Writing a non-relational Django backend
27 | 
28 | Indices and tables
29 | ==================
30 | 
31 | * :ref:`genindex`
32 | * :ref:`modindex`
33 | * :ref:`search`
34 | 
35 | 


--------------------------------------------------------------------------------
/readme.rst:
--------------------------------------------------------------------------------
1 | This is a placeholder with the original Django Non-rel documentation taken from the All Buttons Pressed Blog. To read it here go to the content/ directory, The _build/ directory has the Sphinx version. 
2 | 
3 | As of now no editing has been done on the content. So there are numerous references to the blog that are not appropriate (and probably some other stuff as well..) 
4 | 
5 | I suspect that we will want to put the docs with the project components over time.. we can then pont Sphinx to the right spots.. 
6 | 


--------------------------------------------------------------------------------