├── requirements.txt
├── sharded_polls
├── templates
│ ├── 500.html
│ ├── 404.html
│ └── polls
│ │ ├── results.html
│ │ ├── index.html
│ │ └── detail.html
├── __init__.py
├── urls.py
├── wsgi.py
└── settings.py
├── polls
├── __init__.py
├── admin.py
├── urls.py
├── views.py
├── models.py
└── tests.py
├── sqlshards
├── __init__.py
├── models.py
├── db
│ ├── __init__.py
│ └── shards
│ │ ├── __init__.py
│ │ ├── sql.py
│ │ ├── helpers.py
│ │ ├── routers.py
│ │ ├── fields.py
│ │ ├── manager.py
│ │ └── models.py
├── management
│ ├── __init__.py
│ └── commands
│ │ ├── __init__.py
│ │ └── sqlpartition.py
├── tests
│ ├── sample
│ │ ├── __init__.py
│ │ └── models.py
│ ├── __init__.py
│ └── integration.py
└── utils.py
├── manage.py
└── LICENSE
/requirements.txt:
--------------------------------------------------------------------------------
1 | Django==1.4.5
2 |
--------------------------------------------------------------------------------
/sharded_polls/templates/500.html:
--------------------------------------------------------------------------------
1 | Sorry, It appears something went wrong.
2 |
3 | HTTP 500
4 |
--------------------------------------------------------------------------------
/sharded_polls/templates/404.html:
--------------------------------------------------------------------------------
1 | Sorry, the page you requested is unavailable.
2 |
3 |
4 | HTTP 404
5 |
--------------------------------------------------------------------------------
/sharded_polls/templates/polls/results.html:
--------------------------------------------------------------------------------
1 | {{ poll.question }}
2 |
3 |
4 | {% for choice in poll.choice_set.all %}
5 | - {{ choice.choice_text }} == {{ choice.votes }} vote {{ choice.votes|pluralize }}
6 | {% endfor %}
7 |
8 |
9 | Vote again?
10 |
--------------------------------------------------------------------------------
/sharded_polls/templates/polls/index.html:
--------------------------------------------------------------------------------
1 | {% block content %}
2 |
3 | {% if latest_poll_list %}
4 |
9 | {% else %}
10 | No polls are available.
11 | {% endif %}
12 | {% endblock %}
13 |
--------------------------------------------------------------------------------
/sharded_polls/templates/polls/detail.html:
--------------------------------------------------------------------------------
1 | {{ poll.question }}
2 | {% if error_message %}{{ error_message}}
{% endif %}
3 |
4 |
12 |
--------------------------------------------------------------------------------
/polls/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 |
--------------------------------------------------------------------------------
/sqlshards/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 |
--------------------------------------------------------------------------------
/sqlshards/models.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 |
--------------------------------------------------------------------------------
/sharded_polls/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 |
--------------------------------------------------------------------------------
/sqlshards/db/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 |
--------------------------------------------------------------------------------
/sqlshards/db/shards/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 |
--------------------------------------------------------------------------------
/sqlshards/management/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 |
--------------------------------------------------------------------------------
/sqlshards/tests/sample/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 |
--------------------------------------------------------------------------------
/sqlshards/management/commands/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 |
--------------------------------------------------------------------------------
/sqlshards/tests/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 | from integration import *
18 |
--------------------------------------------------------------------------------
/sharded_polls/urls.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 | from django.conf.urls import patterns, include, url
18 | from django.contrib import admin
19 |
20 | admin.autodiscover()
21 |
22 | urlpatterns = patterns('',
23 | url(r'', include('polls.urls', namespace='polls')),
24 | url(r'^admin/', include(admin.site.urls)),
25 | )
26 |
--------------------------------------------------------------------------------
/manage.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 | #!/usr/bin/env python
18 | import os
19 | import sys
20 |
21 | if __name__ == "__main__":
22 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "sharded_polls.settings")
23 |
24 | from django.core.management import execute_from_command_line
25 |
26 | execute_from_command_line(sys.argv)
27 |
--------------------------------------------------------------------------------
/polls/admin.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 | from django.contrib import admin
18 | from polls.models import Poll, Choice
19 |
20 | class ChoiceInline(admin.TabularInline):
21 | model = Choice
22 | extra = 3
23 |
24 | class PollAdmin(admin.ModelAdmin):
25 | fieldsets = [
26 | (None, {'fields': ['question']}),
27 | ('Date information', {'fields': ['pub_date'], 'classes': ['collapse']}),
28 | ]
29 | inlines = [ChoiceInline]
30 | list_display = ('question', 'pub_date', 'was_published_recently')
31 | list_filter = ['pub_date']
32 | search_fields = ['question']
33 | date_hierarchy = 'pub_date'
34 |
35 | admin.site.register(Poll, PollAdmin)
36 |
--------------------------------------------------------------------------------
/sqlshards/db/shards/sql.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 | import time
18 | from django.conf import settings
19 |
20 | next_sharded_id = """CREATE OR REPLACE FUNCTION next_sharded_id(varchar, int, OUT result bigint) AS $$
21 | DECLARE
22 | sequence_name ALIAS FOR $1;
23 | shard_id ALIAS FOR $2;
24 |
25 | seq_id bigint;
26 | now_millis bigint;
27 | BEGIN
28 | SELECT nextval(sequence_name::regclass) % 1024 INTO seq_id;
29 |
30 | SELECT FLOOR(EXTRACT(EPOCH FROM clock_timestamp()) * 1000) INTO now_millis;
31 | result := (now_millis - {our_epoch}) << 23;
32 | result := result | (shard_id << 10);
33 | result := result | (seq_id);
34 | END;
35 | $$ LANGUAGE PLPGSQL;""".format(our_epoch=settings.SHARD_EPOCH)
36 |
--------------------------------------------------------------------------------
/sqlshards/db/shards/helpers.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 | def get_sharded_id_sequence_name(model):
18 | # XXX: This matches what PostgreSQL would normally generate for a serial
19 | # type. This is needed because the old AutoSequenceField sets up some
20 | # signals which assume the sequence name that @replace_pk overwrites.
21 | return '%s_%s_seq' % (model._meta.db_table, model._meta.pk.column)
22 |
23 |
24 | def get_canonical_model(model):
25 | """
26 | Accepts a model class, returning the canonical parent model if the model is
27 | a child of an abstract partitioned model.
28 | """
29 | if is_partitioned(model) and model._shards.is_child:
30 | model = model._shards.parent
31 |
32 | return model
33 |
34 |
35 | #: Returns ``True`` if the given class is a partitioned model.
36 | is_partitioned = lambda cls: hasattr(cls, '_shards')
37 |
--------------------------------------------------------------------------------
/sqlshards/tests/sample/models.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 | from django.db import models
18 | from sqlshards.db.shards.models import PartitionModel
19 |
20 |
21 | class SimpleModel(models.Model):
22 | pass
23 |
24 |
25 | class PartitionedModel(PartitionModel):
26 | key = models.PositiveIntegerField()
27 |
28 | class Shards:
29 | key = 'key'
30 | num_shards = 2
31 | cluster = 'sharded'
32 |
33 |
34 | class TestModel(PartitionModel):
35 | key = models.IntegerField()
36 | foo = models.CharField(null=True, max_length=32)
37 |
38 | class Shards:
39 | key = 'key'
40 | num_shards = 2
41 | cluster = 'sharded'
42 |
43 | class Meta:
44 | unique_together = (('key', 'foo'),)
45 |
46 |
47 | class CompositeTestModel(PartitionModel):
48 | key = models.IntegerField()
49 | foo = models.IntegerField()
50 |
51 | class Shards:
52 | key = ('key', 'foo')
53 | num_shards = 2
54 | cluster = 'sharded'
55 |
56 | class Meta:
57 | unique_together = (('key', 'foo'),)
58 |
--------------------------------------------------------------------------------
/polls/urls.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 | from django.conf.urls import patterns, url
18 | from django.views.generic import DetailView, ListView
19 | from polls.models import Poll
20 | from django.utils import timezone
21 |
22 | urlpatterns = patterns('',
23 | # Ex: /polls/
24 | url(r'^$',
25 | ListView.as_view(
26 | queryset=Poll.objects.filter(pub_date__lte=timezone.now) \
27 | .order_by('-pub_date')[:5],
28 | context_object_name='latest_poll_list',
29 | template_name='polls/index.html'),
30 | name='index'),
31 | # Ex: /polls/5/
32 | url(r'^(?P\d+)/$',
33 | DetailView.as_view(
34 | queryset=Poll.objects.filter(pub_date__lte=timezone.now),
35 | model=Poll,
36 | template_name='polls/detail.html'),
37 | name='detail'),
38 | # Ex: /polls/5/results/
39 | url(r'^(?P\d+)/results/$',
40 | DetailView.as_view(
41 | model=Poll,
42 | template_name='polls/results.html'),
43 | name='results'),
44 | # Ex: /polls/5/vote/
45 | url(r'^(?P\d+)/vote/$', 'polls.views.vote', name='vote'),
46 | )
47 |
--------------------------------------------------------------------------------
/polls/views.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 | from django.shortcuts import get_object_or_404, render_to_response
18 | from django.http import HttpResponseRedirect, HttpResponse
19 | from django.core.urlresolvers import reverse
20 | from django.template import RequestContext
21 | from polls.models import Choice, Poll
22 |
23 |
24 | def vote(request, poll_id):
25 | p = get_object_or_404(Poll, pk=poll_id)
26 | try:
27 | selected_choice = Choice.objects.get(poll_id=p.pk, pk=request.POST['choice'])
28 | except (KeyError, Choice.DoesNotExist):
29 | # Redisplay the poll voting form.
30 | return render_to_response('polls/detail.html', {
31 | 'poll': p,
32 | 'error_message': "You didn't select a choice.",
33 | }, context_instance=RequestContext(request))
34 | else:
35 | Choice.objects.filter(pk=pk, poll_id=p.pk).update(votes=F('votes') + 1)
36 | # Always return an HttpResponseRedirect after successfully dealing
37 | # with POST data. This prevents data from being posted twice if a
38 | # user hits the Back button.
39 | return HttpResponseRedirect(reverse('polls:results', args=(p.id,)))
40 |
--------------------------------------------------------------------------------
/sqlshards/db/shards/routers.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 | class ShardedRouter(object):
18 | """
19 | Breaks up apps based on their attached shard info.
20 |
21 | This looks for "_shards" on the model (which is defined as part of PartitionBase)
22 | and ensures only child tables get synced, as well as guarantees the correct (master)
23 | database for queries on a given shard.
24 | """
25 | def db_for_read(self, model, **hints):
26 | shard_info = getattr(model, '_shards', None)
27 | if shard_info:
28 | if not shard_info.is_child:
29 | raise ValueError('%r cannot be queried as its a virtual partition model' % model.__name__)
30 | return shard_info.get_database()
31 |
32 | return None
33 |
34 | def db_for_write(self, model, **hints):
35 | hints['is_write'] = True
36 | return self.db_for_read(model, **hints)
37 |
38 | def allow_syncdb(self, db, model):
39 | shard_info = getattr(model, '_shards', None)
40 | if shard_info:
41 | if db == shard_info.cluster:
42 | return True
43 | if shard_info.is_child and db in shard_info.get_all_databases():
44 | return True
45 | return False
46 |
47 | return None
48 |
--------------------------------------------------------------------------------
/polls/models.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 | import datetime
18 | from django.db import models
19 | from django.utils import timezone
20 | from sqlshards.db.shards.models import ShardedAutoField, PartitionModel
21 |
22 | class Poll(models.Model):
23 | def __unicode__(self):
24 | return self.question
25 |
26 | def was_published_recently(self):
27 | now = timezone.now()
28 | return now - datetime.timedelta(days = 1) <= self.pub_date < now
29 |
30 | was_published_recently.admin_order_field = 'pub_date'
31 | was_published_recently.boolean = True
32 | was_published_recently.short_description = 'Published recently?'
33 |
34 | question = models.CharField(max_length=200)
35 | pub_date = models.DateTimeField('date published')
36 |
37 |
38 | class PollPartitionBase(PartitionModel):
39 | id = ShardedAutoField(primary_key=True, auto_created=True)
40 | poll_id = models.PositiveIntegerField(db_index=True)
41 |
42 | class Meta:
43 | abstract = True
44 |
45 | class Shards:
46 | key = 'poll_id'
47 | num_shards = 2
48 | cluster = 'sharded'
49 |
50 |
51 | class Choice(PollPartitionBase):
52 | def __unicode__(self):
53 | return self.choice_text
54 |
55 | choice_text = models.CharField(max_length=200)
56 | votes = models.IntegerField()
57 |
--------------------------------------------------------------------------------
/sharded_polls/wsgi.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 | """
18 | WSGI config for sharded_polls project.
19 |
20 | This module contains the WSGI application used by Django's development server
21 | and any production WSGI deployments. It should expose a module-level variable
22 | named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover
23 | this application via the ``WSGI_APPLICATION`` setting.
24 |
25 | Usually you will have the standard Django WSGI application here, but it also
26 | might make sense to replace the whole Django WSGI application with a custom one
27 | that later delegates to the Django one. For example, you could introduce WSGI
28 | middleware here, or combine a Django application with an application of another
29 | framework.
30 |
31 | """
32 | import os
33 |
34 | # We defer to a DJANGO_SETTINGS_MODULE already in the environment. This breaks
35 | # if running multiple sites in the same mod_wsgi process. To fix this, use
36 | # mod_wsgi daemon mode with each site in its own daemon process, or use
37 | # os.environ["DJANGO_SETTINGS_MODULE"] = "sharded_polls.settings"
38 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "sharded_polls.settings")
39 |
40 | # This application object is used by any WSGI server configured to use this
41 | # file. This includes Django's development server, if the WSGI_APPLICATION
42 | # setting points here.
43 | from django.core.wsgi import get_wsgi_application
44 | application = get_wsgi_application()
45 |
46 | # Apply WSGI middleware here.
47 | # from helloworld.wsgi import HelloWorldApplication
48 | # application = HelloWorldApplication(application)
49 |
--------------------------------------------------------------------------------
/sqlshards/utils.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 | class DatabaseConfigurator(object):
18 | """
19 | Similar to the logging DictConfigurator, DatabaseConfigurator allows simply
20 | inheritance within a dictionary.
21 |
22 | It also looks for a special key (Django is not aware of it) called SHARDS
23 | which contains some logic for automatically creating additional connections
24 | based on the named connection.
25 |
26 | Additionally it handles a field called HOSTS, which is only used in conjuction
27 | with SHARDS. If this is set, it will handle mapping the underlying shards
28 | to other physical machines so that a shard's host is hosts[ % ].
29 | """
30 | def __init__(self, settings, defaults={}):
31 | self.settings = settings
32 | self.defaults = defaults
33 |
34 | def __iter__(self):
35 | for alias in self.settings.iterkeys():
36 | config = self.get_value(alias)
37 |
38 | shard_info = config.get('SHARDS', {})
39 | host_list = config.get('HOSTS')
40 |
41 | if not shard_info and not host_list:
42 | yield alias, config
43 | continue
44 |
45 | # if servers are present this is a virtual connection mapping to N connections
46 | hosts = {}
47 | if host_list:
48 | for num, host_config in host_list.iteritems():
49 | for key, value in config.iteritems():
50 | if key in ('SHARDS', 'HOSTS'):
51 | continue
52 | host_config.setdefault(key, value)
53 | hosts[num] = host_config
54 | else:
55 | hosts[0] = config
56 |
57 | assert not set(hosts.iterkeys()).difference(set(xrange(len(hosts)))), 'Host indexes must not contain any gaps'
58 |
59 | cluster_size = shard_info.get('size') or len(hosts)
60 | assert cluster_size > 0, 'Cluster cannot be empty'
61 |
62 | # we create ``size`` new connections based on the parent connections
63 | for num in xrange(cluster_size):
64 | shard = '%s.shard%d' % (alias, num)
65 | host_num = num % len(hosts)
66 |
67 | shard_n_config = hosts[host_num].copy()
68 |
69 | # test mirror can vary if its referencing a clustered connection
70 | if not shard_n_config.get('TEST_MIRROR'):
71 | shard_n_config['TEST_MIRROR'] = alias
72 |
73 | # ensure test mirror points to parent shard if a mirror is not
74 | # explicitly set
75 | yield shard, shard_n_config
76 |
77 | # maintain the original connection for dev/test environments
78 | yield alias, config
79 |
80 | def get_parent_value(self, key):
81 | defaults = self.defaults.copy()
82 |
83 | if '.' in key:
84 | parts = key.split('.')
85 | for subkey in ('.'.join(parts[:(idx + 1)]) for idx in xrange(len(parts) - 1)):
86 | if subkey in self.settings:
87 | defaults.update(self.settings[subkey])
88 |
89 | return defaults
90 |
91 | def get_value(self, key):
92 | defaults = self.get_parent_value(key)
93 | defaults.update(self.settings[key])
94 |
95 | return defaults
96 |
97 | def to_dict(self):
98 | return dict(self)
99 |
100 |
101 | def wraps(func):
102 | """
103 | Nearly identical to functools.wraps, except that it also
104 | maintains the ``__wraps__`` attribute, which will always
105 | be the origin function.
106 | """
107 | def wrapped(wrapper):
108 | actual = getattr(func, '__wraps__', func)
109 | for attr in ('__module__', '__name__', '__doc__'):
110 | if hasattr(actual, attr):
111 | setattr(wrapper, attr, getattr(actual, attr))
112 | wrapper.__wraps__ = actual
113 | return wrapper
114 | return wrapped
115 |
--------------------------------------------------------------------------------
/polls/tests.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 | import datetime
18 |
19 | from django.utils import timezone
20 | from django.test import TestCase
21 | from django.core.urlresolvers import reverse
22 |
23 | from polls.models import Poll
24 |
25 | def create_poll(question, days):
26 | """
27 | Creates a poll with the given `question` published the given number of
28 | `days` offset to now (negative for polls published in the past,
29 | positive for polls that have yet to be published).
30 | """
31 | return Poll.objects.create(question=question,
32 | pub_date=timezone.now() + datetime.timedelta(days=days))
33 |
34 | class PollMethodTests(TestCase):
35 |
36 | def test_was_published_recently_with_future_poll(self):
37 | """
38 | was_published_recently() should return False for polls whose
39 | pub_date is in the test_was_published_recently_with_future_poll
40 | """
41 | future_poll = Poll(pub_date=timezone.now() + datetime.timedelta(days=30))
42 | self.assertEqual(future_poll.was_published_recently(), False)
43 |
44 | def test_was_published_recently_with_old_poll(self):
45 | """
46 | was_published_recently() should return False for polls whose pub_date
47 | is older than 1 days
48 | """
49 | old_poll = Poll(pub_date=timezone.now() - datetime.timedelta(days=30))
50 | self.assertEqual(old_poll.was_published_recently(), False)
51 |
52 | def test_was_published_recently_with_recent_poll(self):
53 | """
54 | was_published_recently() should return True for polls whose pub_date
55 | is within the last days
56 | """
57 | recent_poll = Poll(pub_date=timezone.now() - datetime.timedelta(hours=1))
58 | self.assertEqual(recent_poll.was_published_recently(), True)
59 |
60 | class PollViewTests(TestCase):
61 | def test_index_view_with_no_polls(self):
62 | """
63 | If no polls exist, an appropriate message should be displayed.
64 | """
65 | response = self.client.get(reverse('polls:index'))
66 | self.assertEqual(response.status_code, 200)
67 | self.assertContains(response, "No polls are available.")
68 | self.assertQuerysetEqual(response.context['latest_poll_list'], [])
69 |
70 | def test_index_view_with_a_past_poll(self):
71 | """
72 | Polls with a pub_date in the past should be displayed on the index page.
73 | """
74 | create_poll(question="Past poll.", days=-30)
75 | response = self.client.get(reverse('polls:index'))
76 | self.assertQuerysetEqual(
77 | response.context['latest_poll_list'],
78 | ['']
79 | )
80 |
81 | def test_index_view_with_a_future_poll(self):
82 | """
83 | Polls with a pub_date in the future should not be displayed on the
84 | index page.
85 | """
86 | create_poll(question="Future poll.", days=30)
87 | response = self.client.get(reverse('polls:index'))
88 | self.assertContains(response, "No polls are available.", status_code=200)
89 | self.assertQuerysetEqual(response.context['latest_poll_list'], [])
90 |
91 | def test_index_view_with_future_poll_and_past_poll(self):
92 | """
93 | Even if both past and future polls exist, only past polls should be
94 | displayed.
95 | """
96 | create_poll(question="Past poll.", days=-30)
97 | create_poll(question="Future poll.", days=30)
98 | response = self.client.get(reverse('polls:index'))
99 | self.assertQuerysetEqual(
100 | response.context['latest_poll_list'],
101 | ['']
102 | )
103 |
104 | def test_index_view_with_two_past_polls(self):
105 | """
106 | The polls index page may display multiple polls.
107 | """
108 | create_poll(question="Past poll 1.", days=-30)
109 | create_poll(question="Past poll 2.", days=-5)
110 | response = self.client.get(reverse('polls:index'))
111 | self.assertQuerysetEqual(
112 | response.context['latest_poll_list'],
113 | ['', '']
114 | )
115 |
116 | class PollIndexDetailTests(TestCase):
117 | def test_detail_view_with_a_future_poll(self):
118 | """
119 | The detail view of a poll with a pub_date in the future should
120 | return a 404 not found.
121 | """
122 | future_poll = create_poll(question='Future poll.', days=5)
123 | response = self.client.get(reverse('polls:detail', args=(future_poll.id,)))
124 | self.assertEqual(response.status_code, 404)
125 |
126 | def test_detail_view_with_a_past_poll(self):
127 | """
128 | The detail view of a poll with a pub_date in the past should display
129 | the poll's question.
130 | """
131 | past_poll = create_poll(question='Past Poll.', days=-5)
132 | response = self.client.get(reverse('polls:detail', args=(past_poll.id,)))
133 | self.assertContains(response, past_poll.question, status_code=200)
134 |
--------------------------------------------------------------------------------
/sqlshards/db/shards/fields.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 | from django.db import connections, transaction
18 | from django.db.models.fields import AutoField, BigIntegerField
19 | from django.db.models.signals import post_syncdb, class_prepared
20 | from django.db.utils import DatabaseError
21 | from django.utils.translation import ugettext_lazy as _
22 |
23 |
24 | class AutoSequenceField(BigIntegerField):
25 | """
26 | A ``BigIntegerField`` that increments using an external PostgreSQL sequence
27 | generator. Used for primary keys on partitioned tables that require a
28 | canonical source for unique IDs.
29 |
30 | ``db_alias`` is the alias to the Django connection to the database
31 | containing the sequence.
32 |
33 | ``sequence`` is the string representation of the sequence table.
34 |
35 | """
36 |
37 | description = _("Integer")
38 |
39 | def __init__(self, db_alias, sequence=None, *args, **kwargs):
40 | self.db_alias = db_alias
41 | self.sequence = sequence
42 |
43 | kwargs['blank'] = True
44 | kwargs['editable'] = False
45 | kwargs['unique'] = True
46 | super(AutoSequenceField, self).__init__(*args, **kwargs)
47 |
48 | def set_sequence_name(self, **kwargs):
49 | self._sequence = self.sequence or '%s_%s_seq' % (self.model._meta.db_table, self.column)
50 |
51 | def create_sequence(self, created_models, **kwargs):
52 | if self.model not in created_models:
53 | return
54 |
55 | if not getattr(self, '_sequence', None):
56 | return
57 |
58 | # if hasattr(self.model, '_shards') and hasattr(self.model._shards, 'parent') and self.model._shards.parent not in created_models:
59 | # return
60 |
61 | cursor = connections[self.db_alias].cursor()
62 | sid = transaction.savepoint(self.db_alias)
63 | try:
64 | cursor.execute("CREATE SEQUENCE %s;" % self._sequence)
65 | except DatabaseError:
66 | transaction.savepoint_rollback(sid, using=self.db_alias)
67 | # Sequence must already exist, ensure it gets reset
68 | cursor.execute("SELECT setval('%s', 1, false)" % (self._sequence,))
69 | else:
70 | print 'Created sequence %r on %r' % (self._sequence, self.db_alias)
71 | transaction.savepoint_commit(sid, using=self.db_alias)
72 | cursor.close()
73 |
74 | def contribute_to_class(self, cls, name):
75 | super(AutoSequenceField, self).contribute_to_class(cls, name)
76 | # parent models still call this method, but dont need sequences
77 | post_syncdb.connect(self.create_sequence, dispatch_uid='create_sequence_%s_%s' % (cls._meta, name), weak=False)
78 | class_prepared.connect(self.set_sequence_name, sender=cls, weak=False)
79 |
80 | def pre_save(self, model_instance, add):
81 | value = getattr(model_instance, self.attname, None)
82 | if add and not value:
83 | value = self.get_next_value()
84 | setattr(model_instance, self.attname, value)
85 | return value
86 |
87 | def south_field_triple(self):
88 | "Returns a suitable description of this field for South."
89 | from south.modelsinspector import introspector
90 | field_class = "django.db.models.fields.PositiveIntegerField"
91 | args, kwargs = introspector(self)
92 | return (field_class, args, kwargs)
93 |
94 | def get_next_value(self):
95 | cursor = connections[self.db_alias].cursor()
96 | try:
97 | cursor.execute("SELECT NEXTVAL(%s)", (self._sequence,))
98 | return cursor.fetchone()[0]
99 | finally:
100 | cursor.close()
101 |
102 |
103 | class ShardedAutoField(AutoField):
104 | def db_type(self, *args, **kwargs):
105 | if not hasattr(self.model, '_shards'):
106 | raise ValueError("ShardedAutoField must be used with a PartitionModel.")
107 |
108 | if self.model._shards.is_master:
109 | return "bigint"
110 |
111 | return "bigint DEFAULT next_sharded_id('%s', %d)" % (
112 | get_sharded_id_sequence_name(self.model),
113 | self.model._shards.num)
114 |
115 | def create_sequence(self, created_models, **kwargs):
116 | # Sequence creation for production is handled by DDL scripts
117 | # (sqlpartition). This is needed to create sequences for
118 | # test models.
119 | if self.model not in created_models:
120 | return
121 |
122 | db_alias = self.model._shards.cluster
123 |
124 | for child in self.model._shards.nodes:
125 | cursor = connections[db_alias].cursor()
126 | sid = transaction.savepoint(db_alias)
127 | sequence_name = get_sharded_id_sequence_name(child)
128 | try:
129 | cursor.execute("CREATE SEQUENCE %s;" % sequence_name)
130 | except DatabaseError:
131 | transaction.savepoint_rollback(sid, using=db_alias)
132 | # Sequence must already exist, ensure it gets reset
133 | cursor.execute("SELECT setval('%s', 1, false)" % (sequence_name,))
134 | else:
135 | print 'Created sequence %r on %r' % (sequence_name, db_alias)
136 | transaction.savepoint_commit(sid, using=db_alias)
137 | cursor.close()
138 |
--------------------------------------------------------------------------------
/sqlshards/management/commands/sqlpartition.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 | from datetime import datetime
18 | from optparse import make_option
19 | import time
20 |
21 | from django.conf import settings
22 | from django.core.management.base import CommandError, BaseCommand
23 | from django.db import connections
24 | from django.db.models.loading import get_app
25 |
26 | from sqlshards.db.shards.helpers import get_sharded_id_sequence_name
27 | from sqlshards.db.shards.models import generate_child_partition
28 |
29 |
30 | class Command(BaseCommand):
31 | help = 'Generates DML for partitioned tables (expects argument in the '\
32 | 'form .).'
33 |
34 | option_list = BaseCommand.option_list + (
35 | make_option('--num', action='store', type='int', dest='num_children', default=settings.DEFAULT_SHARD_COUNT,
36 | help='number of partition tables to create [default: %d]' % (settings.DEFAULT_SHARD_COUNT)),
37 | make_option('--shard', action='store', type='int', dest='shard', default=0,
38 | help='physical shard number to generate DDL for (0-based) [default: 0]'),
39 | make_option('--shards', action='store', type='int', dest='shards', default=1,
40 | help='number of physical shards [default: 1]'),
41 | # TODO: suffix
42 | )
43 |
44 | def get_children_table_sql(self, model, known_models, num_children, shard_range):
45 | output = []
46 | opts = model._meta
47 |
48 | def get_child_table_sql(child_num):
49 | child = generate_child_partition(model, child_num)
50 |
51 | output, references = self.connection.creation.sql_create_model(child, self.style, [model, child])
52 | return output
53 |
54 | for i in shard_range:
55 | output.extend(get_child_table_sql(i))
56 |
57 | # Generate indexes for tables.
58 | original_db_table = opts.db_table
59 | for i in shard_range:
60 | # TODO: suffix
61 | opts.db_table = '%s_%s' % (original_db_table, i)
62 | output.extend(self.connection.creation.sql_indexes_for_model(model, self.style))
63 | opts.db_table = original_db_table
64 |
65 | # ALTERs for check constraint on children table.
66 | migrations = []
67 | for i in shard_range:
68 | child = generate_child_partition(model, i)
69 | if isinstance(child._shards.key, basestring):
70 | shard_key_repr = child._shards.key
71 | shard_key_expr = '"%s"' % shard_key_repr
72 | else:
73 | shard_key_repr = '_'.join(child._shards.key)
74 | # TODO: This sums the keys for the expression right now.
75 | # This needs to match the logic in MasterShardOptions.get_key_from_kwargs.
76 | shard_key_expr = '("' + '" + "'.join(child._shards.key) + '")'
77 |
78 | constraint_name = "%s_%s_check_modulo" % (child._meta.db_table, shard_key_repr)
79 | output.append(self.style.SQL_KEYWORD('ALTER TABLE ') +
80 | self.style.SQL_TABLE('"' + child._meta.db_table + '"') +
81 | self.style.SQL_KEYWORD(' ADD CONSTRAINT ') +
82 | self.style.SQL_FIELD('"' + constraint_name + '"') +
83 | self.style.SQL_KEYWORD(' CHECK ') +
84 | '(%s %% %d = %d);' % (shard_key_expr, num_children, i))
85 |
86 | # Temporary ALTER TABLEs to use new sequences until we've fully
87 | # transitioned all old tables.
88 | migrations.append('ALTER TABLE "{0}" ALTER COLUMN id SET DEFAULT next_sharded_id(\'{0}_id_seq\', {1});'.format(child._meta.db_table, i))
89 |
90 | return output + migrations
91 |
92 | def get_sequences(self, model, num_children, shard_range):
93 | output = []
94 |
95 | our_epoch = int(time.mktime(datetime(2012, 11, 1).timetuple()) * 1000)
96 | proc = """CREATE OR REPLACE FUNCTION next_sharded_id(varchar, int, OUT result bigint) AS $$
97 | DECLARE
98 | sequence_name ALIAS FOR $1;
99 | shard_id ALIAS FOR $2;
100 |
101 | seq_id bigint;
102 | now_millis bigint;
103 | BEGIN
104 | SELECT nextval(sequence_name::regclass) % 1024 INTO seq_id;
105 |
106 | SELECT FLOOR(EXTRACT(EPOCH FROM clock_timestamp()) * 1000) INTO now_millis;
107 | result := (now_millis - {our_epoch}) << 23;
108 | result := result | (shard_id << 10);
109 | result := result | (seq_id);
110 | END;
111 | $$ LANGUAGE PLPGSQL;""".format(our_epoch=our_epoch)
112 | output.append(self.style.SQL_KEYWORD(proc))
113 |
114 | for i in shard_range:
115 | child = generate_child_partition(model, i)
116 | output.append(self.style.SQL_KEYWORD("CREATE SEQUENCE ") +
117 | self.style.SQL_TABLE(get_sharded_id_sequence_name(child)) + ";")
118 |
119 | return output
120 |
121 | def get_partitioned_model(self, app, model):
122 | for obj in (getattr(app, x) for x in dir(app)):
123 | if not hasattr(obj, '_shards') or not obj._shards.is_master:
124 | continue
125 |
126 | if obj._meta.module_name == model:
127 | return obj
128 | raise ValueError
129 |
130 | def handle(self, *args, **options):
131 | try:
132 | app, model = args[0].split('.')
133 | except ValueError:
134 | raise CommandError('Expected argument .')
135 |
136 | self.connection = connections['default']
137 |
138 | # XXX: We cant use get_model because its now an abstract model
139 | # model = get_model(app, model)
140 | app = get_app(app)
141 | model = self.get_partitioned_model(app, model)
142 |
143 | num_children = options['num_children']
144 | shard_range = range(options['shard'], num_children, options['shards'])
145 |
146 | output = self.get_sequences(model, num_children, shard_range)
147 | output.extend(self.get_children_table_sql(model, [model], num_children, shard_range))
148 |
149 | return u'\n\n'.join(output) + '\n'
150 |
--------------------------------------------------------------------------------
/sharded_polls/settings.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 | # Django settings for sharded_polls project.
18 | import os
19 | import time
20 | from datetime import datetime
21 | from sqlshards.utils import DatabaseConfigurator
22 |
23 | DEBUG = True
24 | TEMPLATE_DEBUG = DEBUG
25 | PWD = os.path.dirname(os.path.realpath(__file__))
26 |
27 | ADMINS = (
28 | # ('Your Name', 'your_email@example.com'),
29 | )
30 |
31 | MANAGERS = ADMINS
32 |
33 | DATABASE_CONFIG = {
34 | 'root': {
35 | 'ENGINE': 'django.db.backends.postgresql_psycopg2',
36 | 'USER': '',
37 | 'PASSWORD': '',
38 | 'HOST': '',
39 | 'PORT': '',
40 | },
41 | 'servers': {
42 | 'default': {
43 | 'NAME': 'polls',
44 | },
45 | 'sharded': {
46 | 'NAME': 'sharded_polls',
47 | 'HOSTS': {
48 | 0: {},
49 | 1: {},
50 | }
51 | }
52 | }
53 | }
54 | DATABASES = dict(DatabaseConfigurator(DATABASE_CONFIG['servers'], DATABASE_CONFIG['root']))
55 | DATABASE_ROUTERS = ['sqlshards.db.shards.routers.ShardedRouter']
56 | SHARD_EPOCH = int(time.mktime(datetime(2012, 11, 1).timetuple()) * 1000)
57 | DEFAULT_SHARD_COUNT = 2
58 |
59 | # Hosts/domain names that are valid for this site; required if DEBUG is False
60 | # See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
61 | ALLOWED_HOSTS = []
62 |
63 | # Local time zone for this installation. Choices can be found here:
64 | # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
65 | # although not all choices may be available on all operating systems.
66 | # In a Windows environment this must be set to your system time zone.
67 | TIME_ZONE = 'America/Chicago'
68 |
69 | # Language code for this installation. All choices can be found here:
70 | # http://www.i18nguy.com/unicode/language-identifiers.html
71 | LANGUAGE_CODE = 'en-us'
72 |
73 | SITE_ID = 1
74 |
75 | # If you set this to False, Django will make some optimizations so as not
76 | # to load the internationalization machinery.
77 | USE_I18N = True
78 |
79 | # If you set this to False, Django will not format dates, numbers and
80 | # calendars according to the current locale.
81 | USE_L10N = True
82 |
83 | # If you set this to False, Django will not use timezone-aware datetimes.
84 | USE_TZ = True
85 |
86 | # Absolute filesystem path to the directory that will hold user-uploaded files.
87 | # Example: "/var/www/example.com/media/"
88 | MEDIA_ROOT = ''
89 |
90 | # URL that handles the media served from MEDIA_ROOT. Make sure to use a
91 | # trailing slash.
92 | # Examples: "http://example.com/media/", "http://media.example.com/"
93 | MEDIA_URL = ''
94 |
95 | # Absolute path to the directory static files should be collected to.
96 | # Don't put anything in this directory yourself; store your static files
97 | # in apps' "static/" subdirectories and in STATICFILES_DIRS.
98 | # Example: "/var/www/example.com/static/"
99 | STATIC_ROOT = ''
100 |
101 | # URL prefix for static files.
102 | # Example: "http://example.com/static/", "http://static.example.com/"
103 | STATIC_URL = '/static/'
104 |
105 | # Additional locations of static files
106 | STATICFILES_DIRS = (
107 | # Put strings here, like "/home/html/static" or "C:/www/django/static".
108 | # Always use forward slashes, even on Windows.
109 | # Don't forget to use absolute paths, not relative paths.
110 | )
111 |
112 | # List of finder classes that know how to find static files in
113 | # various locations.
114 | STATICFILES_FINDERS = (
115 | 'django.contrib.staticfiles.finders.FileSystemFinder',
116 | 'django.contrib.staticfiles.finders.AppDirectoriesFinder',
117 | # 'django.contrib.staticfiles.finders.DefaultStorageFinder',
118 | )
119 |
120 | # Make this unique, and don't share it with anybody.
121 | SECRET_KEY = 'ueb!e*@2qm92$3914f#iee!9g(h%y1wnng#pet=kbcgx9m(2w1'
122 |
123 | # List of callables that know how to import templates from various sources.
124 | TEMPLATE_LOADERS = (
125 | 'django.template.loaders.filesystem.Loader',
126 | 'django.template.loaders.app_directories.Loader',
127 | # 'django.template.loaders.eggs.Loader',
128 | )
129 |
130 | MIDDLEWARE_CLASSES = (
131 | 'django.middleware.common.CommonMiddleware',
132 | 'django.contrib.sessions.middleware.SessionMiddleware',
133 | 'django.middleware.csrf.CsrfViewMiddleware',
134 | 'django.contrib.auth.middleware.AuthenticationMiddleware',
135 | 'django.contrib.messages.middleware.MessageMiddleware',
136 | # Uncomment the next line for simple clickjacking protection:
137 | # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
138 | )
139 |
140 | ROOT_URLCONF = 'sharded_polls.urls'
141 |
142 | # Python dotted path to the WSGI application used by Django's runserver.
143 | WSGI_APPLICATION = 'sharded_polls.wsgi.application'
144 |
145 | TEMPLATE_DIRS = (
146 | # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
147 | # Always use forward slashes, even on Windows.
148 | # Don't forget to use absolute paths, not relative paths.
149 | os.path.join(PWD, 'templates'),
150 | )
151 |
152 | INSTALLED_APPS = (
153 | 'django.contrib.auth',
154 | 'django.contrib.contenttypes',
155 | 'django.contrib.sessions',
156 | 'django.contrib.sites',
157 | 'django.contrib.messages',
158 | 'django.contrib.staticfiles',
159 | # Uncomment the next line to enable the admin:
160 | 'django.contrib.admin',
161 | # Uncomment the next line to enable admin documentation:
162 | # 'django.contrib.admindocs',
163 | 'sqlshards',
164 | 'polls'
165 | )
166 |
167 | # A sample logging configuration. The only tangible logging
168 | # performed by this configuration is to send an email to
169 | # the site admins on every HTTP 500 error when DEBUG=False.
170 | # See http://docs.djangoproject.com/en/dev/topics/logging for
171 | # more details on how to customize your logging configuration.
172 | LOGGING = {
173 | 'version': 1,
174 | 'disable_existing_loggers': False,
175 | 'filters': {
176 | 'require_debug_false': {
177 | '()': 'django.utils.log.RequireDebugFalse'
178 | }
179 | },
180 | 'handlers': {
181 | 'mail_admins': {
182 | 'level': 'ERROR',
183 | 'filters': ['require_debug_false'],
184 | 'class': 'django.utils.log.AdminEmailHandler'
185 | }
186 | },
187 | 'loggers': {
188 | 'django.request': {
189 | 'handlers': ['mail_admins'],
190 | 'level': 'ERROR',
191 | 'propagate': True,
192 | },
193 | }
194 | }
195 |
--------------------------------------------------------------------------------
/sqlshards/tests/integration.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 | from unittest import TestCase as UnitTestCase
18 | from django.db.models import signals
19 | from django.test import TestCase
20 | from sqlshards.db.shards.helpers import get_canonical_model, is_partitioned
21 |
22 | from .sample.models import SimpleModel, PartitionedModel, PartitionedModel_Partition0, \
23 | TestModel, CompositeTestModel
24 |
25 |
26 | class CompositeKeyShardTest(TestCase):
27 | def test_get_key_from_kwargs(self):
28 | self.assertEqual(CompositeTestModel._shards.get_key_from_kwargs(key=1, foo=2), 3)
29 |
30 | def test_get_key_from_instance(self):
31 | inst = TestModel(key=1, foo=2)
32 | self.assertEqual(CompositeTestModel._shards.get_key_from_instance(inst), 3)
33 |
34 |
35 | class PartitionShardTest(TestCase):
36 | def test_get_database_master(self):
37 | node = TestModel._shards.nodes[0]
38 | self.assertEqual(node._shards.get_database(), 'sharded.shard0')
39 |
40 | def test_get_database_slave(self):
41 | node = TestModel._shards.nodes[0]
42 | self.assertEqual(node._shards.get_database(slave=True), 'sharded.slave.shard0')
43 |
44 | def test_get_all_databases(self):
45 | node = TestModel._shards.nodes[0]
46 | self.assertEqual(node._shards.get_all_databases(), ('sharded.shard0', 'sharded.slave.shard0'))
47 |
48 | def test_get_key_from_kwargs(self):
49 | self.assertEqual(TestModel._shards.get_key_from_kwargs(key=1), 1)
50 |
51 | def test_get_key_from_instance(self):
52 | inst = TestModel(key=1)
53 | self.assertEqual(TestModel._shards.get_key_from_instance(inst), 1)
54 |
55 | def test_shardinfo(self):
56 | self.assertTrue(hasattr(TestModel, '_shards'))
57 | self.assertTrue(hasattr(TestModel._shards, 'is_master'))
58 | self.assertTrue(TestModel._shards.is_master)
59 | self.assertTrue(hasattr(TestModel._shards, 'is_child'))
60 | self.assertFalse(TestModel._shards.is_child)
61 | self.assertTrue(hasattr(TestModel._shards, 'nodes'))
62 | self.assertNotEquals(len(TestModel._shards.nodes), 0)
63 | self.assertTrue(hasattr(TestModel._shards, 'num_shards'))
64 | self.assertNotEquals(TestModel._shards.num_shards, 0)
65 | self.assertTrue(hasattr(TestModel._shards, 'key'))
66 | self.assertEqual(TestModel._shards.key, ('key',))
67 | self.assertTrue(hasattr(TestModel._shards, 'sequence'))
68 | self.assertEqual(TestModel._shards.sequence, 'sample_testmodel')
69 | self.assertTrue(hasattr(TestModel._shards, 'cluster'))
70 | self.assertEqual(TestModel._shards.cluster, 'sharded')
71 |
72 | node1 = TestModel._shards.nodes[0]
73 | self.assertTrue(hasattr(node1, '_shards'))
74 | self.assertTrue(hasattr(node1._shards, 'is_master'))
75 | self.assertFalse(node1._shards.is_master)
76 | self.assertTrue(hasattr(node1._shards, 'is_child'))
77 | self.assertTrue(node1._shards.is_child)
78 | self.assertTrue(hasattr(node1._shards, 'parent'))
79 | self.assertEqual(node1._shards.parent, TestModel)
80 | self.assertTrue(hasattr(node1._shards, 'num'))
81 | self.assertEqual(node1._shards.num, 0)
82 | self.assertTrue(hasattr(TestModel._shards, 'cluster'))
83 | self.assertEqual(TestModel._shards.cluster, 'sharded')
84 |
85 |
86 | class PartitionTest(TestCase):
87 | def test_get_database_from_key(self):
88 | self.assertEqual(TestModel.objects.get_database_from_key(2), 'sharded.shard0')
89 | self.assertEqual(TestModel.objects.get_database_from_key(3), 'sharded.shard1')
90 |
91 | def test_get_database_from_key_slave(self):
92 | self.assertEqual(TestModel.objects.get_database_from_key(2, slave=True), 'sharded.slave.shard0')
93 | self.assertEqual(TestModel.objects.get_database_from_key(3, slave=True), 'sharded.slave.shard1')
94 |
95 | def test_get_model_from_key(self):
96 | self.assertEqual(TestModel.objects.get_model_from_key(2), TestModel._shards.nodes[0])
97 | self.assertEqual(TestModel.objects.get_model_from_key(3), TestModel._shards.nodes[1])
98 |
99 | def test_get_database(self):
100 | self.assertEqual(TestModel.objects.get_database(0), 'sharded.shard0')
101 | self.assertEqual(TestModel.objects.get_database(1), 'sharded.shard1')
102 |
103 | def test_get_database_slave(self):
104 | self.assertEqual(TestModel.objects.get_database(0, slave=True), 'sharded.slave.shard0')
105 | self.assertEqual(TestModel.objects.get_database(1, slave=True), 'sharded.slave.shard1')
106 |
107 | def test_get_database_invalid_shard(self):
108 | self.assertRaises(ValueError, TestModel.objects.get_database, 2)
109 | self.assertRaises(ValueError, TestModel.objects.get_database, 2, slave=True)
110 |
111 | def test_routing_missing_key(self):
112 | self.assertRaises(AssertionError, TestModel.objects.filter, value='bar')
113 |
114 | def test_shard_with_valid_key(self):
115 | queryset = TestModel.objects.shard(0)
116 | self.assertEqual(queryset.model, TestModel._shards.nodes[0])
117 |
118 | queryset = TestModel.objects.shard(1)
119 | self.assertEqual(queryset.model, TestModel._shards.nodes[1])
120 |
121 | def test_missing_key_on_query(self):
122 | self.assertRaises(AssertionError, TestModel.objects.all)
123 |
124 | def test_module_imports(self):
125 | from sample import models # NOQA
126 | self.assertTrue('TestModel_Partition0' in dir(models), dir(models))
127 |
128 | def test_get_model_on_slaves(self):
129 | # Ensure we're registered as part of the app's models (both abstract and partitions)
130 | from django.db.models import get_model
131 | result = get_model('sample', 'testmodel_partition0', only_installed=False)
132 | self.assertNotEquals(result, None)
133 | self.assertEqual(result.__name__, 'TestModel_Partition0')
134 |
135 | def test_get_model_on_parent(self):
136 | # Ensure we're registered as part of the app's models (both abstract and partitions)
137 | from django.db.models import get_model
138 | result = get_model('sample', 'testmodel', only_installed=False)
139 | self.assertEqual(result, TestModel)
140 |
141 |
142 | class IsPartitionedTestCase(UnitTestCase):
143 | def test(self):
144 | self.assertFalse(is_partitioned(SimpleModel))
145 | self.assertTrue(is_partitioned(PartitionedModel))
146 | self.assertTrue(is_partitioned(PartitionedModel_Partition0))
147 |
148 |
149 | class GetCanonicalModelTestCase(UnitTestCase):
150 | def test(self):
151 | self.assertEqual(get_canonical_model(SimpleModel), SimpleModel)
152 | self.assertEqual(get_canonical_model(PartitionedModel), PartitionedModel)
153 | self.assertEqual(get_canonical_model(PartitionedModel_Partition0), PartitionedModel)
154 |
--------------------------------------------------------------------------------
/sqlshards/db/shards/manager.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 | from django.db import transaction, router, IntegrityError
18 | from django.db.models.manager import Manager
19 | from django.db.models.query import QuerySet, ValuesQuerySet, ValuesListQuerySet
20 |
21 |
22 | class PartitionQuerySetBase(object):
23 | @property
24 | def db(self):
25 | if self._db:
26 | return self._db
27 |
28 | if self._for_write:
29 | return router.db_for_write(self.model, exact_lookups=self._exact_lookups,
30 | instance=getattr(self, '_instance', None))
31 | return router.db_for_read(self.model, exact_lookups=self._exact_lookups)
32 |
33 |
34 | class PartitionQuerySet(PartitionQuerySetBase, QuerySet):
35 | """
36 | QuerySet which helps partitioning by field in the database routers by
37 | providing hints about what fields are being queried against (in
38 | ``exact_lookups``).
39 |
40 | If ``actual_model`` is passed, it will be used to reraise exceptions rather
41 | than ``model``.
42 | """
43 | def __init__(self, model=None, actual_model=None, *args, **kwargs):
44 | super(PartitionQuerySet, self).__init__(model=model, *args, **kwargs)
45 | self.actual_model = actual_model or model
46 | self._exact_lookups = {}
47 |
48 | def __getitem__(self, *args, **kwargs):
49 | try:
50 | return super(PartitionQuerySet, self).__getitem__(*args, **kwargs)
51 | except self.model.DoesNotExist, e:
52 | raise self.actual_model.DoesNotExist(unicode(e).replace(self.model.__name__, self.actual_model.__name__))
53 |
54 | def _clone(self, klass=None, *args, **kwargs):
55 | if klass is QuerySet:
56 | klass = PartitionQuerySet
57 | if klass is ValuesQuerySet:
58 | klass = PartitionValuesQuerySet
59 | elif klass is ValuesListQuerySet:
60 | klass = PartitionValuesListQuerySet
61 | clone = super(PartitionQuerySet, self)._clone(klass, *args, **kwargs)
62 | clone._exact_lookups = self._exact_lookups.copy()
63 | return clone
64 |
65 | def _filter_or_exclude(self, *args, **kwargs):
66 | clone = super(PartitionQuerySet, self)._filter_or_exclude(*args, **kwargs)
67 | if getattr(clone, '_exact_lookups', None) is None:
68 | clone._exact_lookups = {}
69 | clone._exact_lookups.update(dict([(k, v) for k, v in kwargs.items() if '__' not in k]))
70 | return clone
71 |
72 | def create(self, **kwargs):
73 | """
74 | This is a copy of QuerySet.create, except we save the instance we're
75 | about to save for the db_for_write router. This can't use super()
76 | since we'll either be too early (before the instance is created) or
77 | too late (after the ``db`` property is hit).
78 | """
79 | obj = self.model(**kwargs)
80 | self._for_write = True
81 | self._instance = obj
82 | obj.save(force_insert=True, using=self.db)
83 | return obj
84 |
85 | def get(self, *args, **kwargs):
86 | try:
87 | return super(PartitionQuerySet, self).get(*args, **kwargs)
88 | except self.model.DoesNotExist, e:
89 | raise self.actual_model.DoesNotExist(unicode(e).replace(self.model.__name__, self.actual_model.__name__))
90 |
91 | def get_or_create(self, **kwargs):
92 | """
93 | This is a copy of QuerySet.get_or_create, that forces calling our custom
94 | create method when the get fails.
95 | """
96 | assert kwargs, \
97 | 'get_or_create() must be passed at least one keyword argument'
98 | defaults = kwargs.pop('defaults', {})
99 | try:
100 | self._for_write = True
101 | return self.get(**kwargs), False
102 | except self.actual_model.DoesNotExist:
103 | params = dict([(k, v) for k, v in kwargs.items() if '__' not in k])
104 | params.update(defaults)
105 | obj = self.model(**params)
106 | self._for_write = True
107 | self._instance = obj
108 | using = self.db
109 | try:
110 | sid = transaction.savepoint(using=using)
111 | obj.save(force_insert=True, using=using)
112 | except IntegrityError, e:
113 | transaction.savepoint_rollback(sid, using=using)
114 | try:
115 | return self.get(**kwargs), False
116 | except self.actual_model.DoesNotExist, e:
117 | raise self.actual_model.DoesNotExist(unicode(e).replace(self.model.__name__, self.actual_model.__name__))
118 | else:
119 | transaction.savepoint_commit(sid, using=using)
120 | return obj, True
121 |
122 |
123 | def partition_query_set_factory(klass):
124 | class _PartitionQuerySetFromFactory(PartitionQuerySetBase, klass):
125 | def _clone(self, klass=None, *args, **kwargs):
126 | clone = super(_PartitionQuerySetFromFactory, self)._clone(klass, *args, **kwargs)
127 | clone._exact_lookups = self._exact_lookups.copy()
128 | return clone
129 |
130 | return _PartitionQuerySetFromFactory
131 |
132 | PartitionValuesQuerySet = partition_query_set_factory(ValuesQuerySet)
133 | PartitionValuesListQuerySet = partition_query_set_factory(ValuesListQuerySet)
134 |
135 |
136 | class PartitionManager(Manager):
137 | def get_query_set(self):
138 | return PartitionQuerySet(model=self.model)
139 |
140 |
141 | class MasterPartitionManager(Manager):
142 | """
143 | Allows operation of partitions by passing key to get_query_set().
144 | """
145 | def shard(self, key, slave=False):
146 | """
147 | Given a key, which is defined by the partition and used to route queries, returns a QuerySet
148 | that is bound to the correct shard.
149 |
150 | >>> shard(343)
151 |
152 | >>> shard(343, slave=True)
153 | """
154 | queryset = self.get_query_set(key)
155 | return queryset.using(self.get_database_from_key(key, slave=slave))
156 |
157 | def get_database(self, shard, slave=False):
158 | """
159 | Given a shard (numeric index value), returns the correct database alias to query against.
160 |
161 | If ``slave`` is True, returns a read-slave.
162 | """
163 | try:
164 | model = self.model._shards.nodes[shard]
165 | except IndexError:
166 | raise ValueError('Shard %r does not exist on %r' % (shard, self.model.__name__))
167 | return model._shards.get_database(slave=slave)
168 |
169 | def get_database_from_key(self, key, slave=False):
170 | """
171 | Given a key, which is defined by the partition and used to route queries, returns the
172 | database connection alias which the data lives on.
173 | """
174 | return self.get_database(key % self.model._shards.num_shards, slave=slave)
175 |
176 | def get_model_from_key(self, key):
177 | """
178 | Given a key, which is defined by the partition and used to route queries, returns the
179 | Model which represents the shard.
180 | """
181 | shards = self.model._shards
182 | return shards.nodes[key % shards.num_shards]
183 |
184 | def get_query_set(self, key=None):
185 | shards = self.model._shards
186 |
187 | assert key is not None, 'You must filter on %s before expanding a QuerySet on %s models.' % (
188 | shards.key, self.model.__name__)
189 |
190 | model = self.get_model_from_key(key)
191 |
192 | return PartitionQuerySet(model=model, actual_model=self.model)
193 |
194 | def _wrap(func_name):
195 | def wrapped(self, **kwargs):
196 | shards = self.model._shards
197 | try:
198 | key = shards.get_key_from_kwargs(**kwargs)
199 | except KeyError:
200 | raise AssertionError('You must filter on %s before expanding a QuerySet on %s models.' % (
201 | shards.key, self.model.__name__))
202 |
203 | return getattr(self.get_query_set(key=int(key)), func_name)(**kwargs)
204 |
205 | wrapped.__name__ = func_name
206 | return wrapped
207 |
208 | filter = _wrap('filter')
209 | get = _wrap('get')
210 | create = _wrap('create')
211 | get_or_create = _wrap('get_or_create')
212 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "[]"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright 2013 DISQUS
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/sqlshards/db/shards/models.py:
--------------------------------------------------------------------------------
1 | """
2 | Copyright 2013 DISQUS
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | """
16 |
17 | import sys
18 |
19 | from django.conf import settings
20 | from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned, ValidationError
21 | from django.db import connections, transaction
22 | from django.db.models import loading, Manager, Model, signals
23 | from django.db.models.base import ModelBase, subclass_exception
24 | from django.db.models.fields import PositiveIntegerField
25 | from django.db.models.fields.related import ForeignKey, ManyToOneRel, \
26 | RECURSIVE_RELATIONSHIP_CONSTANT, ReverseSingleRelatedObjectDescriptor
27 | from django.db.utils import DatabaseError
28 |
29 | from sqlshards.db.shards.fields import AutoSequenceField
30 | from sqlshards.db.shards.helpers import get_sharded_id_sequence_name
31 | from sqlshards.db.shards.manager import MasterPartitionManager
32 | from sqlshards.utils import wraps
33 |
34 |
35 | class PartitionedForeignKey(ForeignKey):
36 | """
37 | Behaves identical to a ForeignKey except it allows referencing an fkey
38 | that would live on the same cluster.
39 | """
40 | def __init__(self, to, to_field=None, rel_class=ManyToOneRel, **kwargs):
41 | try:
42 | to._meta.object_name.lower()
43 | except AttributeError: # to._meta doesn't exist, so it must be RECURSIVE_RELATIONSHIP_CONSTANT
44 | assert isinstance(to, basestring), "%s(%r) is invalid. First parameter to ForeignKey must be either a model, a model name, or the string %r" % (self.__class__.__name__, to, RECURSIVE_RELATIONSHIP_CONSTANT)
45 | else:
46 | # For backwards compatibility purposes, we need to *try* and set
47 | # the to_field during FK construction. It won't be guaranteed to
48 | # be correct until contribute_to_class is called. Refs #12190.
49 | to_field = to_field or (to._meta.pk and to._meta.pk.name)
50 | kwargs['verbose_name'] = kwargs.get('verbose_name', None)
51 |
52 | kwargs['rel'] = rel_class(to, to_field,
53 | related_name=kwargs.pop('related_name', None),
54 | limit_choices_to=kwargs.pop('limit_choices_to', None),
55 | parent_link=kwargs.pop('parent_link', False))
56 | super(ForeignKey, self).__init__(**kwargs)
57 |
58 | self.db_index = True
59 |
60 | def contribute_to_related_class(self, cls, related):
61 | # Reverse lookups not supported currently
62 | return
63 |
64 | def contribute_to_class(self, cls, name):
65 | # Pull in PartitionedReverseRelatedObjectDescriptor
66 | super(ForeignKey, self).contribute_to_class(cls, name)
67 | setattr(cls, self.name, PartitionedReverseRelatedObjectDescriptor(self))
68 | if isinstance(self.rel.to, basestring):
69 | target = self.rel.to
70 | else:
71 | target = self.rel.to._meta.db_table
72 | cls._meta.duplicate_targets[self.column] = (target, "o2m")
73 |
74 | def south_field_triple(self):
75 | "Returns a suitable description of this field for South."
76 | from south.modelsinspector import introspector
77 | field_class = "django.db.models.PositiveIntegerField"
78 | args, kwargs = introspector(self)
79 | return (field_class, [], {'db_column': "%r" % self.column})
80 |
81 |
82 | class PartitionedReverseRelatedObjectDescriptor(ReverseSingleRelatedObjectDescriptor):
83 | """
84 | Identical to ReverseSingleRelatedObjectDescriptor except that we pass the key
85 | to the manager and relying on outside routing.
86 | """
87 | def __get__(self, instance, instance_type=None):
88 | if instance is None:
89 | return self
90 |
91 | cache_name = self.field.get_cache_name()
92 | try:
93 | return getattr(instance, cache_name)
94 | except AttributeError:
95 | val = getattr(instance, self.field.attname)
96 | if val is None:
97 | # If NULL is an allowed value, return it.
98 | if self.field.null:
99 | return None
100 | raise self.field.rel.to.DoesNotExist
101 | other_field = self.field.rel.get_related_field()
102 | relname = self.field.rel.field_name
103 | if other_field.rel:
104 | params = {'%s__pk' % relname: val}
105 | else:
106 | params = {'%s__exact' % relname: val}
107 |
108 | # Ensure key is sent to the manager
109 | for field_name in instance._shards.key:
110 | params[field_name] = getattr(instance, field_name)
111 |
112 | # If the related manager indicates that it should be used for
113 | # related fields, respect that.
114 | rel_mgr = self.field.rel.to._default_manager
115 | rel_obj = rel_mgr.get(**params)
116 | setattr(instance, cache_name, rel_obj)
117 | return rel_obj
118 |
119 |
120 | def resend_signal(new_sender):
121 | @wraps(new_sender)
122 | def wrapped(**kwargs):
123 | signal = kwargs.pop('signal')
124 | kwargs['sender'] = new_sender
125 | signal.send(**kwargs)
126 | return wrapped
127 |
128 |
129 | def get_cluster_sizes(connections):
130 | """
131 | Returns a dictionary mapping clusters of servers (given
132 | by their naming scheme) and the number of connections in
133 | that cluster.
134 | """
135 | import re
136 | expr = re.compile(r'.*\.shard\d+$')
137 | clusters = {}
138 | for conn in connections:
139 | if not expr.match(conn):
140 | continue
141 | cluster = conn.split('.shard', 1)[0]
142 | if cluster not in clusters:
143 | clusters[cluster] = 1
144 | else:
145 | clusters[cluster] += 1
146 | return clusters
147 |
148 |
149 | DEFAULT_NAMES = ('num_shards', 'key', 'sequence', 'abstract', 'cluster')
150 | CLUSTER_SIZES = get_cluster_sizes(connections)
151 |
152 |
153 | class MasterShardOptions(object):
154 | def __init__(self, options, nodes=[]):
155 | self.options = options
156 | self.nodes = nodes
157 | self.model = None
158 | self.name = None
159 | self.size = None
160 |
161 | def __repr__(self):
162 | return u'<%s: model=%s, options=%s, nodes=%s>' % (
163 | self.__class__.__name__, self.model,
164 | self.options, len(self.nodes))
165 |
166 | @property
167 | def is_child(self):
168 | return False
169 |
170 | @property
171 | def is_master(self):
172 | return True
173 |
174 | def contribute_to_class(self, cls, name):
175 | self.name = name
176 | self.model = cls
177 | setattr(cls, name, self)
178 |
179 | opts = self.options
180 |
181 | if opts:
182 | for k in (k for k in DEFAULT_NAMES if hasattr(opts, k)):
183 | setattr(self, k, getattr(opts, k))
184 |
185 | if not hasattr(self, 'sequence'):
186 | self.sequence = cls._meta.db_table
187 |
188 | if hasattr(self, 'cluster'):
189 | self.size = CLUSTER_SIZES[self.cluster]
190 |
191 | # We support both key = 'field_name' and key = ['field_name']
192 | # style declerations
193 | if hasattr(self, 'key') and isinstance(self.key, basestring):
194 | self.key = (self.key,)
195 |
196 | def get_key_from_instance(self, instance):
197 | """
198 | Return the routing key for an instance.
199 |
200 | >>> shard_key = Model._shards.get_key_from_instance(instance)
201 | """
202 | return self.get_key_from_kwargs(**dict((f, getattr(instance, f)) for f in self.key))
203 |
204 | def get_key_from_kwargs(self, **kwargs):
205 | """
206 | Return the routing key for an object given ``kwargs``.
207 |
208 | >>> shard_key = Model._shards.get_key_from_instance(forum_id=1)
209 | """
210 | return sum(int(kwargs[f]) for f in self.key)
211 |
212 |
213 | class ShardOptions(object):
214 | def __init__(self, parent, num):
215 | self.parent = parent
216 | self.num = num
217 | self.model = None
218 | self.name = None
219 |
220 | def __repr__(self):
221 | return u'<%s: model=%s, parent=%s, num=%s>' % (
222 | self.__class__.__name__, self.model,
223 | self.parent, self.num)
224 |
225 | @property
226 | def is_child(self):
227 | return True
228 |
229 | @property
230 | def is_master(self):
231 | return False
232 |
233 | @property
234 | def key(self):
235 | return self.parent._shards.key
236 |
237 | @property
238 | def cluster(self):
239 | return self.parent._shards.cluster
240 |
241 | @property
242 | def sequence(self):
243 | return self.parent._shards.sequence
244 |
245 | def get_all_databases(self):
246 | """
247 | Returns a list of all database aliases that this shard is
248 | bound to.
249 | """
250 | return (self.get_database(), self.get_database(slave=True))
251 |
252 | def get_database(self, slave=False):
253 | parent = self.parent._shards
254 | if not parent.size:
255 | return
256 | alias = parent.cluster
257 | if slave:
258 | alias += '.slave'
259 | alias += '.shard%d' % (self.num % parent.size,)
260 | return alias
261 |
262 | def get_key_from_instance(self, *args, **kwargs):
263 | return self.parent._shards.get_key_from_instance(*args, **kwargs)
264 |
265 | def get_key_from_kwargs(self, *args, **kwargs):
266 | return self.parent._shards.get_key_from_kwargs(*args, **kwargs)
267 |
268 | def contribute_to_class(self, cls, name):
269 | self.name = name
270 | self.model = cls
271 | setattr(cls, name, self)
272 |
273 |
274 | def generate_child_partition(parent, num):
275 | opts = parent._meta
276 | partition_name = '%s_Partition%s' % (parent.__name__, num)
277 |
278 | # HACK: Attempting to initialize a model twice results in a broken model
279 | # even though ModelBase is supposed to handle this case already. Instead,
280 | # we explicitly check to make sure the model wasn't created yet by
281 | # using get_model to prevent this case.
282 | app_label = parent._meta.app_label
283 | m = loading.get_model(app_label, partition_name, seed_cache=False)
284 | if m is not None:
285 | return m
286 |
287 | partition = ModelBase(partition_name, (parent,), {
288 | '__module__': parent.__module__,
289 | 'objects': Manager(),
290 | 'Meta': type('Meta', (object,), {
291 | 'managed': True,
292 | 'db_table': '%s_%s' % (parent._meta.db_table, num),
293 | 'unique_together': opts.unique_together,
294 | }),
295 | '_shards': ShardOptions(parent=parent, num=num),
296 | })
297 | partition.add_to_class('DoesNotExist', subclass_exception('DoesNotExist', (parent.DoesNotExist,), parent.__module__))
298 | partition.add_to_class('MultipleObjectsReturned', subclass_exception('MultipleObjectsReturned', (parent.MultipleObjectsReturned,), parent.__module__))
299 |
300 | # Connect signals so we can re-send them
301 | signaler = resend_signal(parent)
302 | for signal in (signals.pre_save, signals.post_save, signals.pre_delete, signals.post_delete,
303 | signals.pre_init, signals.post_init, signals.m2m_changed):
304 | signal.connect(signaler, sender=partition, weak=False)
305 |
306 | # Ensure the partition is available within the module scope
307 | module = sys.modules[parent.__module__]
308 | setattr(module, partition.__name__, partition)
309 |
310 | # Register all partitions with Django
311 | loading.register_models(app_label, partition)
312 |
313 | return partition
314 |
315 |
316 | class PartitionDescriptor(ModelBase):
317 | """
318 | Creates partitions from the base model and attaches them to ``cls._shardss``.
319 | All children will also have ``cls._shards`` referencing the parent model.
320 |
321 | Also enforces the base model to be abstract, and assumes it's not a real table.
322 | """
323 | def __new__(cls, name, bases, attrs):
324 | # Force this model to be abstract as it's not a real table
325 | if 'Meta' not in attrs:
326 | is_abstract = False
327 | attrs['Meta'] = type('Meta', (object,), {
328 | 'abstract': True,
329 | })
330 | else:
331 | is_abstract = getattr(attrs['Meta'], 'abstract', False)
332 | attrs['Meta'].abstract = True
333 |
334 | attrs['objects'] = MasterPartitionManager()
335 |
336 | new_cls = super(PartitionDescriptor, cls).__new__(cls, name, bases, attrs)
337 |
338 | attr_shardopts = attrs.pop('Shards', None)
339 |
340 | # HACK: non-abstract inheritance is not supported due to issues with metaclass
341 | # recursion
342 | if not any(b._shards.abstract if hasattr(b, '_shards') else True for b in bases):
343 | return new_cls
344 |
345 | if not attr_shardopts:
346 | shardopts = getattr(new_cls, 'Shards', None)
347 | else:
348 | shardopts = attr_shardopts
349 | base_shardopts = getattr(new_cls, '_shards', None)
350 |
351 | shards = []
352 | new_cls.add_to_class('_shards', MasterShardOptions(shardopts, nodes=shards))
353 |
354 | if base_shardopts:
355 | for k in DEFAULT_NAMES:
356 | if not hasattr(new_cls._shards, k):
357 | setattr(new_cls._shards, k, getattr(base_shardopts, k, None))
358 |
359 | # We record the true abstract switch as part of _shards
360 | new_cls._shards.abstract = is_abstract
361 |
362 | if is_abstract:
363 | return new_cls
364 |
365 | # Some basic validation
366 | for k in DEFAULT_NAMES:
367 | if getattr(new_cls._shards, k, None) is None:
368 | raise ValidationError('Missing shard configuration value for %r on %r.' % (k, new_cls))
369 |
370 | new_cls.add_to_class('DoesNotExist', subclass_exception('DoesNotExist', (ObjectDoesNotExist,), new_cls.__module__))
371 | new_cls.add_to_class('MultipleObjectsReturned', subclass_exception('MultipleObjectsReturned', (MultipleObjectsReturned,), new_cls.__module__))
372 |
373 | # Because we're an abstract model, we must also fake our own registration
374 | app_label = new_cls._meta.app_label
375 | loading.register_models(app_label, new_cls)
376 |
377 | new_cls._really_prepare()
378 |
379 | # We need to create a model for each partition instance which is assigned to the
380 | # appropriate table
381 | for n in xrange(new_cls._shards.num_shards):
382 | partition = generate_child_partition(new_cls, n)
383 |
384 | # Add to list of partitions for this master
385 | shards.append(partition)
386 |
387 | return new_cls
388 |
389 | # Kill off default _prepare function
390 | _really_prepare = ModelBase._prepare
391 | # _prepare = lambda x: None
392 |
393 |
394 | class PartitionModel(Model):
395 | __metaclass__ = PartitionDescriptor
396 |
397 | class Meta:
398 | abstract = True
399 |
--------------------------------------------------------------------------------