├── example ├── project │ ├── __init__.py │ ├── wsgi.py │ ├── urls.py │ └── settings.py ├── toolbox │ ├── __init__.py │ └── management │ │ ├── __init__.py │ │ └── commands │ │ ├── __init__.py │ │ └── exportcalaccesscampaigncandidates.py ├── templates │ ├── 404.html │ └── 500.html └── manage.py ├── calaccess_campaign_browser ├── utils │ ├── __init__.py │ ├── lazyencoder.py │ ├── serializer.py │ └── models.py ├── management │ ├── __init__.py │ └── commands │ │ ├── buildcalaccesscampaignbrowser.py │ │ ├── exportcalaccesscampaigncandidates.py │ │ ├── flushcalaccesscampaignbrowser.py │ │ ├── dropcalaccesscampaignbrowser.py │ │ ├── exportcalaccesscampaignbrowser.py │ │ ├── __init__.py │ │ ├── loadcalaccesscampaignfilings.py │ │ ├── loadcalaccesscampaignsummaries.py │ │ ├── importtosqlserver.py │ │ └── scrapecalaccesscampaigncandidates.py ├── templatetags │ ├── __init__.py │ └── calaccesscampaignbrowser.py ├── templates │ ├── robots.txt │ └── calaccess_campaign_browser │ │ ├── paginator.html │ │ ├── search_list.html │ │ ├── header.html │ │ ├── base.html │ │ ├── latest.html │ │ ├── filer_detail.html │ │ ├── search_contribs_by_name.html │ │ ├── committee_expenditure_list.html │ │ ├── committee_contribution_list.html │ │ ├── expenditure_detail.html │ │ ├── committee_filing_list.html │ │ ├── filer_list.html │ │ ├── committee_nav.html │ │ ├── contribution_detail.html │ │ ├── filing_detail.html │ │ └── party_list.html ├── __init__.py ├── views │ ├── parties.py │ ├── expenditures.py │ ├── contributions.py │ ├── __init__.py │ ├── search.py │ ├── filings.py │ ├── base.py │ └── committees.py ├── apps.py ├── static │ └── calaccess_campaign_browser │ │ ├── images │ │ ├── bear.gif │ │ ├── ccdc-logo.png │ │ ├── bearwalk_small.gif │ │ ├── bear-no-red-line.png │ │ └── party_icons │ │ │ ├── na_icon_white.png │ │ │ ├── democratic_icon.png │ │ │ ├── green_icon_white.png │ │ │ ├── republican_icon.png │ │ │ ├── reform_icon_white.png │ │ │ ├── unknown_icon_white.png │ │ │ ├── democratic_icon_white.png │ │ │ ├── republican_icon_white.png │ │ │ ├── independent_icon_white.png │ │ │ ├── libertarian_icon_white.png │ │ │ ├── natural-law_icon_white.png │ │ │ ├── non-partisan_icon_white.png │ │ │ ├── americans-elect_icon_white.png │ │ │ ├── peace-and-freedom_icon_white.png │ │ │ ├── american-independent_icon_white.png │ │ │ └── no-party-preference_icon_white.png │ │ ├── fonts │ │ ├── FontAwesome.otf │ │ ├── fontawesome-webfont.eot │ │ ├── fontawesome-webfont.ttf │ │ └── fontawesome-webfont.woff │ │ ├── css │ │ ├── home.css │ │ └── main.css │ │ └── js │ │ ├── jquery.sieve.min.js │ │ └── jquery.twbsPagination.min.js ├── models │ ├── __init__.py │ ├── contributions.py │ ├── elections.py │ ├── expenditures.py │ └── filings.py ├── api.py ├── tests.py ├── urls.py ├── admin.py └── managers.py ├── docs ├── models.rst ├── deployment.rst ├── _static │ ├── cir-logo.png │ ├── ccdc-logo.png │ ├── latimes-logo.gif │ ├── opennews-logo.png │ ├── stanford-logo.png │ ├── application-layers.png │ └── los-angeles-times-logo.png ├── changelog.rst ├── exportingthedata.rst ├── howtocontribute.rst ├── sqlserver.rst ├── index.rst ├── howtouseit.rst ├── Makefile └── make.bat ├── MANIFEST.in ├── .gitignore ├── tox.ini ├── requirements_dev.txt ├── .travis.yml ├── LICENSE ├── Makefile ├── .coveragerc ├── README.md └── setup.py /example/project/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /example/toolbox/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /example/templates/404.html: -------------------------------------------------------------------------------- 1 | 404 2 | -------------------------------------------------------------------------------- /example/templates/500.html: -------------------------------------------------------------------------------- 1 | 500 2 | -------------------------------------------------------------------------------- /example/toolbox/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /example/toolbox/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/templatetags/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/models.rst: -------------------------------------------------------------------------------- 1 | Models 2 | ====== 3 | 4 | *coming soon* -------------------------------------------------------------------------------- /docs/deployment.rst: -------------------------------------------------------------------------------- 1 | Deployment 2 | ========== 3 | 4 | Coming soon 5 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/templates/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Disallow: /admin/ -------------------------------------------------------------------------------- /docs/_static/cir-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/docs/_static/cir-logo.png -------------------------------------------------------------------------------- /docs/_static/ccdc-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/docs/_static/ccdc-logo.png -------------------------------------------------------------------------------- /docs/_static/latimes-logo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/docs/_static/latimes-logo.gif -------------------------------------------------------------------------------- /docs/_static/opennews-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/docs/_static/opennews-logo.png -------------------------------------------------------------------------------- /docs/_static/stanford-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/docs/_static/stanford-logo.png -------------------------------------------------------------------------------- /docs/_static/application-layers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/docs/_static/application-layers.png -------------------------------------------------------------------------------- /docs/_static/los-angeles-times-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/docs/_static/los-angeles-times-logo.png -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.md 3 | recursive-include calaccess_campaign_browser/static * 4 | recursive-include calaccess_campaign_browser/templates * 5 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | default_app_config = \ 3 | 'calaccess_campaign_browser.apps.CalAccessCampaignBrowserConfig' 4 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/views/parties.py: -------------------------------------------------------------------------------- 1 | from django.views import generic 2 | 3 | 4 | class PartyListView(generic.TemplateView): 5 | template_name = "calaccess_campaign_browser/party_list.html" 6 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class CalAccessCampaignBrowserConfig(AppConfig): 5 | name = 'calaccess_campaign_browser' 6 | verbose_name = "CAL-ACCESS campaign browser" 7 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/images/bear.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/calaccess_campaign_browser/static/calaccess_campaign_browser/images/bear.gif -------------------------------------------------------------------------------- /calaccess_campaign_browser/views/expenditures.py: -------------------------------------------------------------------------------- 1 | from django.views import generic 2 | from calaccess_campaign_browser.models import Expenditure 3 | 4 | 5 | class ExpenditureDetailView(generic.DetailView): 6 | model = Expenditure 7 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/fonts/FontAwesome.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/calaccess_campaign_browser/static/calaccess_campaign_browser/fonts/FontAwesome.otf -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/images/ccdc-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/calaccess_campaign_browser/static/calaccess_campaign_browser/images/ccdc-logo.png -------------------------------------------------------------------------------- /calaccess_campaign_browser/views/contributions.py: -------------------------------------------------------------------------------- 1 | from django.views import generic 2 | from calaccess_campaign_browser.models import Contribution 3 | 4 | 5 | class ContributionDetailView(generic.DetailView): 6 | model = Contribution 7 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | 0.1.1 (February 2015) 5 | --------------------- 6 | 7 | * Scrapers that collect elections and propositions 8 | 9 | 0.1.0 (2014) 10 | -------------------- 11 | 12 | * Initial release -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/images/bearwalk_small.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/calaccess_campaign_browser/static/calaccess_campaign_browser/images/bearwalk_small.gif -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/calaccess_campaign_browser/static/calaccess_campaign_browser/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/calaccess_campaign_browser/static/calaccess_campaign_browser/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/images/bear-no-red-line.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/calaccess_campaign_browser/static/calaccess_campaign_browser/images/bear-no-red-line.png -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/calaccess_campaign_browser/static/calaccess_campaign_browser/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/na_icon_white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/na_icon_white.png -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/democratic_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/democratic_icon.png -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/green_icon_white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/green_icon_white.png -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/republican_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/republican_icon.png -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/reform_icon_white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/reform_icon_white.png -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/unknown_icon_white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/unknown_icon_white.png -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/democratic_icon_white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/democratic_icon_white.png -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/republican_icon_white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/republican_icon_white.png -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/independent_icon_white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/independent_icon_white.png -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/libertarian_icon_white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/libertarian_icon_white.png -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/natural-law_icon_white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/natural-law_icon_white.png -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/non-partisan_icon_white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/non-partisan_icon_white.png -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/americans-elect_icon_white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/americans-elect_icon_white.png -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/peace-and-freedom_icon_white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/peace-and-freedom_icon_white.png -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/american-independent_icon_white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/american-independent_icon_white.png -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/no-party-preference_icon_white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/palewire/django-calaccess-campaign-browser/HEAD/calaccess_campaign_browser/static/calaccess_campaign_browser/images/party_icons/no-party-preference_icon_white.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | _build/ 3 | .tox/ 4 | example/project/settings_local.py 5 | example/.static/ 6 | *.pyc 7 | *.swp 8 | .DS_Store 9 | *.egg-info 10 | reference 11 | dist 12 | *.swo 13 | docs/_build 14 | .coverage 15 | .tox 16 | cover 17 | bower_components 18 | node_modules 19 | data 20 | CSV 21 | migrations 22 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist=py27 3 | 4 | [testenv] 5 | deps= 6 | pep8 7 | pyflakes 8 | coverage 9 | django 10 | commands= 11 | pep8 calaccess_campaign_browser 12 | pyflakes calaccess_campaign_browser 13 | python setup.py install 14 | coverage run setup.py test 15 | coverage report -m 16 | -------------------------------------------------------------------------------- /requirements_dev.txt: -------------------------------------------------------------------------------- 1 | Django==1.7 2 | Fabric 3 | MySQL-python 4 | Sphinx 5 | beautifulsoup4 6 | coverage 7 | csvkit 8 | django-calaccess-raw-data==0.1.2 9 | django-debug-toolbar 10 | django-haystack 11 | django-tastypie 12 | hurry.filesize 13 | pep8 14 | pyflakes 15 | pypyodbc==1.3.3 16 | python-coveralls 17 | python-dateutil 18 | requests 19 | six 20 | sphinx-autobuild 21 | tox 22 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/utils/lazyencoder.py: -------------------------------------------------------------------------------- 1 | from django.utils.functional import Promise 2 | from django.utils.encoding import force_text 3 | from django.core.serializers.json import DjangoJSONEncoder 4 | 5 | 6 | class LazyEncoder(DjangoJSONEncoder): 7 | def default(self, obj): 8 | if isinstance(obj, Promise): 9 | return force_text(obj) 10 | return super(LazyEncoder, self).default(obj) 11 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | sudo: false 3 | python: 4 | - "2.7" 5 | env: 6 | - DJANGO_VERSION=1.7.3 7 | - DJANGO_VERSION=1.8 8 | install: 9 | - pip install pep8 pyflakes coverage python-coveralls 10 | - pip install -q Django==$DJANGO_VERSION 11 | - python setup.py install 12 | script: 13 | - pep8 calaccess_campaign_browser 14 | - pyflakes calaccess_campaign_browser 15 | - coverage run setup.py test 16 | after_success: 17 | - coveralls 18 | -------------------------------------------------------------------------------- /example/project/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for project project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "project.settings") 12 | 13 | from django.core.wsgi import get_wsgi_application 14 | application = get_wsgi_application() 15 | -------------------------------------------------------------------------------- /example/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | try: 6 | import pymysql 7 | pymysql.install_as_MySQLdb() 8 | except ImportError: 9 | pass 10 | 11 | if __name__ == "__main__": 12 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "project.settings") 13 | sys.path.append(os.path.dirname(os.path.dirname(__file__))) 14 | from django.core.management import execute_from_command_line 15 | execute_from_command_line(sys.argv) 16 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/css/home.css: -------------------------------------------------------------------------------- 1 | .jumbotron { 2 | background: transparent; 3 | text-align: center; 4 | } 5 | 6 | .jumbotron h1 { 7 | /*display: inline-block;*/ 8 | } 9 | 10 | .jumbotron .logo { 11 | width: 200px; 12 | height: 200px; 13 | background-size: cover; 14 | display: inline-block; 15 | } 16 | 17 | .jumbotron .label { 18 | display: block; 19 | width: 400px; 20 | margin: 30px auto 0 auto; 21 | } -------------------------------------------------------------------------------- /calaccess_campaign_browser/templatetags/calaccesscampaignbrowser.py: -------------------------------------------------------------------------------- 1 | import json 2 | from django import template 3 | from django.forms.models import model_to_dict 4 | from calaccess_campaign_browser.utils.lazyencoder import LazyEncoder 5 | register = template.Library() 6 | 7 | 8 | @register.filter 9 | def jsonify(obj): 10 | """ 11 | Renders a Django model object as a JSON object. 12 | """ 13 | d = model_to_dict(obj) 14 | return json.dumps(d, cls=LazyEncoder) 15 | -------------------------------------------------------------------------------- /example/project/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import patterns, include, url 2 | from django.conf import settings 3 | from django.contrib import admin 4 | admin.autodiscover() 5 | 6 | 7 | urlpatterns = patterns('', 8 | url(r'^', include('calaccess_campaign_browser.urls')), 9 | url(r'^admin/', include(admin.site.urls)), 10 | url(r'^static/(?P.*)$', 'django.views.static.serve', { 11 | 'document_root': settings.STATIC_ROOT, 12 | 'show_indexes': True, 13 | }), 14 | ) 15 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/models/__init__.py: -------------------------------------------------------------------------------- 1 | from contributions import Contribution 2 | from elections import ( 3 | Election, 4 | Candidate, 5 | Office, 6 | Proposition, 7 | PropositionFiler 8 | ) 9 | from expenditures import Expenditure 10 | from filers import Filer, Committee 11 | from filings import Filing, Cycle, Summary 12 | 13 | __all__ = ( 14 | 'Contribution', 15 | 'Election', 16 | 'Candidate', 17 | 'Office', 18 | 'Proposition', 19 | 'PropositionFiler', 20 | 'Expenditure', 21 | 'Committee', 22 | 'Filer', 23 | 'Filing', 24 | 'Cycle', 25 | 'Summary' 26 | ) 27 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/api.py: -------------------------------------------------------------------------------- 1 | from tastypie.resources import ModelResource, ALL 2 | from .models import Filer, Filing 3 | from .utils.serializer import CIRCustomSerializer 4 | 5 | 6 | class FilerResource(ModelResource): 7 | class Meta: 8 | queryset = Filer.objects.all() 9 | serializer = CIRCustomSerializer() 10 | filtering = {'filer_id_raw': ALL} 11 | excludes = ['id'] 12 | 13 | 14 | class FilingResource(ModelResource): 15 | class Meta: 16 | queryset = Filing.objects.all() 17 | serializer = CIRCustomSerializer() 18 | filtering = {'filing_id_raw': ALL} 19 | excludes = ['id'] 20 | -------------------------------------------------------------------------------- /docs/exportingthedata.rst: -------------------------------------------------------------------------------- 1 | Exporting the data 2 | ================== 3 | 4 | This walkthrough will show you how to export the database tables for the ``Contribution`` and ``Expenditure`` models as CSV files. 5 | 6 | This assumes that you have installed the project and ran the ``buildcalaccesscampaignbrowser`` command. 7 | 8 | .. code-block:: bash 9 | 10 | $ python manage.py buildcalaccesscampaignbrowser 11 | 12 | From here, you can now run the export command to get a dump of the data from the database. 13 | 14 | .. code-block:: bash 15 | 16 | $ python manage.py exportcalaccesscampaignbrowser 17 | 18 | 19 | This will export the CSVs as ``YYYY-MM-DD-model.csv`` to ``os.path.join(settings.BASE_DIR, 'data')`` 20 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/management/commands/buildcalaccesscampaignbrowser.py: -------------------------------------------------------------------------------- 1 | from django.core.management import call_command 2 | from calaccess_campaign_browser.management.commands import CalAccessCommand 3 | 4 | 5 | class Command(CalAccessCommand): 6 | help = 'Transforms and loads refined data from raw CAL-ACCESS source files' 7 | 8 | def handle(self, *args, **options): 9 | call_command("flushcalaccesscampaignbrowser") 10 | call_command("loadcalaccesscampaignfilers") 11 | call_command("loadcalaccesscampaignfilings") 12 | call_command("loadcalaccesscampaignsummaries") 13 | call_command("loadcalaccesscampaigncontributions") 14 | call_command("loadcalaccesscampaignexpenditures") 15 | call_command("scrapecalaccesscampaigncandidates") 16 | call_command("scrapecalaccesscampaignpropositions") 17 | self.success("Done!") 18 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/views/__init__.py: -------------------------------------------------------------------------------- 1 | from committees import ( 2 | CommitteeDetailView, 3 | CommitteeContributionView, 4 | CommitteeExpenditureView, 5 | CommitteeFilingView, 6 | ) 7 | from contributions import ContributionDetailView 8 | from expenditures import ExpenditureDetailView 9 | from filings import ( 10 | LatestFilingView, 11 | FilerListView, 12 | FilingDetailView, 13 | FilerDetailView, 14 | ) 15 | from search import SearchList 16 | from parties import PartyListView 17 | 18 | __all__ = ( 19 | 'CommitteeDetailView', 20 | 'CommitteeContributionView', 21 | 'CommitteeExpenditureView', 22 | 'CommitteeFilingView', 23 | 'ContributionDetailView', 24 | 'ExpenditureDetailView', 25 | 'LatestFilingView', 26 | 'FilerListView', 27 | 'FilingDetailView', 28 | 'FilerDetailView', 29 | 'PartyListView', 30 | 'SearchList', 31 | ) 32 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/templates/calaccess_campaign_browser/paginator.html: -------------------------------------------------------------------------------- 1 | {% if is_paginated %} 2 |
3 |
4 |
    5 | {% if page_obj.has_previous %} 6 |
  • «
    • 7 | {% else %} 8 |
  • «
    • 9 | {% endif %} 10 | {% for i in paginator.page_range %} 11 |
  • 12 | {{ i }} 13 |
    • 14 | {% endfor %} 15 | {% if page_obj.has_next %} 16 |
  • »
    • 17 | {% else %} 18 |
  • »
    • 19 | {% endif %} 20 |
21 |
22 |
23 | {% endif %} 24 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/js/jquery.sieve.min.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * jQuery Sieve v0.3.0 (2013-04-04) 3 | * http://rmm5t.github.com/jquery-sieve/ 4 | * Copyright (c) 2013 Ryan McGeary; Licensed MIT 5 | */ 6 | (function(){var e;e=jQuery,e.fn.sieve=function(t){var n;return n=function(e){var t,n,r,o;for(o=[],n=0,r=e.length;r>n;n++)t=e[n],t&&o.push(t);return o},this.each(function(){var r,o,i;return r=e(this),i=e.extend({searchInput:null,searchTemplate:"
",itemSelector:"tbody tr",textSelector:null,toggle:function(e,t){return e.toggle(t)},complete:function(){}},t),i.searchInput||(o=e(i.searchTemplate),i.searchInput=o.find("input"),r.before(o)),i.searchInput.on("keyup.sieve change.sieve",function(){var t,o;return o=n(e(this).val().toLowerCase().split(/\s+/)),t=r.find(i.itemSelector),t.each(function(){var t,n,r,c,l,a,u;for(n=e(this),i.textSelector?(t=n.find(i.textSelector),l=t.text().toLowerCase()):l=n.text().toLowerCase(),r=!0,a=0,u=o.length;u>a;a++)c=o[a],r&&(r=l.indexOf(c)>=0);return i.toggle(n,r)}),i.complete()})})}}).call(this); -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 California Civic Data Coalition 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/templates/calaccess_campaign_browser/search_list.html: -------------------------------------------------------------------------------- 1 | {% extends 'calaccess_campaign_browser/base.html' %} 2 | {% load humanize %} 3 | 4 | {% block content %} 5 | 6 |
7 |
8 |

Search

9 |
10 |
11 | 12 |
13 |
14 |
15 |
16 |
17 | 18 |
19 |
20 |

Contributions by name

21 |
22 |
23 | 24 |
25 |
26 |
28 |
29 | 31 |
32 | 33 |
34 |
35 |
36 | 37 |
38 |
39 |
40 |
41 |
42 | {% endblock %} 43 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/templates/calaccess_campaign_browser/header.html: -------------------------------------------------------------------------------- 1 | {% load staticfiles %} 2 | 3 |
4 |
5 | 17 |
18 |
19 | 20 | 29 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/management/commands/exportcalaccesscampaigncandidates.py: -------------------------------------------------------------------------------- 1 | from csvkit import CSVKitWriter 2 | from django.db import connection 3 | from calaccess_campaign_browser import models 4 | from calaccess_campaign_browser.management.commands import CalAccessCommand 5 | 6 | 7 | class Command(CalAccessCommand): 8 | help = 'Export candidates scraped from the state site' 9 | 10 | def handle(self, *args, **options): 11 | self.cursor = connection.cursor() 12 | sql = """ 13 | SELECT 14 | o.name, 15 | o.seat, 16 | f.filer_id_raw, 17 | f.xref_filer_id, 18 | f.name, 19 | f.party 20 | FROM %(candidate)s as c 21 | INNER JOIN %(office)s as o 22 | ON c.office_id = o.id 23 | INNER JOIN %(filer)s as f 24 | ON c.filer_id = f.id 25 | """ % dict( 26 | candidate=models.Candidate._meta.db_table, 27 | office=models.Office._meta.db_table, 28 | filer=models.Filer._meta.db_table, 29 | ) 30 | self.cursor.execute(sql) 31 | writer = CSVKitWriter(open("./candidates.csv", 'wb')) 32 | writer.writerow([ 33 | 'office_name', 34 | 'office_seat', 35 | 'filer_id', 36 | 'xref_filer_id', 37 | 'name', 38 | 'party' 39 | ]) 40 | writer.writerows(self.cursor.fetchall()) 41 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/management/commands/flushcalaccesscampaignbrowser.py: -------------------------------------------------------------------------------- 1 | from django.db import connection 2 | from calaccess_campaign_browser import models 3 | from calaccess_campaign_browser.management.commands import CalAccessCommand 4 | 5 | 6 | class Command(CalAccessCommand): 7 | help = "Flush CAL-ACCESS campaign browser database tables" 8 | 9 | def handle(self, *args, **options): 10 | self.header("Flushing CAL-ACCESS campaign browser database tables") 11 | c = connection.cursor() 12 | c.execute("""SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0;""") 13 | c.execute("""SET FOREIGN_KEY_CHECKS = 0;""") 14 | model_list = [ 15 | models.Filer, 16 | models.Filing, 17 | models.Summary, 18 | models.Cycle, 19 | models.Committee, 20 | models.Contribution, 21 | models.Expenditure, 22 | models.Election, 23 | models.Office, 24 | models.Candidate, 25 | models.Proposition, 26 | models.PropositionFiler, 27 | ] 28 | sql = """TRUNCATE `%s`;""" 29 | for m in model_list: 30 | self.log(" %s" % m.__name__) 31 | c.execute(sql % m._meta.db_table) 32 | # Revert database to default "note" warning behavior 33 | c.execute("""SET SQL_NOTES=@OLD_SQL_NOTES;""") 34 | c.execute("""SET FOREIGN_KEY_CHECKS = 1;""") 35 | -------------------------------------------------------------------------------- /example/toolbox/management/commands/exportcalaccesscampaigncandidates.py: -------------------------------------------------------------------------------- 1 | from csvkit import CSVKitWriter 2 | from django.db import connection 3 | from calaccess_campaign_browser import models 4 | from calaccess_campaign_browser.management.commands import CalAccessCommand 5 | 6 | 7 | class Command(CalAccessCommand): 8 | help = 'Export candidates scraped from the state site' 9 | 10 | def handle(self, *args, **options): 11 | self.cursor = connection.cursor() 12 | sql = """ 13 | SELECT DISTINCT 14 | o.name, 15 | o.seat, 16 | f.filer_id_raw, 17 | f.xref_filer_id, 18 | f.name, 19 | f.party 20 | FROM %(candidate)s as c 21 | INNER JOIN %(office)s as o 22 | ON c.office_id = o.id 23 | INNER JOIN %(filer)s as f 24 | ON c.filer_id = f.id 25 | """ % dict( 26 | candidate=models.Candidate._meta.db_table, 27 | office=models.Office._meta.db_table, 28 | filer=models.Filer._meta.db_table, 29 | ) 30 | self.cursor.execute(sql) 31 | writer = CSVKitWriter(open("./candidates.csv", 'wb')) 32 | writer.writerow([ 33 | 'office_name', 34 | 'office_seat', 35 | 'filer_id', 36 | 'xref_filer_id', 37 | 'name', 38 | 'party' 39 | ]) 40 | writer.writerows(self.cursor.fetchall()) 41 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | .PHONY: bootstrap docs reload rs runserver shell test 3 | 4 | bootstrap: 5 | mysqladmin -h localhost -u root -pmysql drop campaign_finance 6 | mysqladmin -h localhost -u root -pmysql create campaign_finance 7 | python example/manage.py syncdb 8 | python example/manage.py build_campaign_finance 9 | python example/manage.py collectstatic --noinput 10 | python example/manage.py runserver 11 | 12 | docs: 13 | cd docs && make livehtml 14 | 15 | reload: 16 | clear 17 | python example/manage.py dropcalaccesscampaignbrowser 18 | python example/manage.py migrate --noinput 19 | python example/manage.py loadcalaccesscampaignfilers 20 | 21 | rs: 22 | python example/manage.py runserver 23 | 24 | runserver: 25 | python example/manage.py runserver 26 | 27 | shell: 28 | python example/manage.py shell 29 | 30 | sh: 31 | python example/manage.py shell_plus 32 | 33 | test: 34 | clear 35 | pep8 --exclude='*/migrations' calaccess_campaign_browser 36 | pyflakes calaccess_campaign_browser 37 | coverage run setup.py test 38 | coverage report -m 39 | 40 | downloaddb: 41 | echo "Downloading database archive" 42 | curl -O https://dl.dropboxusercontent.com/u/3640647/nicar15/ccdc.sql.gz 43 | echo "Creating local database named 'calaccess'" 44 | mysqladmin -h localhost -u root -p create calaccess 45 | echo "Installing database archive to local database" 46 | gunzip < ccdc.sql.gz | mysql calaccess -u root -p 47 | echo "Deleting database archive" 48 | rm ccdc.sql.gz 49 | echo "Success!" 50 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/management/commands/dropcalaccesscampaignbrowser.py: -------------------------------------------------------------------------------- 1 | from django.db import connection 2 | from calaccess_campaign_browser import models 3 | from calaccess_campaign_browser.management.commands import CalAccessCommand 4 | 5 | 6 | class Command(CalAccessCommand): 7 | help = "Drops all CAL-ACCESS campaign browser database tables" 8 | 9 | def handle(self, *args, **options): 10 | self.header("Dropping CAL-ACCESS campaign browser database tables") 11 | self.cursor = connection.cursor() 12 | 13 | # Ignore MySQL "note" warnings so this can be run with DEBUG=True 14 | self.cursor.execute("""SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0;""") 15 | 16 | # Loop through the models and drop all the tables 17 | model_list = [ 18 | models.Contribution, 19 | models.Expenditure, 20 | models.Summary, 21 | models.Filing, 22 | models.Committee, 23 | models.Filer, 24 | models.Cycle, 25 | models.Election, 26 | models.Office, 27 | models.Candidate, 28 | models.Proposition, 29 | models.PropositionFiler, 30 | ] 31 | sql = """DROP TABLE IF EXISTS `%s`;""" 32 | for m in model_list: 33 | self.log(" %s" % m.__name__) 34 | self.cursor.execute(sql % m._meta.db_table) 35 | 36 | # Revert database to default "note" warning behavior 37 | self.cursor.execute("""SET SQL_NOTES=@OLD_SQL_NOTES;""") 38 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/utils/serializer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tastypie serializer class for CSV exports and pretty priting JSON 3 | Use it as a starting point for other custom serializers in a project 4 | """ 5 | import csv 6 | import json 7 | from django.http import HttpResponse 8 | from django.core.serializers.json import DjangoJSONEncoder 9 | from tastypie.serializers import Serializer 10 | 11 | 12 | class CIRCustomSerializer(Serializer): 13 | json_indent = 2 14 | formats = Serializer.formats + ['csv'] 15 | content_types = dict( 16 | Serializer.content_types.items() + [('csv', 'text/csv')] 17 | ) 18 | 19 | def to_json(self, data, options=None): 20 | options = options or {} 21 | data = self.to_simple(data, options) 22 | return json.dumps( 23 | data, 24 | cls=DjangoJSONEncoder, 25 | sort_keys=True, 26 | ensure_ascii=False, 27 | indent=self.json_indent 28 | ) 29 | 30 | def to_csv(self, data, options=None): 31 | """ 32 | Given some Python data, produces JSON output. 33 | """ 34 | response = HttpResponse() 35 | options = options or {} 36 | data = self.to_simple(data, options) 37 | writer = csv.writer(response) 38 | 39 | writer.writerow(data['objects'][0].keys()) 40 | for item in data['objects']: 41 | writer.writerow( 42 | [unicode(item[key]).encode('utf-8', 'replace') 43 | for key 44 | in item.keys()] 45 | ) 46 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | include = 3 | calaccess_campaign_browser/admin.py 4 | calaccess_campaign_browser/api.py 5 | calaccess_campaign_browser/apps.py 6 | calaccess_campaign_browser/managers.py 7 | calaccess_campaign_browser/models.py 8 | calaccess_campaign_browser/urls.py 9 | calaccess_campaign_browser/views/base.py 10 | calaccess_campaign_browser/views/committees.py 11 | calaccess_campaign_browser/views/contributions.py 12 | calaccess_campaign_browser/views/expenditures.py 13 | calaccess_campaign_browser/views/filings.py 14 | calaccess_campaign_browser/views/__init__.py 15 | calaccess_campaign_browser/views/parties.py 16 | calaccess_campaign_browser/views/search.py 17 | calaccess_campaign_browser/management/commands/buildcalaccesscampaignbrowser.py 18 | calaccess_campaign_browser/management/commands/dropcalaccesscampaignbrowser.py 19 | calaccess_campaign_browser/management/commands/exportcalaccesscampaignbrowser.py 20 | calaccess_campaign_browser/management/commands/flushcalaccesscampaignbrowser.py 21 | calaccess_campaign_browser/management/commands/loadcalaccesscampaigncontributions.py 22 | calaccess_campaign_browser/management/commands/loadcalaccesscampaignexpenditures.py 23 | calaccess_campaign_browser/management/commands/loadcalaccesscampaignfilings.py 24 | calaccess_campaign_browser/management/commands/loadcalaccesscampaignfilers.py 25 | calaccess_campaign_browser/management/commands/loadcalaccesscampaignsummaries.py 26 | calaccess_campaign_browser/management/commands/scrapecalaccesscampaigncandidates.py 27 | calaccess_campaign_browser/management/commands/scrapecalaccesscampaignelectiondates.py 28 | calaccess_campaign_browser/management/commands/scrapecalaccesscampaignpropositions.py 29 | 30 | [report] 31 | exclude_lines = 32 | pragma: no cover 33 | add_introspection_rules 34 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # django-calaccess-campaign-browser 2 | 3 | A Django app to refine, review and investigate campaign finance data drawn from the California Secretary of State’s CAL-ACCESS database. 4 | 5 | **This is a work in progress. Its analysis should be considered as provisional until it is further tested and debugged.** 6 | 7 | [![Docs status](https://readthedocs.org/projects/django-calaccess-campaign-browser/badge/)](http://django-calaccess-campaign-browser.californiacivicdata.org/) 8 | [![Build Status](https://travis-ci.org/california-civic-data-coalition/django-calaccess-campaign-browser.png?branch=master)](https://travis-ci.org/california-civic-data-coalition/django-calaccess-campaign-browser) 9 | [![PyPI version](https://badge.fury.io/py/django-calaccess-campaign-browser.png)](http://badge.fury.io/py/django-calaccess-campaign-browser) 10 | [![Coverage Status](https://coveralls.io/repos/california-civic-data-coalition/django-calaccess-campaign-browser/badge.png?branch=master)](https://coveralls.io/r/california-civic-data-coalition/django-calaccess-campaign-browser?branch=master) 11 | 12 | * Documentation: [django-calaccess-campaign-browser.californiacivicdata.org](http://django-calaccess-campaign-browser.californiacivicdata.org//) 13 | * Issues: [github.com/california-civic-data-coalition/django-calaccess-campaign-browser/issues](https://github.com/california-civic-data-coalition/django-calaccess-campaign-browser/issues) 14 | * Packaging: [pypi.python.org/pypi/django-calaccess-campaign-browser](https://pypi.python.org/pypi/django-calaccess-campaign-browser) 15 | * Testing: [travis-ci.org/california-civic-data-coalition/django-calaccess-campaign-browser](https://travis-ci.org/california-civic-data-coalition/django-calaccess-campaign-browser) 16 | * Coverage: [coveralls.io/r/california-civic-data-coalition/django-calaccess-campaign-browser](https://coveralls.io/r/california-civic-data-coalition/django-calaccess-campaign-browser) 17 | -------------------------------------------------------------------------------- /example/project/settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | BASE_DIR = os.path.dirname(os.path.dirname(__file__)) 3 | SECRET_KEY = 'r269$heh9at2cot+5l$*$4&xzwsfbbg0&&^prr+e&oh)_4-+ga' 4 | DEBUG = True 5 | TEMPLATE_DEBUG = True 6 | LANGUAGE_CODE = 'en-us' 7 | TIME_ZONE = 'UTC' 8 | USE_I18N = True 9 | USE_L10N = True 10 | USE_TZ = True 11 | STATIC_ROOT = os.path.join(BASE_DIR, ".static") 12 | STATIC_URL = '/static/' 13 | ALLOWED_HOSTS = [ 14 | 'localhost', 15 | '127.0.0.1', 16 | ] 17 | ROOT_URLCONF = 'project.urls' 18 | WSGI_APPLICATION = 'project.wsgi.application' 19 | 20 | INSTALLED_APPS = ( 21 | 'django.contrib.admin', 22 | 'django.contrib.auth', 23 | 'django.contrib.contenttypes', 24 | 'django.contrib.sessions', 25 | 'django.contrib.messages', 26 | 'django.contrib.staticfiles', 27 | 'django.contrib.humanize', 28 | 'calaccess_raw', 29 | 'calaccess_campaign_browser', 30 | 'toolbox', 31 | ) 32 | 33 | MIDDLEWARE_CLASSES = ( 34 | 'django.contrib.sessions.middleware.SessionMiddleware', 35 | 'django.middleware.common.CommonMiddleware', 36 | 'django.middleware.csrf.CsrfViewMiddleware', 37 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 38 | 'django.contrib.messages.middleware.MessageMiddleware', 39 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 40 | ) 41 | 42 | DATABASES = { 43 | 'default': { 44 | 'ENGINE': 'django.db.backends.mysql', 45 | 'NAME': 'calaccess', 46 | 'USER': '', 47 | 'PASSWORD': '', 48 | 'HOST': '', 49 | 'PORT': '3306', 50 | 'OPTIONS': { 51 | 'local_infile': 1, 52 | } 53 | } 54 | } 55 | 56 | BUILD_DIR = os.path.join(BASE_DIR, '..', '_build') 57 | BAKERY_VIEWS = ( 58 | 'campaign_finance.views.IndexView', 59 | ) 60 | 61 | LANGUAGE_CODE = 'en-us' 62 | TIME_ZONE = 'UTC' 63 | USE_I18N = True 64 | USE_L10N = True 65 | USE_TZ = True 66 | STATIC_URL = '/static/' 67 | 68 | try: 69 | from settings_local import * 70 | except ImportError: 71 | pass 72 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | from setuptools import setup, find_packages 4 | from distutils.core import Command 5 | 6 | 7 | class TestCommand(Command): 8 | user_options = [] 9 | 10 | def initialize_options(self): 11 | pass 12 | 13 | def finalize_options(self): 14 | pass 15 | 16 | def run(self): 17 | from django.conf import settings 18 | settings.configure( 19 | DATABASES={ 20 | 'default': { 21 | 'NAME': ':memory:', 22 | 'ENGINE': 'django.db.backends.sqlite3' 23 | } 24 | }, 25 | INSTALLED_APPS=('calaccess_campaign_browser',), 26 | MIDDLEWARE_CLASSES=() 27 | ) 28 | from django.core.management import call_command 29 | import django 30 | if django.VERSION[:2] >= (1, 7): 31 | django.setup() 32 | call_command('test', 'calaccess_campaign_browser') 33 | 34 | 35 | setup( 36 | name='django-calaccess-campaign-browser', 37 | version='0.1.1', 38 | license='MIT', 39 | description='A Django app to refine and investigate campaign finance data \ 40 | drawn from the California Secretary of State’s CAL-ACCESS database. \ 41 | This is a work in progress. Its analysis should be considered as \ 42 | provisional until it is further tested and debugged.', 43 | url='http://django-calaccess-campaign-browser.californiacivicdata.org', 44 | author='California Civic Data Coalition', 45 | author_email='awilliams@cironline.org', 46 | packages=find_packages(), 47 | include_package_data=True, 48 | zip_safe=False, # because we're including static files 49 | install_requires=( 50 | 'django-calaccess-raw-data==0.1.2', 51 | 'django==1.7', 52 | 'csvkit>=0.6.1', 53 | 'python-dateutil==2.2', 54 | 'mysqlclient>=1.3.6', 55 | 'hurry.filesize>=0.9', 56 | 'django-tastypie>=0.11.1', 57 | 'beautifulsoup4>=4.3.2', 58 | 'pypyodbc==1.3.3', 59 | ), 60 | cmdclass={'test': TestCommand,} 61 | ) 62 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/templates/calaccess_campaign_browser/base.html: -------------------------------------------------------------------------------- 1 | {% load staticfiles %} 2 | 3 | 4 | 5 | 6 | 7 | 8 | {% block title %}CAL-ACCESS Campaign Browser{% endblock%} 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | {% include 'calaccess_campaign_browser/header.html' %} 18 | 19 | {% block container %} 20 |
21 | {% block content %} 22 | {% endblock %} 23 |
24 | {% endblock %} 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | {% block javascript %} 35 | {% endblock %} 36 | 37 | 38 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/templates/calaccess_campaign_browser/latest.html: -------------------------------------------------------------------------------- 1 | {% extends 'calaccess_campaign_browser/base.html' %} 2 | {% load humanize %} 3 | 4 | {% block title %}Latest - {{ block.super }}{% endblock %} 5 | 6 | {% block content %} 7 |
8 |
9 |

Latest

10 |
11 |
12 | 13 | {% include "calaccess_campaign_browser/paginator.html" %} 14 | 15 |
16 |
17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | {% for object in object_list %} 31 | 32 | 35 | 36 | 39 | 40 | 41 | 44 | 47 | 48 | {% endfor %} 49 | 50 |
FilingFiledCommitteeFormPeriodContributionsExpenditures
33 | {{ object.filing_id_raw }} 34 | {{ object.date_filed|date:"Y-m-d" }} 37 | {{ object.committee.short_name }} 38 | {{ object.get_form_type_display }}{{ object.start_date|date:"Y-m-d" }} - {{ object.end_date|date:"Y-m-d" }} 42 | ${{ object.total_contributions|default:0|floatformat:0|intcomma }} 43 | 45 | ${{ object.total_expenditures|default:0|floatformat:0|intcomma }} 46 |
51 |
52 |
53 | 54 | {% include "calaccess_campaign_browser/paginator.html" %} 55 | {% endblock %} 56 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/tests.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from django.test import TestCase 3 | from calaccess_campaign_browser import models 4 | 5 | 6 | class ModelTest(TestCase): 7 | """ 8 | Create model objects and try out their attributes. 9 | """ 10 | def test_models(self): 11 | obj = models.Filer.objects.create( 12 | name="FooPAC", 13 | filer_id_raw=1, 14 | xref_filer_id=1, 15 | filer_type="PAC", 16 | party='0', 17 | status='A', 18 | effective_date=datetime.now() 19 | ) 20 | obj.__unicode__() 21 | obj.slug 22 | obj.real_filings 23 | obj.total_contributions 24 | obj.meta() 25 | obj.klass() 26 | obj.doc() 27 | obj.to_dict() 28 | obj.to_json() 29 | obj.short_name 30 | obj.clean_name 31 | 32 | def test_committee(self): 33 | filer = models.Filer.objects.create( 34 | name="Foo Nixon", 35 | filer_id_raw=1, 36 | xref_filer_id=1, 37 | filer_type="cand", 38 | party='16002', 39 | status='A', 40 | effective_date=datetime.now() 41 | ) 42 | committee = models.Committee.objects.create( 43 | name='Nixon for Governor', 44 | filer=filer, 45 | filer_id_raw=filer.filer_id_raw, 46 | xref_filer_id=filer.xref_filer_id, 47 | committee_type=filer.filer_type, 48 | party=filer.party, 49 | status='Y', 50 | level_of_government='40502', 51 | effective_date=filer.effective_date, 52 | ) 53 | committee.__unicode__() 54 | 55 | def test_cycle(self): 56 | pass 57 | 58 | def test_filingperiod(self): 59 | pass 60 | 61 | def test_filing(self): 62 | pass 63 | 64 | def test_summary(self): 65 | pass 66 | 67 | def test_contribution(self): 68 | pass 69 | 70 | def test_office(self): 71 | pass 72 | 73 | def test_candidate(self): 74 | pass 75 | 76 | def test_proposition(self): 77 | pass 78 | 79 | def test_propositionfiler(self): 80 | pass 81 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/templates/calaccess_campaign_browser/filer_detail.html: -------------------------------------------------------------------------------- 1 | {% extends 'calaccess_campaign_browser/base.html' %} 2 | {% load humanize %} 3 | 4 | {% block title %}{{ filer }} - Filers - {{ block.super }}{% endblock %} 5 | 6 | {% block content %} 7 | 8 |
9 |
10 |
    11 |
  1. All filers
    2. 12 |
  2. Filer committees
    3. 13 |
14 |

{{ filer }}

15 | 16 |
17 |
18 | 19 |
20 |
21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | {% for committee in filer.committee_set.all %} 33 | 34 | 39 | 40 | 41 | 44 | 47 | 48 | {% endfor %} 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 |
Filer IDCommittee nameTypeContributionsExpenditures
35 | 36 | {{ committee.filer_id_raw }} 37 | 38 | {{ committee.short_name }}{{ committee.get_committee_type_display }} 42 | ${{ committee.total_contributions|default:0|floatformat:0|intcomma }} 43 | 45 | ${{ committee.total_expenditures|default:0|floatformat:0|intcomma }} 46 |
Totals${{ contributions_total|default:0|floatformat:0|intcomma }}${{ expenditures_total|default:0|floatformat:0|intcomma }}
58 |
59 |
60 | {% endblock %} 61 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/views/search.py: -------------------------------------------------------------------------------- 1 | import re 2 | from django.db.models import Q 3 | from django.views.generic import TemplateView 4 | from django.shortcuts import render 5 | from calaccess_campaign_browser.models import Contribution 6 | 7 | findterms = re.compile(r'"([^"]+)"|(\S+)').findall 8 | normspace = re.compile(r'\s{2,}').sub 9 | 10 | 11 | class SearchList(TemplateView): 12 | template_name = "calaccess_campaign_browser/search_list.html" 13 | 14 | 15 | def normalize_query(query_string,): 16 | """ 17 | Splits the query string in invidual keywords, getting rid of unecessary 18 | spaces and grouping quoted words together. 19 | 20 | Example: 21 | 22 | >>> normalize_query(' some random words "with quotes " and spaces') 23 | ['some', 'random', 'words', 'with quotes', 'and', 'spaces'] 24 | """ 25 | return [ 26 | normspace(' ', (t[0] or t[1]).strip()) for t in findterms(query_string) 27 | ] 28 | 29 | 30 | def get_query(query_string, search_fields): 31 | """ 32 | Returns a query, that is a combination of Q objects. That combination 33 | aims to search keywords within a model by testing the given search fields. 34 | """ 35 | query = None # Query to search for every search term 36 | terms = normalize_query(query_string) 37 | for term in terms: 38 | or_query = None # Query to search for a given term in each field 39 | for field_name in search_fields: 40 | q = Q(**{"%s__icontains" % field_name: term}) 41 | if or_query is None: 42 | or_query = q 43 | else: 44 | or_query = or_query | q 45 | if query is None: 46 | query = or_query 47 | else: 48 | query = query & or_query 49 | return query 50 | 51 | 52 | def search_contribs_by_name(request): 53 | query_string = '' 54 | results = None 55 | if ('q' in request.GET) and request.GET['q'].strip(): 56 | query_string = request.GET['q'] 57 | query = get_query(query_string, [ 58 | 'contributor_city', 'contributor_zipcode', 59 | 'contributor_first_name', 'contributor_last_name', 60 | 'contributor_employer', 'contributor_occupation' 61 | ]) 62 | results = Contribution.real.filter(query).order_by("-date_received") 63 | context = { 64 | 'query_string': query_string, 65 | 'results': results 66 | } 67 | template = 'calaccess_campaign_browser/search_contribs_by_name.html' 68 | return render(request, template, context) 69 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/utils/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from django.template.defaultfilters import title 3 | from django.utils.datastructures import SortedDict 4 | from calaccess_campaign_browser.templatetags.calaccesscampaignbrowser import ( 5 | jsonify 6 | ) 7 | 8 | 9 | class BaseModel(models.Model): 10 | 11 | class Meta: 12 | abstract = True 13 | 14 | def meta(self): 15 | return self._meta 16 | 17 | def klass(self): 18 | return self.__class__ 19 | 20 | def doc(self): 21 | return self.__doc__ 22 | 23 | def to_dict(self): 24 | d = SortedDict({}) 25 | for f in self._meta.fields: 26 | d[f.verbose_name] = getattr(self, f.name) 27 | return d 28 | 29 | def to_json(self): 30 | return jsonify(self) 31 | 32 | 33 | class AllCapsNameMixin(BaseModel): 34 | """ 35 | Abstract model with name cleaners we can reuse across models. 36 | """ 37 | class Meta: 38 | abstract = True 39 | 40 | def __unicode__(self): 41 | return self.clean_name 42 | 43 | @property 44 | def short_name(self, character_limit=60): 45 | if len(self.clean_name) > character_limit: 46 | return self.clean_name[:character_limit] + "..." 47 | return self.clean_name 48 | 49 | @property 50 | def clean_name(self): 51 | """ 52 | A cleaned up version of the ALL CAPS names that are provided by 53 | the source data. 54 | """ 55 | n = self.name 56 | n = n.strip() 57 | n = n.lower() 58 | n = title(n) 59 | n = n.replace("A. D.", "A.D.") 60 | force_lowercase = ['Of', 'For', 'To', 'By'] 61 | for fl in force_lowercase: 62 | s = [] 63 | for p in n.split(" "): 64 | if p in force_lowercase: 65 | s.append(p.lower()) 66 | else: 67 | s.append(p) 68 | n = " ".join(s) 69 | force_uppercase = [ 70 | 'Usaf', 'Pac', 'Ca', 'Ad', 'Rcc', 'Cdp', 'Aclu', 71 | 'Cbpa-Pac', 'Aka', 'Aflac', 72 | ] 73 | for fl in force_uppercase: 74 | s = [] 75 | for p in n.split(" "): 76 | if p in force_uppercase: 77 | s.append(p.upper()) 78 | else: 79 | s.append(p) 80 | n = " ".join(s) 81 | n = n.replace("Re-Elect", "Re-elect") 82 | n = n.replace("Political Action Committee", "PAC") 83 | return n 84 | -------------------------------------------------------------------------------- /docs/howtocontribute.rst: -------------------------------------------------------------------------------- 1 | How to contribute 2 | ================= 3 | 4 | This walkthrough will show you how to install the source code of this application 5 | to fix bugs and develop new features. 6 | 7 | First create a new virtualenv. 8 | 9 | .. code-block:: bash 10 | 11 | $ virtualenv django-calaccess-campaign-browser 12 | 13 | Jump in. 14 | 15 | .. code-block:: bash 16 | 17 | $ cd django-calaccess-campaign-browser 18 | $ . bin/activate 19 | 20 | Clone the repository from `GitHub `_. 21 | 22 | .. code-block:: bash 23 | 24 | $ git clone https://github.com/california-civic-data-coalition/django-calaccess-campaign-browser.git repo 25 | 26 | Installing the database 27 | ----------------------- 28 | 29 | Move into the repository and install the Python dependencies. 30 | 31 | .. code-block:: bash 32 | 33 | $ cd repo 34 | $ pip install -r requirements_dev.txt 35 | 36 | Make sure you have MySQL installed. If you don't, now is the time to hit Google and figure out how. If 37 | you're using Apple's OSX operating system, you can `install via Homebrew `_. If you need to clean up after a previous MySQL installation, `this might help `_. 38 | 39 | Then create a new database named ``calaccess``. 40 | 41 | .. code-block:: bash 42 | 43 | $ mysqladmin -h localhost -u root -p create calaccess 44 | 45 | If you have a different username, substitute it above. You'll be prompted for that user's mysql password. 46 | 47 | Create a file at ``example/project/settings_local.py`` to save your custom database credentials. That might look something like this. 48 | 49 | .. code-block:: python 50 | 51 | DATABASES = { 52 | 'default': { 53 | 'ENGINE': 'django.db.backends.mysql', 54 | 'NAME': 'calaccess', 55 | 'USER': 'your-username-here', 56 | 'PASSWORD': 'your-password-here', 57 | 'HOST': 'localhost', 58 | 'PORT': '3306', 59 | 'OPTIONS': { 60 | 'local_infile': 1, 61 | } 62 | } 63 | } 64 | 65 | Finally create your database and get to work. 66 | 67 | .. code-block:: bash 68 | 69 | $ python example/manage.py migrate 70 | 71 | Loading the data 72 | ---------------- 73 | 74 | You might start by loading the data dump from the web. 75 | 76 | .. code-block:: bash 77 | 78 | $ python example/manage.py downloadcalaccessrawdata 79 | 80 | Then you can build the campaign finance models 81 | 82 | .. code-block:: bash 83 | 84 | $ python example/manage.py buildcalaccesscampaignbrowser 85 | 86 | And fire up the Django test server to use the browser 87 | 88 | .. code-block:: bash 89 | 90 | $ python example/manage.py collectstatic 91 | $ python example/manage.py runserver 92 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/templates/calaccess_campaign_browser/search_contribs_by_name.html: -------------------------------------------------------------------------------- 1 | {% extends 'calaccess_campaign_browser/base.html' %} 2 | {% load humanize %} 3 | 4 | {% block content %} 5 |
6 |
7 |

Search contributions by name

8 |
9 |
10 | 11 |
12 |
13 |
14 | 17 | 18 |
19 |
20 |
21 | 22 |
23 |
24 | Results: {{ results|length }} 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | {% for i in results %} 43 | 44 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 62 | 63 | 64 | {% endfor %} 65 | 66 |
IDDateFirst NameLast NameEmployerOccupationCityStateZIP CodeCommitteeAmount
45 | 46 | {{ i.transaction_id }} 47 | 48 | {{ i.date_received|date:'Y-m-d' }}{{ i.contributor_first_name }}{{ i.contributor_last_name }}{{ i.contributor_employer }}{{ i.contributor_occupation }}{{ i.contributor_city }}{{ i.contributor_state }}{{ i.contributor_zipcode }} 58 | 59 | {{ i.committee }} 60 | 61 | ${{ i.amount|default:0|floatformat:0|intcomma }}
67 |
68 |
69 | {% endblock %} 70 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/templates/calaccess_campaign_browser/committee_expenditure_list.html: -------------------------------------------------------------------------------- 1 | {% extends 'calaccess_campaign_browser/base.html' %} 2 | {% load humanize %} 3 | 4 | {% block title %}Expenditures - {{ committee.name }} - Committee - {{ block.super }}{% endblock %} 5 | 6 | 7 | {% block javascript %} 8 | 20 | {% endblock %} 21 | 22 | {% block content %} 23 | 24 | {% include "calaccess_campaign_browser/committee_nav.html" %} 25 | 26 |
27 |
28 |

29 | Expenditures 30 |
31 | 34 | 38 |
39 |

40 |

Total expenditures: ${{committee.total_expenditures | intcomma}}

41 |
42 |
43 | 44 | {% include "calaccess_campaign_browser/paginator.html" %} 45 | 46 |
47 |
48 |
49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | {% for expenditure in committee_expenditures %} 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | {% endfor %} 69 | 70 |
cyclefilingraw_org_nameamount
{{ expenditure.cycle }}{{ expenditure.filing.pk }}{{ expenditure.raw_org_name }}${{ expenditure.amount | intcomma }}
71 |
72 |
73 |
74 | 75 | {% include "calaccess_campaign_browser/paginator.html" %} 76 | {% endblock %} 77 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/views/filings.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from .search import get_query 3 | from django.views import generic 4 | from django.core.urlresolvers import reverse 5 | from django.shortcuts import redirect 6 | from calaccess_campaign_browser.models import Filer, Filing 7 | 8 | 9 | class LatestFilingView(generic.ListView): 10 | template_name = 'calaccess_campaign_browser/latest.html' 11 | 12 | def get_queryset(self, *args, **kwargs): 13 | next_year = datetime.date.today() + datetime.timedelta(days=365) 14 | return Filing.objects.exclude( 15 | date_filed__gt=next_year 16 | ).select_related("committee").order_by("-date_filed")[:500] 17 | 18 | 19 | class FilerListView(generic.ListView): 20 | template_name = "filer_list" 21 | allow_empty = True 22 | paginate_by = 100 23 | 24 | def get_queryset(self): 25 | qs = Filer.objects.exclude(name="") 26 | if ('q' in self.request.GET) and self.request.GET['q'].strip(): 27 | query = get_query(self.request.GET['q'], [ 28 | 'name', 'filer_id_raw', 'xref_filer_id' 29 | ]) 30 | qs = qs.filter(query) 31 | if ('t' in self.request.GET) and self.request.GET['t'].strip(): 32 | qs = qs.filter(filer_type=self.request.GET['t']) 33 | if ('p' in self.request.GET) and self.request.GET['p'].strip(): 34 | qs = qs.filter(party=self.request.GET['p']) 35 | return qs 36 | 37 | def get_context_data(self, **kwargs): 38 | context = super(FilerListView, self).get_context_data(**kwargs) 39 | if ('q' in self.request.GET) and self.request.GET['q'].strip(): 40 | context['query_string'] = self.request.GET['q'] 41 | if ('t' in self.request.GET) and self.request.GET['t'].strip(): 42 | context['type'] = self.request.GET['t'] 43 | if ('p' in self.request.GET) and self.request.GET['p'].strip(): 44 | context['party'] = self.request.GET['p'] 45 | context.update(dict( 46 | base_url=reverse("filer_list"), 47 | type_list=sorted(Filer.FILER_TYPE_CHOICES, key=lambda x: x[1]), 48 | party_list=sorted(Filer.PARTY_CHOICES, key=lambda x: x[1]), 49 | )) 50 | return context 51 | 52 | 53 | class FilingDetailView(generic.DetailView): 54 | model = Filing 55 | 56 | 57 | class FilerDetailView(generic.DetailView): 58 | model = Filer 59 | 60 | def get_context_data(self, **kwargs): 61 | context = super(FilerDetailView, self).get_context_data(**kwargs) 62 | context['contributions_total'] = sum([ 63 | i.total_contributions for i in self.object.committee_set.all() 64 | ]) 65 | context['expenditures_total'] = sum([ 66 | i.total_expenditures for i in self.object.committee_set.all() 67 | ]) 68 | return context 69 | 70 | def render_to_response(self, context): 71 | if context['object'].committee_set.count() == 1: 72 | return redirect( 73 | context['object'].committee_set.all()[0].get_absolute_url() 74 | ) 75 | return super(FilerDetailView, self).render_to_response(context) 76 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/templates/calaccess_campaign_browser/committee_contribution_list.html: -------------------------------------------------------------------------------- 1 | {% extends 'calaccess_campaign_browser/base.html' %} 2 | {% load humanize %} 3 | 4 | {% block title %}Contributions - {{ committee.name }} - Committee - {{ block.super }}{% endblock %} 5 | 6 | {% block javascript %} 7 | 19 | {% endblock %} 20 | 21 | {% block content %} 22 | {% include "calaccess_campaign_browser/committee_nav.html" %} 23 | 24 |
25 |
26 |

27 | Contributions 28 |
29 | 32 | 36 |
37 |

38 |

Total contributions: ${{committee.total_contributions | intcomma}}

39 |
40 |
41 | 42 | {% include "calaccess_campaign_browser/paginator.html" %} 43 | 44 |
45 |
46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | {% for obj in object_list %} 56 | 57 | 58 | 67 | 72 | 73 | {% endfor %} 74 | 75 |
DateNameAmount
{{ obj.date_received|date:"Y-m-d" }} 59 | {% if obj.contributor_committee_id %} 60 | 61 | {% endif %} 62 | {{ obj.contributor_full_name }} 63 | {% if obj.contributor_committee_id %} 64 | 65 | {% endif %} 66 | 68 | 69 | ${{ obj.amount | intcomma }} 70 | 71 |
76 |
77 |
78 | {% include "calaccess_campaign_browser/paginator.html" %} 79 | {% endblock %} 80 | -------------------------------------------------------------------------------- /docs/sqlserver.rst: -------------------------------------------------------------------------------- 1 | Importing to Microsoft SQL Server 2 | =============================== 3 | 4 | This walkthough will show you how to import the exported CSV data to `Microsoft SQL Server `_. This guide assumes you already have SQL server installed and running. 5 | 6 | First build the browser and export the tables: 7 | 8 | .. code-block:: bash 9 | 10 | $ python manage.py buildcalaccesscampaignbrowser 11 | $ python manage.py exportcalaccesscampaignbrowser 12 | 13 | Setting up pypyodbc 14 | ------------------- 15 | In order to connect to SQL Server from the application, we'll need to install `pypyodbc `_, an `open database connectivity` library that allows Python to communicate with multiple databases. 16 | 17 | Installation is simple: 18 | 19 | .. code-block:: bash 20 | 21 | $ pip install pypyodbc 22 | 23 | --------------------------- 24 | Configure ODBC and Free TDS 25 | --------------------------- 26 | You'll need to install ODBC and configure its database drivers. 27 | 28 | **Mac OS X** users can install ODBC and Free TDS with homebrew: 29 | 30 | **`Note:` This has not been tested and should be considered provisional** 31 | 32 | .. code-block:: bash 33 | 34 | $ brew install unixodbc 35 | $ brew install freetds 36 | 37 | **Linux** users can follow these instructions are from `the project wiki `_: 38 | 39 | 1. Install ODBC and Free TDS 40 | 41 | .. code-block:: bash 42 | 43 | $ sudo apt-get install tdsodbc unixodbc 44 | 45 | 2. Modify ``/etc/odbcinst.ini`` 46 | 47 | **From the wiki**: "If ``odbcinst.ini`` doesn't exist under /etc, create the file. Find the path to ``libtdsodbc.so``. If the path to the file is ``/usr/lib/x86_64-linux-gnu/libtdsodbc.so``, make sure you have the below content in the file:" 48 | 49 | .. code-block:: bash 50 | 51 | [FreeTDS] 52 | Driver = /usr/lib/x86_64-linux-gnu/odbc/libtdsodbc.so 53 | 54 | 3. Modify /etc/freetds/freetds.conf 55 | 56 | **From the wiki**: "Make sure there are following lines under the "Global" section in the file:" 57 | 58 | .. code-block:: bash 59 | 60 | [Global] 61 | TDS_Version = 8.0 62 | client charset = UTF-8 63 | 64 | 65 | Connecting to the server 66 | ------------------------ 67 | 68 | Next, you'll need to add the SQL server variables to your ``settings.py``. Since this will contain sensitive information, we suggest storing these variables in a ``local_settings.py`` file that's not tracked by version control and importing it. 69 | 70 | .. code-block:: python 71 | 72 | SQL_SERVER_DRIVER = '' # Use 'FreeTDS' if you followed instructions above 73 | SQL_SERVER_ADDRESS = '' # Your SQL Server IP address 74 | SQL_SERVER_PORT = '' # Your SQL Server port number 75 | SQL_SERVER_USER = '' # Your SQL server username 76 | SQL_SERVER_PASSWORD = '' # Your SQL server password 77 | SQL_SERVER_DATABASE = '' # Your SQL server database name 78 | 79 | 80 | Import data into SQL Server 81 | --------------------------- 82 | 83 | With the above configuration, you should now be able to import the exported CSV data from your local folder to SQL server: 84 | 85 | .. code-block:: bash 86 | 87 | $ python manage.py importtosqlserver 88 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/views/base.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import json 3 | from django.views import generic 4 | from django.http import HttpResponse 5 | from django.utils.encoding import smart_text 6 | 7 | 8 | class DataPrepMixin(object): 9 | """ 10 | Provides a method for preping a context object 11 | for serialization as JSON or CSV. 12 | """ 13 | def prep_context_for_serialization(self, context): 14 | field_names = self.model._meta.get_all_field_names() 15 | values = self.get_queryset().values_list(*field_names) 16 | data_list = [] 17 | for i in values: 18 | d = dict((field_names[i], val) for i, val in enumerate(i)) 19 | data_list.append(d) 20 | 21 | return (data_list, field_names) 22 | 23 | 24 | class JSONResponseMixin(DataPrepMixin): 25 | """ 26 | A mixin that can be used to render a JSON response. 27 | """ 28 | def render_to_json_response(self, context, **response_kwargs): 29 | """ 30 | Returns a JSON response, transforming 'context' to make the payload. 31 | """ 32 | data, fields = self.prep_context_for_serialization(context) 33 | return HttpResponse( 34 | json.dumps(data, default=smart_text), 35 | content_type='application/json', 36 | **response_kwargs 37 | ) 38 | 39 | 40 | class CSVResponseMixin(DataPrepMixin): 41 | """ 42 | A mixin that can be used to render a CSV response. 43 | """ 44 | def render_to_csv_response(self, context, **response_kwargs): 45 | """ 46 | Returns a CSV file response, transforming 'context' 47 | to make the payload. 48 | """ 49 | data, fields = self.prep_context_for_serialization(context) 50 | response = HttpResponse(content_type='text/csv') 51 | response['Content-Disposition'] = 'attachment; filename=download.csv' 52 | writer = csv.DictWriter(response, fieldnames=fields) 53 | writer.writeheader() 54 | [writer.writerow(i) for i in data] 55 | return response 56 | 57 | 58 | class CommitteeDataView(JSONResponseMixin, CSVResponseMixin, generic.ListView): 59 | """ 60 | Custom generic view for our committee specific data pages 61 | """ 62 | allow_empty = False 63 | paginate_by = 100 64 | 65 | def get_context_data(self, **kwargs): 66 | context = super(CommitteeDataView, self).get_context_data(**kwargs) 67 | context['committee'] = self.committee 68 | context['base_url'] = self.committee.get_absolute_url 69 | return context 70 | 71 | def render_to_response(self, context, **kwargs): 72 | """ 73 | Return a normal response, or CSV or JSON depending 74 | on a URL param from the user. 75 | """ 76 | # See if the user has requested a special format 77 | format = self.request.GET.get('format', '') 78 | # If it's a CSV 79 | if 'csv' in format: 80 | return self.render_to_csv_response(context) 81 | 82 | # If it's JSON 83 | if 'json' in format: 84 | return self.render_to_json_response(context) 85 | 86 | # And if it's none of the above return something normal 87 | return super(CommitteeDataView, self).render_to_response( 88 | context, **kwargs 89 | ) 90 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import patterns, include, url 2 | from tastypie.api import Api 3 | from django.views.generic.base import RedirectView 4 | from calaccess_campaign_browser.api import FilerResource, FilingResource 5 | from calaccess_campaign_browser import views 6 | from calaccess_campaign_browser.views import search 7 | from django.views.generic import TemplateView 8 | 9 | # Set up the endpoints for the REST service. 10 | # 11 | # Usage: 12 | # 13 | # http://://api/v1/ 14 | # http://://api/v1/filer/ 15 | # http://://api/v1/filing/?filing_id_raw=1852192' 16 | # 17 | v1_api = Api(api_name='v1') 18 | v1_api.register(FilerResource()) 19 | v1_api.register(FilingResource()) 20 | 21 | # Set up the endpoints for the web application. 22 | # 23 | urlpatterns = patterns( 24 | '', 25 | url(r'^$', RedirectView.as_view(url='/latest/', permanent=False)), 26 | url( 27 | r'^latest/$', 28 | views.LatestFilingView.as_view(), 29 | name='latest_list' 30 | ), 31 | url( 32 | r'^filers/$', 33 | RedirectView.as_view(url='/filers/1/', permanent=False), 34 | name="filer_list" 35 | ), 36 | url( 37 | r'^filers/(?P[\d+]+)/$', 38 | views.FilerListView.as_view(), 39 | name='filer_page' 40 | ), 41 | url( 42 | r'^filer/(?P\d+)/$', 43 | views.FilerDetailView.as_view(), 44 | name='filer_detail' 45 | ), 46 | url( 47 | r'^committee/(?P\d+)/contributions/(?P[\d+]+)/$', 48 | views.CommitteeContributionView.as_view(), 49 | name='committee_contribution_list', 50 | ), 51 | url( 52 | r'^committee/(?P\d+)/expenditures/(?P[\d+]+)/$', 53 | views.CommitteeExpenditureView.as_view(), 54 | name='committee_expenditure_list', 55 | ), 56 | url( 57 | r'^committee/(?P\d+)/filings/(?P[\d+]+)/$', 58 | views.CommitteeFilingView.as_view(), 59 | name='committee_filing_list', 60 | ), 61 | url( 62 | r'^committee/(?P\d+)/$', 63 | views.CommitteeDetailView.as_view(), 64 | name='committee_detail' 65 | ), 66 | url( 67 | r'^filing/(?P\d+)/$', 68 | views.FilingDetailView.as_view(), 69 | name='filing_detail' 70 | ), 71 | url( 72 | r'^contribution/(?P\d+)/$', 73 | views.ContributionDetailView.as_view(), 74 | name='contribution_detail' 75 | ), 76 | url( 77 | r'^expenditure/(?P\d+)/$', 78 | views.ExpenditureDetailView.as_view(), 79 | name='expenditure_detail', 80 | ), 81 | url(r'^search/$', search.SearchList.as_view(), name='search-list'), 82 | url( 83 | r'^search/contribs-by-name/$', 84 | search.search_contribs_by_name, 85 | name='search-contribs-by-name' 86 | ), 87 | url( 88 | r'^parties/$', 89 | views.PartyListView.as_view(), 90 | name='party_list' 91 | ), 92 | 93 | # API 94 | url(r'^api/', include(v1_api.urls)), 95 | 96 | url( 97 | r'^robots\.txt$', 98 | TemplateView.as_view( 99 | template_name='robots.txt', 100 | content_type='text/plain') 101 | ), 102 | ) 103 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/templates/calaccess_campaign_browser/expenditure_detail.html: -------------------------------------------------------------------------------- 1 | {% extends 'calaccess_campaign_browser/base.html' %} 2 | {% load humanize %} 3 | 4 | {% block title %} 5 | {% endblock %} 6 | 7 | {% block content %} 8 |
9 |
10 | 11 |

${{ object.amount }} {{ object.committee }}

12 | 13 |

Election cycle: {{ object.cycle }}

14 |

Expenditure date: {{ object.expn_date }}

15 |

Filing: {{ object.filing }}

16 | 17 |

Basics

18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 |
CityStateZipFirst NameLast NameDescription
{{ object.payee_city }}{{ object.payee_st }}{{ object.payee_zip4 }}{{ object.payee_namf }}{{ object.payee_naml }}{% if object.ctrib_nams %}, {{ object.ctrib_nams }}{% endif %}{{ object.expn_dscr }}
40 | 41 |

Details

42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 |
Backref IDCommittee IDIndividual IDForm typeLine ItemMemo CodeMemo Ref NumberOrganization IDTransaction ID
{{ object.bakref_tid }}{{ object.cmte_id }}{{ object.individual_id }}{{ object.form_type }}{{ object.line_item }}{{ object.memo_code }}{{ object.memo_refno }}{{ object.org_id }}{{ object.tran_id }}
70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 |
entity_cdexpn_chknoexpn_codeexpn_code_displayg_from_e_f
{{ object.entity_cd }}{{ object.expn_chkno }}{{ object.expn_code }}{{ object.expn_code_display }}{{ object.g_from_e_f }}
92 | 93 |
94 |
95 | {% endblock %} 96 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/templates/calaccess_campaign_browser/committee_filing_list.html: -------------------------------------------------------------------------------- 1 | {% extends 'calaccess_campaign_browser/base.html' %} 2 | {% load humanize %} 3 | 4 | {% block title %}Filings - {{ committee.name }} - Committee - {{ block.super }}{% endblock %} 5 | 6 | {% block javascript %} 7 | 19 | {% endblock %} 20 | 21 | {% block content %} 22 | {% include "calaccess_campaign_browser/committee_nav.html" %} 23 | 24 |
25 |
26 |

27 | Filings 28 |
29 | 32 | 36 |
37 |

38 |

39 | Total contributions: ${{committee.total_contributions|floatformat:0|intcomma}} 40 |
41 | Total expenditures: ${{committee.total_expenditures|floatformat:0|intcomma}} 42 |

43 |
44 |
45 | 46 | {% include "calaccess_campaign_browser/paginator.html" %} 47 | 48 |
49 |
50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | {% for filing in committee_filings %} 65 | 66 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 80 | 81 | {% endfor %} 82 | 83 |
No.FiledFormPeriodContributionsExpendituresDebtsPDF
67 | {{ filing.filing_id_raw }} 68 | {{ filing.date_filed|date:"Y-m-d" }}{{ filing.form_type }}{{ filing.period }}${{ filing.total_contributions|default:0|floatformat:0|intcomma }}${{ filing.total_expenditures|default:0|floatformat:0|intcomma }}${{ filing.summary.outstanding_debts|default:0|floatformat:0|intcomma }} 76 | 77 | 78 | 79 |
84 |
85 |
86 | {% include "calaccess_campaign_browser/paginator.html" %} 87 | {% endblock %} 88 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/templates/calaccess_campaign_browser/filer_list.html: -------------------------------------------------------------------------------- 1 | {% extends 'calaccess_campaign_browser/base.html' %} 2 | {% load humanize %} 3 | 4 | {% block title %}Filers - {{ block.super }}{% endblock %} 5 | 6 | {% block content %} 7 |
8 |
9 |

Filers

10 |
11 |
12 | 13 |
14 |
15 |
16 | 19 | 25 | 31 | 32 |
33 |
34 |
35 | 36 | {% include "calaccess_campaign_browser/paginator.html" %} 37 | 38 |
39 |
40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | {% for filer in object_list %} 54 | 55 | {% with filer.committee_set.count as count %} 56 | 63 | 66 | 67 | 68 | 69 | 70 | 71 | {% endwith %} 72 | 73 | {% endfor %} 74 | 75 |
IDNameTypePartyStatusEffective dateCommittees
57 | {% if count %} 58 | {{ filer.filer_id_raw }} 59 | {% else %} 60 | {{ filer.filer_id_raw }} 61 | {% endif %} 62 | 64 | {{ filer.short_name }} 65 | {{ filer.get_filer_type_display }}{{ filer.get_party_display }}{{ filer.get_status_display }}{{ filer.effective_date }}{{ count }}
76 |
77 |
78 | 79 | {% include "calaccess_campaign_browser/paginator.html" %} 80 | 81 | {% endblock %} 82 | 83 | {% block javascript %} 84 | 96 | {% endblock %} 97 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/views/committees.py: -------------------------------------------------------------------------------- 1 | from django.views import generic 2 | from django.db.models import Count, Sum 3 | from .base import CommitteeDataView 4 | from calaccess_campaign_browser.models import ( 5 | Committee, 6 | Filing, 7 | Expenditure, 8 | Contribution 9 | ) 10 | 11 | 12 | class CommitteeDetailView(generic.DetailView): 13 | model = Committee 14 | 15 | def get_context_data(self, **kwargs): 16 | context = super(CommitteeDetailView, self).get_context_data(**kwargs) 17 | 18 | # Filings 19 | filing_qs = Filing.real.by_committee( 20 | self.object, 21 | ).select_related("cycle", "period").order_by( 22 | "-end_date", 23 | "filing_id_raw", 24 | "-amend_id" 25 | ) 26 | context['filing_set'] = filing_qs 27 | context['filing_set_count'] = filing_qs.count() 28 | context['filing_set_short'] = filing_qs[:25] 29 | 30 | # Contributions 31 | contribs_qs = Contribution.real.by_committee_to(self.object) 32 | context['contribs_set_short'] = contribs_qs.order_by('-amount')[:25] 33 | context['contribs_set_count'] = contribs_qs.count() 34 | 35 | context['contribs_set_top_contributors'] = contribs_qs.values( 36 | 'contributor_full_name' 37 | ).annotate( 38 | contributor_total=Sum('amount') 39 | ).annotate( 40 | contribution_count=Count('amount') 41 | ).order_by('-contributor_total')[:10] 42 | 43 | context['contribs_set_frequent_contributors'] = contribs_qs.values( 44 | 'contributor_full_name' 45 | ).annotate( 46 | contributor_total=Sum('amount') 47 | ).annotate( 48 | contribution_count=Count('amount') 49 | ).order_by('-contribution_count')[:10] 50 | 51 | # Transfer to other committees 52 | contribs_out = Contribution.real.by_committee_from(self.object) 53 | context['contribs_out_set'] = contribs_out.order_by('-amount')[:25] 54 | context['contribs_out_set_count'] = contribs_out.count() 55 | 56 | # Close out 57 | return context 58 | 59 | 60 | class CommitteeContributionView(CommitteeDataView): 61 | model = Contribution 62 | template_name = 'calaccess_campaign_browser/committee_\ 63 | contribution_list.html' 64 | context_object_name = 'committee_contributions' 65 | 66 | def get_queryset(self): 67 | """ 68 | Returns the contributions related to this committee. 69 | """ 70 | committee = Committee.objects.get(pk=self.kwargs['pk']) 71 | self.committee = committee 72 | return Contribution.real.by_committee_to( 73 | self.committee, 74 | ).order_by("-date_received") 75 | 76 | 77 | class CommitteeExpenditureView(CommitteeDataView): 78 | model = Expenditure 79 | template_name = 'calaccess_campaign_browser/committee_\ 80 | expenditure_list.html' 81 | context_object_name = 'committee_expenditures' 82 | 83 | def get_queryset(self): 84 | """ 85 | Returns the expends related to this committee. 86 | """ 87 | committee = Committee.objects.get(pk=self.kwargs['pk']) 88 | self.committee = committee 89 | return committee.expenditure_set.all().order_by('-cycle') 90 | 91 | 92 | class CommitteeFilingView(CommitteeDataView): 93 | model = Filing 94 | template_name = 'calaccess_campaign_browser/committee_filing_list.html' 95 | context_object_name = 'committee_filings' 96 | 97 | def get_queryset(self): 98 | """ 99 | Returns the expends related to this committee. 100 | """ 101 | committee = Committee.objects.get(pk=self.kwargs['pk']) 102 | self.committee = committee 103 | return Filing.real.by_committee(committee).order_by('-date_filed') 104 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/templates/calaccess_campaign_browser/committee_nav.html: -------------------------------------------------------------------------------- 1 |
2 |
3 | 4 | 55 | 56 | 57 |

{{ committee }}

58 | 59 |
60 |
61 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/templates/calaccess_campaign_browser/contribution_detail.html: -------------------------------------------------------------------------------- 1 | {% extends 'calaccess_campaign_browser/base.html' %} 2 | {% load humanize %} 3 | 4 | {% block title %}Contribution - {{ block.super }}{% endblock %} 5 | 6 | {% block content %} 7 |
8 |
9 |

Contribution

10 |
11 |
12 | 13 |
14 |
15 |

Basics

16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 40 | 41 | 42 |
Date{{ object.date_received|date:"N j, Y" }}
Amount${{ object.amount|intcomma }}
To{{ object.committee }}
From 32 | {% if object.contributor_committee_id %} 33 | 34 | {% endif %} 35 | {{ object.contributor_full_name }} 36 | {% if object.contributor_committee_id %} 37 | 38 | {% endif %} 39 |
43 | 44 |

Bookkeeping

45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 |
Filing{{ object.filing }}
Transaction ID${{ object.transaction_id }}
Transaction type{{ object.transaction_type }}
Amendment{{ object.amend_id }}
Backreference transaction ID{{ object.backreference_transaction_id }}
Crossreference{{ object.is_crossreference }}
Crossreference schedule{{ object.crossreference_schedule }}
Description{{ object.contribution_description }} 78 |
Duplicate{{ object.is_duplicate }}
Raw record{{ object.raw.id }}
89 | 90 |

Contributor

91 | 92 | 93 | {% for field, value in object.contributor_dict.items %} 94 | 95 | 96 | 97 | 98 | {% endfor %} 99 | 100 |
{{ field|capfirst }}{{ value }}
101 | 102 |

Intermediary

103 | 104 | 105 | {% for field, value in object.intermediary_dict.items %} 106 | 107 | 108 | 109 | 110 | {% endfor %} 111 | 112 |
{{ field|capfirst }}{{ value }}
113 |
114 |
115 | {% endblock %} 116 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/js/jquery.twbsPagination.min.js: -------------------------------------------------------------------------------- 1 | (function(e,d,a,f){var b=e.fn.twbsPagination;var c=function(h,g){this.$element=e(h);this.options=e.extend({},e.fn.twbsPagination.defaults,g);this.init(this.options)};c.prototype={constructor:c,init:function(g){this.options=e.extend({},this.options,g);if(this.options.startPage<1||this.options.startPage>this.options.totalPages){throw new Error("Start page option is incorrect")}if(this.options.totalPages<=0){throw new Error("Total pages option cannot be less 1 (one)!")}if(this.options.totalPages")}this.$listContainer.addClass(this.options.paginationClass);if(h!=="UL"){this.$element.append(this.$listContainer)}this.render(this.getPages(this.options.startPage));this.setupEvents();return this},destroy:function(){this.$element.empty();return this},show:function(g){if(g<1||g>this.options.totalPages){throw new Error("Page is incorrect.")}this.render(this.getPages(g));this.setupEvents();this.$element.trigger("page",g);return this},buildListItems:function(g){var j=e();if(this.options.first){j=j.add(this.buildItem("first",1))}if(this.options.prev){var l=g.currentPage>1?g.currentPage-1:1;j=j.add(this.buildItem("prev",l))}for(var h=0;h=this.options.totalPages?this.options.totalPages:g.currentPage+1;j=j.add(this.buildItem("next",k))}if(this.options.last){j=j.add(this.buildItem("last",this.options.totalPages))}return j},buildItem:function(i,j){var h=e("
  • "),k=e(""),g=null;h.addClass(i);h.data("page",j);switch(i){case"page":g=j;break;case"first":g=this.options.first;break;case"prev":g=this.options.prev;break;case"next":g=this.options.next;break;case"last":g=this.options.last;break;default:break}h.append(k.attr("href",this.href(j)).html(g));return h},getPages:function(j){var g=[];var k=Math.floor(this.options.visiblePages/2);var l=j-k+1-this.options.visiblePages%2;var h=j+k;if(l<=0){l=1;h=this.options.visiblePages}if(h>this.options.totalPages){l=this.options.totalPages-this.options.visiblePages+1;h=this.options.totalPages}var i=l;while(i<=h){g.push(i);i++}return{currentPage:j,numeric:g}},render:function(g){this.$listContainer.children().remove();this.$listContainer.append(this.buildListItems(g));this.$listContainer.find(".page").removeClass("active");this.$listContainer.find(".page").filter(function(){return e(this).data("page")===g.currentPage}).addClass("active");if(g.currentPage===1){this.$listContainer.find(".prev a,.first a").attr("href","javascript:void(0);")}if(g.currentPage===this.options.totalPages){this.$listContainer.find(".next a,.last a").attr("href","javascript:void(0);")}this.$listContainer.find(".first").toggleClass("disabled",g.currentPage===1);this.$listContainer.find(".last").toggleClass("disabled",g.currentPage===this.options.totalPages);this.$listContainer.find(".prev").toggleClass("disabled",g.currentPage===1);this.$listContainer.find(".next").toggleClass("disabled",g.currentPage===this.options.totalPages)},setupEvents:function(){var g=this;this.$listContainer.find("li").each(function(){var h=e(this);h.off();if(h.hasClass("disabled")||h.hasClass("active")){return}h.click(function(){g.show(parseInt(h.data("page"),10))})})},href:function(g){return this.options.href.replace(this.options.hrefVariable,g)}};e.fn.twbsPagination=function(i){var h=Array.prototype.slice.call(arguments,1);var k;var l=e(this);var j=l.data("twbs-pagination");var g=typeof i==="object"&&i;if(!j){l.data("twbs-pagination",(j=new c(this,g)))}if(typeof i==="string"){k=j[i].apply(j,h)}return(k===f)?l:k};e.fn.twbsPagination.defaults={totalPages:0,startPage:1,visiblePages:5,href:"javascript:void(0);",hrefVariable:"{{number}}",first:"First",prev:"Previous",next:"Next",last:"Last",paginationClass:"pagination",onPageClick:null};e.fn.twbsPagination.Constructor=c;e.fn.twbsPagination.noConflict=function(){e.fn.twbsPagination=b;return this}})(jQuery,window,document); -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | django-calaccess-campaign-browser 2 | ================================= 3 | 4 | A Django app to refine and investigate campaign finance data drawn from the 5 | California Secretary of State's `CAL-ACCESS `_ database. 6 | 7 | Intended as a second layer atop `django-calaccess-raw-data `_ 8 | that transforms the source data and loads it into simplified models that serve as a platform 9 | for investigative analysis. 10 | 11 | .. image:: /_static/application-layers.png 12 | 13 | .. warning:: 14 | 15 | This is a work in progress. Its analysis should be considered as provisional 16 | until it is further tested and debugged. 17 | 18 | Documentation 19 | ------------- 20 | 21 | .. toctree:: 22 | :maxdepth: 2 23 | 24 | howtouseit 25 | managementcommands 26 | models 27 | howtocontribute 28 | exportingthedata 29 | sqlserver 30 | changelog 31 | 32 | Open-source resources 33 | --------------------- 34 | 35 | * Code: `github.com/california-civic-data-coalition/django-calaccess-campaign-browser `_ 36 | * Issues: `github.com/california-civic-data-coalition/django-calaccess-campaign-browser/issues `_ 37 | * Packaging: `pypi.python.org/pypi/django-calaccess-campaign-browser `_ 38 | * Testing: `travis-ci.org/california-civic-data-coalition/django-calaccess-campaign-browser `_ 39 | * Coverage: `coveralls.io/r/california-civic-data-coalition/django-calaccess-campaign-browser `_ 40 | 41 | Read more 42 | --------- 43 | 44 | * `'CAL-ACCESS Dreaming' `_, the kickoff presentation from the initial code convening (Aug. 13, 2014) 45 | * `'Introducing the California Civic Data Coalition' `_, a blog post annoucing the first public code release (Sept. 24, 2014) 46 | * `'Package data like software, and the stories will flow like wine' `_, a polemic explaining this project's software design philosophy (Sept. 24, 2014) 47 | * `'Light everywhere: The California Civic Data Coalition wants to make public datasets easier to crunch' `_, a story about the creators by Nieman Journalism Lab (Oct. 20, 2014) 48 | * `'Once a crusader against big money, Gov. Brown is collecting millions' `_, a Los Angeles Times story that utilized this application (Oct. 31, 2014) 49 | 50 | Events 51 | ------ 52 | 53 | Development has been advanced by a series of sprints supported by `Knight-Mozilla OpenNews `_. 54 | 55 | * August 13-15, 2014, at `Mozilla's offices in San Francisco `_ 56 | * January 14-15, 2015, at `USC's Wallis Annenberg Hall `_ 57 | * March 4-8, 2015, the California Code Rush at `NICAR 2015 in Atlanta `_ 58 | 59 | Sponsors 60 | -------- 61 | 62 | .. image:: /_static/ccdc-logo.png 63 | :height: 70px 64 | :target: http://www.californiacivicdata.org/ 65 | 66 | .. image:: /_static/los-angeles-times-logo.png 67 | :height: 70px 68 | :target: http://www.github.com/datadesk/ 69 | 70 | .. image:: /_static/cir-logo.png 71 | :height: 70px 72 | :target: http://cironline.org/ 73 | 74 | .. image:: /_static/stanford-logo.png 75 | :height: 70px 76 | :target: http://journalism.stanford.edu/ 77 | 78 | .. image:: /_static/opennews-logo.png 79 | :height: 80px 80 | :target: http://opennews.org/code.html 81 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/management/commands/exportcalaccesscampaignbrowser.py: -------------------------------------------------------------------------------- 1 | import os 2 | import csv 3 | import datetime 4 | from optparse import make_option 5 | 6 | from django.conf import settings 7 | from django.db.models import get_model 8 | from calaccess_campaign_browser.management.commands import CalAccessCommand 9 | 10 | from calaccess_campaign_browser.models import Cycle 11 | 12 | custom_options = ( 13 | make_option( 14 | "--skip-contributions", 15 | action="store_false", 16 | dest="contributions", 17 | default=True, 18 | help="Skip contributions export" 19 | ), 20 | make_option( 21 | "--skip-expenditures", 22 | action="store_false", 23 | dest="expenditures", 24 | default=True, 25 | help="Skip expenditures export" 26 | ), 27 | make_option( 28 | "--skip-summary", 29 | action="store_false", 30 | dest="summary", 31 | default=True, 32 | help="Skip summary export" 33 | ), 34 | ) 35 | 36 | 37 | class Command(CalAccessCommand): 38 | help = 'Export refined CAL-ACCESS campaign browser data as CSV files' 39 | option_list = CalAccessCommand.option_list + custom_options 40 | 41 | def set_options(self, *args, **kwargs): 42 | self.data_dir = os.path.join( 43 | settings.BASE_DIR, 'data') 44 | os.path.exists(self.data_dir) or os.mkdir(self.data_dir) 45 | 46 | def encoded(self, list_of_lists): 47 | """ 48 | Take a list of lists, and encode each list item as utf-8 49 | http://stackoverflow.com/a/17527101/868724 50 | """ 51 | return [[unicode(s).encode('utf-8') for s in t] for t in list_of_lists] 52 | 53 | def export_to_csv(self, model_name): 54 | self.header('Exporting models ...') 55 | 56 | today = datetime.datetime.today() 57 | model = get_model('calaccess_campaign_browser', model_name) 58 | 59 | fieldnames = [f.name for f in model._meta.fields] + [ 60 | 'committee_name', 'filer_name', 'filer_id', 'filer_id_raw'] 61 | 62 | relation_names = [f.name for f in model._meta.fields] + [ 63 | 'committee__name', 64 | 'committee__filer__name', 65 | 'committee__filer__id', 66 | 'committee__filer__filer_id_raw' 67 | ] 68 | 69 | filename = '{}-{}-{}-{}.csv'.format( 70 | today.year, 71 | today.month, 72 | today.day, 73 | model_name.lower() 74 | ) 75 | filepath = os.path.join(self.data_dir, filename) 76 | 77 | self.header(' Exporting {} model ...'.format(model_name.capitalize())) 78 | 79 | with open(filepath, 'wb') as csvfile: 80 | writer = csv.writer(csvfile, delimiter="\t") 81 | writer.writerow(fieldnames) 82 | 83 | if model_name != 'summary': 84 | for cycle in Cycle.objects.all(): 85 | self.log(' Looking at cycle {} ...'.format(cycle.name)) 86 | rows = model.objects.filter(cycle=cycle)\ 87 | .exclude(is_duplicate=True)\ 88 | .values_list(*relation_names) 89 | 90 | if not rows: 91 | self.failure(' No data for {}'.format(cycle.name)) 92 | 93 | else: 94 | rows = self.encoded(rows) 95 | writer.writerows(rows) 96 | self.success(' Added {} {} data'.format( 97 | cycle.name, model_name)) 98 | else: 99 | rows = self.encoded(model.objects.values_list()) 100 | writer.writerows(rows) 101 | 102 | self.success(' Exported {}!'.format(model_name.capitalize())) 103 | 104 | def handle(self, *args, **options): 105 | self.set_options(*args, **options) 106 | 107 | if options['contributions']: 108 | self.export_to_csv('contribution') 109 | 110 | if options['expenditures']: 111 | self.export_to_csv('expenditure') 112 | 113 | if options['summary']: 114 | self.export_to_csv('summary') 115 | -------------------------------------------------------------------------------- /docs/howtouseit.rst: -------------------------------------------------------------------------------- 1 | How to use it 2 | ============= 3 | 4 | This guide will walk users through the process of installing the latest official release `from the Python Package Index `_. If you want to install the raw source code or contribute as a developer refer to the `"How to contribute" `__ page. 5 | 6 | .. warning:: 7 | 8 | This library is intended to be plugged into a project created with the Django web framework. Before you can begin, you'll need to have one up and running. If you don't know how, `check out the official Django documentation `_. 9 | 10 | Installing the Django app 11 | ------------------------- 12 | 13 | The latest official release of the application can be installed from the Python Package Index using ``pip``. 14 | 15 | .. code-block:: bash 16 | 17 | $ pip install django-calaccess-campaign-browser 18 | 19 | Add this ``calaccess_campaign_browser`` app as well as the underlying ``calaccess_raw`` app 20 | that contains the raw government database to your Django project's ``settings.py`` file. 21 | 22 | .. code-block:: python 23 | 24 | INSTALLED_APPS = ( 25 | ... 26 | 'calaccess_raw', 27 | 'calaccess_campaign_browser', 28 | ) 29 | 30 | Connecting to a local database 31 | ------------------------------ 32 | 33 | Also in the ``settings.py`` file, you will need to configure Django so it can connect to a database. 34 | 35 | Unlike a typical Django project, this application only supports the MySQL database backend. This is because we enlist specialized tools to load the immense amount of source data more quickly than Django typically allows. We haven't worked out those routines for PostgreSQL, SQLite and the other Django backends yet, but we're working on it. 36 | 37 | Preparing MySQL 38 | ~~~~~~~~~~~~~~~ 39 | 40 | Before you begin, make sure you have a MySQL server installed. If you don't, now is the time to hit Google and figure out how. If you're using Apple's OSX operating system, you can `install via Homebrew `_. `Here `_ is the official MySQL documentation for how to get it done. 41 | 42 | Once that is handled, create a new database named ``calaccess``. 43 | 44 | .. code-block:: bash 45 | 46 | $ mysqladmin -h localhost -u your-username-here -p create calaccess 47 | 48 | Also in the Django ``settings.py``, configure a database connection. 49 | 50 | .. code-block:: python 51 | 52 | DATABASES = { 53 | 'default': { 54 | 'ENGINE': 'django.db.backends.mysql', 55 | 'NAME': 'calaccess', 56 | 'USER': 'your-username-here', 57 | 'PASSWORD': 'your-password-here', 58 | 'HOST': 'localhost', 59 | 'PORT': '3306', 60 | # You'll need this to use our data loading tricks 61 | 'OPTIONS': { 62 | 'local_infile': 1, 63 | } 64 | } 65 | } 66 | 67 | Loading the data 68 | ---------------- 69 | 70 | Now you're ready to create the database tables with Django using its ``manage.py`` utility belt. 71 | 72 | .. code-block:: bash 73 | 74 | $ python manage.py migrate 75 | 76 | Once everything is set up, this management command will download the latest bulk data release from the state and load it in the database. It'll take a while. Go grab some coffee. 77 | 78 | .. code-block:: bash 79 | 80 | $ python manage.py downloadcalaccessrawdata 81 | 82 | Next, run the management command that extracts and refines campaign finance data from from the raw CAL-ACCESS data dump. 83 | 84 | .. code-block:: bash 85 | 86 | $ python manage.py buildcalaccesscampaignbrowser 87 | 88 | Exploring the data 89 | ------------------ 90 | 91 | In your project ``urls.py`` file, add this app's URLs: 92 | 93 | .. code-block:: python 94 | 95 | urlpatterns = patterns('', 96 | url(r'^', include('calaccess_campaign_browser.urls')), 97 | ) 98 | 99 | Finally start the development server and visit `localhost:8000 `_ in your browser to start using the app. 100 | 101 | .. code-block:: bash 102 | 103 | $ python manage.py runserver 104 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from time import sleep 3 | from bs4 import BeautifulSoup 4 | from requests.exceptions import HTTPError 5 | from django.utils.termcolors import colorize 6 | from django.core.management.base import BaseCommand 7 | 8 | 9 | class CalAccessCommand(BaseCommand): 10 | 11 | def header(self, string): 12 | self.stdout.write(colorize(string, fg="cyan", opts=("bold",))) 13 | 14 | def log(self, string): 15 | self.stdout.write(colorize("%s" % string, fg="white")) 16 | 17 | def success(self, string): 18 | self.stdout.write(colorize(string, fg="green")) 19 | 20 | def failure(self, string): 21 | self.stdout.write(colorize(string, fg="red")) 22 | 23 | def warn(self, string): 24 | self.stdout.write(colorize(string, fg="yellow")) 25 | 26 | 27 | class ScrapeCommand(CalAccessCommand): 28 | base_url = 'http://cal-access.ss.ca.gov/' 29 | 30 | def handle(self, *args, **options): 31 | self.verbosity = int(options['verbosity']) 32 | results = self.build_results() 33 | self.process_results(results) 34 | 35 | def build_results(self): 36 | """ 37 | This method should perform the actual scraping 38 | and return the structured data. 39 | """ 40 | raise NotImplementedError 41 | 42 | def process_results(self, results): 43 | """ 44 | This method receives the structured data returned 45 | by `build_results` and should process it. 46 | """ 47 | raise NotImplementedError 48 | 49 | def get(self, url, retries=1): 50 | """ 51 | Makes a request for a URL and returns the HTML 52 | as a BeautifulSoup object. 53 | """ 54 | if self.verbosity > 2: 55 | self.log(" Retrieving %s" % url) 56 | tries = 0 57 | while tries < retries: 58 | response = requests.get(url) 59 | if response.status_code == 200: 60 | return BeautifulSoup(response.text) 61 | else: 62 | tries += 1 63 | sleep(2.0) 64 | raise HTTPError 65 | 66 | def parse_election_name(self, name): 67 | """ 68 | Translates a raw election name into 69 | one of our canonical names. 70 | """ 71 | name = name.upper() 72 | if 'PRIMARY' in name: 73 | return 'PRIMARY' 74 | elif 'GENERAL' in name: 75 | return 'GENERAL' 76 | elif 'SPECIAL RUNOFF' in name: 77 | return 'SPECIAL_RUNOFF' 78 | elif 'SPECIAL' in name: 79 | return 'SPECIAL' 80 | elif 'RECALL' in name: 81 | return 'RECALL' 82 | else: 83 | return 'OTHER' 84 | 85 | def parse_office_name(self, raw_name): 86 | """ 87 | Translates a raw office name into one of 88 | our canonical names and a seat (if available). 89 | """ 90 | seat = None 91 | raw_name = raw_name.upper() 92 | if 'LIEUTENANT GOVERNOR' in raw_name: 93 | clean_name = 'LIEUTENANT_GOVERNOR' 94 | elif 'GOVERNOR' in raw_name: 95 | clean_name = 'GOVERNOR' 96 | elif 'SECRETARY OF STATE' in raw_name: 97 | clean_name = 'SECRETARY_OF_STATE' 98 | elif 'CONTROLLER' in raw_name: 99 | clean_name = 'CONTROLLER' 100 | elif 'TREASURER' in raw_name: 101 | clean_name = 'TREASURER' 102 | elif 'ATTORNEY GENERAL' in raw_name: 103 | clean_name = 'ATTORNEY_GENERAL' 104 | elif 'SUPERINTENDENT OF PUBLIC INSTRUCTION' in raw_name: 105 | clean_name = 'SUPERINTENDENT_OF_PUBLIC_INSTRUCTION' 106 | elif 'INSURANCE COMMISSIONER' in raw_name: 107 | clean_name = 'INSURANCE_COMMISSIONER' 108 | elif 'MEMBER BOARD OF EQUALIZATION' in raw_name: 109 | clean_name = 'BOARD_OF_EQUALIZATION' 110 | seat = raw_name.split()[-1] 111 | elif 'SENATE' in raw_name: 112 | clean_name = 'SENATE' 113 | seat = raw_name.split()[-1] 114 | elif 'ASSEMBLY' in raw_name: 115 | clean_name = 'ASSEMBLY' 116 | seat = raw_name.split()[-1] 117 | else: 118 | clean_name = 'OTHER' 119 | return { 120 | 'name': clean_name, 121 | 'seat': seat 122 | } 123 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/templates/calaccess_campaign_browser/filing_detail.html: -------------------------------------------------------------------------------- 1 | {% extends 'calaccess_campaign_browser/base.html' %} 2 | {% load humanize %} 3 | 4 | {% block title %}{{ filing.filing_id_raw }} - Filing - {{ filing.committee.name }} - {{ block.super }}{% endblock %} 5 | 6 | {% block content %} 7 |
    8 |
    9 | 10 |
      11 |
    1. Filer
      2. 12 |
    2. {{ filing.committee.name }}
      3. 13 |
    14 |
    15 |
    16 |

    Filing No. {{ filing.filing_id_raw }} 17 | 18 | View Filing 19 | 20 |

    21 | 22 |
    23 |
    24 |

    Type: {{ filing.get_form_type_display }}

    25 |

    Duplicate: {{ filing.is_duplicate }}

    26 |

    Amendment: {{ filing.is_amendment }}

    27 |
    28 |
    29 |

    Start date: {{ filing.start_date|date:'Y-m-d' }}

    30 |

    End date: {{ filing.end_date|date:'Y-m-d' }}

    31 |

    Cycle: {{ filing.cycle.name }}

    32 |
    33 |
    34 |

    Date filed: {{ filing.date_filed|date:'Y-m-d' }}

    35 |

    Date received: {{ filing.date_received|date:'Y-m-d' }}

    36 |
    37 |
    38 | 39 |
    40 |
    41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 |
    Contributions
    Itemized monetary contributions{{ filing.summary.itemized_monetary_contributions | intcomma }}
    Unitemized monetary contributions{{ filing.summary.unitemized_monetary_contributions | intcomma }}
    Total monetary contributions{{ filing.summary.total_monetary_contributions | intcomma }}
    Non-monetary contributions{{ filing.summary.non_monetary_contributions | intcomma }}
    Total contributions{{ filing.summary.total_contributions | intcomma }}
    65 |
    66 |
    67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 |
    Expenditures
    Itemized expenditures{{ filing.summary.itemized_expenditures | intcomma }}
    Unitemized expenditures{{ filing.summary.unitemized_expenditures | intcomma }}
    Total expenditures{{ filing.summary.total_expenditures | intcomma }}
    85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 |
    Debts
    Outstanding debts{{ filing.summary.outstanding_debts | intcomma }}
    97 |
    98 |
    99 | 100 | {% endblock %} 101 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/management/commands/loadcalaccesscampaignfilings.py: -------------------------------------------------------------------------------- 1 | import MySQLdb 2 | import warnings 3 | from django.db import connection 4 | from optparse import make_option 5 | from calaccess_campaign_browser.models import Filing 6 | from calaccess_campaign_browser.management.commands import CalAccessCommand 7 | 8 | 9 | custom_options = ( 10 | make_option( 11 | "--flush", 12 | action="store_true", 13 | dest="flush", 14 | default=False, 15 | help="Flush table before loading data" 16 | ), 17 | ) 18 | 19 | 20 | class Command(CalAccessCommand): 21 | help = "Load refined CAL-ACCESS campaign filings" 22 | option_list = CalAccessCommand.option_list + custom_options 23 | 24 | def handle(self, *args, **options): 25 | """ 26 | Loads raw filings into consolidated tables 27 | """ 28 | self.header("Loading filings") 29 | # Ignore MySQL warnings so this can be run with DEBUG=True 30 | warnings.filterwarnings("ignore", category=MySQLdb.Warning) 31 | if options['flush']: 32 | self.flush() 33 | self.load_filings() 34 | self.mark_duplicates() 35 | 36 | def flush(self): 37 | self.log(" Flushing filings") 38 | c = connection.cursor() 39 | c.execute("""SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0;""") 40 | c.execute("""SET FOREIGN_KEY_CHECKS = 0;""") 41 | sql = """TRUNCATE `%s`;""" % (Filing._meta.db_table) 42 | c.execute(sql) 43 | c.execute("""SET SQL_NOTES=@OLD_SQL_NOTES;""") 44 | c.execute("""SET FOREIGN_KEY_CHECKS = 1;""") 45 | 46 | def load_filings(self): 47 | self.log(" Loading form 450, 460, 497 filings") 48 | c = connection.cursor() 49 | sql = """ 50 | INSERT INTO %(filing_table)s ( 51 | cycle_id, 52 | committee_id, 53 | filing_id_raw, 54 | form_type, 55 | amend_id, 56 | start_date, 57 | end_date, 58 | date_received, 59 | date_filed, 60 | is_duplicate 61 | ) 62 | SELECT 63 | cycle.name as cycle_id, 64 | c.id as committee_id, 65 | ff.FILING_ID as filing_id_raw, 66 | ff.form_id as form_type, 67 | ff.filing_sequence as amend_id, 68 | ff.rpt_start as start_date, 69 | ff.rpt_end as end_date, 70 | ff.rpt_date as date_received, 71 | ff.filing_date as date_filed, 72 | false 73 | FROM ( 74 | SELECT 75 | *, 76 | CASE 77 | WHEN `session_id` %% 2 = 0 THEN `session_id` 78 | ELSE `session_id` + 1 79 | END as cycle 80 | FROM FILER_FILINGS_CD 81 | ) as ff 82 | INNER JOIN calaccess_campaign_browser_committee as c 83 | ON ff.`filer_id` = c.`filer_id_raw` 84 | INNER JOIN calaccess_campaign_browser_cycle as cycle 85 | ON ff.cycle = cycle.name 86 | WHERE `FORM_ID` IN ('F450', 'F460', 'F497') 87 | """ 88 | sql = sql % dict(filing_table=Filing._meta.db_table) 89 | c.execute(sql) 90 | 91 | def mark_duplicates(self): 92 | self.log(" Marking duplicates") 93 | c = connection.cursor() 94 | 95 | sql = """ 96 | CREATE TEMPORARY TABLE tmp_filing_dupes ( 97 | index(`filing_id_raw`) 98 | ) AS ( 99 | SELECT filing_id_raw 100 | FROM calaccess_campaign_browser_filing 101 | GROUP BY 1 102 | HAVING COUNT(*) > 1 103 | ); 104 | """ 105 | c.execute(sql) 106 | 107 | sql = """ 108 | UPDATE calaccess_campaign_browser_filing as f 109 | INNER JOIN tmp_filing_dupes as d 110 | ON f.`filing_id_raw` = d.`filing_id_raw` 111 | SET is_duplicate = true; 112 | """ 113 | c.execute(sql) 114 | 115 | sql = """DROP TABLE tmp_filing_dupes;""" 116 | c.execute(sql) 117 | 118 | # Unmark all those with the maximum id number among their set 119 | sql = """ 120 | CREATE TEMPORARY TABLE tmp_filing_max_dupes ( 121 | index(`filing_id_raw`), 122 | index(`max_id`) 123 | ) AS ( 124 | SELECT f.`filing_id_raw`, MAX(`amend_id`) as max_id 125 | FROM calaccess_campaign_browser_filing as f 126 | WHERE is_duplicate = true 127 | GROUP BY 1 128 | ); 129 | """ 130 | c.execute(sql) 131 | 132 | sql = """ 133 | UPDATE calaccess_campaign_browser_filing as f 134 | INNER JOIN tmp_filing_max_dupes as d 135 | ON f.`filing_id_raw` = d.`filing_id_raw` 136 | AND f.`amend_id` = d.`max_id` 137 | SET is_duplicate = false; 138 | """ 139 | c.execute(sql) 140 | 141 | sql = """DROP TABLE tmp_filing_max_dupes;""" 142 | c.execute(sql) 143 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from calaccess_campaign_browser import models 3 | 4 | 5 | class BaseAdmin(admin.ModelAdmin): 6 | list_per_page = 500 7 | 8 | def get_readonly_fields(self, *args, **kwargs): 9 | return [f.name for f in self.model._meta.fields] 10 | 11 | def has_add_permission(self, request, obj=None): 12 | return False 13 | 14 | def has_delete_permission(self, request, obj=None): 15 | return False 16 | 17 | 18 | @admin.register(models.Candidate) 19 | class CandidateAdmin(BaseAdmin): 20 | list_display = ( 21 | "filer", 22 | "election_year", 23 | "election_type", 24 | "office" 25 | ) 26 | list_filter = ( 27 | "election", 28 | ) 29 | search_fields = ( 30 | "filer__name", 31 | ) 32 | 33 | 34 | @admin.register(models.Filing) 35 | class FilingAdmin(BaseAdmin): 36 | list_display = ( 37 | "filing_id_raw", 38 | "committee_short_name", 39 | "form_type", 40 | "cycle", 41 | "amend_id", 42 | "is_duplicate", 43 | ) 44 | list_filter = ( 45 | "form_type", 46 | "is_duplicate", 47 | "cycle", 48 | ) 49 | search_fields = ( 50 | "filing_id_raw", 51 | ) 52 | 53 | 54 | @admin.register(models.Filer) 55 | class FilerAdmin(BaseAdmin): 56 | list_display = ( 57 | "filer_id_raw", 58 | "name", 59 | "filer_type", 60 | "party", 61 | "status", 62 | "effective_date", 63 | "xref_filer_id" 64 | ) 65 | list_filter = ( 66 | "filer_type", 67 | "party", 68 | "status", 69 | ) 70 | search_fields = ( 71 | "name", 72 | "filer_id_raw", 73 | "xref_filer_id" 74 | ) 75 | date_hierarchy = "effective_date" 76 | 77 | 78 | @admin.register(models.Committee) 79 | class CommitteeAdmin(BaseAdmin): 80 | list_display = ( 81 | "filer_id_raw", 82 | "short_name", 83 | "filer_short_name", 84 | "level_of_government", 85 | "party", 86 | "status", 87 | "effective_date", 88 | ) 89 | list_filter = ( 90 | "committee_type", 91 | "level_of_government", 92 | "party", 93 | "status", 94 | ) 95 | search_fields = ( 96 | "name", 97 | "filer_id_raw", 98 | "xref_filer_id" 99 | ) 100 | date_hierarchy = "effective_date" 101 | 102 | 103 | @admin.register(models.Cycle) 104 | class CycleAdmin(BaseAdmin): 105 | list_display = ("name",) 106 | 107 | 108 | @admin.register(models.Contribution) 109 | class ContributionAdmin(BaseAdmin): 110 | list_display = ( 111 | "id", 112 | "contributor_full_name", 113 | "committee", 114 | "date_received", 115 | "amount", 116 | "is_duplicate" 117 | ) 118 | list_filter = ( 119 | "is_duplicate", 120 | ) 121 | search_fields = ( 122 | "contributor_full_name", 123 | ) 124 | date_hierarchy = "date_received" 125 | 126 | 127 | @admin.register(models.Expenditure) 128 | class ExpenditureAdmin(BaseAdmin): 129 | list_display = ( 130 | "id", 131 | "candidate_full_name", 132 | "committee", 133 | "date_received", 134 | "amount", 135 | "is_duplicate" 136 | ) 137 | list_filter = ( 138 | "is_duplicate", 139 | ) 140 | search_fields = ( 141 | "candidate_full_name", 142 | ) 143 | date_hierarchy = "date_received" 144 | 145 | 146 | @admin.register(models.Summary) 147 | class SummaryAdmin(BaseAdmin): 148 | list_display = ( 149 | "filing_id_raw", 150 | "amend_id", 151 | "total_contributions", 152 | "total_expenditures", 153 | ) 154 | search_fields = ( 155 | "filing_id_raw", 156 | "amend_id", 157 | ) 158 | 159 | 160 | @admin.register(models.Election) 161 | class ElectionAdmin(BaseAdmin): 162 | list_display = ( 163 | "id_raw", 164 | "year", 165 | "election_type", 166 | "date", 167 | "office_count", 168 | "candidate_count", 169 | ) 170 | list_filter = ( 171 | "year", 172 | "election_type", 173 | ) 174 | 175 | 176 | @admin.register(models.Office) 177 | class OfficeAdmin(BaseAdmin): 178 | list_display = ( 179 | "__unicode__", 180 | "election_count", 181 | "candidate_count", 182 | ) 183 | list_filter = ( 184 | "name", 185 | ) 186 | search_fields = ( 187 | "name", 188 | "seat", 189 | ) 190 | list_per_page = 200 191 | 192 | 193 | @admin.register(models.Proposition) 194 | class PropositionAdmin(BaseAdmin): 195 | list_display = ( 196 | "name", 197 | "short_description", 198 | "id_raw", 199 | "election", 200 | ) 201 | list_filter = ( 202 | "election", 203 | ) 204 | search_fields = ( 205 | "name", 206 | "short_description", 207 | ) 208 | list_per_page = 500 209 | 210 | 211 | @admin.register(models.PropositionFiler) 212 | class PropositionFilerAdmin(BaseAdmin): 213 | list_display = ( 214 | "proposition", 215 | "filer", 216 | "position", 217 | ) 218 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/management/commands/loadcalaccesscampaignsummaries.py: -------------------------------------------------------------------------------- 1 | import os 2 | import csv 3 | import MySQLdb 4 | import warnings 5 | from django.db import connection 6 | from calaccess_raw import get_download_directory 7 | from django.utils.datastructures import SortedDict 8 | from calaccess_campaign_browser.models import Summary 9 | from calaccess_campaign_browser.management.commands import CalAccessCommand 10 | 11 | 12 | class Command(CalAccessCommand): 13 | help = "Load refined CAL-ACCESS campaign filing summaries" 14 | 15 | def handle(self, *args, **options): 16 | self.header("Loading summary totals") 17 | self.data_dir = get_download_directory() 18 | self.source_csv = os.path.join(self.data_dir, 'csv', 'smry_cd.csv') 19 | self.target_csv = os.path.join( 20 | self.data_dir, 21 | 'csv', 22 | 'smry_cd_transformed.csv' 23 | ) 24 | self.transform_csv() 25 | self.load_csv() 26 | 27 | def load_csv(self): 28 | self.log(" Loading transformed CSV") 29 | # Ignore MySQL warnings so this can be run with DEBUG=True 30 | warnings.filterwarnings("ignore", category=MySQLdb.Warning) 31 | c = connection.cursor() 32 | sql = """ 33 | LOAD DATA LOCAL INFILE '%s' 34 | INTO TABLE %s 35 | FIELDS TERMINATED BY ',' 36 | OPTIONALLY ENCLOSED BY '"' 37 | LINES TERMINATED BY '\\r\\n' 38 | IGNORE 1 LINES ( 39 | filing_id_raw, 40 | amend_id, 41 | itemized_monetary_contributions, 42 | unitemized_monetary_contributions, 43 | total_monetary_contributions, 44 | non_monetary_contributions, 45 | total_contributions, 46 | itemized_expenditures, 47 | unitemized_expenditures, 48 | total_expenditures, 49 | ending_cash_balance, 50 | outstanding_debts 51 | ) 52 | """ % (self.target_csv, Summary._meta.db_table) 53 | c.execute(sql) 54 | 55 | def transform_csv(self): 56 | self.log(" Transforming source CSV") 57 | grouped = {} 58 | form2field = { 59 | # F460 60 | 'A-1': 'itemized_monetary_contributions', 61 | 'A-2': 'unitemized_monetary_contributions', 62 | 'A-3': 'total_monetary_contributions', 63 | 'F460-4': 'non_monetary_contributions', 64 | 'F460-5': 'total_contributions', 65 | 'E-1': 'itemized_expenditures', 66 | 'E-2': 'unitemized_expenditures', 67 | 'E-4': 'total_expenditures', 68 | 'F460-16': 'ending_cash_balance', 69 | 'F460-19': 'outstanding_debts', 70 | # F450 71 | 'F450-7': 'total_monetary_contributions', 72 | 'F450-8': 'non_monetary_contributions', 73 | 'F450-10': 'total_contributions', 74 | 'F450-1': 'itemized_expenditures', 75 | 'F450-2': 'unitemized_expenditures', 76 | 'E-6': 'total_expenditures', 77 | } 78 | self.log(" Regrouping") 79 | for r in csv.DictReader(open(self.source_csv, 'rb')): 80 | uid = "%s-%s" % (r['FILING_ID'], r['AMEND_ID']) 81 | formkey = "%s-%s" % (r['FORM_TYPE'], r['LINE_ITEM']) 82 | try: 83 | field = form2field[formkey] 84 | except KeyError: 85 | continue 86 | try: 87 | grouped[uid][field] = self.safeamt(r['AMOUNT_A']) 88 | except KeyError: 89 | grouped[uid] = SortedDict(( 90 | ("itemized_monetary_contributions", "\N"), 91 | ("unitemized_monetary_contributions", "\N"), 92 | ("total_monetary_contributions", "\N"), 93 | ("non_monetary_contributions", "\N"), 94 | ("total_contributions", "\N"), 95 | ("itemized_expenditures", "\N"), 96 | ("unitemized_expenditures", "\N"), 97 | ("total_expenditures", "\N"), 98 | ("ending_cash_balance", "\N"), 99 | ("outstanding_debts", "\N") 100 | )) 101 | grouped[uid][field] = self.safeamt(r['AMOUNT_A']) 102 | self.log(" Writing to filesystem") 103 | out = csv.writer(open(self.target_csv, "wb")) 104 | outheaders = ( 105 | "filing_id_raw", 106 | "amend_id", 107 | "itemized_monetary_contributions", 108 | "unitemized_monetary_contributions", 109 | "total_monetary_contributions", 110 | "non_monetary_contributions", 111 | "total_contributions", 112 | "itemized_expenditures", 113 | "unitemized_expenditures", 114 | "total_expenditures", 115 | "ending_cash_balance", 116 | "outstanding_debts" 117 | ) 118 | out.writerow(outheaders) 119 | for uid, data in grouped.items(): 120 | outrow = uid.split("-") + data.values() 121 | out.writerow(outrow) 122 | 123 | def safeamt(self, num): 124 | if not num: 125 | return "\N" 126 | return num 127 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/models/contributions.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from calaccess_campaign_browser import managers 3 | from django.utils.datastructures import SortedDict 4 | from calaccess_campaign_browser.utils.models import BaseModel 5 | 6 | 7 | class Contribution(BaseModel): 8 | """ 9 | Who gave and how much. 10 | """ 11 | cycle = models.ForeignKey('Cycle') 12 | committee = models.ForeignKey( 13 | 'Committee', 14 | related_name="contributions_to" 15 | ) 16 | filing = models.ForeignKey('Filing') 17 | 18 | # CAL-ACCESS ids 19 | filing_id_raw = models.IntegerField(db_index=True) 20 | transaction_id = models.CharField( 21 | 'transaction ID', 22 | max_length=20, 23 | db_index=True 24 | ) 25 | amend_id = models.IntegerField('amendment', db_index=True) 26 | backreference_transaction_id = models.CharField( 27 | 'backreference transaction ID', 28 | max_length=20, 29 | blank=True, 30 | default='', 31 | db_index=True 32 | ) 33 | is_crossreference = models.CharField(max_length=1, blank=True) 34 | crossreference_schedule = models.CharField(max_length=2, blank=True) 35 | 36 | # Basics about the contrib 37 | is_duplicate = models.BooleanField(default=False) 38 | transaction_type = models.CharField(max_length=1, blank=True) 39 | date_received = models.DateField(null=True) 40 | contribution_description = models.CharField(max_length=90, blank=True) 41 | amount = models.DecimalField(decimal_places=2, max_digits=14) 42 | 43 | # About the contributor 44 | contributor_full_name = models.CharField(max_length=255) 45 | contributor_is_person = models.BooleanField(default=False) 46 | contributor_committee = models.ForeignKey( 47 | 'Committee', 48 | null=True, 49 | related_name="contributions_from" 50 | ) 51 | contributor_prefix = models.CharField(max_length=10, blank=True) 52 | contributor_first_name = models.CharField(max_length=255, blank=True) 53 | contributor_last_name = models.CharField(max_length=200, blank=True) 54 | contributor_suffix = models.CharField(max_length=10, blank=True) 55 | contributor_address_1 = models.CharField(max_length=55, blank=True) 56 | contributor_address_2 = models.CharField(max_length=55, blank=True) 57 | contributor_city = models.CharField(max_length=30, blank=True) 58 | contributor_state = models.CharField(max_length=2, blank=True) 59 | contributor_zipcode = models.CharField(max_length=10, blank=True) 60 | contributor_occupation = models.CharField(max_length=60, blank=True) 61 | contributor_employer = models.CharField(max_length=200, blank=True) 62 | contributor_selfemployed = models.CharField(max_length=1, blank=True) 63 | ENTITY_CODE_CHOICES = ( 64 | ("", "None"), 65 | ("0", "0"), 66 | ("BNM", "BNM"), 67 | ("COM", "Recipient committee"), 68 | ("IND", "Individual"), 69 | ("OFF", "OFF"), 70 | ("OTH", "Other"), 71 | ("PTY", "Political party"), 72 | ("RCP", "RCP"), 73 | ("SCC", "Small contributor committee"), 74 | ) 75 | contributor_entity_type = models.CharField( 76 | max_length=3, 77 | blank=True, 78 | help_text="The type of entity that made that contribution", 79 | choices=ENTITY_CODE_CHOICES 80 | ) 81 | 82 | # About the intermediary 83 | intermediary_prefix = models.CharField(max_length=10, blank=True) 84 | intermediary_first_name = models.CharField(max_length=255, blank=True) 85 | intermediary_last_name = models.CharField(max_length=200, blank=True) 86 | intermediary_suffix = models.CharField(max_length=10, blank=True) 87 | intermediary_address_1 = models.CharField(max_length=55, blank=True) 88 | intermediary_address_2 = models.CharField(max_length=55, blank=True) 89 | intermediary_city = models.CharField(max_length=30, blank=True) 90 | intermediary_state = models.CharField(max_length=2, blank=True) 91 | intermediary_zipcode = models.CharField(max_length=10, blank=True) 92 | intermediary_occupation = models.CharField(max_length=60, blank=True) 93 | intermediary_employer = models.CharField(max_length=200, blank=True) 94 | intermediary_selfemployed = models.CharField(max_length=1, blank=True) 95 | intermediary_committee_id = models.CharField(max_length=9, blank=True) 96 | objects = models.Manager() 97 | real = managers.RealContributionManager() 98 | 99 | class Meta: 100 | app_label = 'calaccess_campaign_browser' 101 | 102 | @models.permalink 103 | def get_absolute_url(self): 104 | return ('contribution_detail', [str(self.pk)]) 105 | 106 | @property 107 | def raw(self): 108 | from calaccess_raw.models import RcptCd 109 | return RcptCd.objects.get( 110 | amend_id=self.amend_id, 111 | filing_id=self.filing.filing_id_raw, 112 | tran_id=self.transaction_id, 113 | bakref_tid=self.backreference_transaction_id 114 | ) 115 | 116 | @property 117 | def contributor_dict(self): 118 | d = SortedDict({}) 119 | for k, v in self.to_dict().items(): 120 | if k.startswith("contributor"): 121 | d[k.replace("contributor ", "")] = v 122 | return d 123 | 124 | @property 125 | def intermediary_dict(self): 126 | d = SortedDict({}) 127 | for k, v in self.to_dict().items(): 128 | if k.startswith("intermediary"): 129 | d[k.replace("intermediary ", "")] = v 130 | return d 131 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/managers.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class BaseRealManager(models.Manager): 5 | """ 6 | Base class with common methods used by managers that exclude duplicate 7 | records from data models. 8 | """ 9 | def get_queryset(self): 10 | """ 11 | Excludes records with an ``is_duplicate`` field set to True. 12 | """ 13 | qs = super(BaseRealManager, self).get_queryset() 14 | return qs.exclude(is_duplicate=True) 15 | 16 | def get_committee(self, obj_or_id): 17 | """ 18 | Returns a Committee model object whether you submit the primary key 19 | from our database or the CAL-ACCESS filing id. 20 | 21 | If a Committee object is submitted it is returns as is. 22 | """ 23 | from .models import Committee 24 | 25 | # Pull the committee object 26 | if isinstance(obj_or_id, Committee): 27 | cmte = obj_or_id 28 | elif isinstance(obj_or_id, int): 29 | try: 30 | cmte = Committee.objects.get(id=obj_or_id) 31 | except Committee.DoesNotExist: 32 | cmte = Committee.objects.get(filing_id_raw=obj_or_id) 33 | else: 34 | raise ValueError("You must submit a committee object or ID") 35 | return cmte 36 | 37 | 38 | class RealFilingManager(BaseRealManager): 39 | """ 40 | Only returns records that are not duplicates 41 | and should be treated for lists and counts as "real." 42 | """ 43 | def by_committee(self, obj_or_id): 44 | """ 45 | Returns the "real" or valid filings for a particular committee. 46 | """ 47 | cmte = self.get_committee(obj_or_id) 48 | 49 | # Filer to only filings by this committee 50 | qs = self.get_queryset().filter(committee=cmte) 51 | 52 | # Get most recent end date for quarterly filings 53 | try: 54 | most_recent_quarterly = qs.filter( 55 | form_type__in=['F450', 'F460'] 56 | ).order_by("-end_date")[0] 57 | except (qs.model.DoesNotExist, IndexError): 58 | # If there are none, just return everything 59 | return qs 60 | 61 | # Exclude all F497 late filings that come before that date 62 | qs = qs.exclude( 63 | form_type='F497', 64 | start_date__lte=most_recent_quarterly.end_date 65 | ) 66 | 67 | # Retun the result 68 | return qs 69 | 70 | 71 | class RealContributionManager(BaseRealManager): 72 | """ 73 | Only returns records that are not duplicates. 74 | """ 75 | def by_committee_to(self, obj_or_id): 76 | """ 77 | Returns the "real" or valid contributions received by 78 | a particular committee. 79 | """ 80 | from .models import Filing 81 | 82 | # Pull the committee object 83 | cmte = self.get_committee(obj_or_id) 84 | 85 | # Get a list of the valid filings for this committee 86 | filing_list = Filing.real.by_committee(cmte) 87 | 88 | # Filer to only contributions from real filings by this committee 89 | qs = self.get_queryset().filter( 90 | committee=cmte, 91 | filing__in=filing_list 92 | ) 93 | 94 | # Retun the result 95 | return qs 96 | 97 | def by_committee_from(self, obj_or_id): 98 | """ 99 | Returns the "real" or valid contributions made by 100 | a particular committee. 101 | """ 102 | from .models import Filing 103 | 104 | # Pull the committee object 105 | cmte = self.get_committee(obj_or_id) 106 | 107 | # Get a list of the valid filings for this committee 108 | filing_list = Filing.real.by_committee(cmte) 109 | 110 | # Filer to only contributions from real filings by this committee 111 | qs = self.get_queryset().filter( 112 | contributor_committee=cmte, 113 | filing__in=filing_list 114 | ) 115 | 116 | # Retun the result 117 | return qs 118 | 119 | 120 | class RealExpenditureManager(models.Manager): 121 | """ 122 | Only returns records that are not duplicates. 123 | """ 124 | def by_committee_to(self, obj_or_id): 125 | """ 126 | Returns the "real" or valid expenditures received by 127 | a particular committee. 128 | """ 129 | from .models import Filing 130 | 131 | # Pull the committee object 132 | cmte = self.get_committee(obj_or_id) 133 | 134 | # Get a list of the valid filings for this committee 135 | filing_list = Filing.real.by_committee(cmte) 136 | 137 | # Filer to only expenditures from real filings by this committee 138 | qs = self.get_queryset().filter( 139 | committee=cmte, 140 | filing__in=filing_list 141 | ) 142 | 143 | # Retun the result 144 | return qs 145 | 146 | def by_committee_from(self, obj_or_id): 147 | """ 148 | Returns the "real" or valid expenditures made by 149 | a particular committee. 150 | """ 151 | from .models import Filing 152 | 153 | # Pull the committee object 154 | cmte = self.get_committee(obj_or_id) 155 | 156 | # Get a list of the valid filings for this committee 157 | filing_list = Filing.real.by_committee(cmte) 158 | 159 | # Filer to only expenditures from real filings by this committee 160 | qs = self.get_queryset().filter( 161 | payee_committee=cmte, 162 | filing__in=filing_list 163 | ) 164 | 165 | # Retun the result 166 | return qs 167 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/models/elections.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from calaccess_campaign_browser.utils.models import BaseModel 3 | 4 | 5 | class Election(BaseModel): 6 | """ 7 | A grouping of election contests administered by the state. 8 | """ 9 | ELECTION_TYPE_CHOICES = ( 10 | ("GENERAL", "General"), 11 | ("PRIMARY", "Primary"), 12 | ("RECALL", "Recall"), 13 | ("SPECIAL", "Special"), 14 | ("SPECIAL_RUNOFF", "Special Runoff"), 15 | ("OTHER", "Other"), 16 | ) 17 | election_type = models.CharField( 18 | choices=ELECTION_TYPE_CHOICES, 19 | max_length=50 20 | ) 21 | year = models.IntegerField() 22 | date = models.DateField(null=True, default=None) 23 | id_raw = models.IntegerField( 24 | verbose_name="UID (CAL-ACCESS)", 25 | help_text="The unique identifer from the CAL-ACCESS site" 26 | ) 27 | sort_index = models.IntegerField( 28 | help_text="The order of the election on the CAL-ACCESS site", 29 | ) 30 | 31 | class Meta: 32 | ordering = ('-sort_index',) 33 | app_label = 'calaccess_campaign_browser' 34 | 35 | def __unicode__(self): 36 | return self.name 37 | 38 | @property 39 | def name(self): 40 | return u'%s %s' % ( 41 | self.year, 42 | self.get_election_type_display(), 43 | ) 44 | 45 | @property 46 | def office_count(self): 47 | """ 48 | The total number of offices with active races this election. 49 | """ 50 | return self.candidate_set.values("office_id").distinct().count() 51 | 52 | @property 53 | def candidate_count(self): 54 | """ 55 | The total number of candidates fundraising for this election. 56 | """ 57 | return self.candidate_set.count() 58 | 59 | 60 | class Office(BaseModel): 61 | """ 62 | An office that is at stake in an election contest. 63 | """ 64 | OFFICE_CHOICES = ( 65 | ("ASSEMBLY", "Assembly"), 66 | ("ATTORNEY_GENERAL", "Attorney General"), 67 | ("BOARD_OF_EQUALIZATION", "Board of Equalization"), 68 | ("CONTROLLER", "Controller"), 69 | ("GOVERNOR", "Governor"), 70 | ("INSURANCE_COMMISSIONER", "Insurance Commissioner"), 71 | ("LIEUTENANT_GOVERNOR", "Lieutenant Governor"), 72 | ("OTHER", "Other"), 73 | ("SECRETARY_OF_STATE", "Secretary of State"), 74 | ("SENATE", "Senate"), 75 | ( 76 | "SUPERINTENDENT_OF_PUBLIC_INSTRUCTION", 77 | "Superintendent of Public Instruction" 78 | ), 79 | ("TREASURER", "Treasurer"), 80 | ) 81 | name = models.CharField( 82 | choices=OFFICE_CHOICES, 83 | max_length=50 84 | ) 85 | seat = models.IntegerField(null=True, default=None) 86 | 87 | class Meta: 88 | ordering = ('name', 'seat',) 89 | app_label = 'calaccess_campaign_browser' 90 | 91 | def __unicode__(self): 92 | s = u'%s' % (self.get_name_display(),) 93 | if self.seat: 94 | s = u'%s (%s)' % (s, self.seat) 95 | return s 96 | 97 | @property 98 | def election_count(self): 99 | """ 100 | The total number of elections with active races this office. 101 | """ 102 | return self.candidate_set.values("election_id").distinct().count() 103 | 104 | @property 105 | def candidate_count(self): 106 | """ 107 | The total number of candidates who have fundraised for this office. 108 | """ 109 | return self.candidate_set.count() 110 | 111 | 112 | class Candidate(BaseModel): 113 | """ 114 | Links filers to the contests and elections where they are on the ballot. 115 | """ 116 | election = models.ForeignKey('Election') 117 | office = models.ForeignKey('Office') 118 | filer = models.ForeignKey('Filer') 119 | 120 | class Meta: 121 | ordering = ("election", "office", "filer") 122 | app_label = 'calaccess_campaign_browser' 123 | 124 | def __unicode__(self): 125 | return u'%s: %s [%s]' % (self.filer, self.office, self.election) 126 | 127 | @property 128 | def election_year(self): 129 | return self.election.year 130 | 131 | @property 132 | def election_type(self): 133 | return self.election.get_election_type_display() 134 | 135 | 136 | class Proposition(BaseModel): 137 | """ 138 | A proposition or ballot measure decided by voters. 139 | """ 140 | name = models.CharField(max_length=255, null=True) 141 | description = models.TextField(blank=True) 142 | id_raw = models.IntegerField(db_index=True) 143 | election = models.ForeignKey('Election', null=True, default=None) 144 | filers = models.ManyToManyField('Filer', through='PropositionFiler') 145 | 146 | class Meta: 147 | ordering = ("election", "name") 148 | app_label = 'calaccess_campaign_browser' 149 | 150 | def __unicode__(self): 151 | return self.name 152 | 153 | @property 154 | def short_description(self, character_limit=60): 155 | if len(self.description) > character_limit: 156 | return self.description[:character_limit] + "..." 157 | return self.description 158 | 159 | 160 | class PropositionFiler(BaseModel): 161 | """ 162 | The relationship between filers and propositions. 163 | """ 164 | POSITION_CHOICES = ( 165 | ('SUPPORT', 'Support'), 166 | ('OPPOSE', 'Oppose'), 167 | ) 168 | proposition = models.ForeignKey('Proposition') 169 | filer = models.ForeignKey('Filer') 170 | position = models.CharField( 171 | choices=POSITION_CHOICES, 172 | max_length=50 173 | ) 174 | 175 | class Meta: 176 | app_label = 'calaccess_campaign_browser' 177 | 178 | def __unicode__(self): 179 | return '%s %s' % (self.proposition, self.filer) 180 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/models/expenditures.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from django.utils.datastructures import SortedDict 3 | 4 | from calaccess_campaign_browser import managers 5 | from calaccess_campaign_browser.utils.models import BaseModel 6 | 7 | 8 | class Expenditure(BaseModel): 9 | """ 10 | Who got paid and how much. 11 | """ 12 | cycle = models.ForeignKey('Cycle') 13 | committee = models.ForeignKey( 14 | 'Committee', 15 | related_name='expenditures_to' 16 | ) 17 | filing = models.ForeignKey('Filing') 18 | 19 | # CAL-ACCESS ids 20 | filing_id_raw = models.IntegerField(db_index=True, null=True) 21 | transaction_id = models.CharField( 22 | 'transaction ID', 23 | max_length=20, 24 | db_index=True, 25 | blank=True, 26 | null=True 27 | ) 28 | amend_id = models.IntegerField('amendment', db_index=True) 29 | backreference_transaction_id = models.CharField( 30 | 'backreference transaction ID', 31 | max_length=50, 32 | db_index=True 33 | ) 34 | is_crossreference = models.CharField(max_length=1, blank=True) 35 | crossreference_schedule = models.CharField(max_length=2, blank=True) 36 | 37 | # Basics about expenditure 38 | is_duplicate = models.BooleanField(default=False) 39 | 40 | date_received = models.DateField(null=True) 41 | expenditure_description = models.CharField(max_length=255, blank=True) 42 | amount = models.DecimalField(max_digits=16, decimal_places=2) 43 | 44 | # About the candidate 45 | candidate_full_name = models.CharField(max_length=255) 46 | candidate_is_person = models.BooleanField(default=False) 47 | candidate_committee = models.ForeignKey( 48 | 'Committee', 49 | null=True, 50 | related_name="expenditures_from" 51 | ) 52 | candidate_prefix = models.CharField(max_length=10, blank=True) 53 | candidate_first_name = models.CharField(max_length=255, blank=True) 54 | candidate_last_name = models.CharField(max_length=200, blank=True) 55 | candidate_suffix = models.CharField(max_length=10, blank=True) 56 | 57 | EXPENDITURE_CODE_CHOICES = ( 58 | ('CMP', 'campaign paraphernalia/misc.'), 59 | ('CNS', 'campaign consultants'), 60 | ('CTB', 'contribution (explain nonmonetary)*'), 61 | ('CVC', 'civic donations'), 62 | ('FIL', 'candidate filing/ballot fees'), 63 | ('FND', 'fundraising events'), 64 | ('IND', 65 | 'independent expenditure supporting/opposing others (explain)*'), 66 | ('LEG', 'legal defense'), 67 | ('LIT', 'campaign literature and mailings'), 68 | ('MBR', 'member communications'), 69 | ('MTG', 'meetings and appearances'), 70 | ('OFC', 'office expenses'), 71 | ('PET', 'petition circulating'), 72 | ('PHO', 'phone banks'), 73 | ('POL', 'polling and survey research'), 74 | ('POS', 'postage, delivery and messenger services'), 75 | ('PRO', 'professional services (legal, accounting)'), 76 | ('PRT', 'print ads'), 77 | ('RAD', 'radio airtime and production costs'), 78 | ('RFD', 'returned contributions'), 79 | ('SAL', "campaign workers' salaries"), 80 | ('TEL', 't.v. or cable airtime and production costs'), 81 | ('TRC', 'candidate travel, lodging, and meals'), 82 | ('TRS', 'staff/spouse travel, lodging, and meals'), 83 | ('TSF', 'transfer between committees of the same candidate/sponsor'), 84 | ('VOT', 'voter registration'), 85 | ('WEB', 'information technology costs (internet, e-mail)'), 86 | ) 87 | candidate_expense_code = models.CharField( 88 | max_length=3, 89 | blank=True, 90 | choices=EXPENDITURE_CODE_CHOICES 91 | ) 92 | ENTITY_CD_CHOICES = ( 93 | ('COM', 'Recipient Committee'), 94 | ('RCP', 'Recipient Committee'), 95 | ('IND', 'Individual'), 96 | ('OTH', 'Other'), 97 | ('PTY', 'PTY - Unknown'), 98 | ('SCC', 'SCC 0 Unknown'), 99 | ('BNM', 'BNM - Unknown'), 100 | ('CAO', 'CAO - Unknown'), 101 | ('OFF', 'OFF - Unknown'), 102 | ('PTH', 'PTH - Unknown'), 103 | ('RFD', 'RFD - Unknown'), 104 | ('MBR', 'MBR - Unknown'), 105 | ('0', '0 - Unknown'), 106 | ) 107 | candidate_entity_type = models.CharField( 108 | max_length=3, 109 | blank=True, 110 | help_text="The type of entity that made that expenditure", 111 | choices=ENTITY_CD_CHOICES 112 | ) 113 | 114 | # About the payee 115 | payee_prefix = models.CharField(max_length=10, blank=True) 116 | payee_first_name = models.CharField(max_length=255, blank=True) 117 | payee_last_name = models.CharField(max_length=200, blank=True) 118 | payee_suffix = models.CharField(max_length=10, blank=True) 119 | payee_city = models.CharField(max_length=30, blank=True) 120 | payee_state = models.CharField(max_length=2, blank=True) 121 | payee_zipcode = models.CharField(max_length=10, blank=True) 122 | payee_committee_id = models.CharField(max_length=9, blank=True) 123 | objects = models.Manager() 124 | real = managers.RealExpenditureManager() 125 | 126 | class Meta: 127 | app_label = 'calaccess_campaign_browser' 128 | 129 | @models.permalink 130 | def get_absolute_url(self): 131 | return ('expenditure_detail', [str(self.pk)]) 132 | 133 | @property 134 | def raw(self): 135 | from calaccess_raw.models import ExpnCd 136 | return ExpnCd.objects.get( 137 | amend_id=self.amend_id, 138 | filing_id=self.filing.filing_id_raw, 139 | tran_id=self.transaction_id, 140 | bakref_tid=self.backreference_transaction_id 141 | ) 142 | 143 | @property 144 | def candidate_dict(self): 145 | d = SortedDict({}) 146 | for k, v in self.to_dict().items(): 147 | if k.startswith("candidate"): 148 | d[k.replace("candidate ", "")] = v 149 | return d 150 | 151 | @property 152 | def payee_dict(self): 153 | d = SortedDict({}) 154 | for k, v in self.to_dict().items(): 155 | if k.startswith("payee"): 156 | d[k.replace("payee ", "")] = v 157 | return d 158 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/templates/calaccess_campaign_browser/party_list.html: -------------------------------------------------------------------------------- 1 | {% extends 'calaccess_campaign_browser/base.html' %} 2 | {% load humanize %} 3 | 4 | {% block content %} 5 |

    Political Parties

    6 | 7 |
    8 |
    9 |

    Party Cashflow In The 2014 Cycle

    10 | 13 |
    14 | 15 |
    16 |
    17 | {% if party.total_contributions_by_year %} 18 |

    Contributions by year

    19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | {% for year, total in party.total_contributions_by_year %} 28 | 29 | 30 | 31 | 32 | {% endfor %} 33 | 34 |
    YearTotal
    {{ party }}${{ total|floatformat:0|intcomma }}
    35 | {% endif %} 36 |
    37 |
    38 | {% if party.total_expenditures_by_year %} 39 |

    Expenditures by year

    40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | {% for year, total in party.total_expenditures_by_year %} 49 | 50 | 51 | 52 | 53 | {% endfor %} 54 | 55 |
    YearTotal
    {{ party }}${{ total|floatformat:0|intcomma }}
    56 | {% endif %} 57 |
    58 |
    59 |
    60 |
    61 |
      62 |
    • Contribution
      • 63 |
    • Expenditure
      • 64 |
    65 |
    66 | 67 | 68 |
    69 |
    70 | 71 |
    72 |

    Big Spenders

    73 |
    74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 |
     CandidateTotal ExpendituresParty
    #---$---
    91 |
    92 |

    93 | 94 |
    95 |
    96 | 97 |
    98 |

    $ Hoarders

    99 |
    100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 |
     CandidateTotal ExpendituresParty
    #---$---
    117 |
    118 |
    119 | 120 |
    121 |
    122 | 123 | 124 | 125 | 126 | 127 | 128 | {% endblock %} 129 | 130 | 131 | 205 | 206 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/management/commands/importtosqlserver.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import csv 4 | import fnmatch 5 | from optparse import make_option 6 | 7 | import pypyodbc 8 | 9 | from django.conf import settings 10 | from django.db import connection 11 | from django.db.models import get_model 12 | from django.core.management.base import AppCommand 13 | 14 | from calaccess_campaign_browser.management.commands import CalAccessCommand 15 | 16 | custom_options = ( 17 | make_option( 18 | "--skip-contributions", 19 | action="store_false", 20 | dest="contributions", 21 | default=True, 22 | help="Skip contributions import" 23 | ), 24 | make_option( 25 | "--skip-expenditures", 26 | action="store_false", 27 | dest="expenditures", 28 | default=True, 29 | help="Skip expenditures import" 30 | ), 31 | make_option( 32 | "--skip-summary", 33 | action="store_false", 34 | dest="summary", 35 | default=True, 36 | help="Skip summary import" 37 | ), 38 | ) 39 | 40 | 41 | def all_files(root, patterns='*', single_level=False, yield_folders=False): 42 | """ 43 | Expand patterns form semicolon-separated string to list 44 | example usage: thefiles = list(all_files('/tmp', '*.py;*.htm;*.html')) 45 | """ 46 | patterns = patterns.split(';') 47 | 48 | for path, subdirs, files in os.walk(root): 49 | if yield_folders: 50 | files.extend(subdirs) 51 | 52 | files.sort() 53 | 54 | for name in files: 55 | for pattern in patterns: 56 | if fnmatch.fnmatch(name, pattern): 57 | yield os.path.join(path, name) 58 | break 59 | 60 | if single_level: 61 | break 62 | 63 | 64 | class Command(CalAccessCommand): 65 | """ 66 | Send CSVs exported from `exportcalaccesscampaignbrowser` to 67 | Microsoft SQL Server 68 | """ 69 | option_list = CalAccessCommand.option_list + custom_options 70 | conn_path = ( 71 | 'Driver=%s;Server=%s;port=%s;uid=%s;pwd=%s;database=%s;autocommit=1' 72 | ) % ( 73 | settings.SQL_SERVER_DRIVER, 74 | settings.SQL_SERVER_ADDRESS, 75 | settings.SQL_SERVER_PORT, 76 | settings.SQL_SERVER_USER, 77 | settings.SQL_SERVER_PASSWORD, 78 | settings.SQL_SERVER_DATABASE 79 | ) 80 | 81 | conn = pypyodbc.connect(conn_path) 82 | cursor = conn.cursor() 83 | app = AppCommand() 84 | 85 | def set_options(self, *args, **kwargs): 86 | self.data_dir = os.path.join( 87 | settings.BASE_DIR, 'data') 88 | os.path.exists(self.data_dir) or os.mkdir(self.data_dir) 89 | 90 | def generate_table_schema(self, model_name): 91 | """ 92 | Take Expenditure, Contribution or Summary models; grab their db schema, 93 | and create MS SQL Server compatible schema 94 | """ 95 | self.log(' Creating database schema for {} ...'.format(model_name)) 96 | style = self.app.style 97 | 98 | model = get_model('calaccess_campaign_browser', model_name) 99 | 100 | table_name = 'dbo.{}'.format(model._meta.db_table) 101 | 102 | raw_statement = connection.creation\ 103 | .sql_create_model(model, style)[0][0] 104 | 105 | # http://stackoverflow.com/a/14693789/868724 106 | ansi_escape = re.compile(r'\x1b[^m]*m') 107 | strip_ansi_statement = (ansi_escape.sub('', raw_statement)) 108 | statement = strip_ansi_statement.replace('\n', '')\ 109 | .replace('`', '')\ 110 | .replace('bool', 'bit')\ 111 | .replace(' AUTO_INCREMENT', '')\ 112 | .replace(model._meta.db_table, table_name)\ 113 | .replace('NOT NULL', '') 114 | 115 | statement = """{}, committee_name varchar(255),\ 116 | filer_name varchar(255), filer_id integer,\ 117 | filer_id_raw integer );""".format(statement[:-3]) 118 | 119 | self.construct_table(model_name, table_name, statement) 120 | 121 | def construct_table(self, model_name, table_name, query): 122 | """ 123 | Create matching MS SQL Server database table 124 | """ 125 | statement = str(query) 126 | self.log(' Creating {} table ...'.format(table_name)) 127 | drop_path = "IF object_id('{}') IS NOT NULL DROP TABLE {}".format( 128 | table_name, table_name) 129 | 130 | self.cursor.execute(drop_path) 131 | self.cursor.execute(statement) 132 | self.cursor.commit() 133 | 134 | self.success(' {} created'.format(table_name)) 135 | 136 | self.load_table(table_name, model_name) 137 | 138 | def load_table(self, table_name, model_name): 139 | """ 140 | Load Table with CSVs generated from `exportcalaccesscampaignbrowser` 141 | See: https://msdn.microsoft.com/en-us/library/ms188609.aspx 142 | """ 143 | self.log(' Loading table {} ...'.format(table_name)) 144 | all_csvs = list(all_files(self.data_dir, '*.csv')) 145 | 146 | csv_ = [f for f in all_csvs if fnmatch.fnmatch(f, '*-{}.csv'.format( 147 | model_name))] 148 | 149 | if len(csv_) > 1: 150 | self.log(' There are multiple files matching {}'.format( 151 | model_name)) 152 | self.log(' We only support one match at the moment. Sorry!') 153 | 154 | raise NotImplementedError 155 | 156 | with open(csv_[0]) as csvfile: 157 | reader = csv.reader(csvfile, delimiter='\t') 158 | reader.next() # skip headers 159 | 160 | for row in reader: 161 | # Remove none values and turn booleans into bit type 162 | row = [r.replace('"', '') for r in row] 163 | row = [r.replace('None', '') for r in row] 164 | row = [r.replace('True', '0') for r in row] 165 | row = [r.replace('False', '1') for r in row] 166 | 167 | sql = """INSERT INTO {} VALUES {};""".format( 168 | table_name, tuple(row)) 169 | 170 | try: 171 | self.cursor.execute(sql) 172 | self.log(' loading {} ID:{} ...'.format( 173 | model_name, row[0])) 174 | except pypyodbc.Error, e: 175 | self.failure(' Encountered an arror') 176 | raise e 177 | 178 | self.cursor.commit() 179 | self.success(' Loaded {} with data from {}'.format( 180 | table_name, os.path.split(csv_[0])[1])) 181 | 182 | def handle(self, *args, **options): 183 | self.header('Importing models ...') 184 | self.set_options(*args, **options) 185 | 186 | if options['contributions']: 187 | self.generate_table_schema('contribution') 188 | 189 | if options['expenditures']: 190 | self.generate_table_schema('expenditure') 191 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/models/filings.py: -------------------------------------------------------------------------------- 1 | import json 2 | from django.db import models 3 | from django.db.models import Sum 4 | from calaccess_campaign_browser import managers 5 | from calaccess_campaign_browser.utils.models import BaseModel 6 | from calaccess_campaign_browser.models import Contribution, Expenditure 7 | from calaccess_campaign_browser.templatetags.calaccesscampaignbrowser import ( 8 | jsonify 9 | ) 10 | 11 | 12 | class Cycle(BaseModel): 13 | name = models.IntegerField(db_index=True, primary_key=True) 14 | 15 | class Meta: 16 | ordering = ("-name",) 17 | app_label = 'calaccess_campaign_browser' 18 | 19 | def __unicode__(self): 20 | return unicode(self.name) 21 | 22 | 23 | class Filing(models.Model): 24 | cycle = models.ForeignKey('Cycle') 25 | committee = models.ForeignKey('Committee') 26 | filing_id_raw = models.IntegerField('filing ID', db_index=True) 27 | amend_id = models.IntegerField('amendment', db_index=True) 28 | FORM_TYPE_CHOICES = ( 29 | ('F497', 'F497: Late filing'), 30 | ('F460', 'F460: Quarterly'), 31 | ('F450', 'F450: Quarterly (Short)'), 32 | ) 33 | form_type = models.CharField( 34 | max_length=7, 35 | db_index=True, 36 | choices=FORM_TYPE_CHOICES 37 | ) 38 | start_date = models.DateField(null=True) 39 | end_date = models.DateField(null=True) 40 | date_received = models.DateField(null=True) 41 | date_filed = models.DateField(null=True) 42 | is_duplicate = models.BooleanField( 43 | default=False, 44 | db_index=True, 45 | help_text="A record that has either been superceded by an amendment \ 46 | or was filed unnecessarily. Should be excluded from most analysis." 47 | ) 48 | objects = models.Manager() 49 | real = managers.RealFilingManager() 50 | 51 | class Meta: 52 | app_label = 'calaccess_campaign_browser' 53 | 54 | def __unicode__(self): 55 | return unicode(self.filing_id_raw) 56 | 57 | @models.permalink 58 | def get_absolute_url(self): 59 | return ('filing_detail', [str(self.pk)]) 60 | 61 | def to_json(self): 62 | js = json.loads(jsonify(self)) 63 | s = self.summary or {} 64 | if s: 65 | s = json.loads(jsonify(s)) 66 | js['summary'] = s 67 | return json.dumps(js) 68 | 69 | def get_calaccess_pdf_url(self): 70 | url = "http://cal-access.ss.ca.gov/PDFGen/pdfgen.prg" 71 | qs = "filingid=%s&amendid=%s" % ( 72 | self.filing_id_raw, 73 | self.amend_id 74 | ) 75 | return "%s?%s" % (url, qs) 76 | 77 | def committee_short_name(self): 78 | return self.committee.short_name 79 | committee_short_name.short_description = "committee" 80 | 81 | @property 82 | def summary(self): 83 | try: 84 | return Summary.objects.get( 85 | filing_id_raw=self.filing_id_raw, 86 | amend_id=self.amend_id 87 | ) 88 | except Summary.DoesNotExist: 89 | return None 90 | 91 | @property 92 | def is_amendment(self): 93 | return self.amend_id > 0 94 | 95 | @property 96 | def is_late(self): 97 | return self.form_type == 'F497' 98 | 99 | @property 100 | def is_quarterly(self): 101 | return self.form_type in ['F450', 'F460'] 102 | 103 | @property 104 | def total_contributions(self): 105 | if self.is_quarterly: 106 | summary = self.summary 107 | if summary: 108 | return summary.total_contributions 109 | else: 110 | return None 111 | elif self.is_late: 112 | return Contribution.real.filter( 113 | filing=self 114 | ).aggregate(total=Sum('amount'))['total'] 115 | 116 | @property 117 | def total_expenditures(self): 118 | if self.is_quarterly: 119 | summary = self.summary 120 | if summary: 121 | return summary.total_expenditures 122 | else: 123 | return None 124 | elif self.is_late: 125 | return Expenditure.real.filter( 126 | filing=self 127 | ).aggregate(total=Sum('amount'))['total'] 128 | 129 | 130 | class Summary(BaseModel): 131 | """ 132 | A set of summary totals provided by a filing's cover sheet. 133 | """ 134 | filing_id_raw = models.IntegerField(db_index=True) 135 | amend_id = models.IntegerField(db_index=True) 136 | itemized_monetary_contributions = models.DecimalField( 137 | max_digits=16, 138 | decimal_places=2, 139 | null=True, 140 | default=None, 141 | ) 142 | unitemized_monetary_contributions = models.DecimalField( 143 | max_digits=16, 144 | decimal_places=2, 145 | null=True, 146 | default=None, 147 | ) 148 | total_monetary_contributions = models.DecimalField( 149 | max_digits=16, 150 | decimal_places=2, 151 | null=True, 152 | default=None, 153 | ) 154 | non_monetary_contributions = models.DecimalField( 155 | max_digits=16, 156 | decimal_places=2, 157 | null=True, 158 | default=None, 159 | ) 160 | total_contributions = models.DecimalField( 161 | max_digits=16, 162 | decimal_places=2, 163 | null=True, 164 | default=None, 165 | ) 166 | itemized_expenditures = models.DecimalField( 167 | max_digits=16, 168 | decimal_places=2, 169 | null=True, 170 | default=None, 171 | ) 172 | unitemized_expenditures = models.DecimalField( 173 | max_digits=16, 174 | decimal_places=2, 175 | null=True, 176 | default=None, 177 | ) 178 | total_expenditures = models.DecimalField( 179 | max_digits=16, 180 | decimal_places=2, 181 | null=True, 182 | default=None, 183 | ) 184 | ending_cash_balance = models.DecimalField( 185 | max_digits=16, 186 | decimal_places=2, 187 | null=True, 188 | default=None, 189 | ) 190 | outstanding_debts = models.DecimalField( 191 | max_digits=16, 192 | decimal_places=2, 193 | null=True, 194 | default=None, 195 | ) 196 | 197 | class Meta: 198 | verbose_name_plural = "summaries" 199 | app_label = 'calaccess_campaign_browser' 200 | 201 | def __unicode__(self): 202 | return unicode(self.filing_id_raw) 203 | 204 | @property 205 | def cycle(self): 206 | try: 207 | return self.filing.cycle 208 | except: 209 | return None 210 | 211 | @property 212 | def committee(self): 213 | try: 214 | return self.filing.committee 215 | except: 216 | return None 217 | 218 | @property 219 | def filing(self): 220 | try: 221 | return Filing.objects.get( 222 | filing_id_raw=self.filing_id_raw, 223 | amend_id=self.amend_id 224 | ) 225 | except Filing.DoesNotExist: 226 | return None 227 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/management/commands/scrapecalaccesscampaigncandidates.py: -------------------------------------------------------------------------------- 1 | import re 2 | import urlparse 3 | from time import sleep 4 | from calaccess_campaign_browser.management.commands import ScrapeCommand 5 | from calaccess_campaign_browser.models import ( 6 | Election, 7 | Office, 8 | Candidate, 9 | Filer 10 | ) 11 | 12 | 13 | class Command(ScrapeCommand): 14 | """ 15 | Scraper to get the list of candidates per election. 16 | """ 17 | help = "Scrape links between filers and elections from CAL-ACCESS site" 18 | 19 | def build_results(self): 20 | self.header("Scraping election candidates") 21 | 22 | url = urlparse.urljoin( 23 | self.base_url, 24 | '/Campaign/Candidates/list.aspx?view=certified&electNav=93' 25 | ) 26 | soup = self.get(url) 27 | 28 | # Get all the links out 29 | links = soup.findAll('a', href=re.compile(r'^.*&electNav=\d+')) 30 | 31 | # Drop the link that says "prior elections" because it's a duplicate 32 | links = [ 33 | l for l in links 34 | if l.find_next_sibling('span').text != 'Prior Elections' 35 | ] 36 | 37 | # Loop through the links... 38 | results = [] 39 | for i, link in enumerate(links): 40 | # .. go and get each page and its data 41 | url = urlparse.urljoin(self.base_url, link["href"]) 42 | data = self.scrape_page(url) 43 | # Parse out the name and year 44 | data['raw_name'] = link.find_next_sibling('span').text.strip() 45 | data['election_type'] = self.parse_election_name(data['raw_name']) 46 | data['year'] = int(data['raw_name'][:4]) 47 | # The index value is used to preserve sorting of elections, 48 | # since multiple elections may occur in a year. 49 | # BeautifulSoup goes from top to bottom, 50 | # but the top most election is the most recent so it should 51 | # have the highest id. 52 | data['sort_index'] = len(links) - i 53 | # Add it to the list 54 | results.append(data) 55 | # Take a rest 56 | sleep(0.5) 57 | 58 | # Pass out the data 59 | return results 60 | 61 | def scrape_page(self, url): 62 | """ 63 | Pull the elections and candidates from a CAL-ACCESS page. 64 | """ 65 | # Go and get the page 66 | soup = self.get(url) 67 | 68 | # Loop through all the election sets on the page 69 | sections = {} 70 | for section in soup.findAll('a', {'name': re.compile(r'[a-z]+')}): 71 | 72 | # Check that this data matches the structure we expect. 73 | section_name_el = section.find('span', {'class': 'hdr14'}) 74 | 75 | # If it doesn't just skip this one 76 | if not section_name_el: 77 | continue 78 | 79 | # Get the name out of page and key it in the data dictionary 80 | section_name = section_name_el.text 81 | sections[section_name] = {} 82 | 83 | # Loop thorugh all the rows in the section table 84 | for office in section.findAll('td'): 85 | 86 | # Check that this data matches the structure we expect. 87 | title_el = office.find('span', {'class': 'hdr13'}) 88 | 89 | # If it doesn't, just quit no 90 | if not title_el: 91 | continue 92 | 93 | # Log what we're up to 94 | if self.verbosity > 2: 95 | self.log(' Scraping office %s' % title_el.text) 96 | 97 | # Pull the candidates out 98 | people = [] 99 | for p in office.findAll('a', {'class': 'sublink2'}): 100 | people.append({ 101 | 'name': p.text, 102 | 'id': re.match(r'.+id=(\d+)', p['href']).group(1) 103 | }) 104 | 105 | for p in office.findAll('span', {'class': 'txt7'}): 106 | people.append({ 107 | 'name': p.text, 108 | 'id': None 109 | }) 110 | 111 | # Add it to the data dictionary 112 | sections[section_name][title_el.text] = people 113 | 114 | return { 115 | 'id': int(re.match(r'.+electNav=(\d+)', url).group(1)), 116 | 'data': sections, 117 | } 118 | 119 | def process_results(self, results): 120 | self.log(' Creating and/or updating models...') 121 | self.log(' Found %s elections.' % len(results)) 122 | 123 | # Loop through all the results 124 | for d in results: 125 | 126 | self.log(' Processing %s' % d['raw_name']) 127 | 128 | election, c = Election.objects.get_or_create( 129 | year=d['year'], 130 | election_type=d['election_type'], 131 | id_raw=d['id'], 132 | sort_index=d['sort_index'] 133 | ) 134 | 135 | if self.verbosity > 2: 136 | if c: 137 | self.log(' Created %s' % election) 138 | 139 | # Loop through the data list from the scraped page 140 | for office_dict in d['data'].values(): 141 | 142 | # Loop through each of the offices we found there 143 | for office_name, candidates in office_dict.items(): 144 | 145 | # Create an office object out of the data 146 | office, c = Office.objects.get_or_create( 147 | **self.parse_office_name(office_name) 148 | ) 149 | 150 | # Log 151 | if self.verbosity > 2: 152 | if c: 153 | self.log(' Created %s' % office) 154 | 155 | # Loop through each of the candidates 156 | for candidate in candidates: 157 | 158 | # If it doesn't have an id, skip it. 159 | if not candidate['id']: 160 | continue 161 | # If it does, try to find a matching Filer object 162 | # and then add it to the database 163 | try: 164 | # Pull the Filer object from the database 165 | filer = Filer.objects.get( 166 | filer_id_raw=int(candidate['id']) 167 | ) 168 | 169 | # Get or create the Candidate object 170 | candidate, c = Candidate.objects.get_or_create( 171 | election=election, 172 | office=office, 173 | filer=filer 174 | ) 175 | 176 | # Log 177 | if self.verbosity > 2: 178 | if c: 179 | self.log(' Created %s' % candidate) 180 | 181 | # And if there isn't filer just rock on 182 | except Filer.DoesNotExist: 183 | pass 184 | -------------------------------------------------------------------------------- /calaccess_campaign_browser/static/calaccess_campaign_browser/css/main.css: -------------------------------------------------------------------------------- 1 | .red-border { 2 | border-bottom: solid 7px #ba0a35; 3 | height: 45px; 4 | } 5 | table { 6 | font-size:13px; 7 | 8 | } 9 | 10 | .bold { font-weight: bold; } 11 | 12 | th { 13 | background-color: #fefefe; 14 | } 15 | th[data-sort="int"]:hover, th[data-sort="currency"]:hover, th[data-sort="date"]:hover, th[data-sort="string"]:hover { 16 | cursor:pointer; 17 | } 18 | 19 | td.right, th.right { 20 | text-align:right; 21 | } 22 | td.center, th.center{ 23 | text-align:center; 24 | } 25 | header { 26 | padding-top: 5px; 27 | } 28 | .totals{ 29 | font-size: 18px; 30 | } 31 | .view-more{ 32 | margin-top:-10px; 33 | } 34 | .section-select{ 35 | width:100%; 36 | z-index: 100; 37 | } 38 | .affix{ 39 | top:0px; 40 | width:100%; 41 | } 42 | .navbar-default { 43 | background-color: #fff; 44 | border-color: #fff; 45 | } 46 | .breadcrumb { 47 | padding: 10px 10px 10px 0; 48 | margin-bottom: 0px; 49 | border-color: #fff; 50 | background-color: #fff; 51 | list-style: none; 52 | border-radius: 4px; 53 | } 54 | .breadcrumb>li>a { 55 | background-color: #ba0a35; 56 | color: #fff; 57 | padding: 2px 4px; 58 | } 59 | .align-right { 60 | float: right; 61 | border: solid 0px #ba0a35; 62 | border-radius: 4px; 63 | margin-right: -30px; 64 | } 65 | } 66 | .navbar-header{ 67 | margin-left:-25px !important; 68 | } 69 | .navbar-nav>li>a { 70 | padding-top: 10px; 71 | padding-bottom: 10px; 72 | line-height: 1; 73 | } 74 | .navbar { 75 | position: relative; 76 | min-height: 20px; 77 | margin-bottom:0px; 78 | } 79 | ol { 80 | line-height:1; 81 | } 82 | .nav-pills>li.active>a, 83 | .nav-pills>li.active>a:hover, 84 | .nav-pills>li.active>a:focus { 85 | color: #ba0a35; 86 | background-color: #fff; 87 | } 88 | .nav>li>a { 89 | position: relative; 90 | display: block; 91 | padding: 10px 15px 8px 15px; 92 | border-radius:0px; 93 | color:#000; 94 | } 95 | .nav .logo { 96 | display:inline; 97 | padding-top:0; 98 | } 99 | .nav .logo:hover, .nav .logo:active, .nav .logo:focus { 100 | background:#fff; 101 | } 102 | .nav>li>a:hover { 103 | color:#fff; 104 | background-color:#ba0a35; 105 | } 106 | .btn-default, .btn-default:hover{ 107 | border-color: #ba0a35; 108 | } 109 | .btn-default:hover{ 110 | background-color: #ba0a35; 111 | color: #ffffff; 112 | } 113 | a, a.btn-default{ 114 | color:#ba0a35; 115 | } 116 | a:hover{ 117 | color:#d87d85; 118 | } 119 | .navbar-default .navbar-nav>li>a { 120 | color: #ba0a35; 121 | } 122 | .pagination>li>a, .pagination>li>span { 123 | color: #ba0a35; 124 | } 125 | .pagination>li>a:hover, .pagination>li>span:hover { 126 | color:#d87d85; 127 | } 128 | .pagination>.active>a, 129 | .pagination>.active>span, 130 | .pagination>.active>a:hover, 131 | .pagination>.active>span:hover, 132 | .pagination>.active>a:focus, 133 | .pagination>.active>span:focus { 134 | z-index: 2; 135 | color: #fff; 136 | cursor: default; 137 | background-color: #ba0a35; 138 | border-color: #ba0a35; 139 | } 140 | 141 | /* Graph Styles */ 142 | .yeartext, .barContainer, .contrib *, .expend * { 143 | display: inline-block; 144 | } 145 | .contrib, .expend, .bar { 146 | height: 2em; 147 | } 148 | .yeargroup { 149 | height: 5em; 150 | } 151 | .yeartext { 152 | height: 4.2em; 153 | font-size: 1.2em; 154 | padding-right: 10px; 155 | vertical-align: middle; 156 | } 157 | .contrib span, .expend span { 158 | vertical-align: bottom; 159 | height: 2em; 160 | padding-left: 10px; 161 | } 162 | ul.legend { 163 | list-style: none; 164 | } 165 | ul.legend li { 166 | display: inline-block; 167 | padding: 15px; 168 | } 169 | .legenditem span{ 170 | width: 1.5em; 171 | height: 1.5em; 172 | display: inline-block; 173 | margin-right: 5px; 174 | } 175 | .series1{ 176 | background-color: gray; 177 | } 178 | .series2{ 179 | background-color: lightgray; 180 | } 181 | .graphheader > *{ 182 | display: inline-block; 183 | margin-right: 20px; 184 | } 185 | .toggle-btn{ 186 | background-color: #ba0a35; 187 | color: #fff; 188 | padding: 2px 4px; 189 | } 190 | p.party-label{ 191 | display:inline; 192 | } 193 | .party-label-container{ 194 | display:inline-block; 195 | } 196 | .party{ 197 | white-space:nowrap; 198 | padding:2px 3px 2px 0px; 199 | border-style:solid; 200 | border-width:1px; 201 | margin-left: -5px; 202 | padding: 1px 3px; 203 | border-radius: 0px 2px 2px 0px; 204 | -moz-border-radius: 0px 2px 2px 0px; 205 | -webkit-border-radius: 0px 2px 2px 0px; 206 | } 207 | .party.republican{ 208 | border-color:#ba0a35; 209 | color:#ba0a35; 210 | } 211 | .party.democratic{ 212 | border-color:#08009C; 213 | color:#08009C; 214 | } 215 | .party.libertarian{ 216 | border-color:#DCB732; 217 | color:#DCB732; 218 | } 219 | .party.americans-elect{ 220 | border-color:#97AEBD; 221 | color:#97AEBD; 222 | } 223 | .party.natural-law{ 224 | border-color:#CC0066; 225 | color:#CC0066; 226 | } 227 | .party.american-independent{ 228 | border-color:#042c72; 229 | color:#042c72; 230 | } 231 | .party.independent{ 232 | border-color:#7000B5; 233 | color:#7000B5; 234 | } 235 | .party.peace-and-freedom{ 236 | border-color:#000000; 237 | color:#000000; 238 | } 239 | .party.green{ 240 | border-color:#17ab5c; 241 | color:#17ab5c; 242 | } 243 | .party.reform{ 244 | border-color:#2a609e; 245 | color:#2a609e; 246 | } 247 | .party.non-partisan{ 248 | border-color:#969696; 249 | color:#969696; 250 | } 251 | .party.unknown{ 252 | border-color:#11A6A3; 253 | color:#11A6A3; 254 | } 255 | .party.no-party-preference{ 256 | border-color:#E6A100; 257 | color:#E6A100; 258 | } 259 | .party.na{ 260 | border-color:#C9C9C9; 261 | color:#C9C9C9; 262 | } 263 | .party-image{ 264 | padding:2px 3px; 265 | vertical-align: top; 266 | border-radius: 2px 0px 0px 2px; 267 | -moz-border-radius: 2px 0px 0px 2px; 268 | -webkit-border-radius: 2px 0px 0px 2px; 269 | } 270 | .party-image.republican{ 271 | background-color:#ba0a35; 272 | } 273 | .party-image.democratic{ 274 | background-color:#08009C; 275 | } 276 | .party-image.libertarian{ 277 | background-color:#DCB732; 278 | } 279 | .party-image.americans-elect{ 280 | background-color:#97AEBD; 281 | } 282 | .party-image.natural-law{ 283 | background-color:#CC0066; 284 | } 285 | .party-image.american-independent{ 286 | background-color:#042c72; 287 | } 288 | .party-image.independent{ 289 | background-color:#7000B5; 290 | } 291 | .party-image.peace-and-freedom{ 292 | background-color:#000000; 293 | } 294 | .party-image.green{ 295 | background-color:#17ab5c; 296 | } 297 | .party-image.reform{ 298 | background-color:#2a609e; 299 | } 300 | .party-image.non-partisan{ 301 | background-color:#969696; 302 | } 303 | .party-image.unknown{ 304 | background-color:#11A6A3; 305 | } 306 | .party-image.no-party-preference{ 307 | background-color:#E6A100; 308 | } 309 | .party-image.na{ 310 | background-color:#C9C9C9; 311 | } 312 | 313 | .view-filing{ 314 | vertical-align: top; 315 | margin-left: 5px; 316 | } 317 | .True{ 318 | color: #438238; 319 | font-weight: 500; 320 | } 321 | .False{ 322 | color: #333; 323 | } 324 | 325 | @media only screen 326 | and (max-width : 992px) { 327 | .align-right{ 328 | float:left; 329 | margin-left: -20px; 330 | } 331 | } 332 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | livehtml: 58 | sphinx-autobuild -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 59 | 60 | dirhtml: 61 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 62 | @echo 63 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 64 | 65 | singlehtml: 66 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 67 | @echo 68 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 69 | 70 | pickle: 71 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 72 | @echo 73 | @echo "Build finished; now you can process the pickle files." 74 | 75 | json: 76 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 77 | @echo 78 | @echo "Build finished; now you can process the JSON files." 79 | 80 | htmlhelp: 81 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 82 | @echo 83 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 84 | ".hhp project file in $(BUILDDIR)/htmlhelp." 85 | 86 | qthelp: 87 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 88 | @echo 89 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 90 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 91 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/django-calaccess-campaign-finance.qhcp" 92 | @echo "To view the help file:" 93 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/django-calaccess-campaign-finance.qhc" 94 | 95 | devhelp: 96 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 97 | @echo 98 | @echo "Build finished." 99 | @echo "To view the help file:" 100 | @echo "# mkdir -p $$HOME/.local/share/devhelp/django-calaccess-campaign-finance" 101 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/django-calaccess-campaign-finance" 102 | @echo "# devhelp" 103 | 104 | epub: 105 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 106 | @echo 107 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 108 | 109 | latex: 110 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 111 | @echo 112 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 113 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 114 | "(use \`make latexpdf' here to do that automatically)." 115 | 116 | latexpdf: 117 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 118 | @echo "Running LaTeX files through pdflatex..." 119 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 120 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 121 | 122 | latexpdfja: 123 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 124 | @echo "Running LaTeX files through platex and dvipdfmx..." 125 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 126 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 127 | 128 | text: 129 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 130 | @echo 131 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 132 | 133 | man: 134 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 135 | @echo 136 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 137 | 138 | texinfo: 139 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 140 | @echo 141 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 142 | @echo "Run \`make' in that directory to run these through makeinfo" \ 143 | "(use \`make info' here to do that automatically)." 144 | 145 | info: 146 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 147 | @echo "Running Texinfo files through makeinfo..." 148 | make -C $(BUILDDIR)/texinfo info 149 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 150 | 151 | gettext: 152 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 153 | @echo 154 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 155 | 156 | changes: 157 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 158 | @echo 159 | @echo "The overview file is in $(BUILDDIR)/changes." 160 | 161 | linkcheck: 162 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 163 | @echo 164 | @echo "Link check complete; look for any errors in the above output " \ 165 | "or in $(BUILDDIR)/linkcheck/output.txt." 166 | 167 | doctest: 168 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 169 | @echo "Testing of doctests in the sources finished, look at the " \ 170 | "results in $(BUILDDIR)/doctest/output.txt." 171 | 172 | xml: 173 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 174 | @echo 175 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 176 | 177 | pseudoxml: 178 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 179 | @echo 180 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 181 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | goto end 41 | ) 42 | 43 | if "%1" == "clean" ( 44 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 45 | del /q /s %BUILDDIR%\* 46 | goto end 47 | ) 48 | 49 | 50 | %SPHINXBUILD% 2> nul 51 | if errorlevel 9009 ( 52 | echo. 53 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 54 | echo.installed, then set the SPHINXBUILD environment variable to point 55 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 56 | echo.may add the Sphinx directory to PATH. 57 | echo. 58 | echo.If you don't have Sphinx installed, grab it from 59 | echo.http://sphinx-doc.org/ 60 | exit /b 1 61 | ) 62 | 63 | if "%1" == "html" ( 64 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 68 | goto end 69 | ) 70 | 71 | if "%1" == "dirhtml" ( 72 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 76 | goto end 77 | ) 78 | 79 | if "%1" == "singlehtml" ( 80 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 84 | goto end 85 | ) 86 | 87 | if "%1" == "pickle" ( 88 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can process the pickle files. 92 | goto end 93 | ) 94 | 95 | if "%1" == "json" ( 96 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 97 | if errorlevel 1 exit /b 1 98 | echo. 99 | echo.Build finished; now you can process the JSON files. 100 | goto end 101 | ) 102 | 103 | if "%1" == "htmlhelp" ( 104 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 105 | if errorlevel 1 exit /b 1 106 | echo. 107 | echo.Build finished; now you can run HTML Help Workshop with the ^ 108 | .hhp project file in %BUILDDIR%/htmlhelp. 109 | goto end 110 | ) 111 | 112 | if "%1" == "qthelp" ( 113 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 117 | .qhcp project file in %BUILDDIR%/qthelp, like this: 118 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\django-calaccess-campaign-finance.qhcp 119 | echo.To view the help file: 120 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\django-calaccess-campaign-finance.ghc 121 | goto end 122 | ) 123 | 124 | if "%1" == "devhelp" ( 125 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished. 129 | goto end 130 | ) 131 | 132 | if "%1" == "epub" ( 133 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 137 | goto end 138 | ) 139 | 140 | if "%1" == "latex" ( 141 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 145 | goto end 146 | ) 147 | 148 | if "%1" == "latexpdf" ( 149 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 150 | cd %BUILDDIR%/latex 151 | make all-pdf 152 | cd %BUILDDIR%/.. 153 | echo. 154 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 155 | goto end 156 | ) 157 | 158 | if "%1" == "latexpdfja" ( 159 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 160 | cd %BUILDDIR%/latex 161 | make all-pdf-ja 162 | cd %BUILDDIR%/.. 163 | echo. 164 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 165 | goto end 166 | ) 167 | 168 | if "%1" == "text" ( 169 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 170 | if errorlevel 1 exit /b 1 171 | echo. 172 | echo.Build finished. The text files are in %BUILDDIR%/text. 173 | goto end 174 | ) 175 | 176 | if "%1" == "man" ( 177 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 178 | if errorlevel 1 exit /b 1 179 | echo. 180 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 181 | goto end 182 | ) 183 | 184 | if "%1" == "texinfo" ( 185 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 186 | if errorlevel 1 exit /b 1 187 | echo. 188 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 189 | goto end 190 | ) 191 | 192 | if "%1" == "gettext" ( 193 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 194 | if errorlevel 1 exit /b 1 195 | echo. 196 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 197 | goto end 198 | ) 199 | 200 | if "%1" == "changes" ( 201 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 202 | if errorlevel 1 exit /b 1 203 | echo. 204 | echo.The overview file is in %BUILDDIR%/changes. 205 | goto end 206 | ) 207 | 208 | if "%1" == "linkcheck" ( 209 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 210 | if errorlevel 1 exit /b 1 211 | echo. 212 | echo.Link check complete; look for any errors in the above output ^ 213 | or in %BUILDDIR%/linkcheck/output.txt. 214 | goto end 215 | ) 216 | 217 | if "%1" == "doctest" ( 218 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 219 | if errorlevel 1 exit /b 1 220 | echo. 221 | echo.Testing of doctests in the sources finished, look at the ^ 222 | results in %BUILDDIR%/doctest/output.txt. 223 | goto end 224 | ) 225 | 226 | if "%1" == "xml" ( 227 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 228 | if errorlevel 1 exit /b 1 229 | echo. 230 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 231 | goto end 232 | ) 233 | 234 | if "%1" == "pseudoxml" ( 235 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 236 | if errorlevel 1 exit /b 1 237 | echo. 238 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 239 | goto end 240 | ) 241 | 242 | :end 243 | --------------------------------------------------------------------------------