├── geodjango ├── geodjango │ ├── __init__.py │ ├── wsgi.py │ ├── urls.py │ └── settings.py ├── countdracula │ ├── __init__.py │ ├── parsers │ │ └── __init__.py │ ├── templatetags │ │ ├── __init__.py │ │ └── jsonify.py │ ├── static │ │ ├── small_blue.png │ │ ├── small_red.png │ │ ├── measle_brown.png │ │ ├── small_green.png │ │ ├── small_purple.png │ │ ├── small_yellow.png │ │ ├── count_db_logo_220w.jpg │ │ ├── count_db_logo_500w.jpg │ │ ├── countdracula.css │ │ └── gmap.css │ ├── tests.py │ ├── admin.py │ ├── forms.py │ ├── models.py │ └── views.py ├── templates │ ├── gis │ │ └── admin │ │ │ ├── googlemap.js │ │ │ └── googlemap.html │ ├── admin │ │ ├── base_site.html │ │ └── countdracula │ │ │ └── upload.html │ └── countdracula │ │ └── gmap.html ├── manage.py └── geodjango.wsgi ├── .gitmodules ├── doc ├── images │ └── PeMS_to_NetworkNodes_Workbook.png ├── index.rst ├── Makefile ├── make.bat ├── setup.rst └── conf.py ├── .gitignore ├── uploads └── Readme.txt ├── Readme.txt ├── Readme.md ├── .project ├── .pydevproject ├── httpd-countdracula.conf ├── scripts ├── querySanFranciscoCounts.py ├── setupSanFranciscoCountDracula.bat ├── insertSanFranciscoCounts.py ├── insertSanFranciscoIntersectionsFromCube.py ├── insertSanFranciscoMTCCounts.py ├── updateCountsWorkbooks.py └── insertSanFranciscoPeMSCounts.py └── COPYING /geodjango/geodjango/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /geodjango/countdracula/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /geodjango/countdracula/parsers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /geodjango/countdracula/templatetags/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "doc/_build/html"] 2 | path = doc/_build/html 3 | url = git@github.com:sfcta/CountDracula.git 4 | -------------------------------------------------------------------------------- /doc/images/PeMS_to_NetworkNodes_Workbook.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sfcta/CountDracula/HEAD/doc/images/PeMS_to_NetworkNodes_Workbook.png -------------------------------------------------------------------------------- /geodjango/countdracula/static/small_blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sfcta/CountDracula/HEAD/geodjango/countdracula/static/small_blue.png -------------------------------------------------------------------------------- /geodjango/countdracula/static/small_red.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sfcta/CountDracula/HEAD/geodjango/countdracula/static/small_red.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .settings/* 3 | doc/_build/* 4 | doc/_generated/* 5 | *.DEBUG.log 6 | *.bak 7 | static/* 8 | uploads/*.xls 9 | uploads/*.xlsx -------------------------------------------------------------------------------- /geodjango/countdracula/static/measle_brown.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sfcta/CountDracula/HEAD/geodjango/countdracula/static/measle_brown.png -------------------------------------------------------------------------------- /geodjango/countdracula/static/small_green.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sfcta/CountDracula/HEAD/geodjango/countdracula/static/small_green.png -------------------------------------------------------------------------------- /geodjango/countdracula/static/small_purple.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sfcta/CountDracula/HEAD/geodjango/countdracula/static/small_purple.png -------------------------------------------------------------------------------- /geodjango/countdracula/static/small_yellow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sfcta/CountDracula/HEAD/geodjango/countdracula/static/small_yellow.png -------------------------------------------------------------------------------- /geodjango/countdracula/static/count_db_logo_220w.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sfcta/CountDracula/HEAD/geodjango/countdracula/static/count_db_logo_220w.jpg -------------------------------------------------------------------------------- /geodjango/countdracula/static/count_db_logo_500w.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sfcta/CountDracula/HEAD/geodjango/countdracula/static/count_db_logo_500w.jpg -------------------------------------------------------------------------------- /uploads/Readme.txt: -------------------------------------------------------------------------------- 1 | 2 | CountDracula will archive uploaded counts workbooks into this directory. 3 | 4 | See UploadCountForm in geodjango/countdracula/forms.py 5 | -------------------------------------------------------------------------------- /Readme.txt: -------------------------------------------------------------------------------- 1 | The subdirectory, static, is automatically updated by running 2 | python manage.py collectstatic in ..\geodjango 3 | 4 | (based on the STATIC_ROOT setting) Thus, I'm not going to check the contents into git. -------------------------------------------------------------------------------- /geodjango/templates/gis/admin/googlemap.js: -------------------------------------------------------------------------------- 1 | {% extends "gis/admin/openlayers.js" %} 2 | {% block base_layer %}new OpenLayers.Layer.Google("Google Base Layer", {'type': G_NORMAL_MAP, 'sphericalMercator' : true});{% endblock %} -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | 2 | See more information about the development of CountDracula in the paper, 3 | "Creating CountDracula: An Open Source Counts Management Tool", 4 | presented at the Transportation Research Board 93rd Annual Meeting 5 | 6 | http://docs.trb.org/prp/14-4625.pdf 7 | -------------------------------------------------------------------------------- /geodjango/templates/gis/admin/googlemap.html: -------------------------------------------------------------------------------- 1 | {% extends "gis/admin/openlayers.html" %} 2 | {% block extrastyle %}{{ block.super }} 3 | 4 | {% endblock %} 5 | {% block openlayers %}{% include "gis/admin/googlemap.js" %}{% endblock %} -------------------------------------------------------------------------------- /geodjango/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "geodjango.settings") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /geodjango/countdracula/static/countdracula.css: -------------------------------------------------------------------------------- 1 | 2 | input#id_sourcefile { 3 | width:600px; 4 | } 5 | 6 | div#upload_errors { 7 | color:red; 8 | border: 1px dotted red; 9 | padding: 20px; 10 | margin: 20px; 11 | } 12 | 13 | div#success { 14 | color:gray; 15 | border: 1px dotted gray; 16 | padding: 20px; 17 | margin: 20px; 18 | } 19 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | CountDracula 4 | 5 | 6 | 7 | 8 | 9 | org.python.pydev.PyDevBuilder 10 | 11 | 12 | 13 | 14 | 15 | org.python.pydev.pythonNature 16 | 17 | 18 | -------------------------------------------------------------------------------- /.pydevproject: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | python27 6 | python 2.7 7 | 8 | /CountDracula 9 | 10 | 11 | -------------------------------------------------------------------------------- /geodjango/countdracula/tests.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file demonstrates writing tests using the unittest module. These will pass 3 | when you run "manage.py test". 4 | 5 | Replace this with more appropriate tests for your application. 6 | """ 7 | 8 | from django.test import TestCase 9 | 10 | 11 | class SimpleTest(TestCase): 12 | def test_basic_addition(self): 13 | """ 14 | Tests that 1 + 1 always equals 2. 15 | """ 16 | self.assertEqual(1 + 1, 2) 17 | -------------------------------------------------------------------------------- /geodjango/templates/admin/base_site.html: -------------------------------------------------------------------------------- 1 | {% extends "admin/base.html" %} 2 | {% load i18n %} 3 | 4 | {% block title %}{{ title }} | {% trans 'Count Dracula site admin' %}{% endblock %} 5 | 6 | {% block extrastyle %} 7 | 8 | {% endblock %} 9 | 10 | {% block branding %} 11 |

{% trans 'Count Dracula administration' %}

12 | {% endblock %} 13 | 14 | {% block nav-global %}{% endblock %} 15 | -------------------------------------------------------------------------------- /geodjango/countdracula/templatetags/jsonify.py: -------------------------------------------------------------------------------- 1 | from django.core.serializers import serialize 2 | from django.db.models.query import QuerySet 3 | from django.utils import simplejson 4 | from django.utils.safestring import mark_safe 5 | from django.template import Library 6 | 7 | register = Library() 8 | 9 | def jsonify(object): 10 | if isinstance(object, QuerySet): 11 | return mark_safe(serialize('json', object)) 12 | 13 | return mark_safe(simplejson.dumps(object)) 14 | 15 | register.filter('jsonify', jsonify, is_safe=True) -------------------------------------------------------------------------------- /httpd-countdracula.conf: -------------------------------------------------------------------------------- 1 | 2 | SetEnv GDAL_DATA "C:\OSGeo4W\share\gdal" 3 | 4 | Alias /robots.txt "C:\CountDracula\geodjango\static\robots.txt" 5 | Alias /favicon.ico "C:\CountDracula\geodjango\static\favicon.ico" 6 | 7 | # This matches MEDIA_ROOT and STATIC_ROOT in settings 8 | Alias /media "C:\CountDracula\media" 9 | Alias /static "C:\CountDracula\static" 10 | 11 | 12 | Order deny,allow 13 | Allow from all 14 | 15 | 16 | 17 | Order deny,allow 18 | Allow from all 19 | 20 | 21 | WSGIScriptAlias /countdracula "C:\CountDracula\geodjango\geodjango.wsgi" 22 | WSGIPythonPath "C:\CountDracula\geodjango" 23 | # WSGIDaemonProcess countdracula threads=1 24 | 25 | 26 | 27 | Order allow,deny 28 | Allow from all 29 | 30 | -------------------------------------------------------------------------------- /geodjango/countdracula/static/gmap.css: -------------------------------------------------------------------------------- 1 | html { height:100% } 2 | 3 | body { 4 | font-family: "Trebuchet MS", Helvetica, sans-serif; 5 | color: #4F4F4F; 6 | font-size: 69%; 7 | height:100%; 8 | } 9 | 10 | #header { 11 | height:40px; 12 | } 13 | 14 | #nav { 15 | float:left; 16 | width:220px; 17 | height:150px; 18 | margin:5px; 19 | } 20 | 21 | #map { 22 | height:95%; 23 | margin:5px; 24 | } 25 | 26 | #footer { 27 | clear:both; 28 | height:20px; 29 | margin-bottom:10px; 30 | text-align:right; 31 | color:#999999; 32 | } 33 | #footer a { 34 | color:#999999; 35 | } 36 | 37 | 38 | .linkOFF { 39 | color: darkblue 40 | } 41 | 42 | .linkON { 43 | color: white; 44 | background-color: darkblue 45 | } 46 | 47 | table#filters { 48 | border-collapse: collapse; 49 | } 50 | table#filters th, 51 | table#filters td { 52 | border-top: 1px solid grey; 53 | border-bottom: 1px solid grey; 54 | } -------------------------------------------------------------------------------- /geodjango/geodjango.wsgi: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import sys 4 | 5 | os.environ['DJANGO_SETTINGS_MODULE'] = 'geodjango.settings' 6 | print sys.path 7 | 8 | project_path=r"C:\CountDracula\geodjango" 9 | if project_path not in sys.path: 10 | sys.path.append(project_path) 11 | geo_path=r"C:\OSGeo4W\bin" 12 | if geo_path not in sys.path: 13 | sys.path.append(geo_path) 14 | # print sys.path 15 | # print os.environ['PATH'] 16 | os.environ['PATH']=r"C:\Python27;C:\OSGeo4W\bin;C:\Program Files (x86)\PostgreSQL\9.0\bin;C:\Windows\SysWoW64;C:\Windows\System32" 17 | 18 | from django.contrib.gis import gdal 19 | print "gdal.HAS_GDAL? " + str(gdal.HAS_GDAL) 20 | # print os.environ 21 | 22 | import django.core.handlers.wsgi 23 | application = django.core.handlers.wsgi.WSGIHandler() 24 | 25 | # log errors to the error log 26 | logger = logging.getLogger('') 27 | logger.setLevel(logging.DEBUG) 28 | handler = logging.StreamHandler(sys.stderr) 29 | handler.setLevel(logging.DEBUG) 30 | formatter = logging.Formatter('%(levelname)-8s %(message)s') 31 | handler.setFormatter(formatter) 32 | logger.addHandler(handler) -------------------------------------------------------------------------------- /geodjango/templates/admin/countdracula/upload.html: -------------------------------------------------------------------------------- 1 | {% extends "admin/base_site.html" %} 2 | 3 | {% block content %} 4 |
5 | 6 | {% if upload_errors %} 7 |
8 | An error occurred:
9 |
{{ upload_errors|safe }}
10 |
11 | {% endif %} 12 | 13 | {% if success_msg %} 14 |
15 | {{ success_msg }}
16 |
{{ success_detail|safe }}
17 |
18 | {% endif %} 19 | 20 | 21 |

Choose an excel workbook with counts to upload below.

22 |
23 | {% csrf_token %} 24 | {# Include the visible fields #} 25 | {% for field in form.visible_fields %} 26 |
27 | {{ field.errors }} 28 | {{ field.label_tag }}: {{ field }} 29 |
30 | {% endfor %} 31 | 32 |
33 |
34 | 35 | {% endblock %} 36 | -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | .. CountDracula documentation master file, created by 2 | sphinx-quickstart on Tue Jul 12 17:13:39 2011. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to CountDracula's documentation! 7 | ======================================== 8 | CountDracula is a counts database framework for storing street traffic counts. 9 | 10 | .. toctree:: 11 | 12 | setup 13 | 14 | CountDracula API 15 | ================ 16 | 17 | .. autosummary:: 18 | :nosignatures: 19 | :toctree: _generated 20 | 21 | countdracula.parsers.CountsWorkbookParser.CountsWorkbookParser 22 | countdracula.models.Node 23 | countdracula.models.VehicleTypes 24 | countdracula.models.Directions 25 | countdracula.models.StreetName 26 | countdracula.models.TurnCountLocation 27 | countdracula.models.TurnCount 28 | countdracula.models.MainlineCountLocation 29 | countdracula.models.MainlineCount 30 | 31 | 32 | Indices and tables 33 | ================== 34 | 35 | * :ref:`genindex` 36 | * :ref:`modindex` 37 | * :ref:`search` 38 | 39 | 40 | TODO List 41 | ========= 42 | .. todolist:: 43 | -------------------------------------------------------------------------------- /geodjango/geodjango/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for geodjango project. 3 | 4 | This module contains the WSGI application used by Django's development server 5 | and any production WSGI deployments. It should expose a module-level variable 6 | named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover 7 | this application via the ``WSGI_APPLICATION`` setting. 8 | 9 | Usually you will have the standard Django WSGI application here, but it also 10 | might make sense to replace the whole Django WSGI application with a custom one 11 | that later delegates to the Django one. For example, you could introduce WSGI 12 | middleware here, or combine a Django application with an application of another 13 | framework. 14 | 15 | """ 16 | import os 17 | 18 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "geodjango.settings") 19 | 20 | # This application object is used by any WSGI server configured to use this 21 | # file. This includes Django's development server, if the WSGI_APPLICATION 22 | # setting points here. 23 | from django.core.wsgi import get_wsgi_application 24 | application = get_wsgi_application() 25 | 26 | # Apply WSGI middleware here. 27 | # from helloworld.wsgi import HelloWorldApplication 28 | # application = HelloWorldApplication(application) 29 | -------------------------------------------------------------------------------- /geodjango/geodjango/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import patterns, include, url 2 | 3 | # Uncomment the next two lines to enable the admin: 4 | from django.contrib.gis import admin 5 | from countdracula.admin import countdracula_admin 6 | admin.autodiscover() 7 | 8 | urlpatterns = patterns('', 9 | # Examples: 10 | # url(r'^$', 'geodjango.views.home', name='home'), 11 | # url(r'^geodjango/', include('geodjango.foo.urls')), 12 | 13 | # Uncomment the admin/doc line below to enable admin documentation: 14 | url(r'^admin/doc/', include('django.contrib.admindocs.urls')), 15 | 16 | # Uncomment the next line to enable the admin: 17 | url(r'^admin_auth/', include(admin.site.urls)), 18 | 19 | # Count Dracula admin - edit count dracula models 20 | url(r'^admin/', include(countdracula_admin.urls)), 21 | 22 | # map view 23 | url(r'^map/', 'countdracula.views.mapview'), 24 | 25 | # for the map view to fetch count information for a location 26 | url(r'^counts_for_location/$', 'countdracula.views.counts_for_location'), 27 | 28 | # for the map view to fetch locations near a point 29 | url(r'^countlocs_for_point/$', 'countdracula.views.countlocs_for_point'), 30 | 31 | # download button from map 32 | url(r'^download/', 'countdracula.views.download') 33 | 34 | ) -------------------------------------------------------------------------------- /scripts/querySanFranciscoCounts.py: -------------------------------------------------------------------------------- 1 | """ 2 | Created on April 3, 2012 3 | @author: lmz 4 | 5 | Quick script to print out a tally of the most popular time slices for counts in the database. 6 | 7 | NOTE: this is broken... 8 | """ 9 | 10 | import countdracula 11 | import logging, os 12 | from operator import itemgetter 13 | 14 | def ignoreDates(count_counts): 15 | """ 16 | Assuming counts is a dictionary of (datetime.datetime, datetime.timedelta) -> counts 17 | 18 | Returns the equivalent but with the datetime.datetime objects converted to datetime.time objects (so summing across dates). 19 | """ 20 | # but we're actually interested in just the times, not the dates -> aggregate 21 | time_counts = {} 22 | for (timeslice,count) in count_counts.iteritems(): 23 | newkey = (timeslice[0].time(), timeslice[1]) 24 | if newkey not in time_counts: time_counts[newkey] = 0 25 | time_counts[newkey] += count 26 | return time_counts 27 | 28 | 29 | if __name__ == '__main__': 30 | logger = logging.getLogger('countdracula') 31 | consolehandler = logging.StreamHandler() 32 | consolehandler.setLevel(logging.DEBUG) 33 | consolehandler.setFormatter(logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')) 34 | logger.addHandler(consolehandler) 35 | logger.setLevel(logging.DEBUG) 36 | 37 | cd_reader = countdracula.CountsDatabaseReader(pw="ReadOnly", logger=logger) 38 | 39 | 40 | count_counts = cd_reader.countCounts(turning=True, mainline=False) 41 | time_counts = ignoreDates(count_counts) 42 | 43 | print "Most Frequent Timeslices for Turn Counts" 44 | print " %10s %10s %10s" % ("start", "duration(min)", "#counts") 45 | 46 | for (timeslice,count) in sorted(time_counts.items(), key=itemgetter(1), reverse=True): 47 | print " %10s %10d %10d" % (timeslice[0].isoformat(), timeslice[1].seconds/60.0, count) 48 | 49 | print 50 | print "Sequential Timeslices for Turn Counts" 51 | print " %10s %10s %10s" % ("start", "duration(min)", "#counts") 52 | 53 | for timeslice in sorted(time_counts.keys(), key=itemgetter(0)): 54 | print " %10s %10d %10d" % (timeslice[0].isoformat(), timeslice[1].seconds/60.0, time_counts[timeslice]) 55 | 56 | count_counts = cd_reader.countCounts(turning=False, mainline=True) 57 | time_counts = ignoreDates(count_counts) 58 | 59 | print 60 | print "Most Frequent Timeslices for Mainline Counts" 61 | print " %10s %10s %10s" % ("start", "duration(min)", "#counts") 62 | 63 | for (timeslice,count) in sorted(time_counts.items(), key=itemgetter(1), reverse=True): 64 | print " %10s %10d %10d" % (timeslice[0].isoformat(), timeslice[1].seconds/60.0, count) 65 | 66 | print 67 | print "Sequential Timeslices for Mainline Counts" 68 | print " %10s %10s %10s" % ("start", "duration(min)", "#counts") 69 | 70 | for timeslice in sorted(time_counts.keys(), key=itemgetter(0)): 71 | print " %10s %10d %10d" % (timeslice[0].isoformat(), timeslice[1].seconds/60.0, time_counts[timeslice]) -------------------------------------------------------------------------------- /scripts/setupSanFranciscoCountDracula.bat: -------------------------------------------------------------------------------- 1 | 2 | 3 | :: RUN THIS IN THE CountDracula directory 4 | :: 5 | :: set PATH to have Python, psql, runtpp, geo (for libraries), git 6 | :: 7 | set PATH=C:\Python27;C:\Python27\Scripts;C:\OSGeo4W\bin;C:\Program Files (x86)\PostgreSQL\9.0\bin;C:\Program Files (x86)\Citilabs\CubeVoyager;C:\Program Files (x86)\Git\bin;C:\Windows\System32 8 | 9 | :: the following assumes "postgres" is the name of your postgres superuser 10 | :: 11 | :: clear out what's in there already 12 | :: 13 | psql -U postgres -c "drop database countdracula_geodjango" 14 | if ERRORLEVEL 1 goto done 15 | 16 | :: 17 | :: create the countdracula user (if you haven't already) 18 | :: This should match the settings.py file 19 | :: 20 | :: This will prompt for the the password of the new user (role), followed by that of the super postgres user (postgres). 21 | :: 22 | psql -U postgres -c "select * from pg_user where usename='countdracula'" | findstr countdracula 23 | :: ERRORLEVEL will be 0 if it already exists 24 | if %ERRORLEVEL% GTR 0 ( 25 | createuser -SDR --username postgres -P countdracula 26 | ) 27 | 28 | :: 29 | :: create the postgis database 30 | :: This will prompt for the password of the super postgres user (postgres). 31 | :: 32 | 33 | createdb --username postgres --owner=countdracula -T template_postgis countdracula_geodjango 34 | 35 | :: 36 | :: change the user for the two tables to countdracula 37 | :: This will prompt for the password of the super postgres user (postgres). 38 | :: 39 | psql -U postgres -d countdracula_geodjango -c "ALTER TABLE spatial_ref_sys OWNER to countdracula;" 40 | psql -U postgres -d countdracula_geodjango -c "ALTER TABLE geometry_columns OWNER to countdracula;" 41 | 42 | :: 43 | :: This is just a historical note for how the project was created 44 | :: django-admin.py startproject geodjango 45 | cd geodjango 46 | 47 | :: This is just a historical note for how the app was created 48 | ::python manage.py startapp countdracula 49 | 50 | :: 51 | :: Verify the countdracula model is AOK 52 | :: This should output a bunch of SQL 53 | python manage.py sqlall countdracula 54 | 55 | :: 56 | :: Setup the database -- you'll need to setup the django superuser (not to be confused with the postgres super user 57 | :: or the postgres countdracula user). 58 | python manage.py syncdb 59 | 60 | :: 61 | :: Setup the static files into STATIC_ROOT 62 | :: This will collect all the css and javascript files into the STATIC_ROOT. 63 | python manage.py collectstatic 64 | 65 | 66 | :: ======================================== setup_complete =============================================== 67 | :: 68 | :: get in place to run scripts 69 | :: 70 | cd ..\scripts 71 | 72 | :: 73 | :: read nodes and intersection streetnames from Cube static network 74 | :: 75 | python insertSanFranciscoIntersectionsFromCube.py Y:\networks\Roads2010\FREEFLOW.net 76 | 77 | :: 78 | :: insert PeMS counts 79 | :: 80 | python insertSanFranciscoPeMSCounts.py -v "Q:\Roadway Observed Data\PeMS\D4_Data_2010\pems_dist4_2010_fullyr.dat" -c "Q:\Roadway Observed Data\PeMS\D4_Data_2010\PeMS_Census" lisa "Q:\Roadway Observed Data\PeMS\PeMs_to_NetworkNodes.xls" 81 | 82 | :: 83 | :: insert MTC counts 84 | :: 85 | python insertSanFranciscoMTCCounts.py lisa "Q:\Roadway Observed Data\MTC\all_MTC_Counts.xls" 86 | 87 | :: 88 | :: insert other counts 89 | :: -> process Q:\Roadway Observed Data\Counts\CountDraculaCounts\CountDraculaToProcess 90 | :: -> move successfully processed files into: Q:\Roadway Observed Data\Counts\CountDraculaCounts\CountDraculaSuccess 91 | :: -> move failures into: Q:\Roadway Observed Data\Counts\CountDraculaCounts\CountDraculaFailed 92 | python insertSanFranciscoCounts.py -f "Q:\Roadway Observed Data\Counts\CountDraculaCounts\CountDraculaFailed" -s "Q:\Roadway Observed Data\Counts\CountDraculaCounts\CountDraculaSuccess" lisa "Q:\Roadway Observed Data\Counts\CountDraculaCounts\CountDraculaToProcess" 93 | 94 | :done -------------------------------------------------------------------------------- /geodjango/countdracula/admin.py: -------------------------------------------------------------------------------- 1 | import logging, os, traceback 2 | from django.conf.urls import patterns 3 | from django.contrib import admin 4 | from django.contrib.gis import admin as gis_admin 5 | from django.contrib.gis.maps.google import GoogleMap 6 | from django.shortcuts import render 7 | 8 | from countdracula.forms import UploadCountForm 9 | from countdracula.models import Node,StreetName,TurnCountLocation,TurnCount,MainlineCountLocation,MainlineCount 10 | 11 | # key associated with sfcta.mapping@gmail.com 12 | GMAP = GoogleMap(key='AIzaSyDSscDrdYK3lENjefyjoBof_JjXY5LJLRo') 13 | 14 | # create custom admin site to add the upload counts view 15 | class CountDraculaAdminSite(admin.sites.AdminSite): 16 | 17 | def get_urls(self): 18 | urls = super(CountDraculaAdminSite, self).get_urls() 19 | my_urls = patterns('', (r'^upload_counts/$', self.admin_view(self.upload_view))) 20 | # print "CountDraculaAdminSite" 21 | # print (my_urls + urls) 22 | # todo: can we put the auth (admin.site.get_urls() here? it'd be nice to have one page) 23 | return my_urls + urls 24 | 25 | def upload_view(self, request): 26 | context_dict = {} 27 | 28 | if request.method == 'POST': 29 | form = UploadCountForm(request.POST, request.FILES) 30 | if form.is_valid(): 31 | (num_processed, log_string) = form.read_sourcefile_and_insert_counts(request, request.FILES['sourcefile']) 32 | if num_processed < 0: 33 | context_dict['upload_errors'] = log_string 34 | else: 35 | # success! 36 | context_dict['success_msg'] = "Successfully uploaded %d counts from %s!" % (num_processed, form.cleaned_data['sourcefile']) 37 | context_dict['success_detail'] = log_string 38 | form = UploadCountForm() 39 | else: 40 | # form is not bound to data 41 | form = UploadCountForm() 42 | 43 | context_dict['form'] = form 44 | return render(request, 'admin/countdracula/upload.html', context_dict) 45 | 46 | 47 | countdracula_admin = CountDraculaAdminSite(name='countdracula') 48 | 49 | class StreetNameAdmin(admin.ModelAdmin): 50 | search_fields = ['street_name'] 51 | 52 | readonly_fields = ('nodes_map',) 53 | 54 | def nodes_map(self, instance): 55 | return '
' + (str(instance.nodes.all())) + "
" 56 | nodes_map.short_description = "Nodes Map" 57 | nodes_map.allow_tags = True 58 | countdracula_admin.register(StreetName, StreetNameAdmin) 59 | 60 | countdracula_admin.register(TurnCountLocation) 61 | 62 | class MainlineCountInline(admin.TabularInline): 63 | model = MainlineCount 64 | 65 | readonly_fields = ('count','count_date','count_year', 66 | 'start_time','period_minutes','vehicle_type', 67 | 'sourcefile','project','reference_position','upload_user',) 68 | 69 | 70 | class MainlineCountLocationAdmin(admin.ModelAdmin): 71 | list_display = ('on_street', 'from_street', 'to_street') 72 | 73 | inlines = [ MainlineCountInline, ] 74 | 75 | countdracula_admin.register(MainlineCountLocation, MainlineCountLocationAdmin) 76 | 77 | class TurnCountAdmin(admin.ModelAdmin): 78 | # let users search by sourcefile 79 | search_fields = ['sourcefile'] 80 | list_filter = ('vehicle_type',) 81 | list_display = ('location', 'period_minutes', 'count_date', 'count_year', 'start_time', 'vehicle_type', 'count') 82 | 83 | countdracula_admin.register(TurnCount, TurnCountAdmin) 84 | 85 | class MainlineCountAdmin(admin.ModelAdmin): 86 | # let users search by sourcefile 87 | search_fields = ['sourcefile'] 88 | list_filter = ('vehicle_type',) 89 | list_display = ('location', 'period_minutes', 'count_date', 'count_year', 'start_time', 'vehicle_type', 'count') 90 | 91 | countdracula_admin.register(MainlineCount, MainlineCountAdmin) 92 | 93 | class StreetnameInline(admin.TabularInline): 94 | model = StreetName.nodes.through 95 | 96 | class NodeAdmin(gis_admin.OSMGeoAdmin): 97 | extra_js = [GMAP.api_url + GMAP.key] 98 | map_template = 'gis/admin/googlemap.html' 99 | 100 | inlines = [ StreetnameInline, ] 101 | 102 | countdracula_admin.register(Node, NodeAdmin) 103 | -------------------------------------------------------------------------------- /geodjango/countdracula/forms.py: -------------------------------------------------------------------------------- 1 | from django import forms 2 | from geodjango import settings 3 | from countdracula.parsers.CountsWorkbookParser import CountsWorkbookParser 4 | import logging 5 | import os 6 | import StringIO 7 | import traceback 8 | 9 | 10 | class UploadCountForm(forms.Form): 11 | 12 | sourcefile = forms.FileField(max_length=150, help_text="Upload a count file to process.") 13 | xl_parser = CountsWorkbookParser() 14 | 15 | def clean_sourcefile(self): 16 | """ 17 | Make sure the sourcefile exists and that it's an xls file, and that the name indicates two streetnames. 18 | """ 19 | sourcefile_name = self.cleaned_data['sourcefile'].name 20 | 21 | # if not os.path.isabs(sourcefile_name): 22 | # raise forms.ValidationError("Sourcefile must be an absolute path to an excel file. Invalid value: %s" % sourcefile_name) 23 | 24 | 25 | if sourcefile_name[-4:].lower() != ".xls" and sourcefile_name[-5:].lower() != ".xlsx": 26 | raise forms.ValidationError("Sourcefile must be have a .xls|.xlsx suffix. Invalid value: %s" % sourcefile_name) 27 | 28 | # set streetnames 29 | self.cleaned_data['streetnames'] = CountsWorkbookParser.parseFilename(sourcefile_name) 30 | 31 | if len(self.cleaned_data['streetnames']) not in [2,3]: 32 | raise forms.ValidationError("Sourcefile name should be of the format streetname1_streetname2.xls or streetname_fromstreetname.tostreetname.xls. Invalid value: %s" % sourcefile_name) 33 | 34 | return self.cleaned_data['sourcefile'] 35 | 36 | def read_sourcefile_and_insert_counts(self, request, file): 37 | """ 38 | Do the work! Read and insert the turn counts into the database. 39 | Returns ( num_processed, error_string ), where num_processed will be -1 on error. 40 | """ 41 | # Figure out a filename 42 | file_suffix_num = 1 43 | new_filename = file.name 44 | # check if the file already exists in uploads 45 | while os.path.exists(os.path.join(settings.UPLOAD_DIR, new_filename)): 46 | if file.name[-4:].lower() == ".xls": 47 | new_filename = "%s_%d%s" % (file.name[:-4],file_suffix_num,file.name[-4:]) 48 | else: 49 | new_filename = "%s_%d%s" % (file.name[:-5],file_suffix_num,file.name[-5:]) 50 | file_suffix_num += 1 51 | 52 | 53 | # for now, save the file to c:\CountDracula\uploads 54 | with open(os.path.join(settings.UPLOAD_DIR, new_filename), 'wb+') as destination: 55 | for chunk in file.chunks(): 56 | destination.write(chunk) 57 | 58 | # catch logs 59 | buffer = StringIO.StringIO() 60 | logHandler = logging.StreamHandler(buffer) 61 | logHandler.setLevel(logging.INFO) 62 | logging.getLogger().addHandler(logHandler) 63 | 64 | logging.info("Saving into uploads as [%s]" % new_filename) 65 | 66 | if len(self.cleaned_data['streetnames']) == 2: 67 | # turn counts 68 | processed = self.xl_parser.readAndInsertTurnCounts(os.path.join(settings.UPLOAD_DIR, new_filename), 69 | self.cleaned_data['streetnames'][0], 70 | self.cleaned_data['streetnames'][1], 71 | request.user, 72 | logging.getLogger()) 73 | else: 74 | # mainline counts 75 | processed = self.xl_parser.readAndInsertMainlineCounts(os.path.join(settings.UPLOAD_DIR, new_filename), 76 | self.cleaned_data['streetnames'][0], 77 | self.cleaned_data['streetnames'][1], 78 | self.cleaned_data['streetnames'][2], 79 | request.user, 80 | logging.getLogger()) 81 | 82 | # stop catching logs 83 | logging.getLogger().removeHandler(logHandler) 84 | logHandler.flush() 85 | buffer.flush() 86 | return_str = buffer.getvalue() 87 | 88 | # remove file on failure 89 | if processed < 0: 90 | os.remove(os.path.join(settings.UPLOAD_DIR, new_filename)) 91 | return_str += "Removed %s" % os.path.join(settings.UPLOAD_DIR,new_filename) 92 | 93 | return_str = return_str.replace("<","<") 94 | return_str = return_str.replace(">",">") 95 | return_str = return_str.replace("\n","
") 96 | return (processed, return_str) 97 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | 15 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest 16 | 17 | help: 18 | @echo "Please use \`make ' where is one of" 19 | @echo " html to make standalone HTML files" 20 | @echo " dirhtml to make HTML files named index.html in directories" 21 | @echo " singlehtml to make a single large HTML file" 22 | @echo " pickle to make pickle files" 23 | @echo " json to make JSON files" 24 | @echo " htmlhelp to make HTML files and a HTML help project" 25 | @echo " qthelp to make HTML files and a qthelp project" 26 | @echo " devhelp to make HTML files and a Devhelp project" 27 | @echo " epub to make an epub" 28 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 29 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 30 | @echo " text to make text files" 31 | @echo " man to make manual pages" 32 | @echo " changes to make an overview of all changed/added/deprecated items" 33 | @echo " linkcheck to check all external links for integrity" 34 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 35 | 36 | clean: 37 | -rm -rf $(BUILDDIR)/* 38 | 39 | html: 40 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 41 | @echo 42 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 43 | 44 | dirhtml: 45 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 48 | 49 | singlehtml: 50 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 51 | @echo 52 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 53 | 54 | pickle: 55 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 56 | @echo 57 | @echo "Build finished; now you can process the pickle files." 58 | 59 | json: 60 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 61 | @echo 62 | @echo "Build finished; now you can process the JSON files." 63 | 64 | htmlhelp: 65 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 66 | @echo 67 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 68 | ".hhp project file in $(BUILDDIR)/htmlhelp." 69 | 70 | qthelp: 71 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 72 | @echo 73 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 74 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 75 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/CountDracula.qhcp" 76 | @echo "To view the help file:" 77 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/CountDracula.qhc" 78 | 79 | devhelp: 80 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 81 | @echo 82 | @echo "Build finished." 83 | @echo "To view the help file:" 84 | @echo "# mkdir -p $$HOME/.local/share/devhelp/CountDracula" 85 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/CountDracula" 86 | @echo "# devhelp" 87 | 88 | epub: 89 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 90 | @echo 91 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 92 | 93 | latex: 94 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 95 | @echo 96 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 97 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 98 | "(use \`make latexpdf' here to do that automatically)." 99 | 100 | latexpdf: 101 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 102 | @echo "Running LaTeX files through pdflatex..." 103 | make -C $(BUILDDIR)/latex all-pdf 104 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 105 | 106 | text: 107 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 108 | @echo 109 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 110 | 111 | man: 112 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 113 | @echo 114 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 115 | 116 | changes: 117 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 118 | @echo 119 | @echo "The overview file is in $(BUILDDIR)/changes." 120 | 121 | linkcheck: 122 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 123 | @echo 124 | @echo "Link check complete; look for any errors in the above output " \ 125 | "or in $(BUILDDIR)/linkcheck/output.txt." 126 | 127 | doctest: 128 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 129 | @echo "Testing of doctests in the sources finished, look at the " \ 130 | "results in $(BUILDDIR)/doctest/output.txt." 131 | -------------------------------------------------------------------------------- /doc/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | if NOT "%PAPER%" == "" ( 11 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 12 | ) 13 | 14 | if "%1" == "" goto help 15 | 16 | if "%1" == "help" ( 17 | :help 18 | echo.Please use `make ^` where ^ is one of 19 | echo. html to make standalone HTML files 20 | echo. dirhtml to make HTML files named index.html in directories 21 | echo. singlehtml to make a single large HTML file 22 | echo. pickle to make pickle files 23 | echo. json to make JSON files 24 | echo. htmlhelp to make HTML files and a HTML help project 25 | echo. qthelp to make HTML files and a qthelp project 26 | echo. devhelp to make HTML files and a Devhelp project 27 | echo. epub to make an epub 28 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 29 | echo. text to make text files 30 | echo. man to make manual pages 31 | echo. changes to make an overview over all changed/added/deprecated items 32 | echo. linkcheck to check all external links for integrity 33 | echo. doctest to run all doctests embedded in the documentation if enabled 34 | goto end 35 | ) 36 | 37 | if "%1" == "clean" ( 38 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 39 | del /q /s %BUILDDIR%\* 40 | goto end 41 | ) 42 | 43 | if "%1" == "html" ( 44 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 45 | if errorlevel 1 exit /b 1 46 | echo. 47 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 48 | goto end 49 | ) 50 | 51 | if "%1" == "dirhtml" ( 52 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 53 | if errorlevel 1 exit /b 1 54 | echo. 55 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 56 | goto end 57 | ) 58 | 59 | if "%1" == "singlehtml" ( 60 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 61 | if errorlevel 1 exit /b 1 62 | echo. 63 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 64 | goto end 65 | ) 66 | 67 | if "%1" == "pickle" ( 68 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 69 | if errorlevel 1 exit /b 1 70 | echo. 71 | echo.Build finished; now you can process the pickle files. 72 | goto end 73 | ) 74 | 75 | if "%1" == "json" ( 76 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 77 | if errorlevel 1 exit /b 1 78 | echo. 79 | echo.Build finished; now you can process the JSON files. 80 | goto end 81 | ) 82 | 83 | if "%1" == "htmlhelp" ( 84 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 85 | if errorlevel 1 exit /b 1 86 | echo. 87 | echo.Build finished; now you can run HTML Help Workshop with the ^ 88 | .hhp project file in %BUILDDIR%/htmlhelp. 89 | goto end 90 | ) 91 | 92 | if "%1" == "qthelp" ( 93 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 94 | if errorlevel 1 exit /b 1 95 | echo. 96 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 97 | .qhcp project file in %BUILDDIR%/qthelp, like this: 98 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\CountDracula.qhcp 99 | echo.To view the help file: 100 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\CountDracula.ghc 101 | goto end 102 | ) 103 | 104 | if "%1" == "devhelp" ( 105 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 106 | if errorlevel 1 exit /b 1 107 | echo. 108 | echo.Build finished. 109 | goto end 110 | ) 111 | 112 | if "%1" == "epub" ( 113 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 117 | goto end 118 | ) 119 | 120 | if "%1" == "latex" ( 121 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 122 | if errorlevel 1 exit /b 1 123 | echo. 124 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 125 | goto end 126 | ) 127 | 128 | if "%1" == "text" ( 129 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 130 | if errorlevel 1 exit /b 1 131 | echo. 132 | echo.Build finished. The text files are in %BUILDDIR%/text. 133 | goto end 134 | ) 135 | 136 | if "%1" == "man" ( 137 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 138 | if errorlevel 1 exit /b 1 139 | echo. 140 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 141 | goto end 142 | ) 143 | 144 | if "%1" == "changes" ( 145 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 146 | if errorlevel 1 exit /b 1 147 | echo. 148 | echo.The overview file is in %BUILDDIR%/changes. 149 | goto end 150 | ) 151 | 152 | if "%1" == "linkcheck" ( 153 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 154 | if errorlevel 1 exit /b 1 155 | echo. 156 | echo.Link check complete; look for any errors in the above output ^ 157 | or in %BUILDDIR%/linkcheck/output.txt. 158 | goto end 159 | ) 160 | 161 | if "%1" == "doctest" ( 162 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 163 | if errorlevel 1 exit /b 1 164 | echo. 165 | echo.Testing of doctests in the sources finished, look at the ^ 166 | results in %BUILDDIR%/doctest/output.txt. 167 | goto end 168 | ) 169 | 170 | :end 171 | -------------------------------------------------------------------------------- /doc/setup.rst: -------------------------------------------------------------------------------- 1 | CountDracula Setup 2 | ================== 3 | 4 | Requirements 5 | ------------ 6 | 7 | CountDracula runs on `Geodjango `_, a GIS-based extension to 8 | `Django `_, which is a high-level python web framework. 9 | 10 | To setup a CountDracula database, you should first install `Geodjango `_ 11 | The best way to do this is to follow the Geodjango `Platform-specific instructions `_. 12 | 13 | 14 | The following were used to develop and test CountDracula: 15 | 16 | * Windows 7 64-bit 17 | * `postgreSQL `_, an open source object-relational database system. 18 | *Tested with postgreSQL 9.0.10* 19 | * `postGIS `_, an open source add-on that "spatially enables" postgreSQL 20 | with GIS functionality. *Tested with postGIS 1.5* 21 | * `Apache `_, the webserver. *Tested with Apache 2.2* 22 | * `Python 2.7.3 32-bit `_ 23 | * `modwsgi `_, a Python WSGI adaptor module for Apache. This doesn't seem to have a version. 24 | The following python modules are also used: 25 | 26 | * `psycopg `_, an PostgreSQL adapter for Python. *Tested with psycopg 2.4.5* 27 | * `xlrd `_, a python library for reading Microsoft Excel files. 28 | *Tested with xlrd 0.9.2* 29 | * `python-memcached `_ This is a memory-based caching 30 | framework that can help with performance. Optional. *Tested with python-memcached 1.48*. 31 | * `django `_ *Tested with django-1.5.1* 32 | 33 | Installation Instructions 34 | ------------------------- 35 | * postgreSQL has a straightforward installer; it asks you to choose a password for the "database superuser" and "service account". 36 | * postGIS has an installer which asks you for the password (presumably from the previous step) 37 | 38 | Depending on how your setup is, you may want to adjust the 39 | `client authentication settings `_ 40 | on your database. For example, if your database is setup on a publicly accessible machine, you might want to 41 | restrict the hosts which can connect to the database to be only localhost, or only the machine from which you'll 42 | run theses scripts. 43 | 44 | Django/GeoDjango Setup for CountDracula 45 | --------------------------------------- 46 | Download the CountDracula code from `CountDracula on GitHub `_. 47 | This should be downloaded to a location that can be served by Apache, or the same local drive. 48 | In our setup, we'll download into ``C:\CountDracula``. From here forward, ``%COUNT_DRACULA_ROOT%`` refers 49 | to the root directory of the code. 50 | 51 | Update the CountDracula settings file, ``%COUNT_DRACULA_ROOT%\geodjango\geodjango\settings.py`` 52 | You should confirm/set the following: 53 | 54 | * Admins: A name and email address. (What's this used for? I'm not sure...) 55 | * The Database information: choose the name of your countdracula database, the postgres user that 56 | will access it, and a password for that user. 57 | * The ``MEDIA_ROOT`` and ``STATIC_ROOT`` directories are wher media (currently none) and static files (js, css) 58 | files will be put. We put them in ``%COUNT_DRACULA_ROOT%/media`` and ``%COUNT_DRACULA_ROOT%/static``, 59 | respectively. 60 | * Comment out ``CACHES`` if you don't want to deal with `Memcache or Caching `_ 61 | (or if you want to deal with it later). 62 | * Set the ``TIME_ZONE`` to the right time zone for you. 63 | 64 | Then run the following setup commands - we recommend doing each on by hand to start with. 65 | 66 | .. literalinclude:: ..\scripts\setupSanFranciscoCountDracula.bat 67 | :linenos: 68 | :language: bat 69 | :end-before: setup_complete 70 | 71 | Now your CountDracula instance is setup! One last thing - you'll need to setup Apache to serve the CountDracula web interface. 72 | 73 | Apache Setup 74 | ------------ 75 | 76 | First, install `modwsgi `_, a python WSGI adapter module for Apache. We followed 77 | the `Windows installation instructions `_. 78 | 79 | Install the following Apache configuration file into the Apache configuration directory; the installation typically includes 80 | an *extra* subdir in the configuration directory. 81 | 82 | In our Windows installation, this file is saved as ``C:\Program Files (x86)\Apache Software Foundation\Apache2.2\conf\extra\httpd-countdracula.conf``. 83 | Note you may have to act as root or the System Administrator to edit Apache configuration. 84 | 85 | .. literalinclude:: ..\httpd-countdracula.conf 86 | 87 | Add a line into the main Apache configuration file to make sure this file is included. I typically do this where other configuration files are also included:: 88 | 89 | LoadModule wsgi_module modules/mod_wsgi.so 90 | 91 | # CountDracula 92 | Include conf/extra/httpd-countdracula.conf 93 | 94 | 95 | Restart Apache. That's it! Hopefully everything worked out and you can now open a browser window 96 | and navigate to http://[your_hostname]/countdracula/admin for the admin interface, and http://[your_hostname]/countdracula/map for the 97 | map view. 98 | 99 | Now you can start to put counts in! More on this later. 100 | 101 | .. literalinclude:: ..\scripts\setupSanFranciscoCountDracula.bat 102 | :linenos: 103 | :language: bat 104 | :start-after: setup_complete 105 | 106 | Troubleshooting 107 | --------------- 108 | Some issues we ran into: 109 | 110 | * When installing `OSGeo4W `_, make sure you include the libraries GDAL18, Geos, zlib, proj, openssl, libjpeg12. 111 | * I had some dynamic library loading errors associated with loading gdal. I found it useful to try to load it from the command line following 112 | `these instructions `_ until it was successful. 113 | For each error I encountered, I would run the OSGeo4W installer again and choose Advanced Install, and pick the libraries that appeared to 114 | be relevant. I also had to rename ``C:\OSGeo4W\bin\proj.dll`` to ``C:\OSGeo4W\bin\proj_fw.dll``. -------------------------------------------------------------------------------- /geodjango/geodjango/settings.py: -------------------------------------------------------------------------------- 1 | # Django settings for geodjango project. 2 | 3 | DEBUG = True 4 | TEMPLATE_DEBUG = DEBUG 5 | 6 | # set this! 7 | ADMINS = ( 8 | ('Lisa Zorn', 'lisa.zorn@sfcta.org'), 9 | ) 10 | 11 | MANAGERS = ADMINS 12 | 13 | DATABASES = { 14 | 'default': { 15 | 'ENGINE': 'django.contrib.gis.db.backends.postgis', 16 | 'NAME': 'countdracula_geodjango', # set this! 17 | 'USER': 'countdracula', # set this! 18 | 'PASSWORD': 'carscookiesbikes', # set this! 19 | 'HOST': '', # Set to empty string for localhost. 20 | 'PORT': '', # Set to empty string for default. 21 | } 22 | } 23 | 24 | # comment this out if you don't want to bother with Caching 25 | CACHES = { 26 | 'default': { 27 | 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', 28 | 'LOCATION': '127.0.0.1:11211', 29 | } 30 | } 31 | 32 | # Local time zone for this installation. Choices can be found here: 33 | # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name 34 | # although not all choices may be available on all operating systems. 35 | # In a Windows environment this must be set to your system time zone. 36 | TIME_ZONE = 'America/Los_Angeles' 37 | 38 | # Language code for this installation. All choices can be found here: 39 | # http://www.i18nguy.com/unicode/language-identifiers.html 40 | LANGUAGE_CODE = 'en-us' 41 | 42 | SITE_ID = 1 43 | 44 | # If you set this to False, Django will make some optimizations so as not 45 | # to load the internationalization machinery. 46 | USE_I18N = True 47 | 48 | # If you set this to False, Django will not format dates, numbers and 49 | # calendars according to the current locale. 50 | USE_L10N = True 51 | 52 | # If you set this to False, Django will not use timezone-aware datetimes. 53 | USE_TZ = True 54 | 55 | # Absolute filesystem path to the directory that will hold user-uploaded files. 56 | # Example: "/home/media/media.lawrence.com/media/" 57 | MEDIA_ROOT = 'C:\CountDracula\media' # set this! 58 | 59 | # URL that handles the media served from MEDIA_ROOT. Make sure to use a 60 | # trailing slash. 61 | # Examples: "http://media.lawrence.com/media/", "http://example.com/media/" 62 | MEDIA_URL = '/media/' 63 | 64 | # Absolute path to the directory static files should be collected to. 65 | # Don't put anything in this directory yourself; store your static files 66 | # in apps' "static/" subdirectories and in STATICFILES_DIRS. 67 | # Example: "/home/media/media.lawrence.com/static/" 68 | STATIC_ROOT = 'C:\CountDracula\static' # set this! 69 | 70 | # URL prefix for static files. 71 | # Example: "http://media.lawrence.com/static/" 72 | STATIC_URL = '/static/' 73 | 74 | # Additional locations of static files 75 | STATICFILES_DIRS = ( 76 | # Put strings here, like "/home/html/static" or "C:/www/django/static". 77 | # Always use forward slashes, even on Windows. 78 | # Don't forget to use absolute paths, not relative paths. 79 | ) 80 | 81 | # List of finder classes that know how to find static files in 82 | # various locations. 83 | STATICFILES_FINDERS = ( 84 | 'django.contrib.staticfiles.finders.FileSystemFinder', 85 | 'django.contrib.staticfiles.finders.AppDirectoriesFinder', 86 | # 'django.contrib.staticfiles.finders.DefaultStorageFinder', 87 | ) 88 | 89 | # Make this unique, and don't share it with anybody. 90 | SECRET_KEY = '7_a_gi(!k1b8m=rn$h7lh%kjufk7s!g_428r)odzu4#%pppri1' 91 | 92 | # List of callables that know how to import templates from various sources. 93 | TEMPLATE_LOADERS = ( 94 | 'django.template.loaders.filesystem.Loader', 95 | 'django.template.loaders.app_directories.Loader', 96 | # 'django.template.loaders.eggs.Loader', 97 | ) 98 | 99 | MIDDLEWARE_CLASSES = ( 100 | 'django.middleware.common.CommonMiddleware', 101 | 'django.contrib.sessions.middleware.SessionMiddleware', 102 | 'django.middleware.csrf.CsrfViewMiddleware', 103 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 104 | 'django.contrib.messages.middleware.MessageMiddleware', 105 | # caching 106 | 'django.middleware.cache.UpdateCacheMiddleware', 107 | 'django.middleware.common.CommonMiddleware', 108 | 'django.middleware.cache.FetchFromCacheMiddleware', 109 | # Uncomment the next line for simple clickjacking protection: 110 | # 'django.middleware.clickjacking.XFrameOptionsMiddleware', 111 | ) 112 | 113 | ROOT_URLCONF = 'geodjango.urls' 114 | 115 | # Python dotted path to the WSGI application used by Django's runserver. 116 | WSGI_APPLICATION = 'geodjango.wsgi.application' 117 | 118 | TEMPLATE_DIRS = ( 119 | # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". 120 | # Always use forward slashes, even on Windows. 121 | # Don't forget to use absolute paths, not relative paths. 122 | "C:/CountDracula/geodjango/templates", 123 | ) 124 | 125 | INSTALLED_APPS = ( 126 | 'django.contrib.auth', 127 | 'django.contrib.contenttypes', 128 | 'django.contrib.sessions', 129 | 'django.contrib.sites', 130 | 'django.contrib.messages', 131 | 'django.contrib.staticfiles', 132 | # Uncomment the next line to enable the admin: 133 | 'django.contrib.admin', 134 | # Uncomment the next line to enable admin documentation: 135 | 'django.contrib.admindocs', 136 | 'django.contrib.gis', 137 | # this is us 138 | 'countdracula' 139 | ) 140 | 141 | # A sample logging configuration. The only tangible logging 142 | # performed by this configuration is to send an email to 143 | # the site admins on every HTTP 500 error when DEBUG=False. 144 | # See http://docs.djangoproject.com/en/dev/topics/logging for 145 | # more details on how to customize your logging configuration. 146 | LOGGING = { 147 | 'version': 1, 148 | 'disable_existing_loggers': False, 149 | 'filters': { 150 | 'require_debug_false': { 151 | '()': 'django.utils.log.RequireDebugFalse' 152 | } 153 | }, 154 | 'handlers': { 155 | 'mail_admins': { 156 | 'level': 'ERROR', 157 | 'filters': ['require_debug_false'], 158 | 'class': 'django.utils.log.AdminEmailHandler' 159 | } 160 | }, 161 | 'loggers': { 162 | 'django.request': { 163 | 'handlers': ['mail_admins'], 164 | 'level': 'ERROR', 165 | 'propagate': True, 166 | }, 167 | } 168 | } 169 | 170 | AUTH_USER_MODEL = 'auth.User' 171 | 172 | # The upload interface will save processed workbooks into this directory 173 | UPLOAD_DIR = r"C:\CountDracula\uploads" -------------------------------------------------------------------------------- /scripts/insertSanFranciscoCounts.py: -------------------------------------------------------------------------------- 1 | """ 2 | Created on Jul 25, 2011 3 | @author: lmz 4 | 5 | This script reads counts data from input Excel workbooks and inserts the info into the CountDracula dataabase. 6 | 7 | """ 8 | 9 | import getopt, logging, os, shutil, sys, time, traceback 10 | 11 | libdir = os.path.realpath(os.path.join(os.path.split(__file__)[0], "..", "geodjango")) 12 | sys.path.append(libdir) 13 | os.environ['DJANGO_SETTINGS_MODULE'] = 'geodjango.settings' 14 | 15 | from django.core.management import setup_environ 16 | from geodjango import settings 17 | from django.contrib.auth.models import User 18 | 19 | import countdracula.models 20 | from countdracula.parsers.CountsWorkbookParser import CountsWorkbookParser 21 | 22 | USAGE = """ 23 | 24 | python insertSanFranciscoCounts.py [-f failDir] [-s successDir] user countsWorkbookFile.xls|countsWorkbookDirectory [STARTFILE] 25 | 26 | The user should be the django user to attribute as the uploader. 27 | 28 | If a workbook file is passed, reads that workbook into the CountDracula database. 29 | If a directory is passed, reads all the workbook files in the given directory into the CountDracula database. 30 | If optional STARTFILE is passed along with directory, starts at the STARTFILE (the filenames are sorted) 31 | 32 | Pass optional failDir as a location to move failed files, and 33 | optional successDir as a location to move successfully parse files. 34 | 35 | example: python insertSanFranciscoCounts.py -f "Q:\Roadway Observed Data\Counts\Standard\v1.0 CountDraculaFailed" 36 | -s "Q:\Roadway Observed Data\Counts\Standard\v1.0 CountDraculaSuccess" 37 | lisa "Q:\Roadway Observed Data\Counts\Standard\v1.0 CountDraculaToProcess" 38 | 39 | 40 | """ 41 | 42 | 43 | if __name__ == '__main__': 44 | if len(sys.argv) < 2: 45 | print USAGE 46 | sys.exit(2) 47 | 48 | optlist, args = getopt.getopt(sys.argv[1:], 'f:s:') 49 | if len(args) < 2: 50 | print USAGE 51 | sys.exit(2) 52 | 53 | USERNAME = args[0] 54 | COUNTS_INPUT = args[1] 55 | STARTFILE = None 56 | if len(args) > 2: 57 | STARTFILE = args[2] 58 | 59 | FAIL_DIR = None 60 | SUCCESS_DIR = None 61 | for (opt,arg) in optlist: 62 | if opt=="-f": 63 | FAIL_DIR = arg 64 | if not os.path.isdir(FAIL_DIR): raise ValueError("FAIL_DIR must be a directory. [%s] is not a directory" % FAIL_DIR) 65 | elif opt=="-s": 66 | SUCCESS_DIR = arg 67 | if not os.path.isdir(SUCCESS_DIR): raise ValueError("SUCCESS_DIR must be a directory. [%s] is not a directory" % SUCCESS_DIR) 68 | 69 | user = User.objects.get(username__exact=USERNAME) 70 | 71 | logger = logging.getLogger('countdracula') 72 | logger.setLevel(logging.DEBUG) 73 | 74 | consolehandler = logging.StreamHandler() 75 | consolehandler.setLevel(logging.DEBUG) 76 | consolehandler.setFormatter(logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')) 77 | logger.addHandler(consolehandler) 78 | 79 | debugFilename = "insertSanFranciscoCounts_%s.DEBUG.log" % time.strftime("%Y%b%d.%H%M%S") 80 | debugloghandler = logging.StreamHandler(open(debugFilename, 'w')) 81 | debugloghandler.setLevel(logging.DEBUG) 82 | debugloghandler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%Y-%m-%d %H:%M')) 83 | logger.addHandler(debugloghandler) 84 | 85 | logger.info("Processing %s%s" % (COUNTS_INPUT, " starting with %s" % STARTFILE if STARTFILE else "")) 86 | if FAIL_DIR: 87 | logger.info("Putting files that CountDracula failed to process into %s" % FAIL_DIR) 88 | if SUCCESS_DIR: 89 | logger.info("Putting files that CountDracula successfully processed into %s" % SUCCESS_DIR) 90 | 91 | xl_parser = CountsWorkbookParser() 92 | 93 | # Read the counts. These reference the above streets and intersections. 94 | mainline_processed_files = 0 95 | mainline_processed_counts = 0 96 | mainline_attempted_files = 0 97 | turns_processed_files = 0 98 | turns_processed_counts = 0 99 | turns_attempted_files = 0 100 | 101 | if os.path.isdir(COUNTS_INPUT): 102 | dir = COUNTS_INPUT 103 | files_to_process = sorted(os.listdir(COUNTS_INPUT)) 104 | else: 105 | (dir,file) = os.path.split(COUNTS_INPUT) 106 | files_to_process = [ file ] 107 | 108 | if STARTFILE: started = False 109 | for file in files_to_process: 110 | if file[-4:] !='.xls' and file[-5:] != '.xlsx': 111 | print "File suffix is not .xls or .xlsx: %s -- skipping" % file 112 | continue 113 | 114 | # given a STARTFILE -- look for it 115 | if STARTFILE and not started: 116 | if file.upper() == STARTFILE.upper(): 117 | started = True 118 | else: 119 | continue 120 | 121 | logger.info("") 122 | logger.info("Processing file %s" % file) 123 | 124 | full_file_path = os.path.join(dir, file) 125 | # optimism: assume success! (ok but also it'll be easier to find) 126 | if SUCCESS_DIR and not os.path.exists(os.path.join(SUCCESS_DIR, file)): 127 | full_file_path = os.path.join(SUCCESS_DIR, file) 128 | shutil.move(os.path.join(dir,file), full_file_path) 129 | 130 | streetlist = CountsWorkbookParser.parseFilename(file) 131 | 132 | if len(streetlist) == 3: 133 | mainline_attempted_files += 1 134 | 135 | processed = xl_parser.readAndInsertMainlineCounts(full_file_path, streetlist[0], streetlist[1], streetlist[2], user, logger) 136 | 137 | mainline_processed_counts += processed 138 | mainline_processed_files += (1 if processed >= 0 else 0) 139 | 140 | if processed < 0 and FAIL_DIR and os.path.exists(os.path.join(FAIL_DIR, file)): shutil.move(full_file_path, os.path.join(FAIL_DIR, file)) 141 | 142 | elif len(streetlist) == 2: 143 | turns_attempted_files += 1 144 | 145 | processed = xl_parser.readAndInsertTurnCounts(full_file_path, streetlist[0], streetlist[1], user, logger) 146 | 147 | turns_processed_counts += processed 148 | turns_processed_files += (1 if processed >= 0 else 0) 149 | 150 | if processed < 0 and FAIL_DIR and os.path.exists(os.path.join(FAIL_DIR, file)): shutil.move(full_file_path, os.path.join(FAIL_DIR, file)) 151 | 152 | else: 153 | logger.info("Didn't understand filename %s" % file) 154 | 155 | if FAIL_DIR and not os.path.exists(os.path.join(FAIL_DIR, file)): shutil.move(full_file_path, os.path.join(FAIL_DIR, file)) 156 | 157 | logger.info("Mainline counts: %4d processed files out of %4d attempts" % (mainline_processed_files, mainline_attempted_files)) 158 | logger.info("Turn counts: %4d processed files out of %4d attempts" % ( turns_processed_files, turns_attempted_files)) 159 | -------------------------------------------------------------------------------- /scripts/insertSanFranciscoIntersectionsFromCube.py: -------------------------------------------------------------------------------- 1 | """ 2 | Created on April 5, 2012 3 | @author: lmz 4 | 5 | This script reads intersection information from a San Francisco Cube Network. 6 | Someday it would be nice to have one that reads data from some sort of API but for now, this is expedient! 7 | 8 | """ 9 | 10 | USAGE = r""" 11 | 12 | python insertSanFranciscoIntersectionsFromCube.py sf_cube_network.net 13 | 14 | e.g. python insertSanFranciscoIntersectionsFromCube.py Y:\networks\Roads2010\FREEFLOW.net 15 | 16 | """ 17 | 18 | import logging, os, shutil, socket, subprocess, sys, tempfile, traceback 19 | 20 | libdir = os.path.realpath(os.path.join(os.path.split(__file__)[0], "..", "geodjango")) 21 | sys.path.append(libdir) 22 | os.environ['DJANGO_SETTINGS_MODULE'] = 'geodjango.settings' 23 | 24 | from django.core.management import setup_environ 25 | from geodjango import settings 26 | 27 | import countdracula.models 28 | from django.contrib.gis.geos import Point 29 | 30 | 31 | EXPORT_SCRIPTNAME = "ExportCubeForCountDracula.s" 32 | EXPORT_SCRIPT = r""" 33 | RUN PGM=NETWORK 34 | 35 | NETI[1]=%s 36 | NODEO=%s\nodes.csv,FORMAT=SDF, INCLUDE=N,X,Y 37 | LINKO=%s\links.csv ,FORMAT=SDF, INCLUDE=A,B,STREETNAME,TYPE 38 | ENDRUN 39 | """ 40 | 41 | def readCubeNetwork(cube_network, logger): 42 | """ 43 | Exports the cube network and reads it in, returning the following a tuple of 44 | ({NodeNum -> (x,y)}, 45 | {(a,b) -> (streetname, type)}) 46 | """ 47 | # export the cube network 48 | tempdir = tempfile.mkdtemp(dir=r"X:\temp") 49 | scriptFilename = os.path.join(tempdir, EXPORT_SCRIPTNAME) 50 | 51 | logger.info("Writing export script to %s" % scriptFilename) 52 | scriptFile = open(scriptFilename, "w") 53 | scriptFile.write(EXPORT_SCRIPT % (cube_network, tempdir, tempdir)) 54 | scriptFile.close() 55 | 56 | # run the script file 57 | cmd = "runtpp " + scriptFilename 58 | env = dict(os.environ) 59 | hostname = socket.gethostname().lower() 60 | if hostname not in ['berry']: 61 | cmd = r'Y:\champ\util\bin\dispatch-one.bat %s' % cmd 62 | env['MACHINES'] = 'vanness' 63 | 64 | logger.info("Running %s" % cmd) 65 | proc = subprocess.Popen( cmd, 66 | cwd = tempdir, 67 | stdout=subprocess.PIPE, 68 | stderr=subprocess.PIPE, 69 | env=env) 70 | for line in proc.stdout: 71 | line = line.strip('\r\n') 72 | logger.info(" stdout: " + line) 73 | 74 | for line in proc.stderr: 75 | line = line.strip('\r\n') 76 | logger.info("stderr: " + line) 77 | retcode = proc.wait() 78 | if retcode ==2: 79 | raise Exception("Failed to export CubeNetwork using %s" % scriptFilename) 80 | 81 | logger.info("Received %d from [%s]" % (retcode, cmd)) 82 | 83 | # read the nodes file 84 | nodes = {} 85 | nodesFile = open(os.path.join(tempdir, "nodes.csv"), "r") 86 | for line in nodesFile: 87 | fields = line.strip().split(",") 88 | nodes[int(fields[0])] = ( float(fields[1]), float(fields[2]) ) 89 | nodesFile.close() 90 | 91 | # read the links file 92 | links = {} 93 | linksFile = open(os.path.join(tempdir, "links.csv"), "r") 94 | for line in linksFile: 95 | fields = line.strip().split(",") 96 | 97 | streetname = fields[2] 98 | if len(streetname) > 0: 99 | if streetname[0]=="'" and streetname[-1]=="'": streetname = streetname[1:-1] 100 | streetname = streetname.strip(" ") 101 | 102 | type = fields[3] 103 | if len(type) > 0: 104 | if type[0]=="'" and type[-1]=="'": type = type[1:-1] 105 | type = type.strip(" ") 106 | 107 | links[ (int(fields[0]),int(fields[1])) ] = ( streetname, type ) 108 | linksFile.close() 109 | shutil.rmtree(tempdir) 110 | 111 | return (nodes, links) 112 | 113 | def coordInSanFrancisco(x,y): 114 | """ 115 | The following query works well enough to create a bounding box for San Francisco 116 | in GIS (Create a new selection on a FREEFLOW_nodes.shp file). The second bit is to include 117 | Treasure Island without the southern tip of Marin. 118 | 119 | (Y > 2085000 And Y < 2129000 And X < 6024550) Or 120 | (Y > 2085000 And Y < 2140000 And X > 6019000 And X < 6024550) 121 | 122 | 7/3/2012 edit: extended south to include some border nodes 123 | """ 124 | if y > 2082000 and y < 2129000 and x < 6024550: 125 | return True 126 | if y > 2082000 and y < 2140000 and x > 6019000 and x < 6024550: 127 | return True 128 | return False 129 | 130 | if __name__ == '__main__': 131 | logger = logging.getLogger('countdracula') 132 | consolehandler = logging.StreamHandler() 133 | consolehandler.setLevel(logging.DEBUG) 134 | consolehandler.setFormatter(logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')) 135 | logger.addHandler(consolehandler) 136 | logger.setLevel(logging.DEBUG) 137 | 138 | if len(sys.argv) != 2: 139 | print USAGE 140 | sys.exit(2) 141 | 142 | CUBE_NETWORK = sys.argv[1] 143 | 144 | (nodes,links) = readCubeNetwork(CUBE_NETWORK, logger) 145 | 146 | # map map nodeid -> [(streetname1,type1), (streetname2,type2) ] 147 | node_to_streets = {} 148 | 149 | # collect set of [(streetname1,type1), (streetname2,type2)] 150 | streetnames = set() 151 | 152 | for linknodes,streetnametuple in links.iteritems(): 153 | # skip if node A isn't in SF 154 | if not coordInSanFrancisco(nodes[linknodes[0]][0], nodes[linknodes[0]][1]): continue 155 | 156 | # skip if node B isn't in SF 157 | if not coordInSanFrancisco(nodes[linknodes[1]][0], nodes[linknodes[1]][1]): continue 158 | 159 | # pass over unnamed links 160 | if len(streetnametuple[0]) == 0: continue 161 | 162 | # collect in the streetname_list if there's a real streetname there 163 | streetnames.add(streetnametuple) 164 | 165 | # make the mapping 166 | if linknodes[0] not in node_to_streets: 167 | node_to_streets[linknodes[0]] = [] 168 | if streetnametuple not in node_to_streets[linknodes[0]]: 169 | node_to_streets[linknodes[0]].append(streetnametuple) 170 | 171 | if linknodes[1] not in node_to_streets: 172 | node_to_streets[linknodes[1]] = [] 173 | if streetnametuple not in node_to_streets[linknodes[1]]: 174 | node_to_streets[linknodes[1]].append(streetnametuple) 175 | 176 | # insert the nodes first 177 | for nodeid in node_to_streets.iterkeys(): 178 | try: 179 | node = countdracula.models.Node(id=nodeid, point=Point(nodes[nodeid][0], nodes[nodeid][1], srid=3494)) 180 | node.save() 181 | except: 182 | print "Unexpected error:", sys.exc_info()[0] 183 | traceback.print_exc() 184 | print node 185 | sys.exit(2) 186 | 187 | # then the streets {(a,b) -> (streetname, type)}) 188 | for street_tuple in streetnames: 189 | combined = street_tuple[0]+((" " +street_tuple[1]) if len(street_tuple[1])>0 else "") 190 | streetname = countdracula.models.StreetName(street_name=combined, 191 | nospace_name=combined.replace(" ",""), 192 | short_name=street_tuple[0], 193 | suffix=street_tuple[1]) 194 | streetname.save() 195 | 196 | # insert the node/street correspondence 197 | # nodeid -> [(streetname1,type1), (streetname2,type2) ] 198 | for nodeid,streetset in node_to_streets.iteritems(): 199 | node = countdracula.models.Node.objects.get(id=nodeid) 200 | for street_tuple in streetset: 201 | combined = street_tuple[0]+((" " +street_tuple[1]) if len(street_tuple[1])>0 else "") 202 | street_name = countdracula.models.StreetName.objects.get(street_name=combined) 203 | 204 | # add the association 205 | street_name.nodes.add(node) 206 | 207 | -------------------------------------------------------------------------------- /scripts/insertSanFranciscoMTCCounts.py: -------------------------------------------------------------------------------- 1 | """ 2 | Created on Jul 25, 2011 3 | 4 | @author: lmz 5 | 6 | This script reads PEMS counts from Excel workbooks and inserts the info into the CountDracula dataabase. 7 | """ 8 | 9 | import decimal, datetime, getopt, logging, os, sys, time, traceback 10 | import pytz 11 | import xlrd 12 | 13 | libdir = os.path.realpath(os.path.join(os.path.split(__file__)[0], "..", "geodjango")) 14 | sys.path.append(libdir) 15 | os.environ['DJANGO_SETTINGS_MODULE'] = 'geodjango.settings' 16 | 17 | from django.core.exceptions import ObjectDoesNotExist 18 | from django.core.management import setup_environ 19 | from geodjango import settings 20 | from django.contrib.auth.models import User 21 | from django.contrib.gis.gdal import SpatialReference, CoordTransform 22 | 23 | from countdracula.models import Node, StreetName, MainlineCountLocation, MainlineCount 24 | 25 | USAGE = """ 26 | 27 | python insertSanFranciscoMTCCounts.py user mtc_counts.xls 28 | 29 | e.g. python insertSanFranciscoMTCCounts.py lisa "Q:\Roadway Observed Data\MTC\all_MTC_Counts.xls" 30 | 31 | The user should be the django user to attribute as the uploader. 32 | 33 | The worksheets in the workbook are assumed to have the following columns: A, B, EA_OBS, AM_OBS, MD_OBS, PM_OBS, EV_OBS, TOT_OBS 34 | 35 | Excludes workbook counts2005 because counts2005_hovdummy is duplicative. (?) 36 | """ 37 | 38 | OBS_COL_TO_MINUTES = { 39 | 'EA_OBS':60*3, 40 | 'AM_OBS':60*3, 41 | 'MD_OBS':60*6.5, 42 | 'PM_OBS':60*3, 43 | 'EV_OBS':60*8.5, 44 | 'TOT_OBS':60*24 45 | } 46 | OBS_COL_TO_STARTTIME = { 47 | 'EA_OBS':datetime.time(hour=3), 48 | 'AM_OBS':datetime.time(hour=6), 49 | 'MD_OBS':datetime.time(hour=9), 50 | 'PM_OBS':datetime.time(hour=15,minute=30), 51 | 'EV_OBS':datetime.time(hour=18,minute=30), 52 | 'TOT_OBS':datetime.time(hour=0) 53 | } 54 | 55 | 56 | if __name__ == '__main__': 57 | 58 | opts, args = getopt.getopt(sys.argv[1:], '') 59 | if len(args) != 2: 60 | print USAGE 61 | sys.exit(2) 62 | 63 | USERNAME = args[0] 64 | MTC_COUNTS_FILE = args[1] 65 | MTC_COUNTS_FILE_FULLNAME = os.path.abspath(MTC_COUNTS_FILE) 66 | 67 | user = User.objects.get(username__exact=USERNAME) 68 | 69 | logger = logging.getLogger('countdracula') 70 | logger.setLevel(logging.DEBUG) 71 | 72 | consolehandler = logging.StreamHandler() 73 | consolehandler.setLevel(logging.DEBUG) 74 | consolehandler.setFormatter(logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')) 75 | logger.addHandler(consolehandler) 76 | 77 | debugFilename = "insertSanFranciscoMTCCounts_%s.DEBUG.log" % time.strftime("%Y%b%d.%H%M%S") 78 | debugloghandler = logging.StreamHandler(open(debugFilename, 'w')) 79 | debugloghandler.setLevel(logging.DEBUG) 80 | debugloghandler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%Y-%m-%d %H:%M')) 81 | logger.addHandler(debugloghandler) 82 | 83 | book = xlrd.open_workbook(MTC_COUNTS_FILE) 84 | 85 | # open the workbook 86 | for sheetname in sorted(book.sheet_names()): 87 | # duplicative with counts2005_hovdummy 88 | if sheetname == "counts2005": continue 89 | 90 | # countsXXXX 91 | count_year = int(sheetname[6:10]) 92 | 93 | logger.info("Processing worksheet %s" % sheetname) 94 | datasheet = book.sheet_by_name(sheetname) 95 | 96 | # figure out the columns 97 | colname_to_colnum = {} 98 | colnames = ['A', 'B', 'EA_OBS', 'AM_OBS', 'MD_OBS', 'PM_OBS', 'EV_OBS', 'TOT_OBS'] 99 | for colnum in range(len(datasheet.row(0))): 100 | if datasheet.cell_value(0,colnum) in colnames: 101 | colname_to_colnum[datasheet.cell_value(0,colnum)] = colnum 102 | if len(colname_to_colnum) != len(colnames): 103 | logger.fatal("Couldn't find all column headings %s: %s" % (str(colnames), str(colname_to_colnum))) 104 | sys.exit(2) 105 | 106 | # read the data 107 | total_rows = 0 108 | saved_rows = 0 109 | for row in range(1, len(datasheet.col(0))): 110 | 111 | total_rows += 1 112 | A = int(datasheet.cell_value(row, colname_to_colnum['A'])) 113 | B = int(datasheet.cell_value(row, colname_to_colnum['B'])) 114 | 115 | try: 116 | A_node = Node.objects.get(id=A) 117 | except: 118 | # logger.error("Couldn't find node %d in CountDracula -- skipping" % A) 119 | continue 120 | 121 | try: 122 | B_node = Node.objects.get(id=B) 123 | except: 124 | # logger.error("Couldn't find node %d in CountDracula -- skipping" % B) 125 | continue 126 | 127 | streetnames_A = StreetName.objects.filter(nodes=A_node) 128 | streetnames_B = StreetName.objects.filter(nodes=B_node) 129 | 130 | on_street_set = set(streetnames_A).intersection(set(streetnames_B)) 131 | if len(on_street_set) != 1: 132 | logger.error("On street not found for %d (%s) - %d (%s)" % 133 | (A, str(streetnames_A), B, str(streetnames_B))) 134 | continue 135 | 136 | # ok, we have on_street 137 | on_street = on_street_set.pop() 138 | 139 | streetnames_A_list = list(streetnames_A) 140 | streetnames_B_list = list(streetnames_B) 141 | streetnames_A_list.remove(on_street) 142 | streetnames_B_list.remove(on_street) 143 | 144 | # and from_street 145 | if len(streetnames_A_list) == 0: 146 | logger.error("From street not found for %d (%s) - %d (%s)" % 147 | (A, str(streetnames_A), B, str(streetnames_B))) 148 | # use it anyway... 149 | from_street = on_street 150 | else: 151 | from_street = streetnames_A_list[0] 152 | 153 | # and to_street 154 | if len(streetnames_B_list) == 0: 155 | logger.error("To street not found for %d (%s) - %d (%s)" % 156 | (A, str(streetnames_A), B, str(streetnames_B))) 157 | # use it anyway... 158 | to_street = on_street 159 | else: 160 | to_street = streetnames_B_list[0] 161 | 162 | # we just need direction - tranform the two points to feet 163 | long_lat = SpatialReference('WGS84') 164 | nad83stateplane_feet = SpatialReference(3494) 165 | ct = CoordTransform(long_lat, nad83stateplane_feet) 166 | A_point_feet = A_node.point.transform(ct, clone=True) 167 | B_point_feet = B_node.point.transform(ct, clone=True) 168 | 169 | diff_x = B_point_feet.x - A_point_feet.x 170 | diff_y = B_point_feet.y - A_point_feet.y 171 | if abs(diff_y) > abs(diff_x): 172 | if diff_y > 0: 173 | on_dir = "NB" 174 | else: 175 | on_dir = "SB" 176 | else: 177 | if diff_x > 0: 178 | on_dir = "EB" 179 | else: 180 | on_dir = "WB" 181 | 182 | logger.info("Mainline count on_street=%s on_dir=%s from_street=%s to_street=%s" % 183 | (on_street, on_dir, from_street, to_street)) 184 | 185 | try: 186 | mainline_count_location = MainlineCountLocation.objects.get(on_street = on_street, 187 | on_dir = on_dir, 188 | from_int = A_node, 189 | to_int = B_node) 190 | except ObjectDoesNotExist: 191 | mainline_count_location = MainlineCountLocation(on_street = on_street, 192 | on_dir = on_dir, 193 | from_street = from_street, 194 | from_int = A_node, 195 | to_street = to_street, 196 | to_int = B_node) 197 | mainline_count_location.save() 198 | 199 | for colname in ['EA_OBS', 'AM_OBS', 'MD_OBS', 'PM_OBS', 'EV_OBS', 'TOT_OBS']: 200 | count = decimal.Decimal(datasheet.cell_value(row, colname_to_colnum[colname])) 201 | 202 | if count == 0: continue # zeros are not real 203 | 204 | mainline_count = MainlineCount(location = mainline_count_location, 205 | count = count, 206 | count_year = count_year, 207 | start_time = OBS_COL_TO_STARTTIME[colname], 208 | period_minutes = OBS_COL_TO_MINUTES[colname], 209 | vehicle_type = 0, 210 | sourcefile = MTC_COUNTS_FILE_FULLNAME, 211 | project = "mtc", 212 | reference_position = -1, 213 | upload_user = user) 214 | mainline_count.save() 215 | saved_rows += 1 216 | 217 | logger.info("Processed %d out of %d rows" % (saved_rows, total_rows)) 218 | -------------------------------------------------------------------------------- /scripts/updateCountsWorkbooks.py: -------------------------------------------------------------------------------- 1 | """ 2 | Created on Jul 25, 2011 3 | @author: lmz 4 | 5 | This script reads counts data from input Excel workbooks and inserts the info into the CountDracula dataabase. 6 | 7 | """ 8 | 9 | import getopt, logging, os, re, shutil, sys, time, traceback, xlrd, xlwt 10 | 11 | libdir = os.path.realpath(os.path.join(os.path.split(__file__)[0], "..", "geodjango")) 12 | sys.path.append(libdir) 13 | os.environ['DJANGO_SETTINGS_MODULE'] = 'geodjango.settings' 14 | 15 | from django.core.management import setup_environ 16 | from geodjango import settings 17 | from django.contrib.auth.models import User 18 | 19 | import countdracula.models 20 | from countdracula.parsers.CountsWorkbookParser import CountsWorkbookParser 21 | 22 | USAGE = """ 23 | 24 | python updateCountsWorkbooks.py v1.0_toprocess_dir v1.0_outdated_dir v1.1_new_dir 25 | 26 | """ 27 | 28 | DATE_REGEX = re.compile(r"(\d\d\d\d)\.(\d{1,2})\.(\d{1,2})") 29 | MAINLINE_NODES = re.compile(r"(\d{5,6}) (\d{5,6})") 30 | 31 | CALIBRI_10PT = xlwt.easyxf('font: name Calibri, height 200;') 32 | 33 | CALIBRI_10PT_RED = xlwt.easyxf('font: name Calibri, height 200, color-index red;') 34 | CALIBRI_10PT_ORANGE_CENTER = xlwt.easyxf('font: name Calibri, height 200; pattern: pattern solid, fore_color 0x33; alignment: horz center;') 35 | CALIBRI_10PT_LIME_CENTER = xlwt.easyxf('font: name Calibri, height 200; pattern: pattern solid, fore_color 0x32; alignment: horz center;') 36 | 37 | def copysheet(rb, r_sheet, wb): 38 | w_sheet = wb.add_sheet(r_sheet.name) 39 | for rownum in range(r_sheet.nrows): 40 | for colnum in range(r_sheet.ncols): 41 | w_sheet.write(rownum, colnum, r_sheet.cell_value(rownum,colnum), CALIBRI_10PT_RED) 42 | 43 | 44 | def isRowEmpty(r_sheet, r_rownum): 45 | """ 46 | Is the row empty? (aside for the first column) 47 | """ 48 | for colnum in range(1,r_sheet.ncols): 49 | # logger.debug("cell_type=%d cell_value=[%s]" % (r_sheet.cell_type(r_rownum,colnum), str(r_sheet.cell_value(r_rownum,colnum)))) 50 | 51 | if r_sheet.cell_type(r_rownum,colnum) in [xlrd.XL_CELL_BLANK,xlrd.XL_CELL_EMPTY]: 52 | continue 53 | 54 | if r_sheet.cell_value(r_rownum,colnum) != "": 55 | # found something! 56 | return False 57 | 58 | return True # didn't find anything 59 | 60 | def isColumnZeros(r_sheet, colnum): 61 | """ 62 | Starts at row 2. Breaks on empty row. 63 | """ 64 | for r_rownum in range(2,r_sheet.nrows): 65 | 66 | if r_sheet.cell_type(r_rownum,colnum) in [xlrd.XL_CELL_BLANK,xlrd.XL_CELL_EMPTY]: break 67 | 68 | elif r_sheet.cell_type(r_rownum,colnum) in [xlrd.XL_CELL_NUMBER]: 69 | if float(r_sheet.cell_value(r_rownum,colnum)) > 0.0: return False 70 | 71 | else: 72 | raise Exception("Didn't understand cell value at (%d,%d)" % (r_rownum, colnum)) 73 | 74 | return True 75 | 76 | 77 | 78 | def updateWorkbook(logger, DIR_TOPROCESS, DIR_OLDV10, DIR_NEWV11, file, mainline_or_turns): 79 | """ 80 | Converts a v1.0 workbook to a v1.1 workbook. For anything unexpected, logs and error and returns. 81 | 82 | For success only, the new workbook will be placed in *DIR_NEWV11* and the old one will be placed in *DIR_OLDV10*. 83 | """ 84 | assert(mainline_or_turns in ["MAINLINE","TURNS"]) 85 | rb = xlrd.open_workbook(os.path.join(DIR_TOPROCESS, file), formatting_info=True) 86 | wb = xlwt.Workbook(encoding='utf-8') 87 | 88 | # go through the sheets 89 | for sheet_idx in range(rb.nsheets): 90 | r_sheet = rb.sheet_by_index(sheet_idx) 91 | 92 | sheet_name = r_sheet.name 93 | logger.info(" Reading sheet [%s]" % sheet_name)\ 94 | 95 | # just copy the source sheet 96 | if sheet_name == "source": 97 | copysheet(rb, r_sheet, wb) 98 | continue 99 | 100 | match_obj = re.match(DATE_REGEX, sheet_name) 101 | if match_obj.group(0) != sheet_name: 102 | logger.error("Sheetname [%s] is not the standard date format! Skipping this workbook." % sheet_name) 103 | return 104 | 105 | w_sheet = wb.add_sheet(sheet_name) 106 | 107 | # check what we're copying over 108 | for colnum in range(r_sheet.ncols): 109 | if mainline_or_turns == "MAINLINE": 110 | # nodes ok 111 | if r_sheet.cell_type(1,colnum) == xlrd.XL_CELL_TEXT and re.match(MAINLINE_NODES, str(r_sheet.cell_value(1,colnum))) != None: 112 | continue 113 | 114 | if r_sheet.cell_value(1,colnum) not in [1.0, 2.0, ""]: 115 | logger.warn("Unexpected MAINLINE row 1 cell value = [%s]! Skipping this workbook." % r_sheet.cell_value(1,colnum)) 116 | return 117 | if mainline_or_turns == "TURNS" and colnum==0 and r_sheet.cell_value(1,colnum) not in [3.0, 4.0, ""]: 118 | logger.warn("Unexpected TURNS row 1 cell value = [%s]! Skipping this workbook." % r_sheet.cell_value(1,colnum)) 119 | return 120 | 121 | # copy first line down; make sure its MAINLINE|TURNS, [dir1], [dir2], ... 122 | for colnum in range(r_sheet.ncols): 123 | if colnum == 0 and r_sheet.cell_value(0, colnum) != mainline_or_turns: 124 | logger.warn("Unexpected row 0 cell value = [%s]! Skipping this workbook." % r_sheet.cell_value(0,colnum)) 125 | return 126 | if mainline_or_turns == "MAINLINE" and colnum > 0 and r_sheet.cell_value(0,colnum) not in ["NB","SB","EB","WB", ""]: 127 | logger.warn("Unexpected mainline row 0 cell value = [%s]! Skipping this workbook." % r_sheet.cell_value(0,colnum)) 128 | return 129 | if mainline_or_turns == "TURNS" and colnum > 0 and r_sheet.cell_value(0,colnum) not in ["NBLT", "NBRT", "NBTH", 130 | "SBLT", "SBRT", "SBTH", 131 | "EBLT", "EBRT", "EBTH", 132 | "WBLT", "WBRT", "WBTH"]: 133 | logger.warn("Unexpected turns row 0 cell value = [%s]! Skipping this workbook." % r_sheet.cell_value(0,colnum)) 134 | return 135 | 136 | w_sheet.write(1, colnum, r_sheet.cell_value(0,colnum), CALIBRI_10PT_ORANGE_CENTER) 137 | if colnum != 0: w_sheet.write(0, colnum, "") 138 | 139 | w_sheet.write(0,0, "All", CALIBRI_10PT_LIME_CENTER) 140 | 141 | # mainline - copy over non-empty rows 142 | if mainline_or_turns == "MAINLINE": 143 | w_rownum = 2 144 | for r_rownum in range(2,r_sheet.nrows): 145 | # don't copy the empty rows 146 | if isRowEmpty(r_sheet, r_rownum): continue 147 | 148 | # copy this row 149 | for colnum in range(r_sheet.ncols): 150 | w_sheet.write(w_rownum, colnum, r_sheet.cell_value(r_rownum,colnum), CALIBRI_10PT) 151 | w_rownum += 1 152 | # turns - error non-zero columns 153 | else: 154 | # look for zero columns and abort if found 155 | for colnum in range(1,r_sheet.ncols): 156 | if isColumnZeros(r_sheet, colnum): 157 | logger.warn("Zero column found! Skipping this workbook.") 158 | return 159 | 160 | # copy over everything 161 | for r_rownum in range(2,r_sheet.nrows): 162 | for colnum in range(r_sheet.ncols): 163 | w_sheet.write(r_rownum, colnum, r_sheet.cell_value(r_rownum,colnum), CALIBRI_10PT) 164 | 165 | if os.path.exists(os.path.join(DIR_NEWV11, file)): 166 | logger.warn("File %s already exists! Skipping." % os.path.join(DIR_NEWV11, file)) 167 | return 168 | 169 | wb.default_style.font.height = 20*10 170 | wb.save(os.path.join(DIR_NEWV11, file)) 171 | 172 | # move the old one to the deprecated dir 173 | shutil.move(os.path.join(DIR_TOPROCESS,file), 174 | os.path.join(DIR_OLDV10,file)) 175 | 176 | if __name__ == '__main__': 177 | optlist, args = getopt.getopt(sys.argv[1:], '') 178 | if len(args) < 2: 179 | print USAGE 180 | sys.exit(2) 181 | 182 | if len(args) != 3: 183 | print USAGE 184 | sys.exit(2) 185 | 186 | DIR_TOPROCESS = args[0] 187 | DIR_OLDV10 = args[1] 188 | DIR_NEWV11 = args[2] 189 | 190 | logger = logging.getLogger('countdracula') 191 | logger.setLevel(logging.DEBUG) 192 | 193 | consolehandler = logging.StreamHandler() 194 | consolehandler.setLevel(logging.DEBUG) 195 | consolehandler.setFormatter(logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')) 196 | logger.addHandler(consolehandler) 197 | 198 | debugFilename = "updateCountsWorkbooks.DEBUG.log" 199 | debugloghandler = logging.StreamHandler(open(debugFilename, 'w')) 200 | debugloghandler.setLevel(logging.DEBUG) 201 | debugloghandler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%Y-%m-%d %H:%M')) 202 | logger.addHandler(debugloghandler) 203 | 204 | files_to_process = sorted(os.listdir(DIR_TOPROCESS)) 205 | 206 | for file in files_to_process: 207 | if file[-4:] !='.xls': 208 | print "File suffix is not .xls: %s -- skipping" % file[-4:] 209 | continue 210 | 211 | logger.info("") 212 | logger.info("Processing file %s" % file) 213 | 214 | streetlist = CountsWorkbookParser.parseFilename(file) 215 | 216 | # mainline 217 | if len(streetlist) in [2,3]: 218 | updateWorkbook(logger, DIR_TOPROCESS, DIR_OLDV10, DIR_NEWV11, file, "MAINLINE" if len(streetlist)==3 else "TURNS") 219 | else: 220 | logger.info(" Invalid workbook name %s" % file) 221 | 222 | -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # CountDracula documentation build configuration file, created by 4 | # sphinx-quickstart on Wed Jul 27 15:56:25 2011. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | 16 | libdir = os.path.realpath(os.path.join(os.path.split(__file__)[0], "..", "geodjango")) 17 | sys.path.append(libdir) 18 | 19 | os.environ['DJANGO_SETTINGS_MODULE'] = 'geodjango.settings' 20 | from django.core.management import setup_environ 21 | from geodjango import settings 22 | import countdracula.models 23 | 24 | # If extensions (or modules to document with autodoc) are in another directory, 25 | # add these directories to sys.path here. If the directory is relative to the 26 | # documentation root, use os.path.abspath to make it absolute, like shown here. 27 | #sys.path.insert(0, os.path.abspath('.')) 28 | sys.path.insert(0,"../countdracula") 29 | 30 | # -- General configuration ----------------------------------------------------- 31 | 32 | # If your documentation needs a minimal Sphinx version, state it here. 33 | #needs_sphinx = '1.0' 34 | 35 | # Add any Sphinx extension module names here, as strings. They can be extensions 36 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 37 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.ifconfig', 'sphinx.ext.autosummary'] 38 | 39 | # Add any paths that contain templates here, relative to this directory. 40 | templates_path = ['_templates'] 41 | 42 | # The suffix of source filenames. 43 | source_suffix = '.rst' 44 | 45 | # The encoding of source files. 46 | #source_encoding = 'utf-8-sig' 47 | 48 | # The master toctree document. 49 | master_doc = 'index' 50 | 51 | # General information about the project. 52 | project = u'CountDracula' 53 | copyright = u'2011-2013, SFCTA Modeling team' 54 | 55 | # The version info for the project you're documenting, acts as replacement for 56 | # |version| and |release|, also used in various other places throughout the 57 | # built documents. 58 | # 59 | # The short X.Y version. 60 | version = '.7' 61 | # The full version, including alpha/beta/rc tags. 62 | release = '.7' 63 | 64 | # The language for content autogenerated by Sphinx. Refer to documentation 65 | # for a list of supported languages. 66 | #language = None 67 | 68 | # There are two options for replacing |today|: either, you set today to some 69 | # non-false value, then it is used: 70 | #today = '' 71 | # Else, today_fmt is used as the format for a strftime call. 72 | #today_fmt = '%B %d, %Y' 73 | 74 | # List of patterns, relative to source directory, that match files and 75 | # directories to ignore when looking for source files. 76 | exclude_patterns = [] 77 | 78 | # The reST default role (used for this markup: `text`) to use for all documents. 79 | #default_role = None 80 | 81 | # If true, '()' will be appended to :func: etc. cross-reference text. 82 | #add_function_parentheses = True 83 | 84 | # If true, the current module name will be prepended to all description 85 | # unit titles (such as .. function::). 86 | add_module_names = False 87 | 88 | # If true, sectionauthor and moduleauthor directives will be shown in the 89 | # output. They are ignored by default. 90 | #show_authors = False 91 | 92 | # The name of the Pygments (syntax highlighting) style to use. 93 | pygments_style = 'sphinx' 94 | 95 | # A list of ignored prefixes for module index sorting. 96 | #modindex_common_prefix = [] 97 | 98 | 99 | # -- Options for HTML output --------------------------------------------------- 100 | 101 | # The theme to use for HTML and HTML Help pages. See the documentation for 102 | # a list of builtin themes. 103 | html_theme = 'default' 104 | 105 | # Theme options are theme-specific and customize the look and feel of a theme 106 | # further. For a list of options available for each theme, see the 107 | # documentation. 108 | #html_theme_options = {} 109 | 110 | # Add any paths that contain custom themes here, relative to this directory. 111 | #html_theme_path = [] 112 | 113 | # The name for this set of Sphinx documents. If None, it defaults to 114 | # " v documentation". 115 | #html_title = None 116 | 117 | # A shorter title for the navigation bar. Default is the same as html_title. 118 | #html_short_title = None 119 | 120 | # The name of an image file (relative to this directory) to place at the top 121 | # of the sidebar. 122 | #html_logo = None 123 | 124 | # The name of an image file (within the static path) to use as favicon of the 125 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 126 | # pixels large. 127 | #html_favicon = None 128 | 129 | # Add any paths that contain custom static files (such as style sheets) here, 130 | # relative to this directory. They are copied after the builtin static files, 131 | # so a file named "default.css" will overwrite the builtin "default.css". 132 | html_static_path = ['_static'] 133 | 134 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 135 | # using the given strftime format. 136 | #html_last_updated_fmt = '%b %d, %Y' 137 | 138 | # If true, SmartyPants will be used to convert quotes and dashes to 139 | # typographically correct entities. 140 | #html_use_smartypants = True 141 | 142 | # Custom sidebar templates, maps document names to template names. 143 | #html_sidebars = {} 144 | 145 | # Additional templates that should be rendered to pages, maps page names to 146 | # template names. 147 | #html_additional_pages = {} 148 | 149 | # If false, no module index is generated. 150 | #html_domain_indices = True 151 | 152 | # If false, no index is generated. 153 | #html_use_index = True 154 | 155 | # If true, the index is split into individual pages for each letter. 156 | #html_split_index = False 157 | 158 | # If true, links to the reST sources are added to the pages. 159 | #html_show_sourcelink = True 160 | 161 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 162 | #html_show_sphinx = True 163 | 164 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 165 | #html_show_copyright = True 166 | 167 | # If true, an OpenSearch description file will be output, and all pages will 168 | # contain a tag referring to it. The value of this option must be the 169 | # base URL from which the finished HTML is served. 170 | #html_use_opensearch = '' 171 | 172 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 173 | #html_file_suffix = None 174 | 175 | # Output file base name for HTML help builder. 176 | htmlhelp_basename = 'CountDraculadoc' 177 | 178 | 179 | # -- Options for LaTeX output -------------------------------------------------- 180 | 181 | # The paper size ('letter' or 'a4'). 182 | #latex_paper_size = 'letter' 183 | 184 | # The font size ('10pt', '11pt' or '12pt'). 185 | #latex_font_size = '10pt' 186 | 187 | # Grouping the document tree into LaTeX files. List of tuples 188 | # (source start file, target name, title, author, documentclass [howto/manual]). 189 | latex_documents = [ 190 | ('index', 'CountDracula.tex', u'CountDracula Documentation', 191 | u'SFCTA Modeling team', 'manual'), 192 | ] 193 | 194 | # The name of an image file (relative to this directory) to place at the top of 195 | # the title page. 196 | #latex_logo = None 197 | 198 | # For "manual" documents, if this is true, then toplevel headings are parts, 199 | # not chapters. 200 | #latex_use_parts = False 201 | 202 | # If true, show page references after internal links. 203 | #latex_show_pagerefs = False 204 | 205 | # If true, show URL addresses after external links. 206 | #latex_show_urls = False 207 | 208 | # Additional stuff for the LaTeX preamble. 209 | #latex_preamble = '' 210 | 211 | # Documents to append as an appendix to all manuals. 212 | #latex_appendices = [] 213 | 214 | # If false, no module index is generated. 215 | #latex_domain_indices = True 216 | 217 | 218 | # -- Options for manual page output -------------------------------------------- 219 | 220 | # One entry per manual page. List of tuples 221 | # (source start file, name, description, authors, manual section). 222 | man_pages = [ 223 | ('index', 'countdracula', u'CountDracula Documentation', 224 | [u'SFCTA Modeling team'], 1) 225 | ] 226 | 227 | 228 | autodoc_default_flags = ['members', 'undoc_members', 'inherited-members', 'show-inheritance'] 229 | 230 | autoclass_content = 'both' 231 | 232 | autosummary_generate = True 233 | 234 | todo_include_todos = True 235 | 236 | # This is from http://djangosnippets.org/snippets/2533/ 237 | import inspect 238 | from geodjango import settings 239 | from django.core.management import setup_environ 240 | from django.utils.html import strip_tags 241 | from django.utils.encoding import force_unicode 242 | 243 | setup_environ(settings) 244 | 245 | def process_docstring(app, what, name, obj, options, lines): 246 | # This causes import errors if left outside the function 247 | from django.db import models 248 | 249 | # Only look at objects that inherit from Django's base model class 250 | if inspect.isclass(obj) and issubclass(obj, models.Model): 251 | # Grab the field list from the meta class 252 | fields = obj._meta._fields() 253 | 254 | for field in fields: 255 | # Decode and strip any html out of the field's help text 256 | help_text = strip_tags(force_unicode(field.help_text)) 257 | 258 | # Decode and capitalize the verbose name, for use if there isn't 259 | # any help text 260 | verbose_name = force_unicode(field.verbose_name).capitalize() 261 | 262 | if help_text: 263 | # Add the model field to the end of the docstring as a param 264 | # using the help text as the description 265 | lines.append(u':param %s: %s' % (field.attname, help_text)) 266 | else: 267 | # Add the model field to the end of the docstring as a param 268 | # using the verbose name as the description 269 | lines.append(u':param %s: %s' % (field.attname, verbose_name)) 270 | 271 | # Add the field's type to the docstring 272 | lines.append(u':type %s: %s' % (field.attname, type(field).__name__)) 273 | 274 | # Return the extended docstring 275 | return lines 276 | 277 | def setup(app): 278 | # Register the docstring processor with sphinx 279 | app.connect('autodoc-process-docstring', process_docstring) -------------------------------------------------------------------------------- /geodjango/countdracula/models.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.contrib.gis.db import models as gis_models 3 | from django.db import models 4 | import math 5 | 6 | # Master list of (intersection) nodes and their coordinates 7 | class Node(gis_models.Model): 8 | 9 | point = gis_models.PointField() # default srid 4326=WGS84 long/lat 10 | # In order to conduct geographic queries, each geographic model requires a GeoManager model manager. 11 | objects = gis_models.GeoManager() 12 | 13 | def long_x(self): 14 | """ Returns the longitude of this node. 15 | """ 16 | return self.point[0] 17 | 18 | def lat_y(self): 19 | """ Returns the latitude of this node. 20 | """ 21 | return self.point[1] 22 | 23 | # Returns the string representation of the model. 24 | def __unicode__(self): 25 | return "ID:%d long_x:%f lat_y:%f" % (self.id, self.long_x(), self.lat_y()) 26 | 27 | class Meta: 28 | ordering = ['id'] 29 | 30 | # http://www.fhwa.dot.gov/policy/ohpi/vehclass.htm 31 | VehicleTypes = \ 32 | ( (-1, 'Unknown'), # Unknown 33 | ('Basic', ( 34 | (0, 'All'), # Motorized Vehicles 35 | (1, 'Pedestrian'), # Wheelchairs? Razr scooters? Skateboarders? Unicycles? 36 | (2, 'Bike'), # Two-wheeled bicycles 37 | (3, 'Truck'), # Generic truck classification 38 | (4, 'Bus'), # Buses -- All vehicles manufactured as traditional passenger-carrying buses with two axles and six tires or three or more axles. This category includes only traditional buses (including school buses) functioning as passenger-carrying vehicles. Modified buses should be considered to be a truck and should be appropriately classified. 39 | (5, 'Cars'), # All sedans, coupes, and station wagons manufactured primarily for the purpose of carrying passengers and including those passenger cars pulling recreational or other light trailers. 40 | ) 41 | ), 42 | ('Truck Detail', ( 43 | (6, '2 Axle Long'), # Other Two-Axle, Four-Tire Single Unit Vehicles -- All two-axle, four-tire, vehicles, other than passenger cars. Included in this classification are pickups, panels, vans, and other vehicles such as campers, motor homes, ambulances, hearses, carryalls, and minibuses. Other two-axle, four-tire single-unit vehicles pulling recreational or other light trailers are included in this classification. Because automatic vehicle classifiers have difficulty distinguishing class 3 from class 2, these two classes may be combined into class 2. 44 | (7, '2 Axle 6 Tire'), # Two-Axle, Six-Tire, Single-Unit Trucks -- All vehicles on a single frame including trucks, camping and recreational vehicles, motor homes, etc., with two axles and dual rear wheels. 45 | (8, '3 Axle Single'), # Three-Axle Single-Unit Trucks -- All vehicles on a single frame including trucks, camping and recreational vehicles, motor homes, etc., with three axles. 46 | (9, '4 Axle Single'), # Four or More Axle Single-Unit Trucks -- All trucks on a single frame with four or more axles. 47 | (10, '<5 Axle Double'), # Four or Fewer Axle Single-Trailer Trucks -- All vehicles with four or fewer axles consisting of two units, one of which is a tractor or straight truck power unit. 48 | (11, '5 Axle Double'), # Five-Axle Single-Trailer Trucks -- All five-axle vehicles consisting of two units, one of which is a tractor or straight truck power unit. 49 | (12, '>6 Axle Double'), # Six or More Axle Single-Trailer Trucks -- All vehicles with six or more axles consisting of two units, one of which is a tractor or straight truck power unit. 50 | (13, '<6 Axle Multi'), # Five or fewer Axle Multi-Trailer Trucks -- All vehicles with five or fewer axles consisting of three or more units, one of which is a tractor or straight truck power unit. 51 | (14, '6 Axle Multi'), # Six-Axle Multi-Trailer Trucks -- All six-axle vehicles consisting of three or more units, one of which is a tractor or straight truck power unit. 52 | (15, '>6 Axle Multi'), # Seven or More Axle Multi-Trailer Trucks -- All vehicles with seven or more axles consisting of three or more units, one of which is a tractor or straight truck power unit.') 53 | ) 54 | ) 55 | ) 56 | 57 | # Direction options 58 | Directions = \ 59 | ( ('NB', 'Northbound'), 60 | ('SB', 'Southbound'), 61 | ('EB', 'Eastbound'), 62 | ('WB', 'Westbound') 63 | ) 64 | 65 | # All streetnames in the network 66 | class StreetName(models.Model): 67 | street_name = models.CharField(max_length=100, primary_key=True, help_text="e.g. CESAR CHAVEZ ST") 68 | nospace_name = models.CharField(max_length=100, help_text="e.g. CESARCHAVEZST") 69 | short_name = models.CharField(max_length=100, help_text="e.g. CESAR CHAVEZ") 70 | suffix = models.CharField(max_length=20, help_text="e.g. ST") 71 | 72 | # many to many relationship with the node 73 | nodes = models.ManyToManyField(Node, related_name="street_to_node") 74 | 75 | def __unicode__(self): 76 | return self.street_name 77 | 78 | # sort by this in the admin UI 79 | class Meta: 80 | ordering = ['street_name'] 81 | 82 | @staticmethod 83 | def getPossibleStreetNames(name): 84 | """ 85 | Given a street name string, looks up the name in the various columns of the :ref:`table-street_names` 86 | and returns a QuerySet of the possible :py:class:`StreetName` instances. 87 | 88 | This search is case-insensitive. 89 | 90 | Returns an empty QuerySet on failure. 91 | """ 92 | street_names = StreetName.objects.filter(street_name__iexact=name) 93 | if len(street_names) > 0: return street_names 94 | 95 | street_names = StreetName.objects.filter(nospace_name__iexact=name.replace(" ","")) 96 | if len(street_names) > 0: return street_names 97 | 98 | street_names = StreetName.objects.filter(short_name__iexact=name) 99 | if len(street_names) > 0: return street_names 100 | 101 | # see if we can match the nospace_name with a wild card for the suffix 102 | street_names = StreetName.objects.filter(nospace_name__istartswith=name.replace(" ","")) 103 | return street_names 104 | 105 | 106 | 107 | class TurnCountLocation(models.Model): 108 | from_street = models.ForeignKey(StreetName, help_text="Street from which turn originates") 109 | from_dir = models.CharField(max_length=5, choices=Directions, help_text="Direction going into turn") 110 | to_street = models.ForeignKey(StreetName, help_text="Street to which the turn is destined", related_name="tc_to_street") 111 | to_dir = models.CharField(max_length=5, choices=Directions, help_text="Direction coming out of turn") 112 | intersection_street = models.ForeignKey(StreetName, help_text="Cross street to identify the intersection", related_name="tc_int_street") 113 | intersection = models.ForeignKey(Node, help_text="Intersection", related_name="tc_int_id") 114 | 115 | class Meta: 116 | unique_together = (('from_street', 'from_dir', 'to_street', 'to_dir', 'intersection')) 117 | 118 | def __unicode__(self): 119 | return "%s %s to %s %s" % \ 120 | (self.from_street, self.from_dir, self.to_street, self.to_dir) 121 | 122 | # Turn counts for an intersection 123 | class TurnCount(models.Model): 124 | location = models.ForeignKey(TurnCountLocation, related_name="turncount") 125 | 126 | count = models.DecimalField(max_digits=10,decimal_places=2) # decimal because it could be an average 127 | count_date = models.DateField(help_text="Date the count was collected", blank=True, null=True) 128 | count_year = models.IntegerField(help_text="Year the count was collected. Will populate from Count Date automatically if provided.", editable=False) 129 | 130 | start_time = models.TimeField(help_text="Start time for the count") 131 | period_minutes = models.IntegerField(help_text="Period (minutes)") 132 | vehicle_type = models.IntegerField(choices=VehicleTypes) 133 | sourcefile = models.CharField(max_length=500, help_text="For tracking where this count came from") 134 | project = models.CharField(max_length=100, help_text="For tracking if this count was collected for a specific project") 135 | 136 | upload_user = models.ForeignKey(settings.AUTH_USER_MODEL, help_text="User that uploaded this Count data") 137 | 138 | def __unicode__(self): 139 | return "%3d-minute at %s at location %s" % \ 140 | (self.period_minutes, self.start_time, self.location) 141 | 142 | def clean(self): 143 | from django.core.exceptions import ValidationError 144 | # populate the count_year from the count_date automatically 145 | if self.count_date is not None: 146 | self.count_year = self.count_date.year 147 | 148 | class MainlineCountLocation(models.Model): 149 | on_street = models.ForeignKey(StreetName, help_text="The street with the count", related_name="mc_on_street") 150 | on_dir = models.CharField(max_length=5, choices=Directions, help_text="Direction of count") 151 | from_street = models.ForeignKey(StreetName, help_text="Cross street before count", related_name="mc_from_street") 152 | from_int = models.ForeignKey(Node, help_text="Intersection ID for that Cross street before count", related_name="mc_from_intid") 153 | to_street = models.ForeignKey(StreetName, help_text="Cross street after count", related_name="mc_to_street") 154 | to_int = models.ForeignKey(Node, help_text="Intersection for that Cross street after count", related_name="mc_to_intid") 155 | 156 | class Meta: 157 | unique_together = (('on_street', 'on_dir', 'from_int', 'to_int')) 158 | 159 | def __unicode__(self): 160 | return "%s %s (from %s to %s)" % \ 161 | (self.on_street, self.on_dir, self.from_street, self.to_street) 162 | 163 | # Mainline counts for an intersection 164 | class MainlineCount(models.Model): 165 | location = models.ForeignKey(MainlineCountLocation, related_name="mainlinecount") 166 | 167 | count = models.DecimalField(max_digits=10,decimal_places=2) # decimal because it could be an average 168 | count_date = models.DateField(help_text="Date the count was collected", blank=True, null=True) 169 | count_year = models.IntegerField(help_text="Year the count was collected. Will populate from Count Date automatically if provided.", editable=False) 170 | 171 | start_time = models.TimeField(help_text="Start time for the count") 172 | period_minutes = models.IntegerField(help_text="Period (minutes)") 173 | vehicle_type = models.IntegerField(choices=VehicleTypes) 174 | sourcefile = models.CharField(max_length=500, help_text="For tracking where this count came from") 175 | project = models.CharField(max_length=100, help_text="For tracking if this count was collected for a specific project") 176 | reference_position = models.FloatField(help_text="How far along the link the count was actually taken; use -1 for unknown. Units?") 177 | 178 | upload_user = models.ForeignKey(settings.AUTH_USER_MODEL, help_text="User that uploaded this Count data") 179 | 180 | def __unicode__(self): 181 | return "%3d-minute at %s at location %s" % \ 182 | (self.period_minutes, self.start_time, self.location) 183 | 184 | def clean(self): 185 | from django.core.exceptions import ValidationError 186 | # populate the count_year from the count_date automatically 187 | if self.count_date is not None: 188 | self.count_year = self.count_date.year 189 | -------------------------------------------------------------------------------- /geodjango/countdracula/views.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | import csv, datetime 3 | from django.contrib.gis.geos import fromstr 4 | from django.core.context_processors import csrf 5 | from django.db.models import Q 6 | from django.http import HttpResponse 7 | from django.shortcuts import render_to_response 8 | from django.template import RequestContext 9 | from django.utils import simplejson 10 | 11 | from countdracula.models import Node, MainlineCountLocation, TurnCountLocation, MainlineCount, TurnCount 12 | 13 | 14 | def mapview(request): 15 | """ 16 | The mapview shows all the counts on one map. 17 | """ 18 | print "mapview starting: " + str(datetime.datetime.now()) 19 | 20 | # get all the TurnCountLocation objects 21 | turn_count_locs = TurnCountLocation.objects.all() 22 | print len(turn_count_locs) 23 | 24 | # get all the MainlineCountLocation objects 25 | mainline_count_locs = MainlineCountLocation.objects.all() 26 | print len(mainline_count_locs) 27 | 28 | # and the nodes 29 | nodes = {} 30 | for node in Node.objects.all(): 31 | nodes[node.id] = (node.point.x, node.point.y) 32 | 33 | # get all the count years 34 | ml_years = MainlineCount.objects.values_list('count_year',flat=True).distinct(); 35 | turn_years = TurnCount.objects.values_list('count_year',flat=True).distinct(); 36 | count_years = sorted(set(ml_years).union(set(turn_years))) 37 | 38 | # get a mapping of loc_id -> { year:vehtype } 39 | turnlocs_to_years = {} 40 | tcqs = TurnCountLocation.objects.values('id','turncount__count_year', 'turncount__vehicle_type').distinct() 41 | for row in tcqs: 42 | # init for id 43 | if row['id'] not in turnlocs_to_years: turnlocs_to_years[row['id']] = {} 44 | # init for id, year 45 | if row['turncount__count_year'] not in turnlocs_to_years[row['id']]: 46 | turnlocs_to_years[row['id']][row['turncount__count_year']] = 0 47 | # bitwise or 48 | if row['turncount__vehicle_type'] >= 6: 49 | turnlocs_to_years[row['id']][row['turncount__count_year']] |= (1 << 3) # truck 50 | elif row['turncount__vehicle_type'] >= 0: 51 | turnlocs_to_years[row['id']][row['turncount__count_year']] |= (1 << row['turncount__vehicle_type']) 52 | else: 53 | print row 54 | del tcqs 55 | 56 | mainlinelocs_to_years = {} 57 | mlqs = MainlineCountLocation.objects.values('id','mainlinecount__count_year','mainlinecount__vehicle_type').distinct() 58 | for row in mlqs: 59 | # init for id 60 | if row['id'] not in mainlinelocs_to_years: mainlinelocs_to_years[row['id']] = {} 61 | # init for id, year 62 | if row['mainlinecount__count_year'] not in mainlinelocs_to_years[row['id']]: 63 | mainlinelocs_to_years[row['id']][row['mainlinecount__count_year']] = 0 64 | # bitwise or 65 | if row['mainlinecount__vehicle_type'] >= 6: 66 | mainlinelocs_to_years[row['id']][row['mainlinecount__count_year']] |= (1 << 3) # truck 67 | elif row['mainlinecount__vehicle_type'] >= 0: 68 | mainlinelocs_to_years[row['id']][row['mainlinecount__count_year']] |= (1 << row['mainlinecount__vehicle_type']) 69 | else: 70 | print row 71 | del mlqs 72 | 73 | print "mapview queries done: " + str(datetime.datetime.now()) 74 | 75 | x= render_to_response('countdracula/gmap.html', 76 | {'turn_count_locs' :turn_count_locs, 77 | 'mainline_count_locs' :mainline_count_locs, 78 | 'nodes' :nodes, 79 | 'count_years' :count_years, 80 | 'turnlocs_to_years' :turnlocs_to_years, 81 | 'mainlinelocs_to_years' :mainlinelocs_to_years}, 82 | context_instance=RequestContext(request)) 83 | print "mapview render done: " + str(datetime.datetime.now()) 84 | return x 85 | 86 | def counts_for_location(request): 87 | """ 88 | This enables the map view to fetch count information for a location. 89 | """ 90 | results = { 91 | 'success':False, 92 | 'period_minutes':defaultdict(int), # period_minutes -> number of counts 93 | 'date_min':3000, # earliest date for qualifying count 94 | 'date_max':0 # latest date for qualifying count 95 | } 96 | try: 97 | count_type = request.GET[u'count_type'] 98 | loc_id = int(request.GET[u'loc_id']) 99 | 100 | if count_type == 'mainline': 101 | count_loc = MainlineCountLocation.objects.get(id=loc_id) 102 | counts = MainlineCount.objects.filter(location=count_loc) 103 | 104 | elif count_type == 'turn': 105 | count_loc = TurnCountLocation.objects.get(id=loc_id) 106 | counts = TurnCount.objects.filter(location=count_loc) 107 | 108 | else: 109 | raise Exception("Don't understand count_type=[%s]" % count_type) 110 | 111 | for count in counts: 112 | # find earliest and last dates 113 | if results['date_min'] > count.count_year: results['date_min'] = count.count_year 114 | if results['date_max'] < count.count_year: results['date_max'] = count.count_year 115 | 116 | # tally counts by period_minutes 117 | results['period_minutes'][str(count.period_minutes)] += 1 118 | 119 | results['count_type'] = count_type 120 | results['loc_id'] = loc_id 121 | results['where_str'] = str(count_loc) 122 | results['success'] = True 123 | 124 | except Exception as inst: 125 | results['error'] = inst 126 | 127 | # convert dates into strings 128 | if 'date_min' in results: results['date_min'] = str(results['date_min']) 129 | if 'date_max' in results: results['date_max'] = str(results['date_max']) 130 | return HttpResponse(simplejson.dumps(results), mimetype='application/json') 131 | 132 | def countlocs_for_point(request): 133 | """ 134 | This enables the map view to fetch count locations near a point. 135 | """ 136 | results = {} 137 | try: 138 | lat = float(request.GET[u'lat']) 139 | lng = float(request.GET[u'lng']) 140 | radius = int(request.GET[u'radius']) 141 | 142 | pinpoint= fromstr('POINT(%.6f %.6f)' % (lng,lat), srid=4326) 143 | 144 | node_qs = Node.objects.filter(point__distance_lte=(pinpoint, radius)) 145 | tcl_qs = TurnCountLocation.objects.filter(intersection__in=node_qs) 146 | results['movement_locs'] = [tcl.id for tcl in tcl_qs] 147 | 148 | ml_qs = MainlineCountLocation.objects.filter( Q(from_int__in=node_qs) | Q(to_int__in=node_qs) ) 149 | results['mainline_locs'] = [ml.id for ml in ml_qs] 150 | 151 | except Exception as inst: 152 | results['error'] = inst 153 | 154 | return HttpResponse(simplejson.dumps(results), mimetype='application/json') 155 | 156 | 157 | def download(request): 158 | 159 | response = HttpResponse(content_type='text/csv') 160 | response['Content-Disposition'] = 'attachment; filename="CountDracula_%s.csv"' % \ 161 | ("Mainline" if request.POST[u'download-type'] in [u'mainlinelocs', u'mainlinefilt'] else "Movement") 162 | writer = csv.writer(response) 163 | 164 | count_years = None 165 | vehicle_types = None 166 | if request.POST[u'download-type'] in [u'mainlinefilt', u'movementfilt']: 167 | # transform the count-year to a list 168 | count_years = [] 169 | for arg in request.POST.getlist(u'count-year'): 170 | count_years.append(int(arg[4:])) 171 | 172 | # transform the vehicle types into a list 173 | vehicle_types = [] 174 | for arg in request.POST.getlist(u'vtype'): 175 | vehicle_types.append(int(arg)) 176 | # truck detail 177 | if int(arg)==3: 178 | vehicle_types.extend([6,7,8,9,10,11,12,13,14,15]) 179 | 180 | location_ids = [] 181 | if request.POST[u'download-type'] in [u'mainlinelocs', u'mainlinefilt']: 182 | 183 | for arg in request.POST[u'mainline_loc_ids'].split(','): location_ids.append(int(arg)) 184 | # fetch mainline data 185 | mlqs = MainlineCount.objects.select_related().filter(location_id__in=location_ids).order_by('location') 186 | 187 | else: 188 | for arg in request.POST[u'movement_loc_ids'].split(','): location_ids.append(int(arg)) 189 | # fetch turn data 190 | mlqs = TurnCount.objects.select_related().filter(location_id__in=location_ids).order_by('location') 191 | 192 | 193 | # log it 194 | writer.writerow(["LocationID in %s" % str(location_ids)]) 195 | 196 | # further filter it 197 | if request.POST[u'download-type'] in [u'mainlinefilt', u'movementfilt']: 198 | mlqs = mlqs.filter(count_year__in=count_years).filter(vehicle_type__in=vehicle_types) 199 | # log it 200 | writer.writerow(["Count Year in %s" % str(count_years)]) 201 | writer.writerow(["Vehicle Type in %s" % str(vehicle_types)]) 202 | 203 | 204 | # header row 205 | if request.POST[u'download-type'] in [u'mainlinelocs', u'mainlinefilt']: 206 | 207 | writer.writerow(["LocationID", "Location OnStreet", "Location OnDir", "Location FromStreet", "Location FromNode", 208 | "Location ToStreet", "Location ToNode", 209 | "Count", "Count Date", "Count Year", "Start Time", "Period Minutes", "Vehicle Type", 210 | "Source File", "Project", "Reference Position", "Upload User"]) 211 | else: 212 | writer.writerow(["LocationID", "Location FromStreet", "Location FromDir", "Location ToStreet", "Location ToDir", 213 | "Location IntStreet", "Location IntNode", 214 | "Count", "Count Date", "Count Year", "Start Time", "Period Minutes", "Vehicle Type", 215 | "Source File", "Project", "Upload User"]) 216 | 217 | for mcount in mlqs: 218 | if request.POST[u'download-type'] in [u'mainlinelocs', u'mainlinefilt']: 219 | writer.writerow([mcount.location.id, 220 | mcount.location.on_street, 221 | mcount.location.on_dir, 222 | mcount.location.from_street, 223 | mcount.location.from_int, 224 | mcount.location.to_street, 225 | mcount.location.to_int, 226 | mcount.count, 227 | mcount.count_date, 228 | mcount.count_year, 229 | mcount.start_time, 230 | mcount.period_minutes, 231 | mcount.vehicle_type, 232 | mcount.sourcefile, 233 | mcount.project, 234 | mcount.reference_position, 235 | mcount.upload_user]) 236 | 237 | else: 238 | writer.writerow([mcount.location.id, 239 | mcount.location.from_street, 240 | mcount.location.from_dir, 241 | mcount.location.to_street, 242 | mcount.location.to_dir, 243 | mcount.location.intersection_street, 244 | mcount.location.intersection, 245 | mcount.count, 246 | mcount.count_date, 247 | mcount.count_year, 248 | mcount.start_time, 249 | mcount.period_minutes, 250 | mcount.vehicle_type, 251 | mcount.sourcefile, 252 | mcount.project, 253 | mcount.upload_user]) 254 | 255 | return response 256 | -------------------------------------------------------------------------------- /geodjango/templates/countdracula/gmap.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Count Dracula Map 6 | 7 | 8 | 9 | 307 | 308 | 309 | 310 | 367 |
368 | 369 | 370 | -------------------------------------------------------------------------------- /scripts/insertSanFranciscoPeMSCounts.py: -------------------------------------------------------------------------------- 1 | """ 2 | Created on Jul 25, 2011 3 | 4 | @author: lmz 5 | 6 | This script reads PEMS counts from Excel workbooks and inserts the info into the CountDracula dataabase. 7 | """ 8 | 9 | import datetime, getopt, logging, os, sys, time, traceback 10 | import pytz 11 | import xlrd 12 | 13 | libdir = os.path.realpath(os.path.join(os.path.split(__file__)[0], "..", "geodjango")) 14 | sys.path.append(libdir) 15 | os.environ['DJANGO_SETTINGS_MODULE'] = 'geodjango.settings' 16 | 17 | from django.core.exceptions import ObjectDoesNotExist 18 | from django.core.management import setup_environ 19 | from geodjango import settings 20 | from django.contrib.auth.models import User 21 | 22 | from countdracula.models import Node, StreetName, MainlineCountLocation, MainlineCount 23 | 24 | TIMEZONE = pytz.timezone('America/Los_Angeles') 25 | USAGE = """ 26 | 27 | python insertSanFranciscoPeMSCounts.py [-v pems_vds_datfile] [-c pems_census_dir] user PeMS_to_NetworkNodes.xls 28 | 29 | e.g. python scripts\insertSanFranciscoPeMSCounts.py -v "Q:\Roadway Observed Data\PeMS\D4_Data_2010\pems_dist4_2010_fullyr.dat" 30 | -c "Q:\Roadway Observed Data\PeMS\D4_Data_2010\PeMS_Census" 31 | lisa "Q:\Roadway Observed Data\PeMS\PeMs_to_NetworkNodes.xls" 32 | 33 | PeMS_to_NetworkNodes.xls is a workbook that tells us how to associate PeMS data with the network. 34 | 35 | The *pems_vds_datfile* is a tab-delimited file. 36 | 37 | The *census_dir* is a directory with Excel workbooks downloaded from PeMS for various Census sensors. 38 | 39 | Note: more than one census dir or vds file can be passed in at once, just use the option tag each time. 40 | """ 41 | 42 | def readMappingWorkbook(workbook_filename): 43 | """ 44 | Reads the workbook that maps PeMS station data to network node numbers. 45 | The workbook looks like this: 46 | 47 | .. image:: /images/PeMS_to_NetworkNodes_Workbook.png 48 | 49 | Returns a dictionary mapping a tuple to a tuple: 50 | ( PeMSType, PeMS_IDdir) -> ( LocationType, Rte, Dir, LocDescription, Model_A_node, Model_B_node, Model_LocDescription ) 51 | 52 | e.g. ``( "VDS", "402553") -> ( "SF Screenline", "Golden Gate", "US-101", "N", "GG Bridge Mainline", 8803, 8806, "N of Alex Rd exit" )`` 53 | 54 | All fields are text except for *Model_A_node* and *Model_B_node*, which are ints. 55 | 56 | """ 57 | book = xlrd.open_workbook(workbook_filename) 58 | sheetnames = book.sheet_names() 59 | assert(len(sheetnames)==1) # there should be only one sheet 60 | activesheet = book.sheet_by_name(sheetnames[0]) 61 | return_dict = {} 62 | 63 | # row 0: source 64 | # row 1: header 1 65 | # row 2: header 2 66 | for row in range(3, len(activesheet.col(0))): 67 | pems_ID_dir = activesheet.cell_value(row, 10) 68 | # floats are just integer IDs but we want a string 69 | if isinstance(pems_ID_dir, float): 70 | pems_ID_dir = "%.0f" % pems_ID_dir 71 | 72 | pems_tuple = (activesheet.cell_value(row, 8), # PeMSType 73 | pems_ID_dir) # PeMS_IDdir 74 | 75 | result_tuple = (activesheet.cell_value(row, 0), # LocationType 76 | activesheet.cell_value(row, 2), # Rte 77 | activesheet.cell_value(row, 3), # Dir 78 | activesheet.cell_value(row, 4), # LocDescription 79 | int(activesheet.cell_value(row,12)),# Model_A_node 80 | int(activesheet.cell_value(row,13)),# Model_B_node 81 | activesheet.cell_value(row,15)) # Model_LocDescription 82 | return_dict[pems_tuple] = result_tuple 83 | return return_dict 84 | 85 | def getIntersectionStreetnamesForPemsKey(mapping, pems_key): 86 | """ 87 | Given *mapping*, which is the result of :py:func:`readMappingWorkbook` 88 | and *pems_key*, a tuple key into *mapping* (e.g. ``( "VDS", "40255")``, 89 | looks up the link in the *mapping* and then looks up the nodes in the Count Dracula database using 90 | *cd_reader* (an instance of :py:class:`CountsDatabaseReader`). 91 | 92 | Returns a tuple: ( on_street, from_street, from_node, to_street, to_node ) 93 | 94 | Raises an Exception for failure. 95 | """ 96 | # grab the model nodes and their streets 97 | loc_desc = mapping[pems_key][3] 98 | model_node_A_num = mapping[pems_key][4] 99 | model_node_B_num = mapping[pems_key][5] 100 | 101 | try: 102 | model_node_A = Node.objects.get(id=model_node_A_num) 103 | except: 104 | raise Exception("Couldn't find node %6d in countdracula. Skipping %s" % 105 | (model_node_A_num, loc_desc)) 106 | 107 | try: 108 | streets_A_set = set(streetname for streetname in model_node_A.streetname_set.all()) 109 | except: 110 | raise Exception("Couldn't find streets for model node A %6d in countdracula. Skipping %s." % 111 | (model_node_A_num, loc_desc)) 112 | # logger.debug("model_node_A = %s streets_A_set = %s" % (model_node_A, streets_A_set)) 113 | 114 | try: 115 | model_node_B = Node.objects.get(id=model_node_B_num) 116 | except: 117 | raise Exception("Couldn't find node %6d in countdracula. Skipping %s" % 118 | (model_node_B_num, loc_desc)) 119 | 120 | try: 121 | streets_B_set = set(streetname for streetname in model_node_B.streetname_set.all()) 122 | except: 123 | raise Exception("Couldn't find streets for model node B %6d in countdracula. Skipping %s." % 124 | (model_node_B_num, loc_desc)) 125 | # logger.debug("model_node_B = %s streets_B_set = %s" % (model_node_B, streets_B_set)) 126 | 127 | # figure out the on_street 128 | on_street_set = streets_A_set & streets_B_set 129 | if len(on_street_set) != 1: 130 | raise Exception("Couldn't find a unique single street for both node A %d (%s) and node B %d (%s)" % 131 | (model_node_A_num, str(streets_A_set), model_node_B_num, str(streets_B_set))) 132 | on_street = on_street_set.pop() 133 | on_street_set.add(on_street) 134 | 135 | # figure out the from_street 136 | from_street_set = streets_A_set 137 | from_street_set -= on_street_set 138 | from_street = "" # try blank... 139 | if len(from_street_set) > 0: 140 | # hack - this is the most descriptive 141 | if 'LAKE ST' in from_street_set: 142 | from_street = 'LAKE ST' 143 | else: 144 | from_street = from_street_set.pop() 145 | if from_street == "": 146 | raise Exception("Couldn't find from street for node A %d (%s) to node B %d (%s)" % 147 | (model_node_A_num, str(streets_A_set), model_node_B_num, str(streets_B_set))) 148 | 149 | # and the to_street 150 | to_street_set = streets_B_set 151 | to_street_set -= on_street_set 152 | to_street = "" 153 | if len(to_street_set) > 0: 154 | # hack - this is the most descriptive 155 | if 'LAKE ST' in to_street_set: 156 | to_street = 'LAKE ST' 157 | else: 158 | to_street = to_street_set.pop() 159 | if to_street == "": 160 | raise Exception("Couldn't find to street for node A %d (%s) to node B %d (%s)" % 161 | (model_node_A_num, str(streets_A_set), model_node_B_num, str(streets_B_set))) 162 | 163 | return (on_street, from_street, model_node_A, to_street, model_node_B) 164 | 165 | def readVDSCounts(mapping, vds_datfilename, user): 166 | """ 167 | Read the VDS (Vehicle Detector Station) data from the given *vds_datafilename*, and insert the mainline 168 | counts into countdracula. 169 | """ 170 | vds_datfilename_abspath = os.path.abspath(vds_datfilename) 171 | vds_datfile = open(vds_datfilename, mode="r") 172 | 173 | # read the header line and parse it 174 | line = vds_datfile.readline() 175 | header_fields = line.strip().split("\t") 176 | fieldname_to_fieldidx = {} 177 | for idx in range(len(header_fields)): 178 | fieldname_to_fieldidx[header_fields[idx]] = idx 179 | first_line_pos = vds_datfile.tell() 180 | 181 | # preprocess the info for the pems_id, VDS mapping lookup stuff first and cache that result 182 | logger.info("Processing VDS locations to CountDracula locations") 183 | pemsid_to_cdlocation = {} 184 | for line in vds_datfile: 185 | line = line.strip() 186 | fields = line.split("\t") 187 | 188 | pems_id = fields[fieldname_to_fieldidx["loc"]] 189 | pems_time = fields[fieldname_to_fieldidx["time"]] 190 | 191 | # once we've finished all the 00:00:00 we're done with the preprocessing 192 | if pems_time != "00:00:00": break 193 | 194 | # is the pems ID in our mapping? if not, we don't care about it 195 | pems_key = ("VDS", pems_id) 196 | if pems_key not in mapping: 197 | # logger.debug("Couldn't find %s in mapping; don't care about this VDS. Skipping." % str(pems_key)) 198 | continue 199 | 200 | # did we already do this? 201 | if pems_key in pemsid_to_cdlocation: continue 202 | 203 | try: 204 | pemsid_to_cdlocation[pems_key] = getIntersectionStreetnamesForPemsKey(mapping, pems_key) 205 | 206 | logger.debug("Mapped key=%s route=[%s] dir=[%s] description=[%s] to on_street=[%s] from_street=[%s] to_street=[%s]" % 207 | (str(pems_key), fields[fieldname_to_fieldidx["route"]], fields[fieldname_to_fieldidx["dir"]], 208 | mapping[pems_key][3], pemsid_to_cdlocation[pems_key][0], pemsid_to_cdlocation[pems_key][1], pemsid_to_cdlocation[pems_key][2])) 209 | 210 | except Exception, e: 211 | logger.error(e) 212 | # logger.exception('Exception received!') 213 | 214 | 215 | for pems_key,intersection in pemsid_to_cdlocation.iteritems(): 216 | logger.debug("%20s -> %s" % (str(pems_key), str(intersection))) 217 | 218 | # create the counts list for countdracula 219 | vds_datfile.seek(first_line_pos) 220 | counts_saved = 0 221 | for line in vds_datfile: 222 | line = line.strip() 223 | fields = line.split("\t") 224 | 225 | # read the fields 226 | pems_id = fields[fieldname_to_fieldidx["loc"]] 227 | # pems_route = fields[fieldname_to_fieldidx["route"]] 228 | pems_dir = fields[fieldname_to_fieldidx["dir"]] 229 | # pems_type = fields[fieldname_to_fieldidx["type"]] 230 | pems_flow = fields[fieldname_to_fieldidx["flow"]] # number of vehicles, or count 231 | # pems_avgspd = fields[fieldname_to_fieldidx["avgspd"]] -- TODO 232 | pems_date = fields[fieldname_to_fieldidx["date"]] 233 | pems_time = fields[fieldname_to_fieldidx["time"]] 234 | 235 | # If we failed to map this pems key to count dracula then skip it. 236 | # We already logged issues in preprocessing 237 | pems_key = ("VDS", pems_id) 238 | if pems_key not in pemsid_to_cdlocation: continue 239 | 240 | # this is the format in count dracula 241 | if pems_dir == "S": pems_dir = "SB" 242 | elif pems_dir == "N": pems_dir = "NB" 243 | elif pems_dir == "E": pems_dir = "EB" 244 | elif pems_dir == "W": pems_dir = "WB" 245 | 246 | # look for the mainline count location 247 | try: 248 | mainline_count_location = MainlineCountLocation.objects.get(from_int = pemsid_to_cdlocation[pems_key][2], 249 | to_int = pemsid_to_cdlocation[pems_key][4], 250 | on_street = pemsid_to_cdlocation[pems_key][0], 251 | on_dir = pems_dir) 252 | except ObjectDoesNotExist: 253 | mainline_count_location = MainlineCountLocation(on_street = pemsid_to_cdlocation[pems_key][0], 254 | on_dir = pems_dir, 255 | from_street = pemsid_to_cdlocation[pems_key][1], 256 | from_int = pemsid_to_cdlocation[pems_key][2], 257 | to_street = pemsid_to_cdlocation[pems_key][3], 258 | to_int = pemsid_to_cdlocation[pems_key][4]) 259 | mainline_count_location.save() 260 | 261 | # required fields for insertMainlineCounts: count, starttime, period, vtype, 262 | # onstreet, ondir, fromstreet, tostreet, refpos, sourcefile, project 263 | 264 | pems_date_fields = pems_date.split(r"/") 265 | pems_time_fields = pems_time.split(r":") 266 | count_date = datetime.date(year=int(pems_date_fields[2]), 267 | month=int(pems_date_fields[0]), 268 | day=int(pems_date_fields[1])) 269 | starttime = datetime.time( hour=int(pems_time_fields[0]), 270 | minute=int(pems_time_fields[1]), 271 | second=int(pems_time_fields[2])) 272 | # tzinfo=TIMEZONE) 273 | project_str = "PeMS VDS %s - %s" % (pems_id, mapping[pems_key][3]) 274 | 275 | 276 | 277 | mainline_count = MainlineCount(location = mainline_count_location, 278 | count = pems_flow, 279 | count_date = count_date, 280 | start_time = starttime, 281 | period_minutes = 60, 282 | vehicle_type = 0, # ALL 283 | reference_position = -1, 284 | sourcefile = vds_datfilename_abspath, 285 | project = project_str, 286 | upload_user = user) 287 | mainline_count.clean() 288 | mainline_count.save() 289 | counts_saved += 1 290 | 291 | logger.info("Saved %d PeMS VDS counts into countdracula" % counts_saved) 292 | vds_datfile.close() 293 | 294 | def readCensusCounts(mapping, census_dirname, user): 295 | """ 296 | Reads the census station count workbooks and inputs those counts into the Count Dracula database. 297 | """ 298 | filenames = sorted(os.listdir(census_dirname)) 299 | for filename in filenames: 300 | 301 | if filename[-4:] != ".xls": 302 | logger.debug("Skipping non-xls file %s" % filename) 303 | continue 304 | 305 | logger.info("Processing PeMS Census file %s" % filename) 306 | 307 | filename_parts = filename[:-4].split("_") 308 | pems_id = filename_parts[0] 309 | pems_dir = pems_id[-2:].upper() 310 | 311 | pems_id = pems_id.replace("nb", "N") 312 | pems_id = pems_id.replace("sb", "S") 313 | 314 | 315 | pems_key = ("Census", pems_id) 316 | if pems_key not in mapping: 317 | logger.debug("Couldn't find %s in mapping; don't care about this VDS. Skipping." % str(pems_key)) 318 | continue 319 | 320 | try: 321 | intersection = getIntersectionStreetnamesForPemsKey(mapping, pems_key) 322 | logger.debug("%20s -> %s" % (str(pems_key), str(intersection))) 323 | except Exception, e: 324 | logger.error(e) 325 | continue 326 | 327 | # look for the mainline count location in countdracula 328 | try: 329 | mainline_count_location = MainlineCountLocation.objects.get(from_int = intersection[2], 330 | to_int = intersection[4], 331 | on_street = intersection[0], 332 | on_dir = pems_dir) 333 | except ObjectDoesNotExist: 334 | mainline_count_location = MainlineCountLocation(on_street = intersection[0], 335 | on_dir = pems_dir, 336 | from_street = intersection[1], 337 | from_int = intersection[2], 338 | to_street = intersection[3], 339 | to_int = intersection[4]) 340 | mainline_count_location.save() 341 | 342 | workbook_filename = os.path.join(census_dirname, filename) 343 | book = xlrd.open_workbook(workbook_filename) 344 | 345 | # open the workbook 346 | assert("Report Data" in book.sheet_names()) # standard PeMS sheetnames 347 | datasheet = book.sheet_by_name("Report Data") 348 | counts_saved = 0 349 | 350 | # for each day 351 | for col in range(1, len(datasheet.row(0))): 352 | pems_date = xlrd.xldate_as_tuple(datasheet.cell_value(0, col), book.datemode) 353 | 354 | # for each time 355 | for row in range(1, len(datasheet.col(0))): 356 | pems_time = xlrd.xldate_as_tuple(datasheet.cell_value(row, 0), book.datemode) 357 | 358 | count_date = datetime.date(year = int(pems_date[0]), 359 | month = int(pems_date[1]), 360 | day = int(pems_date[2])) 361 | starttime = datetime.time( hour = int(pems_time[3]), 362 | minute= int(pems_time[4]), 363 | second= 0) 364 | # tzinfo=TIMEZONE) 365 | 366 | count = datasheet.cell_value(row,col) 367 | if count == "": continue # skip blanks 368 | if count == 0.0: continue # skip zeros, they aren't real zero counts 369 | project_str = "PeMS Census %s - %s" % (pems_id, mapping[pems_key][3]) 370 | 371 | # read the counts 372 | mainline_count = MainlineCount(location = mainline_count_location, 373 | count = count, 374 | count_date = count_date, 375 | start_time = starttime, 376 | period_minutes = 60, 377 | vehicle_type = 0, # ALL 378 | reference_position = -1, 379 | sourcefile = workbook_filename, 380 | project = project_str, 381 | upload_user = user) 382 | mainline_count.clean() 383 | mainline_count.save() 384 | counts_saved += 1 385 | 386 | del book 387 | logger.info("Saved %3d census counts from %s into countdracula" % (counts_saved, workbook_filename)) 388 | 389 | if __name__ == '__main__': 390 | 391 | opts, args = getopt.getopt(sys.argv[1:], 'c:v:') 392 | if len(args) != 2: 393 | print USAGE 394 | sys.exit(2) 395 | 396 | if len(opts) == 0: 397 | print "No PeMS data specified for processing" 398 | print USAGE 399 | sys.exit(2) 400 | 401 | USERNAME = args[0] 402 | MAPPING_FILE = args[1] 403 | 404 | user = User.objects.get(username__exact=USERNAME) 405 | 406 | logger = logging.getLogger('countdracula') 407 | logger.setLevel(logging.DEBUG) 408 | 409 | consolehandler = logging.StreamHandler() 410 | consolehandler.setLevel(logging.DEBUG) 411 | consolehandler.setFormatter(logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')) 412 | logger.addHandler(consolehandler) 413 | 414 | debugFilename = "insertSanFranciscoPeMSCounts_%s.DEBUG.log" % time.strftime("%Y%b%d.%H%M%S") 415 | debugloghandler = logging.StreamHandler(open(debugFilename, 'w')) 416 | debugloghandler.setLevel(logging.DEBUG) 417 | debugloghandler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%Y-%m-%d %H:%M')) 418 | logger.addHandler(debugloghandler) 419 | 420 | mapping = readMappingWorkbook(MAPPING_FILE) 421 | 422 | for (opttype, optarg) in opts: 423 | if opttype == "-v": 424 | readVDSCounts(mapping, optarg, user) 425 | if opttype == "-c": 426 | readCensusCounts(mapping, optarg, user) 427 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | --------------------------------------------------------------------------------