├── geomancer ├── mancers │ ├── __init__.py │ ├── geotype.py │ ├── bea.py │ ├── usa_spending.py │ ├── base.py │ ├── bls.py │ ├── geotype_us.py │ ├── census_reporter.py │ ├── wazimap.py │ └── census_reporter_us.py ├── result_folder │ └── .gitkeep ├── static │ ├── images │ │ ├── ap-logo.png │ │ ├── favicon.ico │ │ ├── red_mage.gif │ │ ├── black_mage.gif │ │ ├── white_mage.gif │ │ ├── cfafrica-logo.png │ │ ├── knight-logo.jpg │ │ ├── geomancer-logo.png │ │ ├── geomancer_format.jpg │ │ ├── red_mage_animated.gif │ │ ├── black_mage_animated.gif │ │ └── white_mage_animated.gif │ ├── css │ │ ├── images │ │ │ ├── spritesheet.png │ │ │ └── spritesheet-2x.png │ │ ├── bootstrap-nav-wizard.css │ │ └── custom.css │ └── js │ │ ├── analytics_lib.js │ │ ├── jquery.spin.js │ │ ├── jquery.cookie.js │ │ ├── spin.min.js │ │ ├── ejs_production.js │ │ └── moment.min.js ├── templates │ ├── 404.html │ ├── error.html │ ├── 413.html │ ├── wizard_base.html │ ├── index.html │ ├── geographies.html │ ├── data-sources.html │ ├── upload-formats.html │ ├── upload.html │ ├── geomance.html │ ├── base.html │ ├── select_geo.html │ ├── select_tables.html │ └── contribute-data.html ├── app_config.py.example ├── __init__.py ├── redis_session.py ├── api.py ├── helpers.py ├── views.py └── worker.py ├── opt_requirements.txt ├── examples ├── act.xlsx ├── FEMARecoupment.xlsx ├── fdic_failed_banks.xlsx ├── Eqi_results_2013July22.xlsx ├── my_fascinating_county_data.xlsx ├── 2014_ca_county_camp_contribs_by_party.xlsx └── test_geographies.csv ├── .gitignore ├── runworker.py ├── requirements.txt ├── runserver.py ├── LICENSE └── README.md /geomancer/mancers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /geomancer/result_folder/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /opt_requirements.txt: -------------------------------------------------------------------------------- 1 | raven==5.1.1 2 | blinker==1.3 3 | -------------------------------------------------------------------------------- /examples/act.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/geomancer/master/examples/act.xlsx -------------------------------------------------------------------------------- /examples/FEMARecoupment.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/geomancer/master/examples/FEMARecoupment.xlsx -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .DS_Store 3 | app_config.py 4 | use_cases 5 | .env 6 | *.csv 7 | dump.rdb 8 | geomancer/result_folder/* -------------------------------------------------------------------------------- /examples/fdic_failed_banks.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/geomancer/master/examples/fdic_failed_banks.xlsx -------------------------------------------------------------------------------- /examples/Eqi_results_2013July22.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/geomancer/master/examples/Eqi_results_2013July22.xlsx -------------------------------------------------------------------------------- /geomancer/static/images/ap-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/geomancer/master/geomancer/static/images/ap-logo.png -------------------------------------------------------------------------------- /geomancer/static/images/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/geomancer/master/geomancer/static/images/favicon.ico -------------------------------------------------------------------------------- /geomancer/static/images/red_mage.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/geomancer/master/geomancer/static/images/red_mage.gif -------------------------------------------------------------------------------- /geomancer/static/images/black_mage.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/geomancer/master/geomancer/static/images/black_mage.gif -------------------------------------------------------------------------------- /geomancer/static/images/white_mage.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/geomancer/master/geomancer/static/images/white_mage.gif -------------------------------------------------------------------------------- /examples/my_fascinating_county_data.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/geomancer/master/examples/my_fascinating_county_data.xlsx -------------------------------------------------------------------------------- /geomancer/static/images/cfafrica-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/geomancer/master/geomancer/static/images/cfafrica-logo.png -------------------------------------------------------------------------------- /geomancer/static/images/knight-logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/geomancer/master/geomancer/static/images/knight-logo.jpg -------------------------------------------------------------------------------- /geomancer/static/css/images/spritesheet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/geomancer/master/geomancer/static/css/images/spritesheet.png -------------------------------------------------------------------------------- /geomancer/static/images/geomancer-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/geomancer/master/geomancer/static/images/geomancer-logo.png -------------------------------------------------------------------------------- /geomancer/static/css/images/spritesheet-2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/geomancer/master/geomancer/static/css/images/spritesheet-2x.png -------------------------------------------------------------------------------- /geomancer/static/images/geomancer_format.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/geomancer/master/geomancer/static/images/geomancer_format.jpg -------------------------------------------------------------------------------- /geomancer/static/images/red_mage_animated.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/geomancer/master/geomancer/static/images/red_mage_animated.gif -------------------------------------------------------------------------------- /runworker.py: -------------------------------------------------------------------------------- 1 | from geomancer.worker import queue_daemon 2 | from geomancer import create_app 3 | 4 | app = create_app() 5 | 6 | queue_daemon(app) 7 | -------------------------------------------------------------------------------- /geomancer/static/images/black_mage_animated.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/geomancer/master/geomancer/static/images/black_mage_animated.gif -------------------------------------------------------------------------------- /geomancer/static/images/white_mage_animated.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/geomancer/master/geomancer/static/images/white_mage_animated.gif -------------------------------------------------------------------------------- /examples/2014_ca_county_camp_contribs_by_party.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/geomancer/master/examples/2014_ca_county_camp_contribs_by_party.xlsx -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Flask==0.10.1 2 | csvkit==0.9.0 3 | redis==2.10.1 4 | scrapelib==0.10.1 5 | gunicorn==19.1.1 6 | xlwt==0.7.5 7 | us==0.9.0 8 | lxml==3.4.0 9 | requests==2.3.0 10 | pandas==0.16.0 -------------------------------------------------------------------------------- /runserver.py: -------------------------------------------------------------------------------- 1 | from geomancer import create_app 2 | 3 | app = create_app() 4 | 5 | if __name__ == "__main__": 6 | import sys 7 | try: 8 | port = int(sys.argv[1]) 9 | except IndexError: 10 | port = 5000 11 | app.run(host='0.0.0.0', port=port) 12 | -------------------------------------------------------------------------------- /geomancer/templates/404.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | {% block title %}Page Not Found - Geomancer{% endblock %} 3 | {% block content %} 4 |

Page Not Found

5 |

What you were looking for is just not there.

6 |

« Go back to the home page

7 | {% endblock %} -------------------------------------------------------------------------------- /geomancer/templates/error.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | {% block title %}Error - Geomancer{% endblock %} 3 | {% block content %} 4 |

Error

5 |

Oops! There was an error.

6 |

« go back and try again

7 |

If this issue persists, contact the system administrator.

8 | {% endblock %} -------------------------------------------------------------------------------- /geomancer/templates/413.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | {% block title %}File too large - Geomancer{% endblock %} 3 | {% block content %} 4 |

File too large

5 |

The file you tried to upload was too large for us to handle

6 |

« Try again with another file

7 | {% endblock %} 8 | -------------------------------------------------------------------------------- /geomancer/app_config.py.example: -------------------------------------------------------------------------------- 1 | from os.path import join, abspath, dirname 2 | 3 | SECRET_KEY = 'your secret key here' 4 | CACHE_DIR = '/tmp' 5 | REDIS_QUEUE_KEY = 'geomancer' 6 | RESULT_FOLDER = abspath(join(dirname(__file__), 'result_folder')) 7 | MAX_CONTENT_LENGTH = 10 * 1024 * 1024 # 10mb 8 | ALLOWED_EXTENSIONS = set(['csv', 'xls', 'xlsx']) 9 | SENTRY_DSN = '' 10 | 11 | MANCERS = ( 12 | 'geomancer.mancers.census_reporter.CensusReporter', 13 | 'geomancer.mancers.bea.BureauEconomicAnalysis', 14 | 'geomancer.mancers.bls.BureauLaborStatistics' 15 | ) 16 | 17 | # key = mancer machine_name, val = API key 18 | MANCER_KEYS = { 19 | 'bureau_economic_analysis' : None, # register at http://bea.gov/API/signup/index.cfm 20 | 'bureau_labor_statistics' : None # register at http://data.bls.gov/registrationEngine/ 21 | } 22 | -------------------------------------------------------------------------------- /geomancer/templates/wizard_base.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | {% block content %} 3 |
4 | 20 |
21 | {% block wizard_content %}{% endblock %} 22 |
23 |
24 | {% endblock %} 25 | 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Associated Press 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /geomancer/__init__.py: -------------------------------------------------------------------------------- 1 | from flask import Flask, render_template 2 | from geomancer.api import api 3 | from geomancer.views import views 4 | from geomancer.redis_session import RedisSessionInterface 5 | 6 | try: 7 | from raven.contrib.flask import Sentry 8 | from geomancer.app_config import SENTRY_DSN 9 | sentry = Sentry(dsn=SENTRY_DSN) 10 | except ImportError: 11 | sentry = None 12 | except KeyError: 13 | sentry = None 14 | 15 | def create_app(): 16 | app = Flask(__name__) 17 | app.config.from_object('geomancer.app_config') 18 | app.session_interface = RedisSessionInterface() 19 | app.register_blueprint(api) 20 | app.register_blueprint(views) 21 | 22 | @app.errorhandler(404) 23 | def page_not_found(e): 24 | return render_template('404.html'), 404 25 | 26 | @app.errorhandler(500) 27 | def server_error(e): 28 | return render_template('error.html'), 500 29 | 30 | @app.errorhandler(413) 31 | def file_too_large(e): 32 | return render_template('413.html'), 413 33 | 34 | @app.template_filter('string_split') 35 | def string_split(val, splitter): 36 | return val.split(splitter) 37 | 38 | if sentry: 39 | sentry.init_app(app) 40 | 41 | return app 42 | -------------------------------------------------------------------------------- /geomancer/templates/index.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | {% block title %}Geomancer{% endblock %} 3 | {% block content %} 4 |
5 |
6 |
7 | Geomancer 8 |

Add geographically-related data to any spreadsheet.

9 | 10 |

Get started >

11 | 12 |

All you need is a spreadsheet that includes a column about a location, like city, state or congressional district.

13 | 14 |

Take a look at the data sources and geographies we support.

15 | 16 |
17 | 18 |
19 |

Geomancer API

20 |

Interested in integrating Geomancer into an application? Check out the API!

21 |
22 | 23 |

24 | Knight Foundation 25 | Associated Press 26 |

27 | 28 |
29 |
30 | 31 | 32 | {% endblock %} 33 | -------------------------------------------------------------------------------- /geomancer/templates/geographies.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | {% block title %}Geographies - Geomancer{% endblock %} 3 | {% block content %} 4 | 5 |
6 |
7 |

Geographies

8 | 13 |
14 |
15 |
16 |

Geomancer currently has data for {{ geographies | length }} kinds of geographies. Take a look at the types below, along with the kinds of data we support.

17 | 18 | {% for g in geographies %} 19 |

{{g['info']['human_name']}}

20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 |
Formatting notes{{g['info']['formatting_notes']}}
Example{{g['info']['formatting_example']}}
32 | 33 |

Data available for {{g['info']['human_name']}}

34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | {% for table in g['tables'] %} 43 | 44 | 45 | 46 | 47 | {% endfor %} 48 | 49 |
Data availableSource
{{ table.human_name }}{{ table.source_name }}
50 | {% endfor %} 51 |
52 | 53 | {% endblock %} 54 | -------------------------------------------------------------------------------- /examples/test_geographies.csv: -------------------------------------------------------------------------------- 1 | Census Tract,City,Congressional District,County,School District,State,State County FIPS,State FIPS,Zip 5,Zip 9 ,Montgomery,,,,Alabama,,01,60001, ,Juneau,,,,Alaska,,02,60002, ,Phoenix,,,,Arizona,,04,60004, ,Little Rock,,,,Arkansas,,05,60005, ,Sacramento,,,,California,,06,60006, ,Denver,,,,Colorado,,08,60007, ,Hartford,,,,Connecticut,,09,60008, ,Dover,,,,Delaware,,10,60009, ,Washington DC,,,,District of Columbia,,11,60010, ,Tallahassee,,,,Florida,,12,60011, ,Atlanta,,,,Georgia,,13,60012, ,Honolulu,,,,Hawaii,,15,60013, ,Boise,,,,Idaho,,16,60014, ,Springfield,,,,Illinois,,17,60015, ,Indianapolis,,,,Indiana,,18,60016, ,Des Moines,,,,Iowa,,19,60017 ,Topeka,,,,Kansas,,20,60018 ,Frankfort,,,,Kentucky,,21,60019 ,Baton Rouge,,,,Louisiana,,22,60020 ,Augusta,,,,Maine,,23,60021 ,Annapolis,,,,Maryland,,24,60022 ,Boston,,,,Massachusetts,,25,60025 ,Lansing,,,,Michigan,,26,60026 ,Saint Paul,,,,Minnesota,,27,60029 ,Jackson,,,,Mississippi,,28,60030 ,Jefferson City,,,,Missouri,,29,60031 ,Helena,,,,Montana,,30,60033 ,Lincoln,,,,Nebraska,,31,60034 ,Carson City,,,,Nevada,,32,60035 ,Concord,,,,New Hampshire,,33,60037 ,Trenton,,,,New Jersey,,34,60038 ,Santa Fe,,,,New Mexico,,35,60039 ,Albany,,,,New York,,36,60040 ,Raleigh,,,,North Carolina,,37,60041 ,Bismarck,,,,North Dakota,,38,60042 ,Columbus,,,,Ohio,,39,60043 ,Oklahoma City,,,,Oklahoma,,40,60044 ,Salem,,,,Oregon,,41,60045 ,Harrisburg,,,,Pennsylvania,,42,60046 ,Providence,,,,Rhode Island,,44,60047 ,Columbia,,,,South Carolina,,45,60048 ,Pierre,,,,South Dakota,,46,60049 ,Nashville,,,,Tennessee,,47,60050 ,Austin,,,,Texas,,48,60051 ,Salt Lake City,,,,Utah,,49,60053 ,Montpelier,,,,Vermont,,50,60055 ,Richmond,,,,Virginia,,51,60056 ,Olympia,,,,Washington,,53,60060 ,Charleston,,,,West Virginia,,54,60061 ,Madison,,,,Wisconsin,,55,60062 ,Cheyenne,,,,Wyoming,,56,60064 -------------------------------------------------------------------------------- /geomancer/static/js/analytics_lib.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * Google Analytics Library 3 | * https://github.com/open-city/google-analytics-lib 4 | * 5 | * Copyright 2012, Nick Rougeux and Derek Eder of Open City 6 | * Licensed under the MIT license. 7 | * https://github.com/open-city/google-analytics-lib/wiki/License 8 | * 9 | * Date: 5/9/2012 10 | * 11 | */ 12 | 13 | var analyticsTrackingCode = 'UA-39054699-9'; //enter your tracking code here 14 | 15 | var _gaq = _gaq || []; 16 | _gaq.push(['_setAccount', analyticsTrackingCode]); 17 | _gaq.push(['_trackPageview']); 18 | 19 | (function() { 20 | var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true; 21 | ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js'; 22 | var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s); 23 | })(); 24 | 25 | _trackClickEventWithGA = function (category, action, label) { 26 | if (typeof(_gaq) != 'undefined') 27 | _gaq.push(['_setAccount', analyticsTrackingCode]); 28 | _gaq.push(['_trackEvent', category, action, label]); 29 | }; 30 | 31 | jQuery(function () { 32 | 33 | jQuery('a').click(function () { 34 | var $a = jQuery(this); 35 | var href = $a.attr("href"); 36 | 37 | //links going to outside sites 38 | if (href.match(/^http/i) && !href.match(document.domain)) { 39 | _trackClickEventWithGA("Outgoing", "Click", href); 40 | } 41 | 42 | //direct links to files 43 | if (href.match(/\.(avi|css|doc|docx|exe|gif|js|jpg|mov|mp3|pdf|png|ppt|pptx|rar|txt|vsd|vxd|wma|wmv|xls|xlsx|zip)$/i)) { 44 | _trackClickEventWithGA("Downloads", "Click", href); 45 | } 46 | 47 | //email links 48 | if (href.match(/^mailto:/i)) { 49 | _trackClickEventWithGA("Emails", "Click", href); 50 | } 51 | }); 52 | }); 53 | -------------------------------------------------------------------------------- /geomancer/templates/data-sources.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | {% block title %}Data sources - Geomancer{% endblock %} 3 | {% block content %} 4 | 5 |
6 |
7 |

Data sources

8 | 13 |
14 |
15 |
16 |

Geomancer currently has data from {{ data_sources | length }} sources.

17 |

Is there data that your newsroom would find useful, that isn't already here? You can adapt Geomancer by adding datasets/columns from existing data sources, or by adding a completely new data source.

18 | Learn how to contribute > 19 |
20 | 21 | {% for d in data_sources %} 22 |

23 | {{ d.name }} 24 | {{ d.info_url }} 25 |

26 |

{{ d.description }}

27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | {% for table in d.data_types %} 36 | 37 | 38 | 43 | 44 | {% endfor %} 45 | 46 |
Datasets availableGeographies
{{ table.human_name }} 39 | {% for geo in table.geo_types %} 40 | {{geo.human_name}}
41 | {% endfor %} 42 |
47 | {% endfor %} 48 |
49 | 50 | {% endblock %} 51 | -------------------------------------------------------------------------------- /geomancer/templates/upload-formats.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | {% block title %}Upload formats - Geomancer{% endblock %} 3 | {% block content %} 4 | 5 |
6 |

« back to Upload

7 | 8 |
9 |

Upload formats

10 |
    11 |
  1. File types and size limits
  2. 12 |
  3. Spreadsheets
  4. 13 |
  5. Text delimited data files (CSV)
  6. 14 |
15 |
16 |
17 |
18 | 19 |

File types and size limits

20 |

You can import a file of up to 10mb of these file types:

21 | 22 | 26 | 27 |

Spreadsheets

28 |

Only the first sheet will be imported into Geomancer. Also, please make sure that the first row contains header information with column names.

29 | 30 | 31 | 32 |

Text delimited data files (CSV)

33 |

Comma-separated value (.csv) files are a standard way to represent tabular data. Cell values with comma, newline or quotes must be quoted and quotes escaped by doubling. For example: "17"" LCD display".

34 | 35 |

By convention, the first row acts as header and defines the names of the columns in the data. However, you can specify which row is the header during import.

36 | 37 |
38 |

Many of these guidelines were lifted from Google Fusion Tables, which has excellent documentation.

39 |
40 | 41 | {% endblock %} -------------------------------------------------------------------------------- /geomancer/templates/upload.html: -------------------------------------------------------------------------------- 1 | {% extends 'wizard_base.html' %} 2 | {% block title %}Upload data - Geomancer{% endblock %} 3 | {% block wizard_content %} 4 |

1. Upload a spreadsheet

5 |
6 |
7 |

Pick a spreadsheet that includes a column about a location, like city, state or congressional district, and then pick additional data to add about that location.
Take a look at the data sources and geographies we have, or learn how to add your own.

8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 | 16 |

17 | You can upload spreadsheets (.xls or .xlsx) and delimited text files (.csv) up to 10mb in size. 18 |

19 |
20 | 21 |
22 |
23 | 24 |
25 |
26 |

Spreadsheet format (learn more)

27 | 28 | 29 | 30 | 31 | 32 |
33 |
34 | {% endblock %} 35 | -------------------------------------------------------------------------------- /geomancer/static/js/jquery.spin.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2011-2013 Felix Gnass 3 | * Licensed under the MIT license 4 | */ 5 | 6 | /* 7 | 8 | Basic Usage: 9 | ============ 10 | 11 | $('#el').spin(); // Creates a default Spinner using the text color of #el. 12 | $('#el').spin({ ... }); // Creates a Spinner using the provided options. 13 | 14 | $('#el').spin(false); // Stops and removes the spinner. 15 | 16 | Using Presets: 17 | ============== 18 | 19 | $('#el').spin('small'); // Creates a 'small' Spinner using the text color of #el. 20 | $('#el').spin('large', '#fff'); // Creates a 'large' white Spinner. 21 | 22 | Adding a custom preset: 23 | ======================= 24 | 25 | $.fn.spin.presets.flower = { 26 | lines: 9 27 | length: 10 28 | width: 20 29 | radius: 0 30 | } 31 | 32 | $('#el').spin('flower', 'red'); 33 | 34 | */ 35 | 36 | (function(factory) { 37 | 38 | if (typeof exports == 'object') { 39 | // CommonJS 40 | factory(require('jquery'), require('spin')) 41 | } 42 | else if (typeof define == 'function' && define.amd) { 43 | // AMD, register as anonymous module 44 | define(['jquery', 'spin'], factory) 45 | } 46 | else { 47 | // Browser globals 48 | if (!window.Spinner) throw new Error('Spin.js not present') 49 | factory(window.jQuery, window.Spinner) 50 | } 51 | 52 | }(function($, Spinner) { 53 | 54 | $.fn.spin = function(opts, color) { 55 | 56 | return this.each(function() { 57 | var $this = $(this), 58 | data = $this.data(); 59 | 60 | if (data.spinner) { 61 | data.spinner.stop(); 62 | delete data.spinner; 63 | } 64 | if (opts !== false) { 65 | opts = $.extend( 66 | { color: color || $this.css('color') }, 67 | $.fn.spin.presets[opts] || opts 68 | ) 69 | data.spinner = new Spinner(opts).spin(this) 70 | } 71 | }) 72 | } 73 | 74 | $.fn.spin.presets = { 75 | tiny: { lines: 8, length: 2, width: 2, radius: 3 }, 76 | small: { lines: 8, length: 4, width: 3, radius: 5 }, 77 | large: { lines: 10, length: 8, width: 4, radius: 8 } 78 | } 79 | 80 | })); 81 | -------------------------------------------------------------------------------- /geomancer/static/js/jquery.cookie.js: -------------------------------------------------------------------------------- 1 | /*jshint eqnull:true */ 2 | /*! 3 | * jQuery Cookie Plugin v1.2 4 | * https://github.com/carhartl/jquery-cookie 5 | * 6 | * Copyright 2011, Klaus Hartl 7 | * Dual licensed under the MIT or GPL Version 2 licenses. 8 | * http://www.opensource.org/licenses/mit-license.php 9 | * http://www.opensource.org/licenses/GPL-2.0 10 | */ 11 | (function ($, document, undefined) { 12 | 13 | var pluses = /\+/g; 14 | 15 | function raw(s) { 16 | return s; 17 | } 18 | 19 | function decoded(s) { 20 | return decodeURIComponent(s.replace(pluses, ' ')); 21 | } 22 | 23 | $.cookie = function (key, value, options) { 24 | 25 | // key and at least value given, set cookie... 26 | if (value !== undefined && !/Object/.test(Object.prototype.toString.call(value))) { 27 | options = $.extend({}, $.cookie.defaults, options); 28 | 29 | if (value === null) { 30 | options.expires = -1; 31 | } 32 | 33 | if (typeof options.expires === 'number') { 34 | var days = options.expires, t = options.expires = new Date(); 35 | t.setDate(t.getDate() + days); 36 | } 37 | 38 | value = String(value); 39 | 40 | return (document.cookie = [ 41 | encodeURIComponent(key), '=', options.raw ? value : encodeURIComponent(value), 42 | options.expires ? '; expires=' + options.expires.toUTCString() : '', // use expires attribute, max-age is not supported by IE 43 | options.path ? '; path=' + options.path : '', 44 | options.domain ? '; domain=' + options.domain : '', 45 | options.secure ? '; secure' : '' 46 | ].join('')); 47 | } 48 | 49 | // key and possibly options given, get cookie... 50 | options = value || $.cookie.defaults || {}; 51 | var decode = options.raw ? raw : decoded; 52 | var cookies = document.cookie.split('; '); 53 | for (var i = 0, parts; (parts = cookies[i] && cookies[i].split('=')); i++) { 54 | if (decode(parts.shift()) === key) { 55 | return decode(parts.join('=')); 56 | } 57 | } 58 | 59 | return null; 60 | }; 61 | 62 | $.cookie.defaults = {}; 63 | 64 | $.removeCookie = function (key, options) { 65 | if ($.cookie(key, options) !== null) { 66 | $.cookie(key, null, options); 67 | return true; 68 | } 69 | return false; 70 | }; 71 | 72 | })(jQuery, document); -------------------------------------------------------------------------------- /geomancer/redis_session.py: -------------------------------------------------------------------------------- 1 | import cPickle 2 | from datetime import timedelta 3 | from uuid import uuid4 4 | from redis import Redis 5 | from werkzeug.datastructures import CallbackDict 6 | from flask.sessions import SessionInterface, SessionMixin 7 | 8 | class RedisSession(CallbackDict, SessionMixin): 9 | 10 | def __init__(self, initial=None, sid=None, new=False): 11 | def on_update(self): 12 | self.modified = True 13 | CallbackDict.__init__(self, initial, on_update) 14 | self.sid = sid 15 | self.new = new 16 | self.modifed = False 17 | 18 | 19 | class RedisSessionInterface(SessionInterface): 20 | serializer = cPickle 21 | session_class = RedisSession 22 | 23 | def __init__(self, redis=None, prefix='session:'): 24 | if redis is None: 25 | redis = Redis() 26 | self.redis = redis 27 | self.prefix = prefix 28 | 29 | def generate_sid(self): 30 | return str(uuid4()) 31 | 32 | def get_redis_expiration_time(self, app, session): 33 | if session.permanent: 34 | return app.permanent_session_lifetime 35 | return timedelta(days=1) 36 | 37 | def open_session(self, app, request): 38 | sid = request.cookies.get(app.session_cookie_name) 39 | if not sid: 40 | sid = self.generate_sid() 41 | return self.session_class(sid=sid, new=True) 42 | val = self.redis.get(self.prefix + sid) 43 | if val is not None: 44 | data = self.serializer.loads(val) 45 | return self.session_class(data, sid=sid) 46 | return self.session_class(sid=sid, new=True) 47 | 48 | def save_session(self, app, session, response): 49 | domain = self.get_cookie_domain(app) 50 | if not session: 51 | self.redis.delete(self.prefix + session.sid) 52 | if session.modified: 53 | response.delete_cookie(app.session_cookie_name, domain=domain) 54 | return 55 | redis_exp = self.get_redis_expiration_time(app, session) 56 | cookie_exp = self.get_expiration_time(app, session) 57 | val = self.serializer.dumps(dict(session), protocol=-1) 58 | self.redis.setex(self.prefix + session.sid, val, 59 | int(redis_exp.total_seconds())) 60 | response.set_cookie(app.session_cookie_name, session.sid, 61 | expires=cookie_exp, httponly=True, domain=domain) 62 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Geomancer 2 | 3 | Geomancer is a project with a simple goal: to create a prototype tool that will help journalists easily mash up data based on shared geography. It’s such a common task for data-savvy journalists that it’s easy to underestimate the difficulty. But on deadline, it can become daunting. It requires locating the right data set to provide context, getting it into shape and into place to join it with the data in hand and then merging the two data sets — all before the reporter can even get started with the analysis. 4 | 5 | We aim to take some of that complexity out of the way by providing an intuitive interface to discover available data sets for a given geography and supplement the data in hand with the relevant values. We’ll know it’s working when it appears too easy. 6 | 7 | Read more: [AP wins Knight grant to build data journalism tool](http://www.ap.org/content/press-release/2013/ap-wins-knight-grant-to-build-data-journalism-tool) 8 | 9 | ### Setup 10 | 11 | **Install OS level dependencies:** 12 | 13 | * Python 2.7 14 | * [Redis](http://redis.io/) 15 | * libxml2 16 | * libxml2-dev 17 | * libxslt1-dev 18 | * zlib1g-dev 19 | 20 | **Install app requirements** 21 | 22 | We recommend using [virtualenv](http://virtualenv.readthedocs.org/en/latest/virtualenv.html) and [virtualenvwrapper](http://virtualenvwrapper.readthedocs.org/en/latest/install.html) for working in a virtualized development environment. [Read how to set up virtualenv](http://docs.python-guide.org/en/latest/dev/virtualenvs/). 23 | 24 | Once you have virtualenvwrapper set up, 25 | 26 | ```bash 27 | mkvirtualenv geomancer 28 | git clone https://github.com/associatedpress/geomancer.git 29 | cd geomancer 30 | pip install -r requirements.txt 31 | cp geomancer/app_config.py.example geomancer/app_config.py 32 | ``` 33 | 34 | NOTE: Mac users might need this [lxml workaround](http://stackoverflow.com/questions/22313407/clang-error-unknown-argument-mno-fused-madd-python-package-installation-fa). 35 | 36 | Afterwards, whenever you want to work on geomancer, 37 | 38 | ```bash 39 | workon geomancer 40 | ``` 41 | 42 | ### Running Geomancer 43 | 44 | There are three components that should be running simultaneously for the app to work: Redis, the Flask app, and the worker process that appends to the spreadsheets. For debugging purposes, it is useful to run these three processes in separate terminal sessions. 45 | 46 | ``` bash 47 | redis-server # This command may differ depending on your OS 48 | python runworker.py # starts the worker for processing files 49 | python runserver.py # starts the web server 50 | ``` 51 | 52 | Open your browser and navigate to `http://localhost:5000` 53 | 54 | ## DataMade Team 55 | 56 | * [Eric van Zanten](https://github.com/evz) 57 | * [Cathy Deng](https://github.com/cathydeng) 58 | * [Derek Eder](https://github.com/derekeder) 59 | 60 | ## Errors / Bugs 61 | 62 | If something is not behaving intuitively, it is a bug, and should be reported. 63 | Report it here: https://github.com/associatedpress/geomancer/issues 64 | 65 | ## Note on Patches/Pull Requests 66 | 67 | * Fork the project. 68 | * Make your feature addition or bug fix. 69 | * Commit, do not mess with version or history. 70 | * Send a pull request. Bonus points for topic branches. 71 | 72 | ## Copyright 73 | 74 | Copyright (c) 2014 Associated Press. Released under the [MIT License](https://github.com/associatedpress/geomancer/blob/master/LICENSE). 75 | -------------------------------------------------------------------------------- /geomancer/mancers/geotype.py: -------------------------------------------------------------------------------- 1 | from json import JSONEncoder 2 | import re 3 | import us 4 | from os.path import join, abspath, dirname 5 | import csv 6 | 7 | GAZDIR = join(dirname(abspath(__file__)), 'gazetteers') 8 | 9 | class GeoType(object): 10 | """ 11 | Base class for defining geographic types. 12 | All four static properties should be defined 13 | """ 14 | human_name = None 15 | machine_name = None 16 | formatting_notes = None 17 | formatting_example = None 18 | validation_regex = None 19 | 20 | def as_dict(self): 21 | fields = [ 22 | 'human_name', 23 | 'machine_name', 24 | 'formatting_notes', 25 | 'formatting_example', 26 | ] 27 | d = {k:getattr(self,k) for k in fields} 28 | for k,v in d.items(): 29 | d[k] = ' '.join(v.split()) 30 | return d 31 | 32 | def validate(self, values): 33 | ''' 34 | Default is to implement a regex on a subclass that gets 35 | used here to validate the format. Optionally override this 36 | method to implement custom validation. If validation_regex 37 | is not defined on the subclass, this will always return True. 38 | 39 | values - A list (or other iterable) of values to evaluate 40 | 41 | Returns a boolean indicating whether all the members of the values 42 | list are valid and an optional user friendly message. 43 | ''' 44 | 45 | if self.validation_regex is None: 46 | return False, None 47 | else: 48 | values = list(set([v for v in values if v])) 49 | for v in values: 50 | if not re.match(self.validation_regex, v): 51 | message = 'The column you selected must be formatted \ 52 | like "%s" to match on %s geographies. Please pick another \ 53 | column or change the format of your data.' % \ 54 | (self.formatting_example, self.human_name) 55 | return False, message 56 | return True, None 57 | 58 | class GeoTypeEncoder(JSONEncoder): 59 | ''' 60 | Custom JSON encoder so we can have nice things. 61 | ''' 62 | def default(self, o): 63 | return o.as_dict() 64 | 65 | class County(GeoType): 66 | human_name = 'County' 67 | machine_name = 'County' 68 | formatting_notes = 'County name' 69 | formatting_example = 'Nairobi' 70 | 71 | def validate(self, values): 72 | values = [v for v in values if v] 73 | non_matches = set() 74 | counties = ['Mombasa', 'Kwale', 'Kilifi', 'Tanariver', 'Lamu', 'Taita Taveta', 'Garissa', 'Wajir', 'Mandera', 'Marsabit', 'Isiolo', 'Meru', 'Tharaka', 'Embu', 'Kitui', 'Machakos', 'Makueni', 'Nyandarua', 'Nyeri', 'Kirinyaga', 'Muranga', 'Kiambu', 'Turkana', 'West Pokot', 'Samburu', 'TransNzoia', 'UasinGishu', 'ElgeyoMarakwet', 'Nandi', 'Baringo', 'Laikipia', 'Nakuru', 'Narok', 'Kajiado', 'Bomet', 'Kericho', 'Kakamega', 'Vihiga', 'Bungoma', 'Busia', 'Siaya', 'Kisumu', 'HomaBay', 'Migori', 'Kisii', 'Nyamira', 'Nairobi'] 75 | for val in values: 76 | if val.lower().replace(' ','').replace('-','') not in counties: 77 | non_matches.add(val) 78 | if non_matches: 79 | return False, '"{0}" do not appear to be valid Census places'\ 80 | .format(', '.join(non_matches)) 81 | else: 82 | return True, None -------------------------------------------------------------------------------- /geomancer/static/css/bootstrap-nav-wizard.css: -------------------------------------------------------------------------------- 1 | ul.nav-wizard { 2 | background-color: #f9f9f9; 3 | border: 1px solid #d4d4d4; 4 | -webkit-border-radius: 6px; 5 | -moz-border-radius: 6px; 6 | border-radius: 6px; 7 | *zoom: 1; 8 | position: relative; 9 | overflow: hidden; 10 | } 11 | ul.nav-wizard:before { 12 | display: block; 13 | position: absolute; 14 | left: 0px; 15 | right: 0px; 16 | top: 46px; 17 | height: 47px; 18 | border-top: 1px solid #d4d4d4; 19 | border-bottom: 1px solid #d4d4d4; 20 | z-index: 11; 21 | content: " "; 22 | } 23 | ul.nav-wizard:after { 24 | display: block; 25 | position: absolute; 26 | left: 0px; 27 | right: 0px; 28 | top: 138px; 29 | height: 47px; 30 | border-top: 1px solid #d4d4d4; 31 | border-bottom: 1px solid #d4d4d4; 32 | z-index: 11; 33 | content: " "; 34 | } 35 | ul.nav-wizard li { 36 | position: relative; 37 | float: left; 38 | height: 46px; 39 | display: inline-block; 40 | text-align: middle; 41 | padding: 0 20px 0 30px; 42 | margin: 0; 43 | font-size: 16px; 44 | line-height: 46px; 45 | } 46 | ul.nav-wizard li a { 47 | color: #468847; 48 | padding: 0; 49 | } 50 | ul.nav-wizard li a:hover { 51 | background-color: transparent; 52 | } 53 | ul.nav-wizard li:before { 54 | position: absolute; 55 | display: block; 56 | border: 24px solid transparent; 57 | border-left: 16px solid #d4d4d4; 58 | border-right: 0; 59 | top: -1px; 60 | z-index: 10; 61 | content: ''; 62 | right: -16px; 63 | } 64 | ul.nav-wizard li:after { 65 | position: absolute; 66 | display: block; 67 | border: 24px solid transparent; 68 | border-left: 16px solid #f9f9f9; 69 | border-right: 0; 70 | top: -1px; 71 | z-index: 10; 72 | content: ''; 73 | right: -15px; 74 | } 75 | ul.nav-wizard li.active { 76 | color: #3a87ad; 77 | background: #d9edf7; 78 | } 79 | ul.nav-wizard li.active:after { 80 | border-left: 16px solid #d9edf7; 81 | } 82 | ul.nav-wizard li.active a, 83 | ul.nav-wizard li.active a:active, 84 | ul.nav-wizard li.active a:visited, 85 | ul.nav-wizard li.active a:focus { 86 | color: #3a87ad; 87 | background: #d9edf7; 88 | } 89 | ul.nav-wizard .active ~ li { 90 | color: #999999; 91 | background: #ededed; 92 | } 93 | ul.nav-wizard .active ~ li:after { 94 | border-left: 16px solid #ededed; 95 | } 96 | ul.nav-wizard .active ~ li a, 97 | ul.nav-wizard .active ~ li a:active, 98 | ul.nav-wizard .active ~ li a:visited, 99 | ul.nav-wizard .active ~ li a:focus { 100 | color: #999999; 101 | background: #ededed; 102 | } 103 | ul.nav-wizard.nav-wizard-backnav li:hover { 104 | color: #468847; 105 | background: #f6fbfd; 106 | } 107 | ul.nav-wizard.nav-wizard-backnav li:hover:after { 108 | border-left: 16px solid #f6fbfd; 109 | } 110 | ul.nav-wizard.nav-wizard-backnav li:hover a, 111 | ul.nav-wizard.nav-wizard-backnav li:hover a:active, 112 | ul.nav-wizard.nav-wizard-backnav li:hover a:visited, 113 | ul.nav-wizard.nav-wizard-backnav li:hover a:focus { 114 | color: #468847; 115 | background: #f6fbfd; 116 | } 117 | ul.nav-wizard.nav-wizard-backnav .active ~ li { 118 | color: #999999; 119 | background: #ededed; 120 | } 121 | ul.nav-wizard.nav-wizard-backnav .active ~ li:after { 122 | border-left: 16px solid #ededed; 123 | } 124 | ul.nav-wizard.nav-wizard-backnav .active ~ li a, 125 | ul.nav-wizard.nav-wizard-backnav .active ~ li a:active, 126 | ul.nav-wizard.nav-wizard-backnav .active ~ li a:visited, 127 | ul.nav-wizard.nav-wizard-backnav .active ~ li a:focus { 128 | color: #999999; 129 | background: #ededed; 130 | } 131 | -------------------------------------------------------------------------------- /geomancer/templates/geomance.html: -------------------------------------------------------------------------------- 1 | {% extends 'wizard_base.html' %} 2 | {% block title %}Download - Geomancer {% endblock %} 3 | {% block wizard_content %} 4 | 5 |
6 |

4. Geomancing …

7 | 8 |
9 |

10 | Geomancing takes Black Wizards! 11 | Geomancing takes White Wizards! 12 | Geomancing takes Red Wizards! 13 |

14 |

This may take a few minutes. We are fetching data for each row in your spreadsheet.

15 |

When finished, your spreadsheet will be available to download below.

16 |
17 | 18 |


19 |
20 | 21 |
22 |
23 | {% endblock %} 24 | {% block extra_javascript %} 25 | 26 | 27 | 81 | {% endblock %} 82 | -------------------------------------------------------------------------------- /geomancer/static/js/spin.min.js: -------------------------------------------------------------------------------- 1 | //fgnass.github.com/spin.js#v1.3.3 2 | !function(a,b){"object"==typeof exports?module.exports=b():"function"==typeof define&&define.amd?define(b):a.Spinner=b()}(this,function(){"use strict";function a(a,b){var c,d=document.createElement(a||"div");for(c in b)d[c]=b[c];return d}function b(a){for(var b=1,c=arguments.length;c>b;b++)a.appendChild(arguments[b]);return a}function c(a,b,c,d){var e=["opacity",b,~~(100*a),c,d].join("-"),f=.01+c/d*100,g=Math.max(1-(1-a)/b*(100-f),a),h=k.substring(0,k.indexOf("Animation")).toLowerCase(),i=h&&"-"+h+"-"||"";return m[e]||(n.insertRule("@"+i+"keyframes "+e+"{0%{opacity:"+g+"}"+f+"%{opacity:"+a+"}"+(f+.01)+"%{opacity:1}"+(f+b)%100+"%{opacity:"+a+"}100%{opacity:"+g+"}}",n.cssRules.length),m[e]=1),e}function d(a,b){var c,d,e=a.style;for(b=b.charAt(0).toUpperCase()+b.slice(1),d=0;d',c)}n.addRule(".spin-vml","behavior:url(#default#VML)"),i.prototype.lines=function(a,d){function f(){return e(c("group",{coordsize:k+" "+k,coordorigin:-j+" "+-j}),{width:k,height:k})}function g(a,g,i){b(m,b(e(f(),{rotation:360/d.lines*a+"deg",left:~~g}),b(e(c("roundrect",{arcsize:d.corners}),{width:j,height:d.width,left:d.radius,top:-d.width>>1,filter:i}),c("fill",{color:h(d.color,a),opacity:d.opacity}),c("stroke",{opacity:0}))))}var i,j=d.length+d.width,k=2*j,l=2*-(d.width+d.length)+"px",m=e(f(),{position:"absolute",top:l,left:l});if(d.shadow)for(i=1;i<=d.lines;i++)g(i,-2,"progid:DXImageTransform.Microsoft.Blur(pixelradius=2,makeshadow=1,shadowopacity=.3)");for(i=1;i<=d.lines;i++)g(i);return b(a,m)},i.prototype.opacity=function(a,b,c,d){var e=a.firstChild;d=d.shadow&&d.lines||0,e&&b+d>1):parseInt(h.left,10)+j)+"px",top:("auto"==h.top?d.y-c.y+(b.offsetHeight>>1):parseInt(h.top,10)+j)+"px"})),i.setAttribute("role","progressbar"),f.lines(i,f.opts),!k){var l,m=0,n=(h.lines-1)*(1-h.direction)/2,o=h.fps,p=o/h.speed,q=(1-h.opacity)/(p*h.trail/100),r=p/h.lines;!function s(){m++;for(var a=0;a>1)+"px"})}for(var i,j=0,l=(f.lines-1)*(1-f.direction)/2;j 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | {% block title %}{% endblock %} 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | {% block extra_styles %}{% endblock %} 19 | 20 | 21 | 22 | 26 | 27 | 28 | 29 | 48 |
49 |
50 | {% with messages = get_flashed_messages() %} 51 | {% if messages %} 52 |
53 | {% for message in messages %} 54 | 61 | {% endfor %} 62 |
63 | {% endif %} 64 | {% endwith %} 65 |
66 | {% block content %}{% endblock %} 67 | 68 |
69 |
70 | 71 | 78 |
79 | 80 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | {% block extra_javascript %}{% endblock %} 89 | 90 | 91 | 92 | -------------------------------------------------------------------------------- /geomancer/templates/select_geo.html: -------------------------------------------------------------------------------- 1 | {% extends 'wizard_base.html' %} 2 | {% block title %}Select geography - Geomancer {% endblock %} 3 | {% block wizard_content %} 4 |

2. Select geography

5 |

Select the column that contain geographic data.

6 |

For more accuracy, you can select State along with either City, County, Congressional district, or School district.

7 |
8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | {% for index,name,values in session['sample_data'] %} 18 | 19 | 24 | 27 | 30 | 31 | {% endfor %} 32 | 33 |
Geography typeColumn nameSample values
20 | 23 | 25 | 26 | 28 | 29 |
34 | 35 |
36 | {% endblock %} 37 | {% block extra_javascript %} 38 | 104 | {% endblock %} 105 | -------------------------------------------------------------------------------- /geomancer/static/css/custom.css: -------------------------------------------------------------------------------- 1 | /* Space out content a bit */ 2 | body { 3 | padding-bottom: 20px; 4 | font-size: 16px; 5 | } 6 | 7 | table, label,.control-label,.help-block,.checkbox,.radio { font-size: 14px; } 8 | .alert { font-size: 1em; font-weight: 500; } 9 | 10 | .label:hover { text-decoration: none; } 11 | 12 | .showmore-content { 13 | overflow: hidden; 14 | height: 165px; 15 | } 16 | 17 | .showmore-content p { 18 | font-size: 14px; 19 | line-height: 20px; 20 | } 21 | 22 | .showmore, .showless { 23 | margin-top: 10px; 24 | } 25 | 26 | /* Everything but the jumbotron gets side spacing for mobile first views */ 27 | .header, 28 | .marketing, 29 | .footer { 30 | padding-left: 15px; 31 | padding-right: 15px; 32 | } 33 | 34 | /* Custom page header */ 35 | .header { 36 | border-bottom: 1px solid #e5e5e5; 37 | } 38 | /* Make the masthead heading the same height as the navigation */ 39 | .header h3 { 40 | margin-top: 0; 41 | margin-bottom: 0; 42 | line-height: 40px; 43 | padding-bottom: 19px; 44 | } 45 | 46 | /* Custom page footer */ 47 | .footer { 48 | padding-top: 19px; 49 | color: #777; 50 | border-top: 1px solid #e5e5e5; 51 | } 52 | 53 | /* Customize container */ 54 | .container-narrow > hr { 55 | margin: 30px 0; 56 | } 57 | 58 | /* Main marketing message and sign up button */ 59 | .jumbotron { 60 | text-align: center; 61 | border-bottom: 1px solid #e5e5e5; 62 | } 63 | .jumbotron .btn { 64 | font-size: 21px; 65 | padding: 14px 24px; 66 | } 67 | 68 | .jumbotron li { 69 | font-size: 16px; 70 | } 71 | 72 | /* Responsive: Portrait tablets and up */ 73 | @media screen and (min-width: 768px) { 74 | /* Remove the padding we set earlier */ 75 | .header, 76 | .marketing, 77 | .footer { 78 | padding-left: 0; 79 | padding-right: 0; 80 | } 81 | /* Space out the masthead */ 82 | .header { 83 | margin-bottom: 30px; 84 | } 85 | /* Remove the bottom border on the jumbotron for visual effect */ 86 | .jumbotron { 87 | border-bottom: 0; 88 | } 89 | } 90 | 91 | .nowrap { white-space:nowrap; } 92 | 93 | /* MAP styles */ 94 | #map { 95 | min-height: 500px; 96 | } 97 | .leaflet-top.leaflet-left{ 98 | margin-left: 20px; 99 | } 100 | .row.filters { 101 | margin-left: 5px; 102 | } 103 | 104 | /* Datepicker styles */ 105 | 106 | .ui-widget { 107 | font-size: 0.9em; 108 | } 109 | 110 | .ui-datepicker { 111 | width: 22.5%; 112 | height: auto; 113 | } 114 | 115 | .ui-datepicker table { 116 | width: 100%; 117 | background-color: #fff; 118 | } 119 | 120 | .ui-datepicker table td, 121 | .ui-datepicker table th { 122 | padding: 0; 123 | } 124 | 125 | .ui-datepicker-header { 126 | color: #333; 127 | background-color: #DDD; 128 | font-weight: bold; 129 | } 130 | 131 | .ui-datepicker-title { 132 | text-align: center; 133 | line-height: 30px; 134 | } 135 | 136 | .ui-datepicker-prev:hover, 137 | .ui-datepicker-next:hover { 138 | cursor: pointer; 139 | } 140 | 141 | .ui-datepicker-prev { 142 | float: left; 143 | } 144 | 145 | .ui-datepicker-next { 146 | float: right; 147 | margin-right: 5px; 148 | } 149 | 150 | .ui-datepicker-next:after { 151 | content: ""; 152 | border-color: rgba(0,0,0,0) rgba(0, 0, 0, 0) rgba(0, 0, 0, 0) #333; 153 | border-image: none; 154 | border-style: solid; 155 | border-width: 9px; 156 | display: block; 157 | height: 0; 158 | margin-top: 5px; 159 | width: 0; 160 | } 161 | .ui-datepicker-prev:before { 162 | content: ""; 163 | border-color: rgba(0,0,0,0) #333 rgba(0, 0, 0, 0) rgba(0, 0, 0, 0); 164 | border-image: none; 165 | border-style: solid; 166 | border-width: 9px; 167 | display: block; 168 | height: 0; 169 | margin-top: 5px; 170 | width: 0; 171 | } 172 | 173 | .ui-datepicker th { 174 | text-transform: uppercase; 175 | font-size: 11px; 176 | text-align: center; 177 | } 178 | 179 | .ui-datepicker tbody td { 180 | border-right: none; 181 | text-align: center; 182 | padding: 3px 0; 183 | } 184 | .ui-datepicker tbody tr { 185 | border-bottom: 1px solid #bbb; 186 | } 187 | 188 | .ui-datepicker tbody tr:last-child { 189 | border-bottom: 0px 190 | } 191 | 192 | .ui-tooltip { 193 | width: 25%; 194 | overflow: hidden; 195 | } 196 | 197 | /* Response */ 198 | .response-table { 199 | width: 100%; 200 | } 201 | 202 | /* Map legend styles */ 203 | .legend { 204 | line-height: 18px; 205 | color: #555; 206 | background: rgba(255,255,255,0.8); 207 | box-shadow: 0 0 15px rgba(0,0,0,0.2); 208 | border-radius: 5px; 209 | } 210 | 211 | .legend div { 212 | padding: 10px; 213 | } 214 | 215 | .legend i { 216 | width: 18px; 217 | height: 18px; 218 | float: left; 219 | margin-right: 8px; 220 | opacity: 0.7; 221 | } 222 | 223 | /* Collapse */ 224 | .panel-heading{ 225 | cursor: pointer; cursor: hand; 226 | } 227 | 228 | /* api alert */ 229 | #api-alert{ 230 | background-color: #dcf1e4; 231 | font-weight: 200; 232 | } -------------------------------------------------------------------------------- /geomancer/api.py: -------------------------------------------------------------------------------- 1 | from flask import Blueprint, make_response, request, jsonify, \ 2 | session as flask_session 3 | from geomancer.worker import DelayedResult, do_the_work 4 | from geomancer.helpers import import_class, get_geo_types, get_data_sources 5 | from geomancer.app_config import MANCERS, MANCER_KEYS 6 | from geomancer.mancers.geotype import GeoTypeEncoder 7 | import json 8 | from redis import Redis 9 | from collections import OrderedDict 10 | 11 | redis = Redis() 12 | 13 | api = Blueprint('api', __name__) 14 | 15 | @api.route('/api/geomance/', methods=['POST', 'GET']) 16 | def geomance_api(): 17 | """ 18 | Needs to get the file as well which fields have 19 | geography and what kind of geography they contain. 20 | Should be a multipart POST with the file in the file part 21 | and the field definitions in the form part. 22 | 23 | Field definitions should be in string encoded JSON blob like so: 24 | 25 | { 26 | 10: { 27 | 'type': 'city_state', 28 | 'append_columns': ['total_population', 'median_age'] 29 | } 30 | } 31 | 32 | The key is the zero-indexed position of the columns within the spreadsheet. 33 | The value is a dict containing the geographic type and the columns to 34 | append. The values in that list should be fetched from one of the other 35 | endpoints. 36 | 37 | To mance on a combination of columns, separate the column indexes and 38 | geotypes with a semicolon like so: 39 | 40 | { 41 | 10;2: { 42 | 'type': 'city;state', 43 | 'append_columns': ['total_population', 'median_age'] 44 | } 45 | } 46 | 47 | In this example, column 10 contains the city info and column 2 contains 48 | the state info. 49 | 50 | Responds with a key that can be used to poll for results 51 | """ 52 | 53 | field_defs = json.loads(request.data) 54 | if request.files: 55 | file_contents = request.files['input_file'].read() 56 | filename = request.files['input_file'].filename 57 | else: 58 | file_contents = flask_session['file'] 59 | filename = flask_session['filename'] 60 | session = do_the_work.delay(file_contents, field_defs, filename) 61 | resp = make_response(json.dumps({'session_key': session.key})) 62 | resp.headers['Content-Type'] = 'application/json' 63 | return resp 64 | 65 | @api.route('/api/geomance-results//') 66 | def geomance_results(session_key): 67 | """ 68 | Looks in the Redis queue to see if the worker has finished yet. 69 | """ 70 | rv = DelayedResult(session_key) 71 | if rv.return_value is None: 72 | return jsonify(ready=False) 73 | redis.delete(session_key) 74 | result = rv.return_value 75 | return jsonify(ready=True, result=result['result'], status=result['status']) 76 | 77 | @api.route('/api/data-sources/') 78 | def data_sources(): 79 | """ 80 | Return a list of data sources 81 | """ 82 | mancers = None 83 | if request.args.get('geo_type'): 84 | mancers, errors = get_data_sources(request.args.get('geo_type')) 85 | else: 86 | mancers, errors = get_data_sources() 87 | resp = make_response(json.dumps(mancers, cls=GeoTypeEncoder)) 88 | resp.headers['Content-Type'] = 'application/json' 89 | return resp 90 | 91 | @api.route('/api/table-info/') 92 | def table_info(): 93 | """ 94 | Return a list of data sources 95 | """ 96 | columns = OrderedDict() 97 | for mancer in MANCERS: 98 | m = import_class(mancer) 99 | api_key = MANCER_KEYS.get(m.machine_name) 100 | try: 101 | m = m(api_key=api_key) 102 | except ImportError, e: 103 | continue 104 | col_info = m.get_metadata() 105 | for col in col_info: 106 | columns[col['table_id']] = { 107 | 'table_id': col['table_id'], 108 | 'human_name': col['human_name'], 109 | 'mancer': m.name, 110 | 'columns': col['columns'], 111 | 'source_url': col['source_url'], 112 | } 113 | response = [] 114 | if request.args.get('table_id'): 115 | table_id = request.args['table_id'] 116 | try: 117 | response.append(columns[table_id]) 118 | except KeyError: 119 | response.append({ 120 | 'status': 'error', 121 | 'message': 'table_id %s not found' % table_id 122 | }) 123 | else: 124 | response.extend(columns.values()) 125 | resp = make_response(json.dumps(response)) 126 | resp.headers['Content-Type'] = 'application/json' 127 | return resp 128 | 129 | @api.route('/api/geo-types/') 130 | def geo_types(): 131 | """ 132 | Return a list of tables grouped by geo_type 133 | """ 134 | ordered_types = None 135 | if request.args.get('geo_type'): 136 | ordered_types, errors = get_geo_types(request.args.get('geo_type')) 137 | else: 138 | ordered_types, errors = get_geo_types() 139 | resp = make_response(json.dumps(ordered_types, cls=GeoTypeEncoder)) 140 | resp.headers['Content-Type'] = 'application/json' 141 | return resp 142 | -------------------------------------------------------------------------------- /geomancer/helpers.py: -------------------------------------------------------------------------------- 1 | from geomancer.app_config import MANCERS, MANCER_KEYS 2 | from collections import OrderedDict 3 | import operator 4 | import re 5 | 6 | from geomancer.mancers.geotype import County 7 | 8 | GEOTYPES = [ 9 | County, 10 | ] 11 | 12 | def encoded_dict(in_dict): 13 | out_dict = {} 14 | for k, v in in_dict.iteritems(): 15 | if isinstance(v, unicode): 16 | v = v.encode('utf8') 17 | elif isinstance(v, str): 18 | v.decode('utf8') 19 | out_dict[k] = v 20 | return out_dict 21 | 22 | def import_class(cl): 23 | d = cl.rfind('.') 24 | classname = cl[d+1:len(cl)] 25 | m = __import__(cl[0:d], globals(), locals(), [classname]) 26 | return getattr(m, classname) 27 | 28 | def get_geo_types(geo_type=None): 29 | types = {} 30 | columns = [] 31 | geo_types = [] 32 | errors = [] 33 | 34 | for mancer in MANCERS: 35 | m = import_class(mancer) 36 | api_key = MANCER_KEYS.get(m.machine_name) 37 | try: 38 | m = m(api_key=api_key) 39 | except ImportError, e: 40 | errors.append(e.message) 41 | continue 42 | for col in m.get_metadata(): 43 | geo_types.extend(col['geo_types']) 44 | columns.extend(m.get_metadata()) 45 | for t in geo_types: 46 | types[t.machine_name] = {} 47 | types[t.machine_name]['info'] = t 48 | 49 | tables = [{'human_name': c['human_name'], 50 | 'table_id': c['table_id'], 51 | 'source_name': c['source_name'], 52 | 'count': c['count'], 53 | 'source_url': c['source_url']} \ 54 | for c in columns if t.machine_name in \ 55 | [i.machine_name for i in c['geo_types']]] 56 | 57 | tables_sorted = sorted(tables, key=lambda x: x['human_name']) 58 | types[t.machine_name]['tables'] = tables_sorted 59 | 60 | if geo_type: 61 | types = {geo_type: types[geo_type]} 62 | 63 | types_sorted = sorted(types.values(), key=lambda x: x['info'].human_name) 64 | 65 | results = [] 66 | for v in types_sorted: 67 | results.append(v) 68 | 69 | return results, errors 70 | 71 | GEO_LOOKUP = { 72 | 'county': ['County', 'county'], 73 | } 74 | 75 | def guess_geotype(header, values): 76 | guess = None 77 | for geotype, vals in GEO_LOOKUP.items(): 78 | if header in vals: 79 | return geotype 80 | for geotype in GEOTYPES: 81 | g = geotype() 82 | valid, message = g.validate(values) 83 | if valid: 84 | guess = g.machine_name 85 | return guess 86 | 87 | def get_data_sources(geo_type=None): 88 | mancer_data = [] 89 | errors = [] 90 | for mancer in MANCERS: 91 | m = import_class(mancer) 92 | api_key = MANCER_KEYS.get(m.machine_name) 93 | try: 94 | m = m(api_key=api_key) 95 | except ImportError, e: 96 | errors.append(e.message) 97 | continue 98 | mancer_obj = { 99 | "name": m.name, 100 | "machine_name": m.machine_name, 101 | "base_url": m.base_url, 102 | "info_url": m.info_url, 103 | "description": m.description, 104 | "data_types": {} 105 | } 106 | info = m.get_metadata() 107 | for col in info: 108 | if geo_type: 109 | col_types = [i.machine_name for i in col['geo_types']] 110 | if geo_type in col_types: 111 | mancer_obj["data_types"][col['table_id']] = col 112 | else: 113 | mancer_obj["data_types"][col['table_id']] = col 114 | try: 115 | mancer_obj["data_types"][col['table_id']]['geo_types'] = \ 116 | sorted(mancer_obj["data_types"][col['table_id']]['geo_types'], 117 | key=lambda x: x.human_name) 118 | except KeyError: 119 | pass 120 | 121 | mancer_obj["data_types"] = sorted(mancer_obj["data_types"].values(), 122 | key=lambda x: x['human_name']) 123 | if mancer_obj['data_types']: 124 | mancer_data.append(mancer_obj) 125 | return mancer_data, errors 126 | 127 | def find_geo_type(geo_type, col_idxs): 128 | if ';' not in geo_type: 129 | return geo_type, col_idxs, u'{0}' 130 | else: 131 | g = None 132 | fmt = u'{0}, {1}' 133 | if 'county' in geo_type: 134 | g = u'county' 135 | fmt = u'{0} County, {1}' 136 | if geo_type.find(g) > 0: 137 | col_idxs = list(reversed(col_idxs.split(';'))) 138 | else: 139 | col_idxs = col_idxs.split(';') 140 | return g, col_idxs, fmt 141 | 142 | SENSICAL_TYPES = { 143 | 'county;state': u'county', 144 | 'state;county': u'county', 145 | } 146 | 147 | def check_combos(combo): 148 | ''' 149 | Return boolean telling us whether the geo type combination 150 | makes sense or not 151 | ''' 152 | if ';' not in combo: 153 | return True 154 | sorted_types = ';'.join(sorted(combo.split(';'))) 155 | if sorted_types in SENSICAL_TYPES.keys(): 156 | return True 157 | return False 158 | -------------------------------------------------------------------------------- /geomancer/mancers/bea.py: -------------------------------------------------------------------------------- 1 | from urllib import urlencode 2 | import json 3 | import os 4 | from geomancer.app_config import MANCER_KEYS 5 | from geomancer.helpers import encoded_dict 6 | from geomancer.mancers.geotype import State, StateFIPS 7 | from geomancer.mancers.base import BaseMancer, MancerError 8 | from string import punctuation 9 | import re 10 | from urlparse import urlparse 11 | import us 12 | 13 | class BureauEconomicAnalysis(BaseMancer): 14 | """ 15 | Subclassing the main BaseMancer class 16 | """ 17 | 18 | name = 'Bureau of Economic Analysis' 19 | machine_name = 'bureau_economic_analysis' 20 | base_url = 'http://www.bea.gov/api/data' 21 | info_url = 'http://www.bea.gov' 22 | description = """ 23 | GDP & Personal Income Data (2013) from the Bureau of Economic Analysis 24 | """ 25 | api_key_required = True 26 | 27 | def __init__(self, api_key=None): 28 | self.api_key = api_key 29 | BaseMancer.__init__(self) 30 | 31 | def get_metadata(self): 32 | datasets = [ 33 | { 34 | 'table_id': 'GDP_SP', 35 | 'human_name': 'Nominal GDP', 36 | 'description': '2013 Gross Domestic Product (GDP) (state annual product)', 37 | 'source_name': self.name, 38 | 'source_url': 'http://bea.gov/regional/index.htm', 39 | 'geo_types': [State()], 40 | 'columns': ['2013 GDP'], 41 | 'count': 1 42 | }, 43 | { 44 | 'table_id': 'RGDP_SP', 45 | 'human_name': 'Real GDP', 46 | 'description': '2013 Real GDP (state annual product)', 47 | 'source_name': self.name, 48 | 'source_url': 'http://bea.gov/regional/index.htm', 49 | 'geo_types': [State()], 50 | 'columns': ['2013 Real GDP'], 51 | 'count': 1 52 | }, 53 | { 54 | 'table_id': 'PCRGDP_SP', 55 | 'human_name': 'Real GDP - Per Capita', 56 | 'description': '2013 Per capita Real GDP (state annual product)', 57 | 'source_name': self.name, 58 | 'source_url': 'http://bea.gov/regional/index.htm', 59 | 'geo_types': [State()], 60 | 'columns': ['2013 Per Capita Real GDP'], 61 | 'count': 1 62 | }, 63 | { 64 | 'table_id': 'TPI_SI', 65 | 'human_name': 'Personal Income - Total', 66 | 'description': '2013 Total Personal Income (state annual income)', 67 | 'source_name': self.name, 68 | 'source_url': 'http://bea.gov/regional/index.htm', 69 | 'geo_types': [State()], 70 | 'columns': ['2013 Total Personal Income'], 71 | 'count': 1 72 | }, 73 | { 74 | 'table_id': 'PCPI_SI', 75 | 'human_name': 'Personal Income - Per Capita', 76 | 'description': '2013 Per Capita personal income (state annual income)', 77 | 'source_name': self.name, 78 | 'source_url': 'http://bea.gov/regional/index.htm', 79 | 'geo_types': [State()], 80 | 'columns': ['2013 Per Capita Personal Income'], 81 | 'count': 1 82 | } 83 | ] 84 | 85 | return datasets 86 | 87 | def lookup_state_name(self, term): 88 | st = us.states.lookup(term) 89 | if not st: 90 | st = [s for s in us.STATES if getattr(s, 'ap_abbr') == term] 91 | if st: 92 | return st.name 93 | else: 94 | return term 95 | 96 | def geo_lookup(self, search_term, geo_type=None): 97 | regex = re.compile('[%s]' % re.escape(punctuation)) 98 | search_term = regex.sub('', search_term) 99 | if geo_type == 'state': 100 | return {'term': search_term, 'geoid': self.lookup_state_name(search_term)} 101 | else: 102 | return {'term': search_term, 'geoid': search_term} 103 | 104 | def search(self, geo_ids=None, columns=None): 105 | 106 | column_names = { 107 | 'GDP_SP': '2013 GDP (millions)', 108 | 'RGDP_SP': '2013 Real GDP (millions of chained 2009 dollars)', 109 | 'PCRGDP_SP': '2013 Per Capita Real GDP (chained 2009 dollars)', 110 | 'TPI_SI': '2013 Total Personal Income (thousands of dollars)', 111 | 'PCPI_SI': '2013 Per Capita Personal Income (dollars)' 112 | } 113 | 114 | results = {'header':[]} 115 | 116 | for col in columns: 117 | url = self.base_url+'/?UserID=%s&method=GetData&datasetname=RegionalData&KeyCode=%s&Year=2013&ResultFormat=json' %(self.api_key, col) 118 | try: 119 | response = self.urlopen(url) 120 | except scrapelib.HTTPError, e: 121 | try: 122 | body = json.loads(e.body.json()['error']) 123 | except ValueError: 124 | body = None 125 | except AttributeError: 126 | body = e.body 127 | raise MancerError('BEA API returned an error', body=body) 128 | raw_results = json.loads(response) 129 | raw_data = raw_results['BEAAPI']['Results']['Data'] 130 | results['header'].append(column_names[col]) 131 | for geo_type, geo_id in geo_ids: 132 | if not results.get(geo_id): 133 | results[geo_id] = [] 134 | if geo_type == 'state': #### handle state fips? 135 | for geo_data in raw_data: #this is not efficient...make this better 136 | if geo_data['GeoName'] == geo_id: 137 | results[geo_id].append(geo_data['DataValue']) 138 | break 139 | return results 140 | -------------------------------------------------------------------------------- /geomancer/mancers/usa_spending.py: -------------------------------------------------------------------------------- 1 | import scrapelib 2 | import us 3 | from urllib import urlencode 4 | import json 5 | import os 6 | from geomancer.mancers.base import BaseMancer 7 | from geomancer.mancers.geotype import City, State, Zip5, County, \ 8 | CongressionalDistrict 9 | from geomancer.helpers import encoded_dict 10 | from lxml import etree 11 | import re 12 | from collections import OrderedDict 13 | from datetime import datetime 14 | 15 | TABLE_PARAMS = { 16 | 'fpds': { 17 | 'state': 'stateCode', 18 | 'zip_5': 'placeOfPerformanceZIPCode', 19 | 'congress_district': 'pop_cd' 20 | }, 21 | 'faads': { 22 | 'state': 'principal_place_state_code', 23 | 'city': 'principal_place_cc', 24 | 'county': 'principal_place_cc', 25 | }, 26 | 'fsrs': { 27 | 'state': 'subawardee_pop_state', 28 | 'zip_5': 'subawardee_pop_zip', 29 | 'congress_district': 'subawardee_pop_cd', 30 | }, 31 | } 32 | 33 | class USASpending(BaseMancer): 34 | """ 35 | Subclassing BaseMancer 36 | """ 37 | 38 | name = 'USA Spending' 39 | machine_name = 'usa_spending' 40 | base_url = "http://www.usaspending.gov" 41 | info_url = 'http://www.usaspending.gov' 42 | description = """ 43 | Data from the U.S. Office of Management and Budget on federal contracts awarded. 44 | """ 45 | 46 | def get_metadata(self): 47 | datasets = [ 48 | { 49 | 'table_id': 'fpds', 50 | 'human_name': 'Federal Contracts', 51 | 'description': '', 52 | 'source_name': self.name, 53 | 'source_url': 'http://www.usaspending.gov/data', 54 | 'geo_types': [State(), Zip5(), CongressionalDistrict()], 55 | 'count': 1, # probably a lot more, 56 | 'columns': '', 57 | }, 58 | { 59 | 'table_id': 'faads', 60 | 'human_name': 'Federal Assistance', 61 | 'description': '', 62 | 'source_name': self.name, 63 | 'source_url': 'http://www.usaspending.gov/data', 64 | 'geo_types': [State(), City(), County()], 65 | 'count': 1, # probably a lot more 66 | 'columns': '', 67 | }, 68 | { 69 | 'table_id': 'fsrs', 70 | 'human_name': 'Federal sub-awards', 71 | 'description': '', 72 | 'source_name': self.name, 73 | 'source_url': 'http://www.usaspending.gov/data', 74 | 'geo_types': [State(), Zip5(), CongressionalDistrict()], 75 | 'count': 1, # probably a lot more 76 | 'columns': '', 77 | }, 78 | ] 79 | for dataset in datasets: 80 | table_id = dataset['table_id'] 81 | url = '%s/%s/%s.php' % (self.base_url, table_id, table_id) 82 | param = TABLE_PARAMS[table_id]['state'] 83 | query = {param: 'IL', 'detail': 's'} 84 | params = urlencode(query) 85 | table = self.fetch_xml(url, params) 86 | dataset['count'] = len(table) 87 | dataset['columns'] = [' '.join(c.split('_')).title() for c in table.keys()] 88 | return datasets 89 | 90 | def fetch_xml(self, url, params): 91 | table = OrderedDict() 92 | response = self.urlopen('%s?%s' % (url, params)) 93 | tree = etree.fromstring(str(response)) 94 | xml_schema = tree.nsmap[None] 95 | tables = tree\ 96 | .find('{%s}data' % xml_schema)\ 97 | .find('{%s}record' % xml_schema)\ 98 | .getchildren() 99 | for t in tables: 100 | table_name = t.tag.replace('{%s}' % xml_schema, '') 101 | child_nodes = t.getchildren() 102 | for column in child_nodes: 103 | key = column.tag.replace('{%s}' % xml_schema, '') 104 | value = column.text 105 | if column.attrib: 106 | for k,v in column.attrib.items(): 107 | if k in ['rank', 'year']: 108 | header_val = '%s_%s_%s' % (table_name,k,v.zfill(2)) 109 | table[header_val] = value 110 | if k in ['total_obligatedAmount', 'id', 'name']: 111 | rank = column.attrib['rank'] 112 | header_val = '%s_rank_%s_%s' % (table_name,rank.zfill(2),k) 113 | table[header_val] = v 114 | else: 115 | header_val = '%s_%s' % (table_name,key) 116 | table[header_val] = value 117 | return OrderedDict(sorted(table.items())) 118 | 119 | def lookup_state(self, term): 120 | st = us.states.lookup(term) 121 | if not st: 122 | st = [s for s in us.STATES if getattr(s, 'ap_abbr') == term] 123 | if st: 124 | return st.abbr 125 | else: 126 | return term 127 | 128 | def geo_lookup(self, search_term, geo_type=None): 129 | if geo_type == 'state': 130 | return {'term': search_term, 'geoid': self.lookup_state(search_term)} 131 | elif geo_type == 'congress_district': 132 | parts = search_term.split(' ') 133 | district = search_term 134 | if len(parts) > 1: 135 | st_abbr = self.lookup_state(parts[0]) 136 | dist_code = parts[1].zfill(2) 137 | district = st_abbr + dist_code 138 | return {'term': search_term, 'geoid': district} 139 | else: 140 | return {'term': search_term, 'geoid': search_term.zfill(5)} 141 | 142 | def search(self, geo_ids=None, columns=None): 143 | result = {'header': []} 144 | table_ds = {} 145 | for geo_type, geo_id in geo_ids: 146 | result[geo_id] = [] 147 | for col in columns: 148 | url = '%s/%s/%s.php' % (self.base_url, col, col) 149 | param = TABLE_PARAMS[col][geo_type] 150 | query = {param: geo_id, 'detail': 's'} 151 | params = urlencode(query) 152 | table = self.fetch_xml(url, params) 153 | if not result['header']: 154 | result['header'] = table.keys() 155 | else: 156 | diff = set(table.keys()).difference(set(result['header'])) 157 | positions = [(i,c,) for i,c in enumerate(table.keys()) if c in diff] 158 | for idx, col in positions: 159 | result['header'].insert(idx,col) 160 | table_ds[geo_id] = table 161 | for geo_type, geo_id in geo_ids: 162 | d = OrderedDict() 163 | for key in result['header']: 164 | try: 165 | d[key] = table_ds[geo_id][key] 166 | except KeyError: 167 | d[key] = '' 168 | result[geo_id] = d.values() 169 | result['header'] = [' '.join(c.split('_')).title() for c in result['header']] 170 | return result 171 | 172 | -------------------------------------------------------------------------------- /geomancer/mancers/base.py: -------------------------------------------------------------------------------- 1 | import scrapelib 2 | from urllib import urlencode 3 | import json 4 | import os 5 | from geomancer.app_config import CACHE_DIR 6 | from geomancer.helpers import encoded_dict 7 | from string import punctuation 8 | import re 9 | from urlparse import urlparse 10 | 11 | class MancerError(Exception): 12 | def __init__(self, message, body=None): 13 | Exception.__init__(self, message) 14 | self.message = message 15 | self.body = body 16 | 17 | class BaseMancer(scrapelib.Scraper): 18 | """ 19 | Subclassing scrapelib here mainly to take advantage of pluggable caching backend. 20 | """ 21 | name = None # this is the name that will show up on the /select-tables page 22 | machine_name = None # a slugified machine name 23 | base_url = None # base url for the api 24 | info_url = None # this will show up next to the name on the /select-tables page 25 | description = None # this will show up under the name on the /select-tables page 26 | 27 | # If True, geomancer will check that an API key is passed into the constructor. 28 | # If it's not present, the mancer will be disabled and an error will display. 29 | api_key_required = False 30 | 31 | def __init__(self, 32 | raise_errors=True, 33 | requests_per_minute=0, 34 | retry_attempts=5, 35 | retry_wait_seconds=1, 36 | header_func=None, 37 | cache_dir=CACHE_DIR, 38 | api_key=None): 39 | 40 | super(BaseMancer, self).__init__(raise_errors=raise_errors, 41 | requests_per_minute=requests_per_minute, 42 | retry_attempts=retry_attempts, 43 | retry_wait_seconds=retry_wait_seconds, 44 | header_func=header_func) 45 | 46 | # We might want to talk about configuring an S3 backed cache for this 47 | # so we don't run the risk of running out of disk space. 48 | self.cache_dir = cache_dir 49 | self.cache_storage = scrapelib.cache.FileCache(self.cache_dir) 50 | self.cache_write_only = False 51 | 52 | # If subclass declares that an API Key is required and an API Key is not given, 53 | # raise an ImportError 54 | if self.api_key_required and not self.api_key: 55 | raise ImportError('The %s mancer requires an API key and is disabled.' % self.name) 56 | 57 | 58 | def flush_cache(self): 59 | host = urlparse(self.base_url).netloc 60 | count = 0 61 | for f in os.listdir(self.cache_dir): 62 | if f.startswith(host): 63 | os.remove(os.path.join(self.cache_dir, f)) 64 | count += 1 65 | return count 66 | 67 | def get_metadata(self): 68 | """ 69 | This returns a list of dicts containing info about datasets that can be 70 | returned by the API. This needs to be a static method so that the 71 | application layer can use it to compile a list of columns that can be 72 | appended to incoming spreadsheets. 73 | 74 | Should look like this: 75 | 76 | [ 77 | { 78 | 'table_id': '', 79 | 'human_name': '', 80 | 'description': '', 81 | 'source_name': '', 82 | 'source_url': '', 83 | 'geo_types': ['list', 'of', 'instances', 'of', GeoType()], 84 | 'count': '', 85 | 'columns': ['list', 'of', 'column', 'names', 'that', 'will', 'be', 'appended'] 86 | }, 87 | { 88 | 'table_id': # table id for api lookups, 89 | 'human_name': # this shows up as a table row on the /select-tables page, the /data-sources page, & the /geographies page, 90 | 'description': '', 91 | 'source_name': # this shows up as link text under 'Source' on the /geographies page, 92 | 'source_url': # this is the link url under 'Source' on the /geographies page, 93 | 'geo_types': # this determines the geographies that can be matched, 94 | 'count': # this shows up under 'Columns that will be added' on the /select-tables page, 95 | 'columns': # each list item shows up in the popup when clicking the column info link on the /select-tables page 96 | }, 97 | ...etc... 98 | ] 99 | 100 | """ 101 | 102 | raise NotImplementedError 103 | 104 | def geo_lookup(self, search_term, geo_type=None): 105 | """ 106 | Method for looking up geographies through specific APIs, if needed 107 | Should be implemented by subclasses 108 | 109 | 'search_term' is the string that will be used to search 110 | 'geo_type' is one of the 13 geographic types that we support 111 | ('city', 'state', 'congress_district', ...etc...) 112 | This can be used by subclasses to narrow the search in a way that 113 | is specific to that API 114 | 115 | Returns a response that maps the incoming search term to the 116 | geographic identifier to be used with the search method: 117 | 118 | { 119 | 'term': , 120 | 'geoid': '' 121 | } 122 | 123 | Default behavior is to just echo back the search_term as the geoid. 124 | This makes it possible to create a common interface for all subclasses 125 | without needing to figure out if you need to search or not. 126 | """ 127 | 128 | return {'term': search_term, 'geoid': search_term} 129 | 130 | def search(self, geo_ids=None, columns=None): 131 | """ 132 | This method should send the search request to the API endpoint(s). 133 | 'geo_ids' is a list of tuples with the geography type and geo_id: 134 | 135 | [ 136 | ('state', 'IL',), 137 | ('state', 'CA',), 138 | ...etc... 139 | ] 140 | 141 | 'columns' is a list of columns to 142 | return. Child classes should be capable of looking these up in a way 143 | that makes sense to the API. 144 | 145 | The response should be a dict: 146 | - keys consist of the header and geo_ids 147 | - values are a list, with length = len(columns) 148 | 149 | { 150 | 'header': [ 151 | '', 152 | '', 153 | '...etc...' 154 | ], 155 | '': [ 156 | , 157 | , 158 | , 159 | , 160 | ...etc..., 161 | ], 162 | '': [ 163 | , 164 | , 165 | , 166 | , 167 | ...etc..., 168 | ], 169 | } 170 | 171 | One should be able to call the python zip function on the header list 172 | and any of the lists with data about the geographies and have it work. 173 | """ 174 | raise NotImplementedError 175 | -------------------------------------------------------------------------------- /geomancer/templates/select_tables.html: -------------------------------------------------------------------------------- 1 | {% extends 'wizard_base.html' %} 2 | {% block title %}Add data - Geomancer {% endblock %} 3 | {% block wizard_content %} 4 |

3. Add data

5 |
6 | {% for field_name, info in session['fields'].items() %} 7 |

Column(s) to match on: {{field_name|string_split(';')|join(', ')}}

8 | 9 | {% for mancer in session['mancer_data'] %} 10 | {% if mancer['data_types'] | length > 0 %} 11 |

{{mancer.name}} {{mancer.info_url}}

12 |

{{mancer.description}}

13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | {% for val in mancer['data_types'] %} 23 | 24 | 29 | 34 | 45 | 46 | {% endfor %} 47 | 48 |
SelectData from {{mancer.name}}Columns that will be added
25 | 28 | 30 | 33 | 35 | {% if val.count %} 36 | 37 | 38 | {{val.count}} 39 | {%- if val.count == 1 %} column 40 | {%- else %} columns 41 | {%- endif %} 42 | 43 | {% endif %} 44 |
49 | {% endif %} 50 | {% endfor %} 51 | {% endfor %} 52 | 53 |
54 | 55 | {% endblock %} 56 | {% block extra_javascript %} 57 | 58 | 63 | 86 | 154 | {% endblock %} 155 | -------------------------------------------------------------------------------- /geomancer/views.py: -------------------------------------------------------------------------------- 1 | from flask import Blueprint, make_response, request, redirect, url_for, \ 2 | session, render_template, current_app, send_from_directory, flash, jsonify 3 | import json 4 | import sys 5 | import os 6 | import gzip 7 | import requests 8 | import json 9 | from uuid import uuid4 10 | from werkzeug import secure_filename 11 | from csvkit import convert 12 | from csvkit.unicsv import UnicodeCSVReader 13 | from csvkit.cleanup import RowChecker 14 | from cStringIO import StringIO 15 | from geomancer.helpers import import_class, get_geo_types, get_data_sources, \ 16 | guess_geotype, check_combos, SENSICAL_TYPES 17 | from geomancer.app_config import ALLOWED_EXTENSIONS, \ 18 | MAX_CONTENT_LENGTH 19 | from werkzeug.exceptions import RequestEntityTooLarge 20 | 21 | 22 | 23 | views = Blueprint('views', __name__) 24 | 25 | def allowed_file(filename): 26 | return '.' in filename and \ 27 | filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS 28 | 29 | # primary pages 30 | @views.route('/', methods=['GET', 'POST']) 31 | def index(): 32 | return render_template('index.html') 33 | 34 | @views.route('/about', methods=['GET', 'POST']) 35 | def about(): 36 | return render_template('about.html') 37 | 38 | @views.route('/upload-formats', methods=['GET', 'POST']) 39 | def upload_formats(): 40 | return render_template('upload-formats.html') 41 | 42 | @views.route('/contribute-data', methods=['GET', 'POST']) 43 | def contribute_data(): 44 | return render_template('contribute-data.html') 45 | 46 | @views.route('/geographies', methods=['GET', 'POST']) 47 | def geographies(): 48 | geographies, errors = get_geo_types() 49 | for error in errors: 50 | flash(error) 51 | return render_template('geographies.html', geographies=geographies) 52 | 53 | @views.route('/data-sources', methods=['GET', 'POST']) 54 | def data_sources(): 55 | data_sources, errors = get_data_sources() 56 | for error in errors: 57 | flash(error) 58 | return render_template('data-sources.html', data_sources=data_sources) 59 | 60 | # routes for geomancin' 61 | @views.route('/upload/', methods=['GET', 'POST']) 62 | def upload(): 63 | context = {} 64 | if request.method == 'POST': 65 | big_file = False 66 | try: 67 | files = request.files 68 | except RequestEntityTooLarge, e: 69 | files = None 70 | big_file = True 71 | current_app.logger.info(e) 72 | if files: 73 | f = files['input_file'] 74 | if allowed_file(f.filename): 75 | inp = StringIO(f.read()) 76 | file_format = convert.guess_format(f.filename) 77 | try: 78 | converted = convert.convert(inp, file_format) 79 | except UnicodeDecodeError: 80 | context['errors'] = ['We had a problem with reading your file. \ 81 | This could have to do with the file encoding or format'] 82 | converted = None 83 | f.seek(0) 84 | if converted: 85 | outp = StringIO(converted) 86 | reader = UnicodeCSVReader(outp) 87 | session['header_row'] = reader.next() 88 | rows = [] 89 | columns = [[] for c in session['header_row']] 90 | column_ids = range(len(session['header_row'])) 91 | for row in range(100): 92 | try: 93 | rows.append(reader.next()) 94 | except StopIteration: 95 | break 96 | for i, row in enumerate(rows): 97 | for j,d in enumerate(row): 98 | columns[j].append(row[column_ids[j]]) 99 | sample_data = [] 100 | guesses = {} 101 | for index, header_val in enumerate(session['header_row']): 102 | guesses[index] = guess_geotype(header_val, columns[index]) 103 | sample_data.append((index, header_val, columns[index])) 104 | session['sample_data'] = sample_data 105 | session['guesses'] = json.dumps(guesses) 106 | outp.seek(0) 107 | session['file'] = outp.getvalue() 108 | session['filename'] = f.filename 109 | return redirect(url_for('views.select_geo')) 110 | else: 111 | context['errors'] = ['Only .xls or .xlsx and .csv files are allowed.'] 112 | else: 113 | context['errors'] = ['You must provide a file to upload.'] 114 | if big_file: 115 | context['errors'] = ['Uploaded file must be 10mb or less.'] 116 | return render_template('upload.html', **context) 117 | 118 | @views.route('/select-geography/', methods=['GET', 'POST']) 119 | def select_geo(): 120 | if not session.get('file'): 121 | return redirect(url_for('views.index')) 122 | context = {} 123 | if request.method == 'POST': 124 | inp = StringIO(session['file']) 125 | reader = UnicodeCSVReader(inp) 126 | header = reader.next() 127 | fields = {} 128 | valid = True 129 | geotype_val = None 130 | if not request.form: 131 | valid = False 132 | context['errors'] = ['Select a field that contains a geography type'] 133 | else: 134 | geotypes = [] 135 | indexes = [] 136 | for k,v in request.form.items(): 137 | if k.startswith("geotype"): 138 | geotypes.append(v) 139 | indexes.append(k.split('_')[1]) 140 | if len(indexes) > 2: 141 | valid = False 142 | context['errors'] = ['We can only merge geographic information from 2 columns'] 143 | else: 144 | fields_key = ';'.join([header[int(i)] for i in indexes]) 145 | geotype_val = ';'.join([g for g in geotypes]) 146 | if not check_combos(geotype_val): 147 | valid = False 148 | types = [t.title() for t in geotype_val.split(';')] 149 | context['errors'] = ['The geographic combination of {0} and {1} does not work'.format(*types)] 150 | else: 151 | fields[fields_key] = { 152 | 'geo_type': geotype_val, 153 | 'column_index': ';'.join(indexes) 154 | } 155 | 156 | # found_geo_type = get_geo_types(geo_type)[0]['info'] 157 | # sample_list = session['sample_data'][index][2] 158 | # valid, message = found_geo_type.validate(sample_list) 159 | # context['errors'] = [message] 160 | if valid: 161 | try: 162 | geo_type = SENSICAL_TYPES[geotype_val] 163 | except KeyError: 164 | geo_type = geotype_val 165 | mancer_data, errors = get_data_sources(geo_type=geo_type) 166 | session['fields'] = fields 167 | session['mancer_data'] = mancer_data 168 | for error in errors: 169 | flash(error) 170 | return redirect(url_for('views.select_tables')) 171 | return render_template('select_geo.html', **context) 172 | 173 | @views.route('/select-tables/', methods=['POST', 'GET']) 174 | def select_tables(): 175 | if not session.get('file'): 176 | return redirect(url_for('views.index')) 177 | context = {} 178 | if request.method == 'POST' and not request.form: 179 | valid = False 180 | context['errors'] = ['Select at least on table to join to your spreadsheet'] 181 | return render_template('select_tables.html', **context) 182 | 183 | @views.route('/geomance//') 184 | def geomance_view(session_key): 185 | return render_template('geomance.html', session_key=session_key) 186 | 187 | @views.route('/download/') 188 | def download_results(filename): 189 | return send_from_directory(current_app.config['RESULT_FOLDER'], filename) 190 | 191 | @views.route('/413.html') 192 | def file_too_large(): 193 | return make_response(render_template('413.html'), 413) 194 | -------------------------------------------------------------------------------- /geomancer/mancers/bls.py: -------------------------------------------------------------------------------- 1 | from urllib import urlencode 2 | import json 3 | import os 4 | from geomancer.app_config import MANCER_KEYS 5 | from geomancer.helpers import encoded_dict 6 | from geomancer.mancers.geotype import State, StateFIPS 7 | from geomancer.mancers.base import BaseMancer, MancerError 8 | from string import punctuation 9 | import re 10 | from urlparse import urlparse 11 | import us 12 | import requests 13 | import pandas as pd 14 | 15 | class BureauLaborStatistics(BaseMancer): 16 | """ 17 | Subclassing the main BaseMancer class 18 | """ 19 | 20 | name = 'Bureau of Labor Statistics' 21 | machine_name = 'bureau_labor_statistics' 22 | base_url = 'http://api.bls.gov/publicAPI/v2/timeseries/data' 23 | info_url = 'http://www.bls.gov/' 24 | description = """ 25 | Data from the Bureau of Labor Statistics 26 | """ 27 | api_key_required = True 28 | 29 | # store the data for each column 30 | # b/c bls api has low limit & it doesn't take long to grab all states 31 | oes_column_data = {} 32 | # a mapping of bls oes series id data codes to geomancer column names 33 | oes_column_lookup = { '13': '2014 Annual Wages - Median', 34 | '12': '2014 Annual Wages - 25th Percentile', 35 | '14': '2014 Annual Wages - 75th Percentile'} 36 | 37 | qcew_column_lookup = { 38 | 'annual_avg_estabs_count':'2013 Annual Average of 4 Quarterly Establishment Counts', 39 | 'annual_avg_emplvl': '2013 Annual Average of Monthly Employment Levels', 40 | 'total_annual_wages': '2013 Total Annual Wages (Sum of 4 quarterly total wage levels)', 41 | 'taxable_annual_wages':'2013 Taxable Annual Wages (Sum of the 4 quarterly taxable wage totals)', 42 | 'annual_contributions':'2013 Annual Contributions (Sum of the 4 quarterly contribution totals)', 43 | 'annual_avg_wkly_wage':'2013 Average Weekly Wage (based on the 12-monthly employment levels and total annual wage levels)', 44 | 'avg_annual_pay':'2013 Average Annual Pay (based on employment and wage levels)' 45 | } 46 | 47 | def __init__(self, api_key=None): 48 | self.api_key = MANCER_KEYS[self.machine_name] 49 | BaseMancer.__init__(self) 50 | 51 | def get_metadata(self): 52 | datasets = [ 53 | { 54 | 'table_id': 'oes', 55 | 'human_name': 'Occupational Employment Statistics', 56 | 'description': 'Occupational Employment Statistics', 57 | 'source_name': self.name, 58 | 'source_url': 'http://www.bls.gov/oes/', 59 | 'geo_types': [State(), StateFIPS()], 60 | 'columns': [self.oes_column_lookup[col] for col in self.oes_column_lookup], 61 | 'count': 3 62 | }, 63 | { 64 | 'table_id': 'qcew', 65 | 'human_name': 'Quarterly Census of Employment & Wages', 66 | 'description': 'Quarterly Census of Employment & Wages', 67 | 'source_name': self.name, 68 | 'source_url': 'http://www.bls.gov/cew/home.htm', 69 | 'geo_types': [State(), StateFIPS()], 70 | 'columns': [self.qcew_column_lookup[col] for col in self.qcew_column_lookup], 71 | 'count': 7 72 | } 73 | ] 74 | 75 | return datasets 76 | 77 | def search(self, geo_ids=None, columns=None): 78 | # columns is a list consisting of table_ids from the possible values in get_metadata? 79 | results = {'header':[]} 80 | 81 | all_state_fips = ['01', '02', '04', '05', '06', '08', '09', '10', 82 | '12', '13', '15', '16', '17', '18', '19', '20', '21', '22', 83 | '23', '24', '25', '26', '27', '28', '29', '30', '31', '32', '33', 84 | '34', '35', '36', '37', '38', '39', '40', '41', '42', '44', 85 | '45', '46', '47', '48', '49', '50', '51', '53', '54', '55', '56'] 86 | 87 | for table_id in columns: 88 | if table_id == 'oes': 89 | # only grab data when oes_column_data is not populated 90 | if len(self.oes_column_data)==0: 91 | for col in self.oes_column_lookup: 92 | self.oes_column_data[col] = {} 93 | 94 | self.grab_oes_data(all_state_fips) 95 | 96 | # looping through columns in OES data 97 | for col in self.oes_column_lookup: 98 | results['header'].append(self.oes_column_lookup[col]) 99 | 100 | # compiling matched geo data for results 101 | for geo_type, geo_id in geo_ids: 102 | if not results.get(geo_id): 103 | results[geo_id] = [] 104 | if geo_type == 'state' or geo_type =='state_fips': 105 | if geo_id in self.oes_column_data[col]: 106 | results[geo_id].append(self.oes_column_data[col][geo_id]) 107 | else: 108 | results[geo_id].append("") 109 | 110 | elif table_id == 'qcew': 111 | for col in self.qcew_column_lookup: 112 | results['header'].append(self.qcew_column_lookup[col]) 113 | 114 | for geo_type, geo_id in geo_ids: 115 | if not results.get(geo_id): 116 | results[geo_id] = [] 117 | if geo_type == 'state' or geo_type == 'state_fips': 118 | 119 | summary_df = self.qcewGetSummaryData(geo_id) 120 | for col in self.qcew_column_lookup: 121 | results[geo_id].append(summary_df[col][0]) 122 | 123 | return results 124 | 125 | def geo_lookup(self, search_term, geo_type=None): 126 | regex = re.compile('[%s]' % re.escape(punctuation)) 127 | search_term = regex.sub('', search_term) 128 | if geo_type == 'state' or geo_type == 'state_fips': 129 | return {'term': search_term, 'geoid': self.lookup_state_name(search_term)} 130 | else: 131 | return {'term': search_term, 'geoid': search_term} 132 | 133 | # given a search term, returns state fips code 134 | def lookup_state_name(self, term): 135 | st = us.states.lookup(term) 136 | if not st: 137 | st = [s for s in us.STATES if getattr(s, 'ap_abbr') == term] 138 | if st: 139 | return st.fips 140 | else: 141 | return search_term 142 | 143 | def bls_oes_series_id(self, geo_id, stat_id): 144 | # documentation on constructing series ids at http://www.bls.gov/help/hlpforma.htm#OE 145 | # geo_id is state FIPS code as string 146 | prefix = 'OEU' 147 | area_type = 'S' 148 | area_code = geo_id + '00000' 149 | industry_code = '000000' # this is the code for all industries 150 | occupation_code = '000000' # this is the code for all occupations 151 | datatype_code = stat_id 152 | 153 | return prefix+area_type+area_code+industry_code+occupation_code+datatype_code 154 | 155 | def grab_oes_data(self, geo_ids=None): 156 | # geo_ids is a list of state fips code strings 157 | for col in self.oes_column_lookup: 158 | series_ids = [] 159 | for geo_id in geo_ids: 160 | series_id = self.bls_oes_series_id(geo_id, col) 161 | series_ids.append(series_id) 162 | 163 | # make the request 164 | headers = {'Content-type': 'application/json'} 165 | data = json.dumps({"seriesid": series_ids,"startyear":"2014", "endyear":"2014", "registrationKey":self.api_key}) 166 | p = requests.post('http://api.bls.gov/publicAPI/v2/timeseries/data/', data=data, headers=headers) 167 | json_data = json.loads(p.text) 168 | 169 | self.oes_column_data[col] = {} 170 | # loop through the json data and add it to oes_column_data[col][geo_id] 171 | for result in json_data['Results']['series']: 172 | # grab state id from results series id 173 | this_geo_id = result['seriesID'][4:6] 174 | this_val = result['data'][0]['value'] 175 | self.oes_column_data[col][this_geo_id] = this_val 176 | 177 | def qcewGetSummaryData(self, state_fips): 178 | urlPath = "http://www.bls.gov/cew/data/api/2013/a/area/"+state_fips+"000.csv" 179 | df = pd.read_csv(urlPath) 180 | summary_df = df[(df['industry_code']=='10') & (df['own_code']==0)] # industry code 10 is all industries, own code 0 is all ownership 181 | return summary_df 182 | -------------------------------------------------------------------------------- /geomancer/mancers/geotype_us.py: -------------------------------------------------------------------------------- 1 | from json import JSONEncoder 2 | import re 3 | import us 4 | from os.path import join, abspath, dirname 5 | import csv 6 | 7 | GAZDIR = join(dirname(abspath(__file__)), 'gazetteers') 8 | 9 | class GeoType(object): 10 | """ 11 | Base class for defining geographic types. 12 | All four static properties should be defined 13 | """ 14 | human_name = None 15 | machine_name = None 16 | formatting_notes = None 17 | formatting_example = None 18 | validation_regex = None 19 | 20 | def as_dict(self): 21 | fields = [ 22 | 'human_name', 23 | 'machine_name', 24 | 'formatting_notes', 25 | 'formatting_example', 26 | ] 27 | d = {k:getattr(self,k) for k in fields} 28 | for k,v in d.items(): 29 | d[k] = ' '.join(v.split()) 30 | return d 31 | 32 | def validate(self, values): 33 | ''' 34 | Default is to implement a regex on a subclass that gets 35 | used here to validate the format. Optionally override this 36 | method to implement custom validation. If validation_regex 37 | is not defined on the subclass, this will always return True. 38 | 39 | values - A list (or other iterable) of values to evaluate 40 | 41 | Returns a boolean indicating whether all the members of the values 42 | list are valid and an optional user friendly message. 43 | ''' 44 | 45 | if self.validation_regex is None: 46 | return False, None 47 | else: 48 | values = list(set([v for v in values if v])) 49 | for v in values: 50 | if not re.match(self.validation_regex, v): 51 | message = 'The column you selected must be formatted \ 52 | like "%s" to match on %s geographies. Please pick another \ 53 | column or change the format of your data.' % \ 54 | (self.formatting_example, self.human_name) 55 | return False, message 56 | return True, None 57 | 58 | class GeoTypeEncoder(JSONEncoder): 59 | ''' 60 | Custom JSON encoder so we can have nice things. 61 | ''' 62 | def default(self, o): 63 | return o.as_dict() 64 | 65 | class City(GeoType): 66 | human_name = 'City' 67 | machine_name = 'city' 68 | formatting_notes = 'City name followed by state name, postal abbreviation or \ 69 | AP abbreviation.' 70 | formatting_example = 'Chicago, Illinois; Chicago, IL or Chicago, Ill.' 71 | 72 | #def validate(self, values): 73 | # ''' 74 | # Uses the US Census 2014 Place Name Gazetteer. 75 | # https://www.census.gov/geo/maps-data/data/gazetteer2014.html 76 | # ''' 77 | # gazetteer = set() 78 | # with open(join(GAZDIR, 'place_names.csv'), 'rb') as f: 79 | # reader = csv.reader(f) 80 | # for row in reader: 81 | # gazetteer.add(row[0].lower()) 82 | # values = set([v.split(',')[0].lower() for v in values if v]) 83 | # if values <= gazetteer: 84 | # return True, None 85 | # else: 86 | # diffs = values - gazetteer 87 | # return False, '"{0}" do not appear to be valid Census places'\ 88 | # .format(', '.join(diffs)) 89 | 90 | class State(GeoType): 91 | human_name = 'State' 92 | machine_name = 'state' 93 | formatting_notes = 'State name, postal abbreviation, or AP abbreviation.' 94 | formatting_example = 'Illinois, IL or Ill.' 95 | 96 | def validate(self, values): 97 | values = [v for v in values if v] 98 | non_matches = set() 99 | for val in values: 100 | st = us.states.lookup(val) 101 | if not st: 102 | st = [s for s in us.STATES if getattr(s, 'ap_abbr') == val] 103 | if not st: 104 | non_matches.add(val) 105 | if non_matches: 106 | return False, '"{0}" do not appear to be valid Census places'\ 107 | .format(', '.join(non_matches)) 108 | else: 109 | return True, None 110 | 111 | class County(GeoType): 112 | human_name = 'County' 113 | machine_name = 'county' 114 | formatting_notes = 'Name of a U.S. County and state abbreviation.' 115 | formatting_example = 'Cook County, IL' 116 | 117 | def validate(self, values): 118 | ''' 119 | Uses the US Census 2014 County Gazetteer. 120 | https://www.census.gov/geo/maps-data/data/gazetteer2014.html 121 | ''' 122 | gazetteer = set() 123 | with open(join(GAZDIR, 'county_names.csv'), 'rb') as f: 124 | reader = csv.reader(f) 125 | for row in reader: 126 | gazetteer.add(row[0].lower()) 127 | vals = set() 128 | for val in values: 129 | val = val.lower() 130 | if 'county' not in val: 131 | val = u'{0} county'.format(val) 132 | vals.add(val) 133 | if vals <= gazetteer: 134 | return True, None 135 | else: 136 | diffs = vals - gazetteer 137 | return False, u'"{0}" do not appear to be valid Counties'\ 138 | .format(u', '.join(diffs)) 139 | 140 | class SchoolDistrict(GeoType): 141 | human_name = 'School district' 142 | machine_name = 'school_district' 143 | formatting_notes = 'Name of an elementary, secondary or unified school district.' 144 | formatting_example = 'Chicago Public School District 299, IL' 145 | 146 | def validate(self, values): 147 | ''' 148 | Uses the US Census 2014 Elementary, Secondary and Unified 149 | School District Gazetteers. 150 | https://www.census.gov/geo/maps-data/data/gazetteer2014.html 151 | ''' 152 | gazetteer = set() 153 | with open(join(GAZDIR,'school_dists.csv'), 'rb') as f: 154 | reader = csv.reader(f) 155 | for row in reader: 156 | gazetteer.add(row[0].lower()) 157 | values = set([v.split(',')[0].lower() for v in values if v]) 158 | if values <= gazetteer: 159 | return True, None 160 | else: 161 | diffs = values - gazetteer 162 | return False, u'"{0}" do not appear to be valid School Districts'\ 163 | .format(u', '.join(diffs)) 164 | 165 | class CongressionalDistrict(GeoType): 166 | human_name = 'Congressional district' 167 | machine_name = 'congress_district' 168 | formatting_notes = 'U.S Congressional District.' 169 | formatting_example = 'Congressional District 7, IL' 170 | validation_regex = r'Congressional District \d+,.+' 171 | 172 | class Zip5(GeoType): 173 | human_name = '5 digit zip code' 174 | machine_name = 'zip_5' 175 | formatting_notes = 'Five-digit U.S. Postal Service Zip Code.' 176 | formatting_example = '60601' 177 | validation_regex = r'\d{5}$' 178 | 179 | class Zip9(GeoType): 180 | human_name = '9 digit zip code' 181 | machine_name = 'zip_9' 182 | formatting_notes = 'Five-digit U.S. Postal Service Zip Code plus a four digit \ 183 | geographic identifier.' 184 | formatting_example = '60601-3013' 185 | validation_regex = r'(\d{5})-(\d{4})$' 186 | 187 | class StateFIPS(GeoType): 188 | human_name = 'FIPS: State' 189 | machine_name = 'state_fips' 190 | formatting_notes = 'Federal Information Processing (FIPS) code for a U.S. State.' 191 | formatting_example = '17' 192 | validation_regex = r'\d{2}$' 193 | 194 | class StateCountyFIPS(GeoType): 195 | human_name = 'FIPS: County' 196 | machine_name = 'state_county_fips' 197 | formatting_notes = 'Federal Information Processing (FIPS) code for a U.S. County \ 198 | which includes the FIPS code for the state.' 199 | formatting_example = '17031' 200 | 201 | def validate(self, values): 202 | ''' 203 | Uses the US Census 2014 County Gazetteers. 204 | https://www.census.gov/geo/maps-data/data/gazetteer2014.html 205 | ''' 206 | gazetteer = set() 207 | with open(join(GAZDIR, 'county_names.csv'), 'rb') as f: 208 | reader = csv.reader(f) 209 | for row in reader: 210 | gazetteer.add(row[2].lower()) 211 | values = set([v.split(',')[0].lower() for v in values if v]) 212 | if values <= gazetteer: 213 | return True, None 214 | else: 215 | diffs = values - gazetteer 216 | return False, u'"{0}" do not appear to be valid County FIPS codes'\ 217 | .format(u', '.join(diffs)) 218 | 219 | class CensusTract(GeoType): 220 | human_name = 'FIPS: Census Tract' 221 | machine_name = 'census_tract' 222 | formatting_notes = 'Federal Information Processing (FIPS) code for a U.S Census Tract' 223 | formatting_example = '17031330100' 224 | validation_regex = r'\d{11}$' 225 | 226 | -------------------------------------------------------------------------------- /geomancer/worker.py: -------------------------------------------------------------------------------- 1 | from flask import current_app 2 | from pickle import loads, dumps 3 | from redis import Redis 4 | from uuid import uuid4 5 | import sys 6 | import os 7 | import re 8 | from cStringIO import StringIO 9 | from csvkit.unicsv import UnicodeCSVReader, UnicodeCSVWriter 10 | from geomancer.mancers.base import MancerError 11 | from geomancer.helpers import import_class, find_geo_type, get_geo_types 12 | from geomancer.app_config import RESULT_FOLDER, MANCERS, MANCER_KEYS 13 | from datetime import datetime 14 | import xlwt 15 | from openpyxl import Workbook 16 | from openpyxl.cell import get_column_letter 17 | from itertools import izip_longest 18 | import traceback 19 | 20 | redis = Redis() 21 | 22 | try: 23 | from raven import Client 24 | from geomancer.app_config import SENTRY_DSN 25 | client = Client(dsn=SENTRY_DSN) 26 | except ImportError: 27 | client = None 28 | except KeyError: 29 | client = None 30 | 31 | class DelayedResult(object): 32 | def __init__(self, key): 33 | self.key = key 34 | self._rv = None 35 | 36 | @property 37 | def return_value(self): 38 | if self._rv is None: 39 | rv = redis.get(self.key) 40 | if rv is not None: 41 | self._rv = loads(rv) 42 | return self._rv 43 | 44 | def queuefunc(f): 45 | def delay(*args, **kwargs): 46 | qkey = current_app.config['REDIS_QUEUE_KEY'] 47 | key = '%s:result:%s' % (qkey, str(uuid4())) 48 | s = dumps((f, key, args, kwargs)) 49 | redis.rpush(current_app.config['REDIS_QUEUE_KEY'], s) 50 | return DelayedResult(key) 51 | f.delay = delay 52 | return f 53 | 54 | @queuefunc 55 | def do_the_work(file_contents, field_defs, filename): 56 | """ 57 | field_defs looks like: 58 | { 59 | 10: { 60 | 'type': 'city_state', 61 | 'append_columns': ['total_population', 'median_age'] 62 | } 63 | } 64 | 65 | or like this: 66 | 67 | { 68 | 10;2: { 69 | 'type': 'city;state', 70 | 'append_columns': ['total_population', 'median_age'] 71 | } 72 | } 73 | 74 | where the semicolon separated values represent a multicolumn geography 75 | 76 | file_contents is a string containing the contents of the uploaded file. 77 | """ 78 | contents = StringIO(file_contents) 79 | reader = UnicodeCSVReader(contents) 80 | header = reader.next() 81 | result = None 82 | geo_ids = set() 83 | mancer_mapper = {} 84 | fields_key = field_defs.keys()[0] 85 | errors = [] 86 | 87 | geo_type, col_idxs, val_fmt = find_geo_type(field_defs[fields_key]['type'], 88 | fields_key) 89 | geo_name = get_geo_types(geo_type=geo_type)[0][0]['info'].human_name 90 | for mancer in MANCERS: 91 | m = import_class(mancer) 92 | api_key = MANCER_KEYS.get(m.machine_name) 93 | try: 94 | m = m(api_key=api_key) 95 | except ImportError, e: 96 | errors.append(e.message) 97 | continue 98 | mancer_cols = [c['table_id'] for c in m.get_metadata()] 99 | for k, v in field_defs.items(): 100 | field_cols = v['append_columns'] 101 | for f in field_cols: 102 | if f in mancer_cols: 103 | mancer_mapper[f] = { 104 | 'mancer': m, 105 | 'geo_id_map': {}, 106 | 'geo_ids': set(), 107 | 'geo_type': geo_type, 108 | } 109 | for row_idx, row in enumerate(reader): 110 | vals = [re.sub(r'(?i)county', '', unicode(row[int(i)])).strip() \ 111 | for i in col_idxs] 112 | val = val_fmt.format(*vals) 113 | for column in field_cols: 114 | mancer = mancer_mapper[column]['mancer'] 115 | try: 116 | if val: 117 | geoid_search = mancer.geo_lookup(val, geo_type=geo_type) 118 | else: 119 | continue 120 | except MancerError, e: 121 | return 'Error message: %s, Body: %s' % (e.message, e.body) 122 | row_geoid = geoid_search['geoid'] 123 | if row_geoid: 124 | mancer_mapper[column]['geo_ids'].add(row_geoid) 125 | try: 126 | mancer_mapper[column]['geo_id_map'][row_geoid].append(row_idx) 127 | except KeyError: 128 | mancer_mapper[column]['geo_id_map'][row_geoid] = [row_idx] 129 | all_data = {'header': []} 130 | contents.seek(0) 131 | all_rows = list(reader) 132 | included_idxs = set() 133 | header_row = all_rows.pop(0) 134 | output = [[] for i in range(len(all_rows) + 1)] 135 | 136 | response = { 137 | 'download_url': None, 138 | 'geo_col': field_defs.values()[0]['type'], 139 | 'num_rows': len(all_rows), 140 | 'num_matches': 0, 141 | 'num_missing': 0, 142 | 'cols_added': header_row[:], 143 | 'errors': errors, 144 | } 145 | 146 | for column, defs in mancer_mapper.items(): 147 | geo_ids = defs['geo_ids'] 148 | all_data.update({gid:[] for gid in geo_ids}) 149 | geoid_mapper = defs['geo_id_map'] 150 | geo_type = defs['geo_type'] 151 | if geo_ids: 152 | mancer = defs['mancer'] 153 | try: 154 | gids = [(geo_type, g,) for g in list(geo_ids)] 155 | data = mancer.search(geo_ids=gids, columns=[column]) 156 | except MancerError, e: 157 | if client: 158 | client.captureException() 159 | raise e 160 | header_vals = ['{0} ({1})'.format(h, geo_name) for h in data['header']] 161 | all_data['header'].extend(header_vals) 162 | for gid in geo_ids: 163 | try: 164 | all_data[gid].extend(data[gid]) 165 | except KeyError: 166 | all_data[gid].extend(['' for i in data.values()[0]]) 167 | else: 168 | raise MancerError('No geographies matched') 169 | for col in all_data['header']: 170 | if col not in header_row: 171 | header_row.append(col) 172 | for geoid, row_ids in geoid_mapper.items(): 173 | for row_id in row_ids: 174 | included_idxs.add(row_id) 175 | row = all_rows[row_id] 176 | row.extend(all_data[geoid]) 177 | output[row_id] = row 178 | output.insert(0, header_row) 179 | all_row_idxs = set(list(range(len(all_rows)))) 180 | missing_rows = all_row_idxs.difference(included_idxs) 181 | for idx in sorted(missing_rows): 182 | row = all_rows[idx] 183 | diff = len(output[0]) - len(row) 184 | row.extend(['' for i in range(diff)]) 185 | output[idx] = row 186 | response['num_missing'] = len(missing_rows) # store away missing rows 187 | name, ext = os.path.splitext(filename) 188 | fname = '%s_%s%s' % (name, datetime.now().isoformat(), ext) 189 | fpath = '%s/%s' % (RESULT_FOLDER, fname) 190 | if ext == '.xlsx': 191 | writeXLSX(fpath, output) 192 | elif ext == '.xls': 193 | writeXLS(fpath, output) 194 | else: 195 | writeCSV(fpath, output) 196 | response['download_url'] = '/download/%s' % fname 197 | response['num_matches'] = response['num_rows'] - response['num_missing'] 198 | response['cols_added'] = list(set(header_row) - set(response['cols_added'])) 199 | return response 200 | 201 | def writeXLS(fpath, output): 202 | with open(fpath, 'wb') as f: 203 | workbook = xlwt.Workbook(encoding='utf-8') 204 | sheet = workbook.add_sheet('Geomancer Output') 205 | for r, row in enumerate(output): 206 | if row: 207 | for c, col in enumerate(output[0]): 208 | sheet.write(r, c, row[c]) 209 | workbook.save(fpath) 210 | 211 | def writeXLSX(fpath, output): 212 | with open(fpath, 'wb') as f: 213 | workbook = Workbook() 214 | sheet = workbook.active 215 | sheet.title = 'Geomancer Output' 216 | numcols = len(output[0]) 217 | for r, row in enumerate(output): 218 | if row: 219 | sheet.append([row[col_idx] for col_idx in range(numcols)]) 220 | workbook.save(fpath) 221 | 222 | def writeCSV(fpath, output): 223 | with open(fpath, 'wb') as f: 224 | writer = UnicodeCSVWriter(f) 225 | writer.writerows(output) 226 | 227 | def queue_daemon(app, rv_ttl=500): 228 | print 'Mancing commencing...' 229 | while 1: 230 | msg = redis.blpop(app.config['REDIS_QUEUE_KEY']) 231 | func, key, args, kwargs = loads(msg[1]) 232 | try: 233 | rv = func(*args, **kwargs) 234 | rv = {'status': 'ok', 'result': rv} 235 | except Exception, e: 236 | if client: 237 | client.captureException() 238 | tb = traceback.format_exc() 239 | print tb 240 | try: 241 | if e.body: 242 | rv = {'status': 'error', 'result': '{0} message: {1}'.format(e.message, e.body)} 243 | else: 244 | rv = {'status': 'error', 'result': e.message} 245 | except AttributeError: 246 | rv = {'status': 'error', 'result': 'Error: {0}'.format(e.message)} 247 | if rv is not None: 248 | redis.set(key, dumps(rv)) 249 | redis.expire(key, rv_ttl) 250 | -------------------------------------------------------------------------------- /geomancer/mancers/census_reporter.py: -------------------------------------------------------------------------------- 1 | import scrapelib 2 | from urllib import urlencode 3 | import json 4 | import os 5 | import us 6 | from geomancer.helpers import encoded_dict 7 | from geomancer.mancers.base import BaseMancer, MancerError 8 | from geomancer.mancers.geotype import County 9 | from geomancer.app_config import CACHE_DIR 10 | from string import punctuation 11 | import re 12 | 13 | SUMLEV_LOOKUP = { 14 | "county": "050", 15 | } 16 | 17 | class CensusReporter(BaseMancer): 18 | """ 19 | Subclassing the main BaseMancer class 20 | """ 21 | 22 | name = 'Kenya National Bureau of Statistics' 23 | machine_name = 'knbs' 24 | base_url = 'http://api.censusreporter.org/1.0' 25 | info_url = 'http://www.knbs.or.ke/' 26 | description = """ 27 | Demographic data from the 2009 Kenya Census. 28 | """ 29 | 30 | def get_metadata(self): 31 | datasets = [ 32 | { 33 | 'table_id': 'households', 34 | 'human_name': 'Number of housholds', 35 | 'description': 'Number of households', 36 | 'source_name': self.name, 37 | 'source_url': 'http://www.knbs.or.ke/index.php?option=com_content&view=article&id=176&Itemid=645', 38 | 'geo_types': [County()], 39 | 'columns': ['Number of households'], 40 | 'count': 1 41 | }, 42 | { 43 | 'table_id': 'area', 44 | 'human_name': 'Area in Square KM', 45 | 'description': 'Area in square km', 46 | 'source_name': self.name, 47 | 'source_url': 'http://www.knbs.or.ke/index.php?option=com_content&view=article&id=176&Itemid=645', 48 | 'geo_types': [County()], 49 | 'columns': ['Area in sq km'], 50 | 'count': 1 51 | }, 52 | { 53 | 'table_id': 'density', 54 | 'human_name': 'Population density', 55 | 'description': 'Population density', 56 | 'source_name': self.name, 57 | 'source_url': 'http://www.knbs.or.ke/index.php?option=com_content&view=article&id=176&Itemid=645', 58 | 'geo_types': [County()], 59 | 'columns': ['Population density'], 60 | 'count': 1 61 | }, 62 | { 63 | 'table_id': 'male', 64 | 'human_name': 'Male', 65 | 'description': 'male', 66 | 'source_name': self.name, 67 | 'source_url': 'http://www.knbs.or.ke/index.php?option=com_content&view=article&id=176&Itemid=645', 68 | 'geo_types': [County()], 69 | 'columns': ['male'], 70 | 'count': 1 71 | }, 72 | { 73 | 'table_id': 'female', 74 | 'human_name': 'Female', 75 | 'description': 'female', 76 | 'source_name': self.name, 77 | 'source_url': 'http://www.knbs.or.ke/index.php?option=com_content&view=article&id=176&Itemid=645', 78 | 'geo_types': [County()], 79 | 'columns': ['female'], 80 | 'count': 1 81 | }, 82 | { 83 | 'table_id': 'total', 84 | 'human_name': 'Total', 85 | 'description': 'total', 86 | 'source_name': self.name, 87 | 'source_url': 'http://www.knbs.or.ke/index.php?option=com_content&view=article&id=176&Itemid=645', 88 | 'geo_types': [County()], 89 | 'columns': ['total'], 90 | 'count': 1 91 | }, 92 | 93 | ] 94 | return datasets 95 | 96 | def lookup_state(self, term, attr='name'): 97 | return term 98 | 99 | def geo_lookup(self, search_term, geo_type=None): 100 | """ 101 | Search for geoids based upon name of geography 102 | 103 | Returns a response that maps the incoming search term to the geoid: 104 | 105 | { 106 | 'term': , 107 | 'geoid': '', 108 | } 109 | 110 | """ 111 | results = { 112 | 'term': search_term, 113 | 'geoid': search_term 114 | } 115 | return results 116 | 117 | def search(self, geo_ids=None, columns=None): 118 | """ 119 | Response should look like: 120 | { 121 | 'header': [ 122 | 'Sex by Educational Attainment for the Population 25 Years and Over, 5th and 6th grade', 123 | 'Sex by Educational Attainment for the Population 25 Years and Over, 7th and 8th grade' 124 | '...etc...' 125 | ], 126 | '04000US55': [ 127 | 1427.0, 128 | 723.0, 129 | 3246.0, 130 | 760.0, 131 | ...etc..., 132 | ], 133 | '04000US56': [ 134 | 1567.0, 135 | 743.0, 136 | 4453.0, 137 | 657.0, 138 | ...etc..., 139 | ] 140 | } 141 | 142 | The keys are CensusReporter 'geo_ids' and the value is a list that you 143 | should be able to call the python 'zip' function on with the 'header' key. 144 | """ 145 | # these are the tables where we want to leave the table name out 146 | # of the header cell name in output, for prettiness, b/c 147 | # there is redundant info in table_title & detail_title 148 | 149 | DATA = [{'mombasa': ['268,700', '3,079.00', '305.1', '486,924', '452,446', '939,370']}, {'kwale': ['122,047', '1,265.00', '513.8', '315,997', '333,934', '649,931']}, {'kilifi': ['199,764', '2,343.00', '473.6', '535,526', '574,209', '1,109,735']}, {'tanariver': ['47,414', '626', '383.5', '119,853', '120,222', '240,075']}, {'lamu': ['22,184', '265', '383.2', '53,045', '48,494', '101,539']}, {'taitataveta': ['71,090', '971', '293.2', '145,334', '139,323', '284,657']}, {'garissa': ['98,590', '861', '723.7', '334,939', '288,121', '623,060']}, {'wajir': ['88,574', '815', '812.2', '363,766', '298,175', '661,941']}, {'mandera': ['125,497', '1,038.00', '988.2', '559,943', '465,813', '1,025,756']}, {'marsabit': ['56,941', '653', '445.9', '151,112', '140,054', '291,166']}, {'isiolo': ['31,326', '397', '360.9', '73,694', '69,600', '143,294']}, {'meru': ['319,616', '3,196.00', '424.4', '670,656', '685,645', '1,356,301']}, {'tharakanithi': ['88,803', '1,102.00', '331.5', '178,451', '186,879', '365,330']}, {'embu': ['131,683', '1,296.00', '398.3', '254,303', '261,909', '516,212']}, {'kitui': ['205,491', '3,587.00', '282.3', '481,282', '531,427', '1,012,709']}, {'machakos': ['264,500', '3,052.00', '360', '543,139', '555,445', '1,098,584']}, {'makueni': ['186,478', '2,344.00', '377.4', '430,710', '453,817', '884,527']}, {'nyandarua': ['143,879', '1,259.00', '473.6', '292,155', '304,113', '596,268']}, {'nyeri': ['201,703', '2,077.00', '333.9', '339,725', '353,833', '693,558']}, {'kirinyaga': ['154,220', '1,401.00', '376.9', '260,630', '267,424', '528,054']}, {'muranga': ['255,696', '2,517.00', '374.5', '457,864', '484,717', '942,581']}, {'kiambu': ['469,244', '4,946.00', '328.2', '802,609', '820,673', '1,623,282']}, {'turkana': ['123,191', '1,520.00', '562.8', '445,069', '410,330', '855,399']}, {'westpokot': ['93,777', '1,407.00', '364.4', '254,827', '257,863', '512,690']}, {'samburu': ['47,354', '542', '413.2', '112,007', '111,940', '223,947']}, {'transnzoia': ['170,117', '1,611.00', '508.2', '407,172', '411,585', '818,757']}, {'uasingishu': ['202,291', '2,112.00', '423.4', '448,994', '445,185', '894,179']}, {'elgiyomarakwet': ['77,555', '1,107.00', '334.2', '183,738', '186,260', '369,998']}, {'nandi': ['154,073', '1,777.00', '423.7', '376,488', '376,477', '752,965']}, {'baringo': ['110,649', '1,970.00', '282', '279,081', '276,480', '555,561']}, {'laikipia': ['103,114', '1,023.00', '390.3', '198,625', '200,602', '399,227']}, {'nakuru': ['409,836', '4,650.00', '344.8', '804,582', '798,743', '1,603,325']}, {'narok': ['169,220', '1,852.00', '459.5', '429,026', '421,894', '850,920']}, {'kajiado': ['173,464', '1,105.00', '351.6', '345,146', '342,166', '687,312']}, {'bomet': ['142,361', '1,630.00', '444.3', '359,727', '364,459', '724,186']}, {'kericho': ['160,134', '1,886.00', '402.1', '381,980', '376,359', '758,339']}, {'kakamega': ['355,679', '3,343.00', '496.8', '800,989', '859,662', '1,660,651']}, {'vihiga': ['123,347', '1,271.00', '436.4', '262,716', '291,906', '554,622']}, {'bungoma': ['321,628', '3,123.00', '522.2', '710,510', '835,339', '1,630,934']}, {'busia': ['103,421', '1,171.00', '416.8', '232,075', '256,000', '488,075']}, {'siaya': ['199,034', '2,183.00', '385.9', '398,652', '443,652', '842,304']}, {'kisumu': ['226,719', '2,407.00', '402.5', '474,760', '494,149', '968,909']}, {'homabay': ['160,935', '1,754.00', '427.2', '357,273', '392,058', '749,331']}, {'migori': ['41,800', '489', '523.7', '125,938', '130,148', '256,086']}, {'kisii': ['245,029', '2,588.00', '445.2', '550,464', '601,818', '1,152,282']}, {'nyamira': ['131,039', '1,291.00', '463.4', '287,048', '311,204', '598,252']}, {'nairobi': ['985,016', '10,323.00', '304', '1,605,230', '1,533,139', '3,138,369']}] 150 | COLUMNS = ['households', 'area', 'density', 'male', 'female', 'total'] 151 | results = {'header': []} 152 | for column in columns: 153 | results['header'].append(column) 154 | for geo_type, geo_id in geo_ids: 155 | if not results.get(geo_id): 156 | results[geo_id] = [] 157 | try: 158 | results[geo_id].append((item for item in DATA if item.keys()[0].lower().replace('-','').replace(' ','') == geo_id.lower().replace('-','').replace(' ','')).next().values()[0][COLUMNS.index(column)]) 159 | except Exception, e: 160 | print e 161 | return results 162 | -------------------------------------------------------------------------------- /geomancer/mancers/wazimap.py: -------------------------------------------------------------------------------- 1 | import scrapelib 2 | from urllib import urlencode 3 | import json 4 | import os 5 | import us 6 | from geomancer.helpers import encoded_dict 7 | from geomancer.mancers.base import BaseMancer, MancerError 8 | from geomancer.mancers.geotype import County 9 | from geomancer.app_config import CACHE_DIR 10 | from string import punctuation 11 | import re 12 | 13 | SUMLEV_LOOKUP = { 14 | "county": "050", 15 | } 16 | 17 | class Wazimap(BaseMancer): 18 | """ 19 | Subclassing the main BaseMancer class 20 | """ 21 | 22 | name = 'Wazimap Kenya' 23 | machine_name = 'wazimap_ke' 24 | base_url = 'https://kenya.wazimap.org' 25 | info_url = 'https://kenya.wazimap.org' 26 | description = """ 27 | Kenya data based on counties 28 | """ 29 | datasets = None 30 | 31 | def get_metadata(self): 32 | datasets = [ 33 | { 34 | 'table_id': 'waste', 35 | 'human_name': 'Main mode of waste disposal', 36 | 'description': 'Main mode of waste disposal', 37 | 'source_name': self.name, 38 | 'source_url': 'https://kenya.wazimap.org/api/1.0/data/show/latest?table_ids=mainmodeofhumanwastedisposal&geo_ids=', 39 | 'geo_types': [County()], 40 | 'columns': ['total', 'septic tank', 'bucket', 'bush', 'other', 'main sewer', 'cess pool'], 41 | 'count': 7, 42 | 'key': 'mainmodeofhumanwastedisposal' 43 | }, 44 | { 45 | 'table_id': 'water', 46 | 'human_name': 'Main source of water', 47 | 'description': 'Main source of water', 48 | 'source_name': self.name, 49 | 'source_url': 'https://kenya.wazimap.org/api/1.0/data/show/latest?table_ids=mainsourceofwater&geo_ids=', 50 | 'geo_types': [County()], 51 | 'columns': ['water vendor', 'stream', 'jabia/rain/harvested', 'spring/well/borehole', 'pond/dam', 'lake', 'other','piped into dwelling', 'piped', 'total'], 52 | 'count': 10, 53 | 'key': 'mainsourceofwater' 54 | }, 55 | { 56 | 'table_id': 'lighting', 57 | 'human_name': 'Main type of lighting fuel', 58 | 'description': 'Main type of lighting fuel', 59 | 'source_name': self.name, 60 | 'source_url': 'https://kenya.wazimap.org/api/1.0/data/show/latest?table_ids=maintypeoflightingfuel&geo_ids=', 61 | 'geo_types': [County()], 62 | 'columns': ['total', 'electricity', 'gas lamps', 'lanterns', 'other', 'pressure lamps', 'solar', 'tin lamps', 'wood'], 63 | 'count': 9, 64 | 'key': 'maintypeoflightingfuel' 65 | }, 66 | { 67 | 'table_id': 'flooring', 68 | 'human_name': 'Main type of floor material', 69 | 'description': 'Main type of floor material', 70 | 'source_name': self.name, 71 | 'source_url': 'https://kenya.wazimap.org/api/1.0/data/show/latest?table_ids=maintypeoffloormaterial&geo_ids=', 72 | 'geo_types': [County()], 73 | 'columns': ['total', 'cement', 'earth', 'other', 'tiles', 'wood'], 74 | 'count': 6, 75 | 'key': 'maintypeoffloormaterial' 76 | }, 77 | { 78 | 'table_id': 'wall', 79 | 'human_name': 'Main type of wall material', 80 | 'description': 'Main type of wall material', 81 | 'source_name': self.name, 82 | 'source_url': 'https://kenya.wazimap.org/api/1.0/data/show/latest?table_ids=maintypeofwallmaterial&geo_ids=', 83 | 'geo_types': [County()], 84 | 'columns': ['total', 'brick/block', 'corrugated iron sheets', 'grass/reeds', 'mud/cement', 'mud/wood', 'other', 85 | 'stone', 'tin', 'wood only'], 86 | 'count': 10, 87 | 'key': 'maintypeofwallmaterial' 88 | }, 89 | { 90 | 'table_id': 'roofing', 91 | 'human_name': 'Main type of roofing material', 92 | 'description': 'Main type of roofing material', 93 | 'source_name': self.name, 94 | 'source_url': 'https://kenya.wazimap.org/api/1.0/data/show/latest?table_ids=maintypeofroofingmaterial&geo_ids=', 95 | 'geo_types': [County()], 96 | 'columns': ['total', 'asbestos sheets', 'concrete', 'corrugated iron sheets', 'grass', 'makuti', 97 | 'other', 98 | 'mud/dung', 'tin', 'tiles'], 99 | 'count': 10, 100 | 'key': 'maintypeofroofingmaterial' 101 | }, 102 | { 103 | 'table_id': 'education', 104 | 'human_name': 'Highest education level reached', 105 | 'description': 'Highest education level reached', 106 | 'source_name': self.name, 107 | 'source_url': 'https://kenya.wazimap.org/api/1.0/data/show/latest?table_ids=highesteducationlevelreached&geo_ids=', 108 | 'geo_types': [County()], 109 | 'columns': ['total', 'basic literacy', 'madrassa', 'none', 'pre-primary', 'primary', 110 | 'secondary', 111 | 'university', 'tertiary', 'youth polytechnic'], 112 | 'count': 10, 113 | 'key': 'highesteducationlevelreached' 114 | }, 115 | 116 | ] 117 | self.datasets = datasets 118 | return datasets 119 | 120 | def lookup_state(self, term, attr='name'): 121 | return term 122 | 123 | def geo_lookup(self, search_term, geo_type=None): 124 | lookup_table = [ 125 | ["county","1","Mombasa"], 126 | ["county","2","Kwale"], 127 | ["county","3","Kilifi"], 128 | ["county","4","TanaRiver"], 129 | ["county","5","Lamu"], 130 | ["county","6","TaitaTaveta"], 131 | ["county","7","Garissa"], 132 | ["county","8","Wajir"], 133 | ["county","9","Mandera"], 134 | ["county","10","Marsabit"], 135 | ["county","11","Isiolo"], 136 | ["county","12","Meru"], 137 | ["county","13","TharakaNithi"], 138 | ["county","14","Embu"], 139 | ["county","15","Kitui"], 140 | ["county","16","Machakos"], 141 | ["county","17","Makueni"], 142 | ["county","18","Nyandarua"], 143 | ["county","19","Nyeri"], 144 | ["county","20","Kirinyaga"], 145 | ["county","21","Murang'a"], 146 | ["county","22","Kiambu"], 147 | ["county","23","Turkana"], 148 | ["county","24","WestPokot"], 149 | ["county","25","Samburu"], 150 | ["county","26","TransNzoia"], 151 | ["county","27","UasinGishu"], 152 | ["county","28","ElgeyoMarakwet"], 153 | ["county","29","Nandi"], 154 | ["county","30","Baringo"], 155 | ["county","31","Laikipia"], 156 | ["county","32","Nakuru"], 157 | ["county","33","Narok"], 158 | ["county","34","Kajiado"], 159 | ["county","35","Kericho"], 160 | ["county","36","Bomet"], 161 | ["county","37","Kakamega"], 162 | ["county","38","Vihiga"], 163 | ["county","39","Bungoma"], 164 | ["county","40","Busia"], 165 | ["county","41","Siaya"], 166 | ["county","42","Kisumu"], 167 | ["county","43","HomaBay"], 168 | ["county","44","Migori"], 169 | ["county","45","Kisii"], 170 | ["county","46","Nyamira"], 171 | ["county","47","Nairobi"], 172 | ["country","KE","Kenya"], 173 | ] 174 | geoid = None 175 | for l in lookup_table: 176 | if search_term.lower().replace(' ', '').replace('-', '') == l[2].lower(): 177 | geoid = l[0] + '-' + l[1] 178 | results = { 179 | 'term': search_term, 180 | 'geoid': geoid 181 | } 182 | return results 183 | 184 | def search(self, geo_ids=None, columns=None): 185 | """ 186 | Response should look like: 187 | { 188 | 'header': [ 189 | 'Sex by Educational Attainment for the Population 25 Years and Over, 5th and 6th grade', 190 | 'Sex by Educational Attainment for the Population 25 Years and Over, 7th and 8th grade' 191 | '...etc...' 192 | ], 193 | '04000US55': [ 194 | 1427.0, 195 | 723.0, 196 | 3246.0, 197 | 760.0, 198 | ...etc..., 199 | ], 200 | '04000US56': [ 201 | 1567.0, 202 | 743.0, 203 | 4453.0, 204 | 657.0, 205 | ...etc..., 206 | ] 207 | } 208 | 209 | The keys are CensusReporter 'geo_ids' and the value is a list that you 210 | should be able to call the python 'zip' function on with the 'header' key. 211 | """ 212 | # these are the tables where we want to leave the table name out 213 | # of the header cell name in output, for prettiness, b/c 214 | # there is redundant info in table_title & detail_title 215 | results = {'header': []} 216 | for geo_type, geo_id in geo_ids: 217 | table_id = columns[0] 218 | details = [] 219 | key = None 220 | human_name = None 221 | for y in self.datasets: 222 | if y['table_id'] == table_id: 223 | details = y['columns'] 224 | key = y['key'] 225 | human_name = y['human_name'] 226 | break 227 | if not results.get(geo_id): 228 | results[geo_id] = [] 229 | # try: 230 | info = self.urlopen('%s/api/1.0/data/show/latest?table_ids=%s&geo_ids=%s' % (self.base_url, key, geo_id)) 231 | data_info = json.loads(info)['data'][geo_id][key.upper()]['estimate'] 232 | for k in details: 233 | if k not in results['header']: 234 | h = k 235 | if k == 'total': h = 'total ('+ human_name +')' 236 | results['header'].append(h.title()) 237 | results[geo_id].append(data_info[k]) 238 | # except Exception, e: 239 | # print e 240 | return results 241 | -------------------------------------------------------------------------------- /geomancer/mancers/census_reporter_us.py: -------------------------------------------------------------------------------- 1 | import scrapelib 2 | from urllib import urlencode 3 | import json 4 | import os 5 | import us 6 | from geomancer.helpers import encoded_dict 7 | from geomancer.mancers.base import BaseMancer, MancerError 8 | from geomancer.mancers.geotype import City, State, StateFIPS, StateCountyFIPS, \ 9 | Zip5, Zip9, County, SchoolDistrict, CongressionalDistrict, CensusTract 10 | from geomancer.app_config import CACHE_DIR 11 | from string import punctuation 12 | import re 13 | 14 | SUMLEV_LOOKUP = { 15 | "city": "160,170,060", 16 | "state": "040", 17 | "state_fips": "040", 18 | "state_county_fips":"050", 19 | "zip_5": "850,860", 20 | "zip_9": "850,860", 21 | #"state_postal": "040", 22 | "county": "050", 23 | "school_district": "950,960,970", 24 | "congress_district": "500", # Assuming US Congressional District 25 | "census_tract": "140", 26 | } 27 | 28 | class CensusReporter(BaseMancer): 29 | """ 30 | Subclassing the main BaseMancer class 31 | """ 32 | 33 | name = 'Census Reporter' 34 | machine_name = 'census_reporter' 35 | base_url = 'http://api.censusreporter.org/1.0' 36 | info_url = 'http://censusreporter.org' 37 | description = """ 38 | Demographic data from the 2013 American Community Survey. 39 | """ 40 | 41 | def get_metadata(self): 42 | table_ids = [ 43 | "B01003", # Total Population 44 | "B19013", # Median Household Income 45 | "B19301", # Per Capita Income 46 | "B02001", # Race 47 | "B01002", # Median Age by Sex" 48 | "B25077", # Median Value (Dollars) 49 | "B26001", # Group Quarters Population 50 | "B11009", # Unmarried-partner Households by Sex of Partner 51 | "B05006", # Place of Birth for the Foreign-born Population in the United States 52 | "B19083", # Gini Index of Income Inequality 53 | "B15003", # Educational Attainment 54 | "B03002", # Hispanic or Latino Origin by Race 55 | ] 56 | columns = [] 57 | for table in table_ids: 58 | info = self.urlopen('%s/table/%s' % (self.base_url, table)) 59 | table_info = json.loads(info) 60 | d = { 61 | 'table_id': table, 62 | 'human_name': table_info['table_title'], 63 | 'description': '', 64 | 'source_name': self.name, 65 | 'source_url': 'http://censusreporter.org/tables/%s/' % table, 66 | 'geo_types': [City(), State(), StateFIPS(), StateCountyFIPS(), Zip5(), 67 | Zip9(), County(), SchoolDistrict(), 68 | CongressionalDistrict(), CensusTract()], 69 | 'columns': [v['column_title'] for v in table_info['columns'].values() if v['indent'] is not None] 70 | } 71 | 72 | d['columns'].extend(['%s (error margin)' % v for v in d['columns']]) 73 | d['columns'] = sorted(d['columns']) 74 | 75 | if table == 'B25077': # Overriding the name for "Median Value" table 76 | d['human_name'] = 'Median Value, Owner-Occupied Housing Units' 77 | d['columns'] = ['Median Value, Owner-Occupied Housing Units', 78 | 'Median Value, Owner-Occupied Housing Units (error margin)'] 79 | 80 | d['count'] = len(d['columns']) 81 | columns.append(d) 82 | return columns 83 | 84 | def lookup_state(self, term, attr='name'): 85 | st = us.states.lookup(term) 86 | if not st: 87 | st = [s for s in us.STATES if getattr(s, 'ap_abbr') == term] 88 | if st: 89 | return getattr(st, attr) 90 | else: 91 | return term 92 | 93 | def geo_lookup(self, search_term, geo_type=None): 94 | """ 95 | Search for geoids based upon name of geography 96 | 97 | Returns a response that maps the incoming search term to the geoid: 98 | 99 | { 100 | 'term': , 101 | 'geoid': '', 102 | } 103 | 104 | """ 105 | if geo_type == 'congress_district': 106 | geoid = None 107 | dist, st = search_term.rsplit(',', 1) 108 | fips = self.lookup_state(st.strip(), attr='fips') 109 | try: 110 | dist_num = str(int(dist.split(' ')[-1])) 111 | except ValueError: 112 | dist_num = '00' 113 | if fips and dist_num: 114 | geoid = '50000US{0}{1}'\ 115 | .format(fips, dist_num.zfill(2)) 116 | return { 117 | 'term': search_term, 118 | 'geoid': geoid 119 | } 120 | regex = re.compile('[%s]' % re.escape(punctuation)) 121 | search_term = regex.sub('', search_term) 122 | if geo_type in ['census_tract', 'state_fips']: 123 | return { 124 | 'term': search_term, 125 | 'geoid': '%s00US%s' % (SUMLEV_LOOKUP[geo_type], search_term) 126 | } 127 | if geo_type == 'state_county_fips': 128 | resp = { 129 | 'term': search_term, 130 | 'geoid': None 131 | } 132 | g = StateCountyFIPS() 133 | valid, message = g.validate([search_term]) 134 | if valid: 135 | resp['geoid'] = '05000US%s' % search_term 136 | return resp 137 | q_dict = {'q': search_term} 138 | if geo_type: 139 | q_dict['sumlevs'] = SUMLEV_LOOKUP[geo_type] 140 | if geo_type == 'zip_5': 141 | q_dict['q'] = search_term.zfill(5) 142 | if geo_type == 'state': 143 | q_dict['q'] = self.lookup_state(search_term) 144 | q_dict = encoded_dict(q_dict) 145 | 146 | params = urlencode(q_dict) 147 | try: 148 | response = self.urlopen('%s/geo/search?%s' % (self.base_url, params)) 149 | except scrapelib.HTTPError, e: 150 | try: 151 | body = json.loads(e.body.json()['error']) 152 | except ValueError: 153 | body = None 154 | raise MancerError('Census Reporter API returned a %s status' \ 155 | % response.status_code, body=body) 156 | results = json.loads(response) 157 | try: 158 | results = { 159 | 'term': search_term, 160 | 'geoid': results['results'][0]['full_geoid'] 161 | } 162 | except IndexError: 163 | results = { 164 | 'term': search_term, 165 | 'geoid': None, 166 | } 167 | return results 168 | 169 | def _chunk_geoids(self, geo_ids): 170 | for i in xrange(0, len(geo_ids), 100): 171 | yield geo_ids[i:i+100] 172 | 173 | def _try_search(self, gids, columns, bad_gids=[]): 174 | query = { 175 | 'table_ids': ','.join(columns), 176 | 'geo_ids': ','.join(sorted([g[1] for g in gids])), 177 | } 178 | params = urlencode(query) 179 | try: 180 | response = self.urlopen('%s/data/show/latest?%s' % (self.base_url, params)) 181 | except scrapelib.HTTPError, e: 182 | try: 183 | body = json.loads(e.body.json()['error']) 184 | except ValueError: 185 | body = None 186 | except AttributeError: 187 | body = e.body 188 | if 'The ACS 2013 5-year release doesn\'t include GeoID(s)' in body: 189 | error = json.loads(body) 190 | bad_gids.append(error['error'].rsplit(' ',1)[1].replace('.', '')) 191 | for idx,gid in enumerate(gids): 192 | if gid[1] in bad_gids: 193 | gids.pop(idx) 194 | response = self._try_search(gids, columns, bad_gids=bad_gids) 195 | else: 196 | raise MancerError('Census Reporter API returned an error', body=body) 197 | return response 198 | 199 | 200 | def search(self, geo_ids=None, columns=None): 201 | """ 202 | Response should look like: 203 | { 204 | 'header': [ 205 | 'Sex by Educational Attainment for the Population 25 Years and Over, 5th and 6th grade', 206 | 'Sex by Educational Attainment for the Population 25 Years and Over, 7th and 8th grade' 207 | '...etc...' 208 | ], 209 | '04000US55': [ 210 | 1427.0, 211 | 723.0, 212 | 3246.0, 213 | 760.0, 214 | ...etc..., 215 | ], 216 | '04000US56': [ 217 | 1567.0, 218 | 743.0, 219 | 4453.0, 220 | 657.0, 221 | ...etc..., 222 | ] 223 | } 224 | 225 | The keys are CensusReporter 'geo_ids' and the value is a list that you 226 | should be able to call the python 'zip' function on with the 'header' key. 227 | """ 228 | # these are the tables where we want to leave the table name out 229 | # of the header cell name in output, for prettiness, b/c 230 | # there is redundant info in table_title & detail_title 231 | table_name_exceptions = [ 'Median Household Income in the Past 12 Months (In 2013 Inflation-adjusted Dollars)', 232 | 'Per Capita Income in the Past 12 Months (In 2013 Inflation-adjusted Dollars)', 233 | ] 234 | 235 | results = {'header': []} 236 | for gids in self._chunk_geoids(geo_ids): 237 | raw_results = self._try_search(gids, columns) 238 | raw_results = json.loads(raw_results) 239 | for geo_type, geo_id in gids: 240 | if not results.get(geo_id): 241 | results[geo_id] = [] 242 | for table_id in columns: 243 | table_info = raw_results['tables'][table_id] 244 | title = table_info['title'] 245 | detail_ids = [k for k in table_info['columns'].keys() \ 246 | if table_info['columns'][k].get('indent') is not None] 247 | denominator = table_info['denominator_column_id'] 248 | for detail_id in detail_ids: 249 | table_title = table_info['title'] 250 | column_title = None 251 | detail_title = table_info['columns'][detail_id]['name'] 252 | if table_title in table_name_exceptions: 253 | column_title = detail_title 254 | elif table_id == 'B25077': 255 | column_title = 'Median Value, Owner-Occupied Housing Units' 256 | else: 257 | column_title = '%s, %s' % (table_title, detail_title,) 258 | if column_title not in results['header']: 259 | results['header'].extend([column_title, '%s (error margin)' % column_title]) 260 | 261 | detail_info = raw_results['data'][geo_id][table_id] 262 | results[geo_id].extend([ 263 | detail_info['estimate'][detail_id], 264 | detail_info['error'][detail_id], 265 | ]) 266 | return results 267 | -------------------------------------------------------------------------------- /geomancer/static/js/ejs_production.js: -------------------------------------------------------------------------------- 1 | (function(){var rsplit=function(string,regex){var result=regex.exec(string),retArr=new Array(),first_idx,last_idx,first_bit;while(result!=null){first_idx=result.index;last_idx=regex.lastIndex;if((first_idx)!=0){first_bit=string.substring(0,first_idx);retArr.push(string.substring(0,first_idx));string=string.slice(first_idx)}retArr.push(result[0]);string=string.slice(result[0].length);result=regex.exec(string)}if(!string==""){retArr.push(string)}return retArr},chop=function(string){return string.substr(0,string.length-1)},extend=function(d,s){for(var n in s){if(s.hasOwnProperty(n)){d[n]=s[n]}}};EJS=function(options){options=typeof options=="string"?{view:options}:options;this.set_options(options);if(options.precompiled){this.template={};this.template.process=options.precompiled;EJS.update(this.name,this);return }if(options.element){if(typeof options.element=="string"){var name=options.element;options.element=document.getElementById(options.element);if(options.element==null){throw name+"does not exist!"}}if(options.element.value){this.text=options.element.value}else{this.text=options.element.innerHTML}this.name=options.element.id;this.type="["}else{if(options.url){options.url=EJS.endExt(options.url,this.extMatch);this.name=this.name?this.name:options.url;var url=options.url;var template=EJS.get(this.name,this.cache);if(template){return template}if(template==EJS.INVALID_PATH){return null}try{this.text=EJS.request(url+(this.cache?"":"?"+Math.random()))}catch(e){}if(this.text==null){throw ({type:"EJS",message:"There is no template at "+url})}}}var template=new EJS.Compiler(this.text,this.type);template.compile(options,this.name);EJS.update(this.name,this);this.template=template};EJS.prototype={render:function(object,extra_helpers){object=object||{};this._extra_helpers=extra_helpers;var v=new EJS.Helpers(object,extra_helpers||{});return this.template.process.call(object,object,v)},update:function(element,options){if(typeof element=="string"){element=document.getElementById(element)}if(options==null){_template=this;return function(object){EJS.prototype.update.call(_template,element,object)}}if(typeof options=="string"){params={};params.url=options;_template=this;params.onComplete=function(request){var object=eval(request.responseText);EJS.prototype.update.call(_template,element,object)};EJS.ajax_request(params)}else{element.innerHTML=this.render(options)}},out:function(){return this.template.out},set_options:function(options){this.type=options.type||EJS.type;this.cache=options.cache!=null?options.cache:EJS.cache;this.text=options.text||null;this.name=options.name||null;this.ext=options.ext||EJS.ext;this.extMatch=new RegExp(this.ext.replace(/\./,"."))}};EJS.endExt=function(path,match){if(!path){return null}match.lastIndex=0;return path+(match.test(path)?"":this.ext)};EJS.Scanner=function(source,left,right){extend(this,{left_delimiter:left+"%",right_delimiter:"%"+right,double_left:left+"%%",double_right:"%%"+right,left_equal:left+"%=",left_comment:left+"%#"});this.SplitRegexp=left=="["?/(\[%%)|(%%\])|(\[%=)|(\[%#)|(\[%)|(%\]\n)|(%\])|(\n)/:new RegExp("("+this.double_left+")|(%%"+this.double_right+")|("+this.left_equal+")|("+this.left_comment+")|("+this.left_delimiter+")|("+this.right_delimiter+"\n)|("+this.right_delimiter+")|(\n)");this.source=source;this.stag=null;this.lines=0};EJS.Scanner.to_text=function(input){if(input==null||input===undefined){return""}if(input instanceof Date){return input.toDateString()}if(input.toString){return input.toString()}return""};EJS.Scanner.prototype={scan:function(block){scanline=this.scanline;regex=this.SplitRegexp;if(!this.source==""){var source_split=rsplit(this.source,/\n/);for(var i=0;i0){for(var i=0;i0){buff.push(put_cmd+'"'+clean(content)+'")')}content="";break;case scanner.double_left:content=content+scanner.left_delimiter;break;default:content=content+token;break}}else{switch(token){case scanner.right_delimiter:switch(scanner.stag){case scanner.left_delimiter:if(content[content.length-1]=="\n"){content=chop(content);buff.push(content);buff.cr()}else{buff.push(content)}break;case scanner.left_equal:buff.push(insert_cmd+"(EJS.Scanner.to_text("+content+")))");break}scanner.stag=null;content="";break;case scanner.double_right:content=content+scanner.right_delimiter;break;default:content=content+token;break}}});if(content.length>0){buff.push(put_cmd+'"'+clean(content)+'")')}buff.close();this.out=buff.script+";";var to_be_evaled="/*"+name+"*/this.process = function(_CONTEXT,_VIEW) { try { with(_VIEW) { with (_CONTEXT) {"+this.out+" return ___ViewO.join('');}}}catch(e){e.lineNumber=null;throw e;}};";try{eval(to_be_evaled)}catch(e){if(typeof JSLINT!="undefined"){JSLINT(this.out);for(var i=0;i").replace(/''/g,"'")}return""}};EJS.newRequest=function(){var factories=[function(){return new ActiveXObject("Msxml2.XMLHTTP")},function(){return new XMLHttpRequest()},function(){return new ActiveXObject("Microsoft.XMLHTTP")}];for(var i=0;i")};EJS.Helpers.prototype.start_tag_for=function(A,B){return this.tag(A,B)};EJS.Helpers.prototype.submit_tag=function(A,B){B=B||{};B.type=B.type||"submit";B.value=A||"Submit";return this.single_tag_for("input",B)};EJS.Helpers.prototype.tag=function(C,E,D){if(!D){var D=">"}var B=" ";for(var A in E){if(E[A]!=null){var F=E[A].toString()}else{var F=""}if(A=="Class"){A="class"}if(F.indexOf("'")!=-1){B+=A+'="'+F+'" '}else{B+=A+"='"+F+"' "}}return"<"+C+B+D};EJS.Helpers.prototype.tag_end=function(A){return""};EJS.Helpers.prototype.text_area_tag=function(A,C,B){B=B||{};B.id=B.id||A;B.name=B.name||A;C=C||"";if(B.size){B.cols=B.size.split("x")[0];B.rows=B.size.split("x")[1];delete B.size}B.cols=B.cols||50;B.rows=B.rows||4;return this.start_tag_for("textarea",B)+C+this.tag_end("textarea")};EJS.Helpers.prototype.text_tag=EJS.Helpers.prototype.text_area_tag;EJS.Helpers.prototype.text_field_tag=function(A,C,B){return this.input_field_tag(A,C,"text",B)};EJS.Helpers.prototype.url_for=function(A){return'window.location="'+A+'";'};EJS.Helpers.prototype.img_tag=function(B,C,A){A=A||{};A.src=B;A.alt=C;return this.single_tag_for("img",A)} -------------------------------------------------------------------------------- /geomancer/templates/contribute-data.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | {% block title %}Contribute data - Geomancer{% endblock %} 3 | {% block content %} 4 | 5 |
6 |

How to contribute to Geomancer

7 |

8 | Is there data that you'd like to add to Geomancer? 9 | We built this as an open, extensible platform, so anyone can contribute. 10 | If you know how to program in Python, you can adapt Geomancer to your needs using the 11 | instructions below. Otherwise, post your request to our 12 | Google Group. 13 |

14 |
15 | 16 |
17 |
18 |
19 | 20 | 21 | Setup 22 | 23 |
24 |
25 |
26 |
27 |

28 | 1. Fork & clone Geomancer 29 |

30 |

First, fork the geomancer repository. This requires a GitHub account.

31 |

Then, clone your Geomancer fork:

32 |
 33 | $ git clone https://github.com/YOURGITHUBUSERNAME/geomancer.git
 34 | $ cd geomancer
35 | 36 |

37 | 2. Install requirements 38 |

39 |

Make sure OS level dependencies are installed:

40 |
    41 |
  • Python 2.7
  • 42 |
  • Redis
  • 43 |
  • libxml2
  • 44 |
  • libxml2-dev
  • 45 |
  • libxslt1-dev
  • 46 |
  • zlib1g-dev
  • 47 |
48 |

Install required python libraries. We recommend using virtualenv and virtualenvwrapper for working in a virtualized development environment. Read how to set up virtualenv.

49 |

Once you have virtualenv set up:

50 |
 51 | $ mkvirtualenv geomancer
 52 | $ pip install -r requirements.txt
53 |

NOTE: Mac users might need this lxml workaround.

54 |

Afterwards, whenever you want to work on geomancer using the virtual environment you created:

55 |
$ workon geomancer
56 | 57 |

3. Configure Geomancer

58 |
 59 | $ cp geomancer/app_config.py.example geomancer/app_config.py
60 | In your newly created app_config.py file, the active data sources are defined by MANCERS. Add any relevant API keys to MANCER_KEYS. 61 | 62 |

4. Run Geomancer locally

63 |

There are three components that should be running simultaneously for the app to work: Redis, the Flask app, and the worker process that appends to the spreadsheets. For debugging purposes, it is useful to run each of these commands in a separate terminal session.

64 |
 65 | $ redis-server # This command may differ depending on your OS
 66 | $ python runworker.py # starts the worker for processing files
 67 | $ python runserver.py # starts the web server
68 | 69 |

Open your browser and navigate to http://localhost:5000

70 |

5. Make your changes!

71 |

If you'd like to share your work with the rest of the world, submit a pull request with your changes!

72 | 73 | 74 | 75 |
76 |
77 |
78 |
79 | 80 |
81 |
82 |
83 | 84 | 85 | Overview of the Extensible Design 86 | 87 |
88 |
89 |
90 |
91 |

92 | Each data source corresponds to a 'mancer' in geomancer/mancers/. For example, the BureauLaborStatistics class in geomancer/mancers/bls.py is the 'mancer' for BLS. 93 |

94 |

95 | The datasets and columns that are available to a mancer are defined manually. Mancer datasets usually correspond to tables at the source, and mancer columns correspond to the columns within a table. 96 |

97 |

98 | The methods defined for all mancers are: 99 |

    100 |
  • 101 | get_metadata
    102 | This method returns information about the datasets and columns included in the data source, and is used to populate the data sources page. This is a static method. 103 |
  • 104 |
  • 105 | geo_lookup
    106 | Given a search term and a geography type, this method returns a dictionary with the keys term (the original search term) and geoid (the full geographic id to be used by the search method). If need be, this method can look up geographic ids through specific APIs. 107 |
  • 108 |
  • 109 | search
    110 | Given a list of geography ids and a list of columns to append, the search method returns all the data to be appended to the original spreadsheet: the appropriate values for each column & each geography, as well as a header. 111 |
  • 112 |
113 |

114 | 115 | 116 |
117 |
118 |
119 |
120 | 121 |
122 |
123 |
124 | 125 | 126 | Add columns or datasets to an existing data source 127 | 128 |
129 |
130 |
131 |
132 |

133 | Because the datasets and columns are defined manually, and because of the high granularity of available data, the mancers don't include all possible data from a data source. For example, BLS QCEW data is available for a wide range of geographies and many industries, but the BLS mancer currently only has data at the state level and at the highest industry summary level (all industries). 134 |

135 |

Adding Data

136 |

137 | If you're interested in adding data to an existing mancer (say, adding columns with statistics for specific industries to the BLS mancer), all you'll need to do is modify the mancer metadata (defined in the get_metadata method) and ensure that the search method knows how to return the data you've added. 138 |

139 |
140 |
141 |
142 |
143 | 144 |
145 |
146 |
147 | 148 | 149 | Add a new data source 150 | 151 |
152 |
153 |
154 |
155 | Each data source corresponds to a 'mancer' in geomancer/mancers/. 156 | 157 |

1. Create a new class for your data source

158 |

159 | Geomancer implements a base class that establishes a pattern for setting up a new data source. 160 | In a new .py file in geomancer/mancers/, inherit from the BaseMancer class like so: 161 |

162 |
163 | from geomancer.mancers.base import BaseMancer
164 | 
165 | class MyGreatMancer(BaseMancer):
166 | 
167 |     name = 'My Great Mancer'
168 |     machine_name = 'my_great_mancer'
169 |     base_url = 'http://lotsadata.gov/api'
170 |     info_url = 'http://lotsadata.gov'
171 |     description = 'This is probably the best mancer ever written'
172 | 
173 |     def get_metadata(self):
174 |         return 'woo'
175 | 
176 |     def search(self, geo_ids=None, columns=None):
177 |         return 'woo'
178 | 
179 |     def geo_lookup(self, search_term, geo_type=None):
180 |         return 'woo'
181 | 182 |

Override the name, machine_name, base_url, info_url, & description properties accordingly.

183 | 184 |

2. Implement class methods for your mancer

185 |

This is the bulk of the work. You will need to implement the get_metadata, search, & geo_lookup methods. 186 |

187 | Detailed information about how the responses from these 188 | methods should be structured as well as two example mancers 189 | can be found in the 190 | Github repository. 191 |

192 | 193 |

3. Register your mancer in the application cofiguration

194 |

The basic configuration options for Geomancer exist in app_config.py. Add the import path to the module where you wrote your mancer and you should start seeing it as an option when you run the app. 195 |

196 |
197 | MANCERS = (
198 |     'geomancer.mancers.census_reporter.CensusReporter',
199 |     'geomancer.mancers.usa_spending.USASpending',
200 |     'geomancer.mancers.my_mancer.MyMancer',
201 | )
202 |

If your data source requires an API key, add the API key to app_config.py:

203 |
204 | MANCER_KEYS = {
205 |     'my_great_mancer' : 'biGl0ngUu1dT4ing',
206 | }
207 |

208 | The key (i.e. my_great_mancer) should match the value you used for the machine_name property of your mancer class. 209 |

210 | 211 | 212 | 213 |
214 |
215 |
216 |
217 | 218 |
219 |
220 |
221 | 222 | 223 | Add a new geography type 224 | 225 |
226 |
227 |
228 |
229 | 230 |

231 | Do you have data with geography types that are not 232 | currently offered by Geomancer? The geography types are built in an extensible way - each 233 | geography is implemented as a GeoType subclass that expects 234 | a few static properties to be overridden: 235 |

236 |
237 | from geomancer.mancers.geotype import GeoType
238 | 
239 | class WizardSchoolDistrict(GeoType):
240 |     human_name = 'Wizard school district'
241 |     machine_name = 'wizard_school_district'
242 |     formatting_notes = 'Full name of a school district for wizards and witches'
243 |     formatting_example = 'Hogwarts School of Witchcraft and Wizardry'
244 |

245 | More details on how to implement a new GeoType can be found 246 | here. 247 |

248 | 249 | 250 | 251 | 252 |
253 |
254 |
255 |
256 | 257 | 258 |
259 | 260 | {% endblock %} 261 | 262 | {% block extra_javascript %} 263 | 274 | {% endblock %} 275 | -------------------------------------------------------------------------------- /geomancer/static/js/moment.min.js: -------------------------------------------------------------------------------- 1 | // moment.js 2 | // version : 2.1.0 3 | // author : Tim Wood 4 | // license : MIT 5 | // momentjs.com 6 | !function(t){function e(t,e){return function(n){return u(t.call(this,n),e)}}function n(t,e){return function(n){return this.lang().ordinal(t.call(this,n),e)}}function s(){}function i(t){a(this,t)}function r(t){var e=t.years||t.year||t.y||0,n=t.months||t.month||t.M||0,s=t.weeks||t.week||t.w||0,i=t.days||t.day||t.d||0,r=t.hours||t.hour||t.h||0,a=t.minutes||t.minute||t.m||0,o=t.seconds||t.second||t.s||0,u=t.milliseconds||t.millisecond||t.ms||0;this._input=t,this._milliseconds=u+1e3*o+6e4*a+36e5*r,this._days=i+7*s,this._months=n+12*e,this._data={},this._bubble()}function a(t,e){for(var n in e)e.hasOwnProperty(n)&&(t[n]=e[n]);return t}function o(t){return 0>t?Math.ceil(t):Math.floor(t)}function u(t,e){for(var n=t+"";n.lengthn;n++)~~t[n]!==~~e[n]&&r++;return r+i}function f(t){return t?ie[t]||t.toLowerCase().replace(/(.)s$/,"$1"):t}function l(t,e){return e.abbr=t,x[t]||(x[t]=new s),x[t].set(e),x[t]}function _(t){if(!t)return H.fn._lang;if(!x[t]&&A)try{require("./lang/"+t)}catch(e){return H.fn._lang}return x[t]}function m(t){return t.match(/\[.*\]/)?t.replace(/^\[|\]$/g,""):t.replace(/\\/g,"")}function y(t){var e,n,s=t.match(E);for(e=0,n=s.length;n>e;e++)s[e]=ue[s[e]]?ue[s[e]]:m(s[e]);return function(i){var r="";for(e=0;n>e;e++)r+=s[e]instanceof Function?s[e].call(i,t):s[e];return r}}function M(t,e){function n(e){return t.lang().longDateFormat(e)||e}for(var s=5;s--&&N.test(e);)e=e.replace(N,n);return re[e]||(re[e]=y(e)),re[e](t)}function g(t,e){switch(t){case"DDDD":return V;case"YYYY":return X;case"YYYYY":return $;case"S":case"SS":case"SSS":case"DDD":return I;case"MMM":case"MMMM":case"dd":case"ddd":case"dddd":return R;case"a":case"A":return _(e._l)._meridiemParse;case"X":return B;case"Z":case"ZZ":return j;case"T":return q;case"MM":case"DD":case"YY":case"HH":case"hh":case"mm":case"ss":case"M":case"D":case"d":case"H":case"h":case"m":case"s":return J;default:return new RegExp(t.replace("\\",""))}}function p(t){var e=(j.exec(t)||[])[0],n=(e+"").match(ee)||["-",0,0],s=+(60*n[1])+~~n[2];return"+"===n[0]?-s:s}function D(t,e,n){var s,i=n._a;switch(t){case"M":case"MM":i[1]=null==e?0:~~e-1;break;case"MMM":case"MMMM":s=_(n._l).monthsParse(e),null!=s?i[1]=s:n._isValid=!1;break;case"D":case"DD":case"DDD":case"DDDD":null!=e&&(i[2]=~~e);break;case"YY":i[0]=~~e+(~~e>68?1900:2e3);break;case"YYYY":case"YYYYY":i[0]=~~e;break;case"a":case"A":n._isPm=_(n._l).isPM(e);break;case"H":case"HH":case"h":case"hh":i[3]=~~e;break;case"m":case"mm":i[4]=~~e;break;case"s":case"ss":i[5]=~~e;break;case"S":case"SS":case"SSS":i[6]=~~(1e3*("0."+e));break;case"X":n._d=new Date(1e3*parseFloat(e));break;case"Z":case"ZZ":n._useUTC=!0,n._tzm=p(e)}null==e&&(n._isValid=!1)}function Y(t){var e,n,s=[];if(!t._d){for(e=0;7>e;e++)t._a[e]=s[e]=null==t._a[e]?2===e?1:0:t._a[e];s[3]+=~~((t._tzm||0)/60),s[4]+=~~((t._tzm||0)%60),n=new Date(0),t._useUTC?(n.setUTCFullYear(s[0],s[1],s[2]),n.setUTCHours(s[3],s[4],s[5],s[6])):(n.setFullYear(s[0],s[1],s[2]),n.setHours(s[3],s[4],s[5],s[6])),t._d=n}}function w(t){var e,n,s=t._f.match(E),i=t._i;for(t._a=[],e=0;eo&&(u=o,s=n);a(t,s)}function v(t){var e,n=t._i,s=K.exec(n);if(s){for(t._f="YYYY-MM-DD"+(s[2]||" "),e=0;4>e;e++)if(te[e][1].exec(n)){t._f+=te[e][0];break}j.exec(n)&&(t._f+=" Z"),w(t)}else t._d=new Date(n)}function T(e){var n=e._i,s=G.exec(n);n===t?e._d=new Date:s?e._d=new Date(+s[1]):"string"==typeof n?v(e):d(n)?(e._a=n.slice(0),Y(e)):e._d=n instanceof Date?new Date(+n):new Date(n)}function b(t,e,n,s,i){return i.relativeTime(e||1,!!n,t,s)}function S(t,e,n){var s=W(Math.abs(t)/1e3),i=W(s/60),r=W(i/60),a=W(r/24),o=W(a/365),u=45>s&&["s",s]||1===i&&["m"]||45>i&&["mm",i]||1===r&&["h"]||22>r&&["hh",r]||1===a&&["d"]||25>=a&&["dd",a]||45>=a&&["M"]||345>a&&["MM",W(a/30)]||1===o&&["y"]||["yy",o];return u[2]=e,u[3]=t>0,u[4]=n,b.apply({},u)}function F(t,e,n){var s,i=n-e,r=n-t.day();return r>i&&(r-=7),i-7>r&&(r+=7),s=H(t).add("d",r),{week:Math.ceil(s.dayOfYear()/7),year:s.year()}}function O(t){var e=t._i,n=t._f;return null===e||""===e?null:("string"==typeof e&&(t._i=e=_().preparse(e)),H.isMoment(e)?(t=a({},e),t._d=new Date(+e._d)):n?d(n)?k(t):w(t):T(t),new i(t))}function z(t,e){H.fn[t]=H.fn[t+"s"]=function(t){var n=this._isUTC?"UTC":"";return null!=t?(this._d["set"+n+e](t),H.updateOffset(this),this):this._d["get"+n+e]()}}function C(t){H.duration.fn[t]=function(){return this._data[t]}}function L(t,e){H.duration.fn["as"+t]=function(){return+this/e}}for(var H,P,U="2.1.0",W=Math.round,x={},A="undefined"!=typeof module&&module.exports,G=/^\/?Date\((\-?\d+)/i,Z=/(\-)?(\d*)?\.?(\d+)\:(\d+)\:(\d+)\.?(\d{3})?/,E=/(\[[^\[]*\])|(\\)?(Mo|MM?M?M?|Do|DDDo|DD?D?D?|ddd?d?|do?|w[o|w]?|W[o|W]?|YYYYY|YYYY|YY|gg(ggg?)?|GG(GGG?)?|e|E|a|A|hh?|HH?|mm?|ss?|SS?S?|X|zz?|ZZ?|.)/g,N=/(\[[^\[]*\])|(\\)?(LT|LL?L?L?|l{1,4})/g,J=/\d\d?/,I=/\d{1,3}/,V=/\d{3}/,X=/\d{1,4}/,$=/[+\-]?\d{1,6}/,R=/[0-9]*['a-z\u00A0-\u05FF\u0700-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF]+|[\u0600-\u06FF\/]+(\s*?[\u0600-\u06FF]+){1,2}/i,j=/Z|[\+\-]\d\d:?\d\d/i,q=/T/i,B=/[\+\-]?\d+(\.\d{1,3})?/,K=/^\s*\d{4}-\d\d-\d\d((T| )(\d\d(:\d\d(:\d\d(\.\d\d?\d?)?)?)?)?([\+\-]\d\d:?\d\d)?)?/,Q="YYYY-MM-DDTHH:mm:ssZ",te=[["HH:mm:ss.S",/(T| )\d\d:\d\d:\d\d\.\d{1,3}/],["HH:mm:ss",/(T| )\d\d:\d\d:\d\d/],["HH:mm",/(T| )\d\d:\d\d/],["HH",/(T| )\d\d/]],ee=/([\+\-]|\d\d)/gi,ne="Date|Hours|Minutes|Seconds|Milliseconds".split("|"),se={Milliseconds:1,Seconds:1e3,Minutes:6e4,Hours:36e5,Days:864e5,Months:2592e6,Years:31536e6},ie={ms:"millisecond",s:"second",m:"minute",h:"hour",d:"day",w:"week",M:"month",y:"year"},re={},ae="DDD w W M D d".split(" "),oe="M D H h m s w W".split(" "),ue={M:function(){return this.month()+1},MMM:function(t){return this.lang().monthsShort(this,t)},MMMM:function(t){return this.lang().months(this,t)},D:function(){return this.date()},DDD:function(){return this.dayOfYear()},d:function(){return this.day()},dd:function(t){return this.lang().weekdaysMin(this,t)},ddd:function(t){return this.lang().weekdaysShort(this,t)},dddd:function(t){return this.lang().weekdays(this,t)},w:function(){return this.week()},W:function(){return this.isoWeek()},YY:function(){return u(this.year()%100,2)},YYYY:function(){return u(this.year(),4)},YYYYY:function(){return u(this.year(),5)},gg:function(){return u(this.weekYear()%100,2)},gggg:function(){return this.weekYear()},ggggg:function(){return u(this.weekYear(),5)},GG:function(){return u(this.isoWeekYear()%100,2)},GGGG:function(){return this.isoWeekYear()},GGGGG:function(){return u(this.isoWeekYear(),5)},e:function(){return this.weekday()},E:function(){return this.isoWeekday()},a:function(){return this.lang().meridiem(this.hours(),this.minutes(),!0)},A:function(){return this.lang().meridiem(this.hours(),this.minutes(),!1)},H:function(){return this.hours()},h:function(){return this.hours()%12||12},m:function(){return this.minutes()},s:function(){return this.seconds()},S:function(){return~~(this.milliseconds()/100)},SS:function(){return u(~~(this.milliseconds()/10),2)},SSS:function(){return u(this.milliseconds(),3)},Z:function(){var t=-this.zone(),e="+";return 0>t&&(t=-t,e="-"),e+u(~~(t/60),2)+":"+u(~~t%60,2)},ZZ:function(){var t=-this.zone(),e="+";return 0>t&&(t=-t,e="-"),e+u(~~(10*t/6),4)},z:function(){return this.zoneAbbr()},zz:function(){return this.zoneName()},X:function(){return this.unix()}};ae.length;)P=ae.pop(),ue[P+"o"]=n(ue[P],P);for(;oe.length;)P=oe.pop(),ue[P+P]=e(ue[P],2);for(ue.DDDD=e(ue.DDD,3),s.prototype={set:function(t){var e,n;for(n in t)e=t[n],"function"==typeof e?this[n]=e:this["_"+n]=e},_months:"January_February_March_April_May_June_July_August_September_October_November_December".split("_"),months:function(t){return this._months[t.month()]},_monthsShort:"Jan_Feb_Mar_Apr_May_Jun_Jul_Aug_Sep_Oct_Nov_Dec".split("_"),monthsShort:function(t){return this._monthsShort[t.month()]},monthsParse:function(t){var e,n,s;for(this._monthsParse||(this._monthsParse=[]),e=0;12>e;e++)if(this._monthsParse[e]||(n=H([2e3,e]),s="^"+this.months(n,"")+"|^"+this.monthsShort(n,""),this._monthsParse[e]=new RegExp(s.replace(".",""),"i")),this._monthsParse[e].test(t))return e},_weekdays:"Sunday_Monday_Tuesday_Wednesday_Thursday_Friday_Saturday".split("_"),weekdays:function(t){return this._weekdays[t.day()]},_weekdaysShort:"Sun_Mon_Tue_Wed_Thu_Fri_Sat".split("_"),weekdaysShort:function(t){return this._weekdaysShort[t.day()]},_weekdaysMin:"Su_Mo_Tu_We_Th_Fr_Sa".split("_"),weekdaysMin:function(t){return this._weekdaysMin[t.day()]},weekdaysParse:function(t){var e,n,s;for(this._weekdaysParse||(this._weekdaysParse=[]),e=0;7>e;e++)if(this._weekdaysParse[e]||(n=H([2e3,1]).day(e),s="^"+this.weekdays(n,"")+"|^"+this.weekdaysShort(n,"")+"|^"+this.weekdaysMin(n,""),this._weekdaysParse[e]=new RegExp(s.replace(".",""),"i")),this._weekdaysParse[e].test(t))return e},_longDateFormat:{LT:"h:mm A",L:"MM/DD/YYYY",LL:"MMMM D YYYY",LLL:"MMMM D YYYY LT",LLLL:"dddd, MMMM D YYYY LT"},longDateFormat:function(t){var e=this._longDateFormat[t];return!e&&this._longDateFormat[t.toUpperCase()]&&(e=this._longDateFormat[t.toUpperCase()].replace(/MMMM|MM|DD|dddd/g,function(t){return t.slice(1)}),this._longDateFormat[t]=e),e},isPM:function(t){return"p"===(t+"").toLowerCase()[0]},_meridiemParse:/[ap]\.?m?\.?/i,meridiem:function(t,e,n){return t>11?n?"pm":"PM":n?"am":"AM"},_calendar:{sameDay:"[Today at] LT",nextDay:"[Tomorrow at] LT",nextWeek:"dddd [at] LT",lastDay:"[Yesterday at] LT",lastWeek:"[Last] dddd [at] LT",sameElse:"L"},calendar:function(t,e){var n=this._calendar[t];return"function"==typeof n?n.apply(e):n},_relativeTime:{future:"in %s",past:"%s ago",s:"a few seconds",m:"a minute",mm:"%d minutes",h:"an hour",hh:"%d hours",d:"a day",dd:"%d days",M:"a month",MM:"%d months",y:"a year",yy:"%d years"},relativeTime:function(t,e,n,s){var i=this._relativeTime[n];return"function"==typeof i?i(t,e,n,s):i.replace(/%d/i,t)},pastFuture:function(t,e){var n=this._relativeTime[t>0?"future":"past"];return"function"==typeof n?n(e):n.replace(/%s/i,e)},ordinal:function(t){return this._ordinal.replace("%d",t)},_ordinal:"%d",preparse:function(t){return t},postformat:function(t){return t},week:function(t){return F(t,this._week.dow,this._week.doy).week},_week:{dow:0,doy:6}},H=function(t,e,n){return O({_i:t,_f:e,_l:n,_isUTC:!1})},H.utc=function(t,e,n){return O({_useUTC:!0,_isUTC:!0,_l:n,_i:t,_f:e})},H.unix=function(t){return H(1e3*t)},H.duration=function(t,e){var n,s,i=H.isDuration(t),a="number"==typeof t,o=i?t._input:a?{}:t,u=Z.exec(t);return a?e?o[e]=t:o.milliseconds=t:u&&(n="-"===u[1]?-1:1,o={y:0,d:~~u[2]*n,h:~~u[3]*n,m:~~u[4]*n,s:~~u[5]*n,ms:~~u[6]*n}),s=new r(o),i&&t.hasOwnProperty("_lang")&&(s._lang=t._lang),s},H.version=U,H.defaultFormat=Q,H.updateOffset=function(){},H.lang=function(t,e){return t?(e?l(t,e):x[t]||_(t),H.duration.fn._lang=H.fn._lang=_(t),void 0):H.fn._lang._abbr},H.langData=function(t){return t&&t._lang&&t._lang._abbr&&(t=t._lang._abbr),_(t)},H.isMoment=function(t){return t instanceof i},H.isDuration=function(t){return t instanceof r},H.fn=i.prototype={clone:function(){return H(this)},valueOf:function(){return+this._d+6e4*(this._offset||0)},unix:function(){return Math.floor(+this/1e3)},toString:function(){return this.format("ddd MMM DD YYYY HH:mm:ss [GMT]ZZ")},toDate:function(){return this._offset?new Date(+this):this._d},toISOString:function(){return M(H(this).utc(),"YYYY-MM-DD[T]HH:mm:ss.SSS[Z]")},toArray:function(){var t=this;return[t.year(),t.month(),t.date(),t.hours(),t.minutes(),t.seconds(),t.milliseconds()]},isValid:function(){return null==this._isValid&&(this._isValid=this._a?!c(this._a,(this._isUTC?H.utc(this._a):H(this._a)).toArray()):!isNaN(this._d.getTime())),!!this._isValid},utc:function(){return this.zone(0)},local:function(){return this.zone(0),this._isUTC=!1,this},format:function(t){var e=M(this,t||H.defaultFormat);return this.lang().postformat(e)},add:function(t,e){var n;return n="string"==typeof t?H.duration(+e,t):H.duration(t,e),h(this,n,1),this},subtract:function(t,e){var n;return n="string"==typeof t?H.duration(+e,t):H.duration(t,e),h(this,n,-1),this},diff:function(t,e,n){var s,i,r=this._isUTC?H(t).zone(this._offset||0):H(t).local(),a=6e4*(this.zone()-r.zone());return e=f(e),"year"===e||"month"===e?(s=432e5*(this.daysInMonth()+r.daysInMonth()),i=12*(this.year()-r.year())+(this.month()-r.month()),i+=(this-H(this).startOf("month")-(r-H(r).startOf("month")))/s,i-=6e4*(this.zone()-H(this).startOf("month").zone()-(r.zone()-H(r).startOf("month").zone()))/s,"year"===e&&(i/=12)):(s=this-r,i="second"===e?s/1e3:"minute"===e?s/6e4:"hour"===e?s/36e5:"day"===e?(s-a)/864e5:"week"===e?(s-a)/6048e5:s),n?i:o(i)},from:function(t,e){return H.duration(this.diff(t)).lang(this.lang()._abbr).humanize(!e)},fromNow:function(t){return this.from(H(),t)},calendar:function(){var t=this.diff(H().startOf("day"),"days",!0),e=-6>t?"sameElse":-1>t?"lastWeek":0>t?"lastDay":1>t?"sameDay":2>t?"nextDay":7>t?"nextWeek":"sameElse";return this.format(this.lang().calendar(e,this))},isLeapYear:function(){var t=this.year();return 0===t%4&&0!==t%100||0===t%400},isDST:function(){return this.zone()+H(t).startOf(e)},isBefore:function(t,e){return e="undefined"!=typeof e?e:"millisecond",+this.clone().startOf(e)<+H(t).startOf(e)},isSame:function(t,e){return e="undefined"!=typeof e?e:"millisecond",+this.clone().startOf(e)===+H(t).startOf(e)},min:function(t){return t=H.apply(null,arguments),this>t?this:t},max:function(t){return t=H.apply(null,arguments),t>this?this:t},zone:function(t){var e=this._offset||0;return null==t?this._isUTC?e:this._d.getTimezoneOffset():("string"==typeof t&&(t=p(t)),Math.abs(t)<16&&(t=60*t),this._offset=t,this._isUTC=!0,e!==t&&h(this,H.duration(e-t,"m"),1,!0),this)},zoneAbbr:function(){return this._isUTC?"UTC":""},zoneName:function(){return this._isUTC?"Coordinated Universal Time":""},daysInMonth:function(){return H.utc([this.year(),this.month()+1,0]).date()},dayOfYear:function(t){var e=W((H(this).startOf("day")-H(this).startOf("year"))/864e5)+1;return null==t?e:this.add("d",t-e)},weekYear:function(t){var e=F(this,this.lang()._week.dow,this.lang()._week.doy).year;return null==t?e:this.add("y",t-e)},isoWeekYear:function(t){var e=F(this,1,4).year;return null==t?e:this.add("y",t-e)},week:function(t){var e=this.lang().week(this);return null==t?e:this.add("d",7*(t-e))},isoWeek:function(t){var e=F(this,1,4).week;return null==t?e:this.add("d",7*(t-e))},weekday:function(t){var e=(this._d.getDay()+7-this.lang()._week.dow)%7;return null==t?e:this.add("d",t-e)},isoWeekday:function(t){return null==t?this.day()||7:this.day(this.day()%7?t:t-7)},lang:function(e){return e===t?this._lang:(this._lang=_(e),this)}},P=0;P