you attribute any public use of the data
37 | or publicly shared works produced from the data to DC Metro Metrics.
38 |
any publicly shared adaptations of this data are shared under the ODbL.
39 |
40 |
41 |
42 |
Please see the license for full details.
43 |
44 |
45 |
Download
46 |
47 |
48 |
Click Here to download all of the DC Metro Metrics data. This zip file is regenerated once per day.
49 |
50 |
For information on the meaning of the columns,
51 | check out the README.md included in the zip file. Please note that all timestamps are in the ISO 8601 format in UTC.
52 |
12 |
13 |
14 |
--------------------------------------------------------------------------------
/client/bower.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "dcmetrometrics",
3 | "version": "0.0.1",
4 | "dependencies": {
5 | "angular": "1.3.13",
6 | "json3": "~3.3.1",
7 | "es5-shim": "~3.1.0",
8 | "bootstrap": "~3.2.0",
9 | "angular-resource": "1.3.13",
10 | "angular-cookies": "1.3.13",
11 | "angular-sanitize": "1.3.13",
12 | "angular-animate": "1.3.13",
13 | "angular-touch": "1.3.13",
14 | "angular-route": "1.3.13",
15 | "angular-strap": "2.2.0",
16 | "angular-bootstrap": "~0.11.0",
17 | "angular-ui-router": "~0.2.10",
18 | "ng-table": "~0.3.3",
19 | "angular-spinner": "~0.5.1",
20 | "angular-ui-utils": "bower",
21 | "bootswatch-dist": "3.2.0-yeti",
22 | "d3": "~3.4.11",
23 | "cal-heatmap": "~3.5.2",
24 | "moment": "~2.9.0",
25 | "moment-timezone": "~0.3.0",
26 | "angular-loading-bar": "~0.7.0",
27 | "angular-motion": "~0.3.4",
28 | "angular-tablesort": "~1.0.5"
29 | },
30 | "devDependencies": {
31 | "angular-mocks": "1.3.13",
32 | "angular-scenario": "1.3.13"
33 | },
34 | "appPath": "app"
35 | }
36 |
--------------------------------------------------------------------------------
/client/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "dcmetrometrics",
3 | "version": "2.0.0",
4 | "dependencies": {},
5 | "devDependencies": {
6 | "connect-modrewrite": "^0.7.9",
7 | "grunt": "^0.4.1",
8 | "grunt-autoprefixer": "^0.7.3",
9 | "grunt-concurrent": "^0.5.0",
10 | "grunt-contrib-clean": "^0.5.0",
11 | "grunt-contrib-concat": "^0.4.0",
12 | "grunt-contrib-connect": "^0.7.1",
13 | "grunt-contrib-copy": "^0.5.0",
14 | "grunt-contrib-cssmin": "^0.9.0",
15 | "grunt-contrib-htmlmin": "^0.3.0",
16 | "grunt-contrib-imagemin": "^0.7.0",
17 | "grunt-contrib-jshint": "^0.10.0",
18 | "grunt-contrib-uglify": "^0.4.0",
19 | "grunt-contrib-watch": "^0.6.1",
20 | "grunt-filerev": "^0.2.1",
21 | "grunt-google-cdn": "^0.4.0",
22 | "grunt-html-snapshot": "^0.6.1",
23 | "grunt-karma": "^0.8.3",
24 | "grunt-newer": "^0.7.0",
25 | "grunt-ngmin": "^0.0.3",
26 | "grunt-svgmin": "^0.4.0",
27 | "grunt-usemin": "^2.1.1",
28 | "grunt-wiredep": "^1.7.0",
29 | "jshint-stylish": "^0.2.0",
30 | "karma": "^0.12.19",
31 | "karma-jasmine": "^0.1.5",
32 | "karma-phantomjs-launcher": "^0.1.4",
33 | "load-grunt-tasks": "^0.4.0",
34 | "time-grunt": "^0.3.1"
35 | },
36 | "engines": {
37 | "node": ">=0.10.0"
38 | },
39 | "scripts": {
40 | "test": "grunt test"
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/client/peg/pegBuild.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | import sys, os, subprocess, shutil
3 |
4 | this_dir = os.path.split(os.path.abspath(__file__))[0]
5 | target_dir = os.path.abspath(os.path.join(this_dir, "..", "app", "scripts"))
6 |
7 | input_file_name = "rankingsSearchString.pegjs"
8 | output_file_name = "rankingsSearchString.js"
9 |
10 | input_file = os.path.join(this_dir, input_file_name)
11 | output_file = os.path.join(this_dir, output_file_name)
12 | temp_file = os.path.join(this_dir, ".tmp.out")
13 |
14 | cmd = "pegjs -e searchStringParser %s %s"%(input_file, temp_file)
15 | ret = subprocess.call(cmd.split())
16 | assert(ret == 0)
17 |
18 | script = open(temp_file).read().strip()
19 |
20 | # Rewrite the file to use strict.
21 | template = """"use strict";
22 | var {script}"""
23 |
24 | # Write to output js file.
25 | output = template.format(script = script)
26 | with open(output_file, 'w') as fout:
27 | fout.write(output)
28 | print "Wrote to %s"%output_file
29 |
30 | # Copy the output file
31 | shutil.copy(output_file, target_dir)
32 |
33 | # Delete the temporary file
34 | os.remove(temp_file)
35 |
36 | print "Copied to %s"%target_dir
37 |
38 |
39 |
--------------------------------------------------------------------------------
/client/test/.jshintrc:
--------------------------------------------------------------------------------
1 | {
2 | "node": true,
3 | "browser": true,
4 | "esnext": true,
5 | "bitwise": true,
6 | "camelcase": true,
7 | "curly": true,
8 | "eqeqeq": true,
9 | "immed": true,
10 | "indent": 2,
11 | "latedef": true,
12 | "newcap": true,
13 | "noarg": true,
14 | "quotmark": "single",
15 | "regexp": true,
16 | "undef": true,
17 | "unused": true,
18 | "strict": true,
19 | "trailing": true,
20 | "smarttabs": true,
21 | "globals": {
22 | "after": false,
23 | "afterEach": false,
24 | "angular": false,
25 | "before": false,
26 | "beforeEach": false,
27 | "browser": false,
28 | "describe": false,
29 | "expect": false,
30 | "inject": false,
31 | "it": false,
32 | "jasmine": false,
33 | "spyOn": false
34 | }
35 | }
36 |
37 |
--------------------------------------------------------------------------------
/client/test/karma.conf.js:
--------------------------------------------------------------------------------
1 | // Karma configuration
2 | // http://karma-runner.github.io/0.12/config/configuration-file.html
3 | // Generated on 2014-07-27 using
4 | // generator-karma 0.8.3
5 |
6 | module.exports = function(config) {
7 | 'use strict';
8 |
9 | config.set({
10 | // enable / disable watching file and executing tests whenever any file changes
11 | autoWatch: true,
12 |
13 | // base path, that will be used to resolve files and exclude
14 | basePath: '../',
15 |
16 | // testing framework to use (jasmine/mocha/qunit/...)
17 | frameworks: ['jasmine'],
18 |
19 | // list of files / patterns to load in the browser
20 | files: [
21 | 'bower_components/angular/angular.js',
22 | 'bower_components/angular-mocks/angular-mocks.js',
23 | 'bower_components/angular-animate/angular-animate.js',
24 | 'bower_components/angular-cookies/angular-cookies.js',
25 | 'bower_components/angular-resource/angular-resource.js',
26 | 'bower_components/angular-route/angular-route.js',
27 | 'bower_components/angular-sanitize/angular-sanitize.js',
28 | 'bower_components/angular-touch/angular-touch.js',
29 | 'app/scripts/**/*.js',
30 | 'test/mock/**/*.js',
31 | 'test/spec/**/*.js'
32 | ],
33 |
34 | // list of files / patterns to exclude
35 | exclude: [],
36 |
37 | // web server port
38 | port: 8080,
39 |
40 | // Start these browsers, currently available:
41 | // - Chrome
42 | // - ChromeCanary
43 | // - Firefox
44 | // - Opera
45 | // - Safari (only Mac)
46 | // - PhantomJS
47 | // - IE (only Windows)
48 | browsers: [
49 | 'PhantomJS'
50 | ],
51 |
52 | // Which plugins to enable
53 | plugins: [
54 | 'karma-phantomjs-launcher',
55 | 'karma-jasmine'
56 | ],
57 |
58 | // Continuous Integration mode
59 | // if true, it capture browsers, run tests and exit
60 | singleRun: false,
61 |
62 | colors: true,
63 |
64 | // level of logging
65 | // possible values: LOG_DISABLE || LOG_ERROR || LOG_WARN || LOG_INFO || LOG_DEBUG
66 | logLevel: config.LOG_INFO,
67 |
68 | // Uncomment the following lines if you are using grunt's server to run the tests
69 | // proxies: {
70 | // '/': 'http://localhost:9000/'
71 | // },
72 | // URL root prevent conflicts with the site root
73 | // urlRoot: '_karma_'
74 | });
75 | };
76 |
--------------------------------------------------------------------------------
/client/test/spec/controllers/about.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | describe('Controller: AboutCtrl', function () {
4 |
5 | // load the controller's module
6 | beforeEach(module('clientApp'));
7 |
8 | var AboutCtrl,
9 | scope;
10 |
11 | // Initialize the controller and a mock scope
12 | beforeEach(inject(function ($controller, $rootScope) {
13 | scope = $rootScope.$new();
14 | AboutCtrl = $controller('AboutCtrl', {
15 | $scope: scope
16 | });
17 | }));
18 |
19 | it('should attach a list of awesomeThings to the scope', function () {
20 | expect(scope.awesomeThings.length).toBe(3);
21 | });
22 | });
23 |
--------------------------------------------------------------------------------
/client/test/spec/controllers/main.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | describe('Controller: MainCtrl', function () {
4 |
5 | // load the controller's module
6 | beforeEach(module('clientApp'));
7 |
8 | var MainCtrl,
9 | scope;
10 |
11 | // Initialize the controller and a mock scope
12 | beforeEach(inject(function ($controller, $rootScope) {
13 | scope = $rootScope.$new();
14 | MainCtrl = $controller('MainCtrl', {
15 | $scope: scope
16 | });
17 | }));
18 |
19 | it('should attach a list of awesomeThings to the scope', function () {
20 | expect(scope.awesomeThings.length).toBe(3);
21 | });
22 | });
23 |
--------------------------------------------------------------------------------
/client/test/spec/controllers/station.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | describe('Controller: StationCtrl', function () {
4 |
5 | // load the controller's module
6 | beforeEach(module('dcmetrometricsApp'));
7 |
8 | var StationCtrl,
9 | scope;
10 |
11 | // Initialize the controller and a mock scope
12 | beforeEach(inject(function ($controller, $rootScope) {
13 | scope = $rootScope.$new();
14 | StationCtrl = $controller('StationCtrl', {
15 | $scope: scope
16 | });
17 | }));
18 |
19 | it('should attach a list of awesomeThings to the scope', function () {
20 | expect(scope.awesomeThings.length).toBe(3);
21 | });
22 | });
23 |
--------------------------------------------------------------------------------
/client/test/spec/controllers/stationdirectory.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | describe('Controller: StationdirectoryCtrl', function () {
4 |
5 | // load the controller's module
6 | beforeEach(module('dcmetrometricsApp'));
7 |
8 | var StationdirectoryCtrl,
9 | scope;
10 |
11 | // Initialize the controller and a mock scope
12 | beforeEach(inject(function ($controller, $rootScope) {
13 | scope = $rootScope.$new();
14 | StationdirectoryCtrl = $controller('StationdirectoryCtrl', {
15 | $scope: scope
16 | });
17 | }));
18 |
19 | it('should attach a list of awesomeThings to the scope', function () {
20 | expect(scope.awesomeThings.length).toBe(3);
21 | });
22 | });
23 |
--------------------------------------------------------------------------------
/client/test/spec/controllers/unit.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | describe('Controller: UnitCtrl', function () {
4 |
5 | // load the controller's module
6 | beforeEach(module('dcmetrometricsApp'));
7 |
8 | var UnitCtrl,
9 | scope;
10 |
11 | // Initialize the controller and a mock scope
12 | beforeEach(inject(function ($controller, $rootScope) {
13 | scope = $rootScope.$new();
14 | UnitCtrl = $controller('UnitCtrl', {
15 | $scope: scope
16 | });
17 | }));
18 |
19 | it('should attach a list of awesomeThings to the scope', function () {
20 | expect(scope.awesomeThings.length).toBe(3);
21 | });
22 | });
23 |
--------------------------------------------------------------------------------
/client/test/spec/directives/linecolors.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | describe('Directive: lineColors', function () {
4 |
5 | // load the directive's module
6 | beforeEach(module('dcmetrometricsApp'));
7 |
8 | var element,
9 | scope;
10 |
11 | beforeEach(inject(function ($rootScope) {
12 | scope = $rootScope.$new();
13 | }));
14 |
15 | it('should make hidden element visible', inject(function ($compile) {
16 | element = angular.element('');
17 | element = $compile(element)(scope);
18 | expect(element.text()).toBe('this is the lineColors directive');
19 | }));
20 | });
21 |
--------------------------------------------------------------------------------
/client/test/spec/filters/duration.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | describe('Filter: duration', function () {
4 |
5 | // load the filter's module
6 | beforeEach(module('dcmetrometricsApp'));
7 |
8 | // initialize a new instance of the filter before each test
9 | var duration;
10 | beforeEach(inject(function ($filter) {
11 | duration = $filter('duration');
12 | }));
13 |
14 | it('should return the input prefixed with "duration filter:"', function () {
15 | var text = 'angularjs';
16 | expect(duration(text)).toBe('duration filter: ' + text);
17 | });
18 |
19 | });
20 |
--------------------------------------------------------------------------------
/client/test/spec/filters/unitidtohuman.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | describe('Filter: unitIdToHuman', function () {
4 |
5 | // load the filter's module
6 | beforeEach(module('dcmetrometricsApp'));
7 |
8 | // initialize a new instance of the filter before each test
9 | var unitIdToHuman;
10 | beforeEach(inject(function ($filter) {
11 | unitIdToHuman = $filter('unitIdToHuman');
12 | }));
13 |
14 | it('should return the input prefixed with "unitIdToHuman filter:"', function () {
15 | var text = 'angularjs';
16 | expect(unitIdToHuman(text)).toBe('unitIdToHuman filter: ' + text);
17 | });
18 |
19 | });
20 |
--------------------------------------------------------------------------------
/client/test/spec/services/directory.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | describe('Service: directory', function () {
4 |
5 | // load the service's module
6 | beforeEach(module('dcmetrometricsApp'));
7 |
8 | // instantiate service
9 | var directory;
10 | beforeEach(inject(function (_directory_) {
11 | directory = _directory_;
12 | }));
13 |
14 | it('should do something', function () {
15 | expect(!!directory).toBe(true);
16 | });
17 |
18 | });
19 |
--------------------------------------------------------------------------------
/client/test/spec/services/statustableutils.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | describe('Service: statusTableUtils', function () {
4 |
5 | // load the service's module
6 | beforeEach(module('dcmetrometricsApp'));
7 |
8 | // instantiate service
9 | var statusTableUtils;
10 | beforeEach(inject(function (_statusTableUtils_) {
11 | statusTableUtils = _statusTableUtils_;
12 | }));
13 |
14 | it('should do something', function () {
15 | expect(!!statusTableUtils).toBe(true);
16 | });
17 |
18 | });
19 |
--------------------------------------------------------------------------------
/client/test/spec/services/unitservice.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | describe('Service: unitService', function () {
4 |
5 | // load the service's module
6 | beforeEach(module('dcmetrometricsApp'));
7 |
8 | // instantiate service
9 | var unitService;
10 | beforeEach(inject(function (_unitService_) {
11 | unitService = _unitService_;
12 | }));
13 |
14 | it('should do something', function () {
15 | expect(!!unitService).toBe(true);
16 | });
17 |
18 | });
19 |
--------------------------------------------------------------------------------
/data/codes.tsv:
--------------------------------------------------------------------------------
1 | 3294 TURNED OFF/WALKER
2 | 3430 CALLBACK/REPAIR
3 | 2734 PREV. MAINT. INSPECTION
4 | 3366 WEATHER RELATED
5 | 3434 REHAB/MODERNIZATION
6 | 2096 INCIDENT/ACCIDENT
7 | 2720 FIRE ALARM/DELUGE SYSTEMS
8 | 2914 SCHEDULED SUPPORT
9 | 2723 POWER SURGE/OUTAGE
10 | 4300 SAFETY WORK ORDER
11 | 0003 MINOR REPAIR
12 | 3359 WATER LEAK/INTRUSION
13 | 2907 SAFETY INSPECTION
14 | 5050 HANDRAIL
15 | 0005 MAJOR REPAIR
16 | 2733 PREV. MAINT. REPAIRS
17 | 3433 PREV. MAINT. COMPLIANCE INSPECTION
18 |
--------------------------------------------------------------------------------
/data/test_data/data.ods:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LeeMendelowitz/DCMetroMetrics/73b28117fbdb35886cec0e05328a19416550c6a7/data/test_data/data.ods
--------------------------------------------------------------------------------
/data/test_data/data.tsv:
--------------------------------------------------------------------------------
1 | UnitName UnitType StationCode StationName LocationDescription SymptomCode SymptomDescription TickOffset
2 | A03N01 ESCALATOR A03 Dupont Circle, North Escalator to street 2 Lazy 10
3 | A03N02 ESCALATOR A03 Dupont Circle, North Escalator to street 3 Slippery 10
4 | A02S01 ESCALATOR A02 Farragut North, 17th Street Escalator to street 4 Power 20
5 | A03N02 ESCALATOR A03 Dupont Circle, North Escalator to street 3 Slippery 20
6 | A02S01 ESCALATOR A02 Farragut North, 17th Street Escalator to street 4 Power 30
7 | A03N02 ESCALATOR A03 Dupont Circle, North Escalator to street 3 Slippery 30
8 | A02S01 ESCALATOR A02 Farragut North, 17th Street Escalator to street 4 Power 40
9 | A02S01 ESCALATOR A02 Farragut North, 17th Street Escalator to street 5 PowerStillOff 50
10 | A03N01 ESCALATOR A03 Dupont Circle, North Escalator to street 2 Lazy 60
11 | A03N02 ESCALATOR A03 Dupont Circle, North Escalator to street 3 Slippery 60
12 | A02S01 ESCALATOR A02 Farragut North, 17th Street Escalator to street 4 Power 70
13 | A03N02 ESCALATOR A03 Dupont Circle, North Escalator to street 3 Slippery 70
14 | A02S01 ESCALATOR A02 Farragut North, 17th Street Escalator to street 4 Power 80
15 | A03N02 ESCALATOR A03 Dupont Circle, North Escalator to street 3 Slippery 80
16 | A02S01 ESCALATOR A02 Farragut North, 17th Street Escalator to street 4 Power 90
17 | A02S01 ESCALATOR A02 Farragut North, 17th Street Escalator to street 5 PowerStillOff 100
18 |
--------------------------------------------------------------------------------
/data/test_data/data.txt:
--------------------------------------------------------------------------------
1 | UnitName UnitType StationCode StationName LocationDescription SymptomCode SymptomDescription Group
A03N01ESCALATOR ESCALATOR A03 "Dupont Circle, North" Escalator to street 2 Lazy 1
A03N02ESCALATOR ESCALATOR A03 "Dupont Circle, North" Escalator to street 3 Slippery 1
A02X012ESCALATOR ESCALATOR A02 "Farragut North, 17th Street" Escalator to street 4 Power 2
A03N02ESCALATOR ESCALATOR A03 "Dupont Circle, North" Escalator to street 3 Slippery 2
A02X012ESCALATOR ESCALATOR A02 "Farragut North, 17th Street" Escalator to street 4 Power 3
A03N02ESCALATOR ESCALATOR A03 "Dupont Circle, North" Escalator to street 3 Slippery 3
A02X012ESCALATOR ESCALATOR A02 "Farragut North, 17th Street" Escalator to street 4 Power 4
--------------------------------------------------------------------------------
/data/test_data/data.xlsx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LeeMendelowitz/DCMetroMetrics/73b28117fbdb35886cec0e05328a19416550c6a7/data/test_data/data.xlsx
--------------------------------------------------------------------------------
/dcmetrometrics/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Module: dcmetrometrics
3 |
4 | This python package defines classes/functions:
5 | - to pull WMATA data from the WMATA API and Twitter
6 | - to process and store this data in a MongoDB database
7 | - to generate the DC Metro Metrics website
8 | - to run the automated Twitter accounts @MetroHotCars, @MetroEscalators, @MetroElevators.
9 |
10 | The package is organized as follows:
11 | - eles: package for escalator/elevator data
12 | - common: package for common utility classes/functions used throughout
13 | the application.
14 | - hotcars: package for #wmata #hotcar data
15 | - web: package for generating the DC Metro Metrics website
16 | - third_party: third party packages used by this application.
17 | """
18 |
19 | from .common import *
20 |
--------------------------------------------------------------------------------
/dcmetrometrics/common/DataWriteable.py:
--------------------------------------------------------------------------------
1 | from datetime import date, datetime
2 |
3 |
4 | def convert(k, v):
5 | """Convert a k,v pair into a dictionary.
6 | If v is a dictionary, it will be flattened
7 | """
8 | if isinstance(v, (int, float, str, unicode, list)) or v is None:
9 | return {k: v}
10 | elif isinstance(v, long):
11 | return {k: unicode(v)} # Return as unicode to prevent loss of precision
12 | elif isinstance(v, (date, datetime)):
13 | return {k: v.isoformat()}
14 | elif isinstance(v, dict):
15 | ret = {}
16 | for k2, v2 in v.iteritems():
17 | k3 = '%s_%s'%(k, k2) # flatten the keys
18 | ret.update(convert(k3, v2))
19 | return ret
20 | elif isinstance(v, DataWriteable):
21 | ret = {}
22 | dr = v.to_data_record()
23 | for k2, v2 in dr.iteritems():
24 | k3 = '%s_%s'%(k, k2) # flatten the keys
25 | ret.update(convert(k3, v2))
26 | return ret
27 |
28 | raise TypeError("Cannot convert value of %s"%type(v))
29 |
30 | class DataWriteable(object):
31 | """This will flatten objects into a single dictionary
32 | """
33 |
34 | def to_data_record(self, keep_na = True):
35 |
36 | fields = getattr(self, 'data_fields', [])
37 | ret = {}
38 |
39 | for k in fields:
40 |
41 | v = getattr(self, k, None)
42 |
43 | if keep_na:
44 | ret.update(convert(k, v))
45 | elif v is not None:
46 | ret.update(convert(k, v))
47 |
48 | return ret
--------------------------------------------------------------------------------
/dcmetrometrics/common/WebJSONMixin.py:
--------------------------------------------------------------------------------
1 | class WebJSONMixin(object):
2 |
3 | def to_web_json(self):
4 |
5 | fields = getattr(self, 'web_json_fields', [])
6 |
7 | return dict((k, getattr(self, k, None)) for k in fields)
--------------------------------------------------------------------------------
/dcmetrometrics/common/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | This module provides defines classes and functions that
3 | are used throughout the application.
4 | """
5 |
--------------------------------------------------------------------------------
/dcmetrometrics/common/descriptors.py:
--------------------------------------------------------------------------------
1 | # descriptors.py
2 | # Defines class descriptors
3 | #
4 | # setOnce: descriptor for a class attribute that can be set
5 | # once, and then is read-only thereafter.
6 | #
7 | # computeOnce: descriptor for a class attribute that should be
8 | # computed once on the first access. All future accesses
9 | # returned the computed value. This is similar to using
10 | # the @property decorator.
11 |
12 |
13 | ######################################################
14 | # Descriptor to define a read only attribute
15 | # which can only be set once.
16 | class setOnce(object):
17 | def __init__(self, name, default = None):
18 | self.name = "_" + name # Need leading underscore
19 | self.default = default
20 |
21 | def __get__(self, instance, cls):
22 | if not instance:
23 | raise AttributeError('Can only access property through instance')
24 | return getattr(instance, self.name, self.default)
25 |
26 | # Only allow the attribute to be set once
27 | def __set__(self, instance, value):
28 | if not getattr(instance, self.name, None):
29 | setattr(instance, self.name, value)
30 | return
31 | raise AttributeError('Attribute is read-only.')
32 |
33 | def __delete__(self, instance):
34 | raise AttributeError('Attribute is read-only')
35 |
36 | ######################################################
37 | # Descriptor to define a read only attribute
38 | # which should only be computed once on the first access.
39 | #
40 | # The advantage of using this descriptor is that if the
41 | # class attribute is never used, it is never computed, and
42 | # if it is used, it is only computed once.
43 | class computeOnce(object):
44 | count = 0 # used to assign a unique variable name to store
45 | # value within an instance
46 |
47 | def __init__(self, fget):
48 | self.name = "_" + 'computeOnce%i'%computeOnce.count
49 | computeOnce.count = computeOnce.count + 1
50 | self.fget = fget
51 |
52 | def __get__(self, instance, cls):
53 | if not instance:
54 | raise AttributeError('Can only access property through instance')
55 |
56 | retVal = getattr(instance, self.name, None)
57 | if retVal is None:
58 | retVal = self.fget(instance)
59 | setattr(instance, self.name, retVal)
60 | return retVal
61 |
62 | def __set__(self, instance, value):
63 | raise AttributeError('Attribute is read-only.')
64 |
65 | def __delete__(self, instance):
66 | raise AttributeError('Attribute is read-only')
67 |
--------------------------------------------------------------------------------
/dcmetrometrics/common/globals.py:
--------------------------------------------------------------------------------
1 | """
2 | Define global variables.
3 | """
4 |
5 | import os
6 |
7 | PY_DIR = os.environ['PYTHON_DIR']
8 | REPO_DIR = os.environ['REPO_DIR']
9 | SCRIPT_DIR = os.path.join(REPO_DIR, 'scripts')
10 | DATA_DIR = os.environ['DATA_DIR']
11 | WWW_DIR = os.environ['WWW_DIR']
12 |
13 | MONGODB_HOST = os.environ["MONGODB_HOST"]
14 | MONGODB_PORT = int(os.environ["MONGODB_PORT"])
15 | MONGODB_USERNAME = os.environ.get("MONGODB_USERNAME", None)
16 | MONGODB_PASSWORD = os.environ.get("MONGODB_PASSWORD", None)
17 |
18 | INTERNAL_SERVE_IP = os.environ["INTERNAL_SERVE_IP"] # Internal IP Address to serve app through.
19 | INTERNAL_SERVE_PORT = os.environ["INTERNAL_SERVE_PORT"] # Internal Port to serve app through.
--------------------------------------------------------------------------------
/dcmetrometrics/common/logging_utils.py:
--------------------------------------------------------------------------------
1 | """
2 | Utilities for creating logger objects.
3 | """
4 | import logging, sys
5 |
6 |
7 | def create_logger(name):
8 |
9 | # Get the logger
10 | logger = logging.getLogger(name)
11 |
12 | # Reset handlers on the logger.
13 | logger.handlers = []
14 |
15 | # Create a stream handler
16 | sh = logging.StreamHandler(stream=sys.stderr)
17 |
18 | # create formatter
19 | formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
20 |
21 | # add formatter to stream handler
22 | sh.setFormatter(formatter)
23 |
24 | # add stream handler to logger
25 | logger.addHandler(sh)
26 |
27 | # set logging level
28 | logger.setLevel(logging.DEBUG)
29 |
30 | return logger
--------------------------------------------------------------------------------
/dcmetrometrics/common/restartingGreenlet.py:
--------------------------------------------------------------------------------
1 | """
2 | common.restartingGreenlet
3 |
4 | Extend the gevent Greenlet class so that the Greenlet restarts whenever it finishes.
5 | """
6 |
7 | import sys
8 | from gevent import Greenlet, sleep
9 | from datetime import datetime
10 |
11 | def makeNewGreenlet(g):
12 | args = g.rsargs
13 | kwargs = g.rskwargs
14 | newg = g.__class__(*args, **kwargs)
15 | t = type(newg)
16 | gid = str(newg)
17 | dateStr = str(datetime.now())
18 | sys.stderr.write('%s: Making new greenlet %s of type %s\n'%(dateStr, gid, str(t)))
19 |
20 | # Impose a short sleep. This is to prevent a buggy RestartingGreenlet
21 | # from perpetually throwing an exception and continually restarting
22 | # TO DO: Consider a max restarts parameter
23 | sleep(1)
24 | return newg
25 |
26 | class RestartingGreenlet(Greenlet):
27 |
28 | def __init__(self, *args, **kwargs):
29 | Greenlet.__init__(self)
30 |
31 | # Save the constructer arguments
32 | # so we can recreate the Greenlet
33 | self.rsargs = args
34 | self.rskwargs = kwargs
35 |
36 | # Set up this Greenlet to use the restarter
37 | self.link(self.restart)
38 |
39 | @staticmethod
40 | def restart(g):
41 | newg = makeNewGreenlet(g)
42 | newg.start()
43 |
--------------------------------------------------------------------------------
/dcmetrometrics/common/twitterUtils.py:
--------------------------------------------------------------------------------
1 | """
2 | common.twitterUtils
3 |
4 | """
5 | import twitter
6 | from twitter import TwitterError
7 |
8 | TWITTER_TIMEOUT = 10
9 |
10 | def getApi(keys):
11 | """
12 | Construct a Twitter API instance.
13 | """
14 | api = twitter.Api(consumer_key = keys.consumer_key,
15 | consumer_secret = keys.consumer_secret,
16 | access_token_key = keys.access_token,
17 | access_token_secret = keys.access_token_secret,
18 | cache = None,
19 | requests_timeout = TWITTER_TIMEOUT)
20 | return api
21 |
--------------------------------------------------------------------------------
/dcmetrometrics/eles/WMATA_API.py:
--------------------------------------------------------------------------------
1 | """
2 | eles.wmataAPI
3 |
4 | Request data from the WMATA API
5 | """
6 |
7 | import requests
8 |
9 | class WMATA_API_ERROR(Exception):
10 | def __init__(self, requestObj):
11 | self.requestObj = requestObj
12 | def __str__(self):
13 | return 'Url=%s, status_code=%i'%(self.requestObj.url, self.requestObj.status_code)
14 |
15 | class WMATA_API(object):
16 | """
17 | (partial) implementation of the WMATA API.
18 | """
19 |
20 | def __init__(self, key):
21 | if not isinstance(key, str) or not key:
22 | raise TypeError('WMATA_API key should be str')
23 |
24 | self.API_KEY = key
25 | self.URL_BASE = 'http://api.wmata.com'
26 | self.TIMEOUT = 10 # timeout requests after 10 seconds.
27 |
28 | # Check if a request is okay. If it isn't raise WMATA_API_ERROR
29 | def checkRequest(self, req):
30 | if req.status_code != requests.codes.ok:
31 | raise WMATA_API_ERROR(req)
32 |
33 | def request(self, url, params):
34 | payload = { 'api_key' : self.API_KEY }
35 | headers = { 'api_key' : self.API_KEY }
36 | if params is not None:
37 | payload.update(params)
38 | r = requests.get(url, params=payload, timeout = self.TIMEOUT, headers = headers)
39 | self.checkRequest(r)
40 | return r
41 |
42 | # Request the static webpage with elevator/escalator status
43 | def getEscalatorWebpageStatus(self):
44 | url = 'http://www.wmata.com/rider_tools/metro_service_status/elevator_escalator.cfm'
45 | r = requests.get(url, timeout = self.TIMEOUT)
46 | return r
47 |
48 | def getStations(self, params = None):
49 | base = '%s/Rail.svc/json'%self.URL_BASE
50 | url = '{base}/JStations'.format(base=base)
51 | return self.request(url, params)
52 |
53 | def getLines(self, params = None):
54 | base = '%s/Rail.svc/json'%self.URL_BASE
55 | url = '{base}/JLines'.format(base=base)
56 | return self.request(url, params)
57 |
58 | def getStationInfo(self, params = None):
59 | base = '%s/Rail.svc/json'%self.URL_BASE
60 | url = '{base}/JStationInfo'.format(base=base)
61 | return self.request(url, params)
62 |
63 | def getIncidents(self, params = None):
64 | base = '%s/Incidents.svc/json'%(self.URL_BASE)
65 | url = '{base}/Incidents'.format(url)
66 | return self.request(url, params)
67 |
68 | def getEscalator(self, params = None):
69 | base = '%s/Incidents.svc/json'%(self.URL_BASE)
70 | url = '{base}/ElevatorIncidents'.format(base=base)
71 | return self.request(url, params)
72 |
--------------------------------------------------------------------------------
/dcmetrometrics/eles/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Modules for working with WMATA escalator & elevator data.
3 | """
4 |
--------------------------------------------------------------------------------
/dcmetrometrics/eles/defs.py:
--------------------------------------------------------------------------------
1 | """
2 | eles.defs
3 |
4 | Define constants for escalator/elevator status codes
5 | """
6 |
7 | from collections import defaultdict
8 |
9 | OPERATIONAL_CODE = -1
10 | NUM_ESCALATORS = 588
11 | NUM_ELEVATORS = 238
12 |
13 | ###################################################
14 | # Define symptomToCategory. Consider adding these
15 | # to the symptom_codes database
16 | symptomToCategory = defaultdict(lambda: 'BROKEN')
17 |
18 | offStatuses = ['SAFETY WORK ORDER',
19 | 'TURNED OFF/WALKER',
20 | 'SCHEDULED SUPPORT',
21 | 'PREV. MAINT. REPAIRS',
22 | 'Inspection Repair',
23 | 'Walker',
24 | 'Scheduled Support',
25 | 'Preventive Maintenance Repairs']
26 |
27 | inspectStatuses = ['SAFETY INSPECTION',
28 | 'PREV. MAINT. INSPECTION',
29 | 'PREV. MAINT. COMPLIANCE INSPECTION',
30 | 'Preventive Maintenance Inspection',
31 | 'Safety Inspection']
32 |
33 | rehabStatuses = ['REHAB/MODERNIZATION', 'Modernization']
34 |
35 |
36 | symptomToCategory['OPERATIONAL'] = 'ON'
37 | symptomToCategory.update((status, 'OFF') for status in offStatuses)
38 | symptomToCategory.update((status, 'INSPECTION') for status in inspectStatuses)
39 | symptomToCategory.update((status, 'REHAB') for status in rehabStatuses)
40 |
41 | SYMPTOM_CHOICES = ('ON', 'OFF', 'INSPECTION', 'REHAB', 'BROKEN')
42 |
43 |
--------------------------------------------------------------------------------
/dcmetrometrics/eles/escalatorRequest.py:
--------------------------------------------------------------------------------
1 | """
2 | eles.esclatorRequest
3 |
4 | Get escalator incidents through the wmata API
5 | """
6 |
7 | # Python modules
8 | import time
9 | import os
10 | import sys
11 | import cPickle
12 | from collections import defaultdict, Counter
13 | from datetime import datetime
14 |
15 | # Custom modules
16 | from ..common import stations
17 | from ..keys import WMATA_API_KEY
18 | from .WMATA_API import WMATA_API, WMATA_API_ERROR
19 | from .Incident import Incident
20 |
21 | api = None
22 | def getAPI():
23 | global api
24 | if WMATA_API_KEY is None:
25 | return None
26 | if api is not None:
27 | return api
28 | api = WMATA_API(key=WMATA_API_KEY)
29 | return api
30 |
31 | # Summarize results to standard output
32 | def summarize(result):
33 | """
34 | Summarize the escalator/elevator statuses to standard output.
35 | """
36 | incidents = result['incidents']
37 | numIncidents = len(incidents)
38 | timeStr = time.strftime('%d_%b_%Y-%H_%M_%S', result['requestTime'])
39 | sys.stdout.write('Time: %s\n'%timeStr)
40 |
41 | sys.stdout.write('\n\n')
42 | sys.stdout.write('Num Incidents: %i\n'%numIncidents)
43 |
44 | symptomCounts = Counter(i['SymptomDescription'] for i in incidents)
45 | maxWidth = max(len(s) for s in symptomCounts.keys())
46 | formatStr = '{symptom:%is} {count:d}\n'%maxWidth
47 | for symptom, count in symptomCounts.most_common():
48 | outS = formatStr.format(symptom=symptom, count=count)
49 | sys.stdout.write(outS)
50 | sys.stdout.write('\n\n')
51 |
52 | # Gather incidents by station Code
53 | stationToIncident = defaultdict(list)
54 | for d in incidents:
55 | stationToIncident[d['StationCode']].append(d)
56 | for code in sorted(stations.allCodes):
57 | stationIncidents = stationToIncident.get(code, [])
58 | sys.stdout.write('%s (%s): %i\n'%(stations.codeToName[code], code, len(stationIncidents)))
59 |
60 | def run():
61 | """
62 | Dump all incidents to a pickle file.
63 | """
64 | requestTime = time.localtime()
65 | timeStr = time.strftime('%d_%b_%Y-%H_%M_%S', requestTime)
66 | api = getAPI()
67 | res = api.getEscalator()
68 | incidents = res.json()['ElevatorIncidents']
69 |
70 | websiteTxt = Req.getEscalatorWebpageStatus().text
71 |
72 | result = { 'incidents' : incidents,
73 | 'requestTime' : time.localtime(),
74 | 'webpage' : websiteTxt }
75 |
76 | summarize(result)
77 |
78 | fname = '%s.pickle'%(timeStr)
79 | fout = open(fname, 'w')
80 | cPickle.dump(result, fout)
81 | fout.close()
82 |
83 | # Make a request for the twitter app
84 |
85 | if __name__ == '__main__':
86 | run()
87 |
--------------------------------------------------------------------------------
/dcmetrometrics/eles/misc_utils.py:
--------------------------------------------------------------------------------
1 | """
2 | Miscellaneous utility functions
3 | """
4 |
5 | import itertools
6 | from operator import itemgetter, attrgetter
7 |
8 | from ..common.metroTimes import isNaive
9 |
10 | def get_one(cursor):
11 | """
12 | Get one item from the iterable.
13 | Return None if there is None.
14 | """
15 | try:
16 | return next(cursor)
17 | except StopIteration:
18 | return None
19 |
20 | def get_some(cursor, N):
21 | """
22 | Get at most N items from the cursor, and return as a list
23 | """
24 | return [i for i in itertools.slice(cursor, N)]
25 |
26 | def get_first_status_since(statusList, time):
27 | """
28 | Get the first status in the status list that starts after time.
29 | """
30 | statusList = sorted(statusList, key = attrgetter('time')) # in time ascending
31 | myRecs = (rec for rec in statusList if rec.time > time)
32 | return get_one(myRecs)
33 |
34 | #############################################################
35 | def checkAllTimesNotNaive(statusList):
36 | for s in statusList:
37 | if isNaive(s.time):
38 | raise RuntimeError('Times cannot be naive')
39 | end_time = getattr(s, 'end_time', None)
40 | if end_time and isNaive(end_time):
41 | raise RuntimeError('Times cannot be naive')
42 |
43 | #############################################################
44 | # Check that each status has a 'time' and 'end_time' defined
45 | # and that the list is sorted
46 | def checkStatusListSane(statusList):
47 | lastTime = None
48 | for s in statusList:
49 | if not getattr(s, 'end_time', None):
50 | raise RuntimeError('Status missing end_time')
51 | if not getattr(s, 'time', None):
52 | raise RuntimeError('Status missing time')
53 | if s.time > s.end_time:
54 | raise RuntimeError('Status has bad starting/ending time')
55 | if lastTime and s.time < lastTime:
56 | raise RuntimeError('Status not sorted properly')
57 | lastTime = s.time
58 |
59 | def yieldNothing():
60 | return
61 | yield
--------------------------------------------------------------------------------
/dcmetrometrics/hotcars/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LeeMendelowitz/DCMetroMetrics/73b28117fbdb35886cec0e05328a19416550c6a7/dcmetrometrics/hotcars/__init__.py
--------------------------------------------------------------------------------
/dcmetrometrics/hotcars/twitter_api.py:
--------------------------------------------------------------------------------
1 | """
2 | getTwitterAPI: Return the python_twitter
3 | """
4 | from ..common import twitterUtils
5 | from twitter import TwitterError
6 |
7 | T = None
8 | def getTwitterAPI():
9 | global T
10 |
11 | if T is None:
12 | from ..keys import HotCarKeys, MissingKeyError
13 |
14 | # Check that the HotCar Twitter API Keys have been set.
15 | if not HotCarKeys.isSet():
16 | msg = \
17 | """
18 |
19 | The HotCar App requires that the HotCarKeys be set.
20 | Check your keys.py.
21 |
22 | For more information, see:
23 | https://github.com/LeeMendelowitz/DCMetroMetrics/wiki/API-Keys
24 |
25 | """
26 | raise MissingKeyError(msg)
27 |
28 | T = twitterUtils.getApi(HotCarKeys)
29 | return T
--------------------------------------------------------------------------------
/dcmetrometrics/keys/__init__.py:
--------------------------------------------------------------------------------
1 | from .key_utils import keyModuleError, MissingKeyError
2 |
3 | try:
4 | from .keys import MetroEscalatorKeys,\
5 | MetroElevatorKeys,\
6 | HotCarKeys,\
7 | WUNDERGROUND_API_KEY,\
8 | WMATA_API_KEY,\
9 | MissingKeyError,\
10 | TwitterKeyError
11 |
12 | except ImportError as e:
13 |
14 | # The key module is not properly defined.
15 | keyModuleError()
16 |
17 |
--------------------------------------------------------------------------------
/dcmetrometrics/keys/key_utils.py:
--------------------------------------------------------------------------------
1 | class MissingKeyError(Exception):
2 | pass
3 |
4 | class TwitterKeyError(MissingKeyError):
5 | pass
6 |
7 | class TwitterKeys(object):
8 | @classmethod
9 | def checkKeys(cls):
10 | """
11 | Check that the twitter keys have been properly set.
12 | """
13 | req = ['consumer_key', 'access_token',
14 | 'consumer_secret', 'access_token_secret']
15 | for k in req:
16 | val = getattr(cls, k, None)
17 | if (not val) or (not isinstance(val, str)):
18 | msg = 'Twitter Keys {name} attribute {attr} is not properly set. Check your keys.py'
19 | msg = msg.format(name = cls.__name__, attr=k)
20 | raise TwitterKeyError(msg)
21 | @classmethod
22 | def isSet(cls):
23 | req = ['consumer_key', 'access_token',
24 | 'consumer_secret', 'access_token_secret']
25 | keysAreSet = True
26 | for k in req:
27 | val = getattr(cls, k, None)
28 | if (not val) or (not isinstance(val, str)):
29 | keysAreSet = False
30 | return keysAreSet
31 |
32 | def keyModuleError():
33 | """
34 | Raise a RuntimeError due to a missing keys module.
35 | """
36 |
37 | msg = \
38 | """
39 |
40 | Could not import dcmetrometrics.keys module because the file is missing
41 | or is malformed.
42 |
43 | You must create the keys module using the provided template:
44 |
45 | cp dcmetrometrics/keys/keys_default.py dcmetrometrics/keys/keys.py
46 |
47 | For more information, see:
48 | https://github.com/LeeMendelowitz/DCMetroMetrics/wiki/API-Keys
49 |
50 | """
51 | raise RuntimeError(msg)
52 |
--------------------------------------------------------------------------------
/dcmetrometrics/keys/keys_default.py:
--------------------------------------------------------------------------------
1 | """
2 | This file is a template for storing the API Keys used
3 | by this application.
4 |
5 | Copy this file into keys.py, and replace the None values with
6 | str values of your own keys.
7 |
8 | Required Keys
9 | =============
10 | - The EscalatorApp and the ElevatorApp require the WMATA_API_KEY to run.
11 | - The HotCarsApp requires the HotCarKeys to seach for hot car tweets and to live tweet.
12 |
13 | Optional keys
14 | =============
15 | - The EscalatorApp needs the MetroEscalatorKeys to live tweet, otherwise
16 | tweet messages will only be written to a log file.
17 | - The ElevatorApp needs MetroElevatorKeys to live tweet, otherwise
18 | tweet message will only be written to a log file.
19 | - The WebPageGeneratorApp needs the WundergroundAPI to include DC temperature
20 | history on the hotcars webpage, otherwise default temperature values are used.
21 | """
22 |
23 | from .key_utils import *
24 |
25 | class MetroEscalatorKeys(TwitterKeys):
26 | """
27 | Key for @MetroEscalators Twitter acccount
28 | """
29 | consumer_key = None
30 | access_token = None
31 | consumer_secret = None
32 | access_token_secret = None
33 |
34 | class MetroElevatorKeys(TwitterKeys):
35 | """
36 | Key for @MetroElevators Twitter acccount
37 | """
38 | consumer_key = None
39 | access_token = None
40 | consumer_secret = None
41 | access_token_secret = None
42 |
43 | class HotCarKeys(TwitterKeys):
44 | """
45 | Key for @MetroHotCars Twitter account
46 | """
47 | consumer_key = None
48 | consumer_secret = None
49 | access_token = None
50 | access_token_secret = None
51 |
52 | WUNDERGROUND_API_KEY = None
53 |
54 | WMATA_API_KEY = None
55 |
--------------------------------------------------------------------------------
/dcmetrometrics/third_party/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Third party modules.
3 | """
4 |
5 | from . import twitter, gviz_api
6 |
--------------------------------------------------------------------------------
/elevatorApp.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | """
3 | apps.runElevatorApp
4 |
5 | # This runs the elevatorApp instance.
6 | # This app downloads elevator data from the WMATA API,
7 | # stores elevator statuses in the database, and
8 | # generates tweets for @MetroElevators.
9 |
10 | # This module can be executed directly for local testing
11 | """
12 |
13 | if __name__ == "__main__":
14 | # Local Testing
15 | import test.setup
16 |
17 | # python imports
18 | import os
19 | import sys
20 | import subprocess
21 | from datetime import datetime
22 | import gevent
23 | from gevent import Greenlet
24 |
25 | # custom imports
26 | from dcmetrometrics.common.globals import DATA_DIR, REPO_DIR, DATA_DIR
27 | from dcmetrometrics.common.restartingGreenlet import RestartingGreenlet
28 | from dcmetrometrics.eles.ElevatorApp import ElevatorApp as App
29 |
30 | OUTPUT_DIR = DATA_DIR
31 | if OUTPUT_DIR is None:
32 | OUTPUT_DIR = os.getcwd()
33 |
34 | if REPO_DIR is None:
35 | SCRIPT_DIR = os.getcwd()
36 | else:
37 | SCRIPT_DIR = os.path.join(REPO_DIR, 'scripts')
38 |
39 | SLEEP = 30
40 |
41 | ##########################################
42 | # Run the Twitter App as a Greenlet.
43 | class ElevatorApp(RestartingGreenlet):
44 |
45 | def __init__(self, SLEEP=SLEEP, LIVE=False):
46 | RestartingGreenlet.__init__(self, SLEEP=SLEEP, LIVE=LIVE)
47 | self.LIVE = LIVE # Tweet only if Live
48 | self.SLEEP = SLEEP # Sleep time after each tick
49 | self.logFileName = os.path.join(DATA_DIR, 'runElevatorApp.log')
50 |
51 | def _run(self):
52 | while True:
53 | try:
54 | self.tick()
55 | except Exception as e:
56 | import traceback
57 | logFile = open(self.logFileName, 'a')
58 | logFile.write('ElevatorApp caught Exception: %s\n'%(str(e)))
59 | tb = traceback.format_exc()
60 | logFile.write('Traceback:\n%s\n\n'%tb)
61 | logFile.close()
62 | gevent.sleep(self.SLEEP)
63 |
64 | def tick(self):
65 |
66 | # Run MetroElevators twitter App
67 | with open(self.logFileName, 'a') as logFile:
68 |
69 | n = datetime.now()
70 | timeStr = n.strftime('%d-%B-%Y %H:%M:%S')
71 |
72 | msg = '*'*50 + '\n'
73 | msg += '%s Elevator App Tick\n'%timeStr
74 | msg += 'App Mode: %s\n'%('LIVE' if self.LIVE else 'NOT LIVE')
75 |
76 | logFile.write(msg)
77 | logFile.flush()
78 |
79 | app = App(logFile, LIVE=self.LIVE)
80 | app.tick()
81 |
82 | if __name__ == "__main__":
83 | print 'Running the elevator app locally....'
84 | escApp = ElevatorApp()
85 | escApp.start()
86 | escApp.join()
87 |
--------------------------------------------------------------------------------
/escalatorApp.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | """
3 | apps.runEscalatorApp
4 |
5 | # This runs the escalatorApp instance.
6 | # This app downloads escalator data from the WMATA API,
7 | # stores escalator statuses in the database, and
8 | # generates tweets for @MetroEscalators.
9 |
10 | # This module can be executed directly for local testing
11 | """
12 |
13 | if __name__ == "__main__":
14 | # Local Testing
15 | import test.setup
16 |
17 | import os
18 | import sys
19 | import subprocess
20 | from datetime import datetime
21 | import gevent
22 | from gevent import Greenlet
23 |
24 | from dcmetrometrics.common.restartingGreenlet import RestartingGreenlet
25 | from dcmetrometrics.eles.EscalatorApp import EscalatorApp as App
26 | from dcmetrometrics.common.globals import DATA_DIR, REPO_DIR
27 |
28 | OUTPUT_DIR = DATA_DIR
29 | if OUTPUT_DIR is None:
30 | OUTPUT_DIR = os.getcwd()
31 |
32 | if REPO_DIR is None:
33 | SCRIPT_DIR = os.getcwd()
34 | else:
35 | SCRIPT_DIR = os.path.join(REPO_DIR, 'scripts')
36 |
37 | SLEEP = 30
38 |
39 | ##########################################
40 | # Run the Twitter App as a Greenlet.
41 | class EscalatorApp(RestartingGreenlet):
42 |
43 | def __init__(self, SLEEP=SLEEP, LIVE=False):
44 | RestartingGreenlet.__init__(self, SLEEP=SLEEP, LIVE=LIVE)
45 | self.LIVE = LIVE # Tweet only if Live
46 | self.SLEEP = SLEEP # Sleep time after each tick
47 | self.logFileName = os.path.join(DATA_DIR, 'runEscalatorApp.log')
48 |
49 | def _run(self):
50 | while True:
51 | try:
52 | self.tick()
53 | except Exception as e:
54 | import traceback
55 | logFile = open(self.logFileName, 'a')
56 | logFile.write('EscalatorApp caught Exception: %s\n'%(str(e)))
57 | tb = traceback.format_exc()
58 | logFile.write('Traceback:\n%s\n\n'%tb)
59 | logFile.close()
60 | gevent.sleep(self.SLEEP)
61 |
62 | def tick(self):
63 |
64 | # Run MetroEsclaators twitter App
65 | with open(self.logFileName, 'a') as logFile:
66 |
67 | n = datetime.now()
68 | timeStr = n.strftime('%d-%B-%Y %H:%M:%S')
69 |
70 | msg = '*'*50 + '\n'
71 | msg += '%s Escalator App Tick\n'%timeStr
72 | msg += 'App Mode: %s\n'%('LIVE' if self.LIVE else 'NOT LIVE')
73 |
74 | logFile.write(msg)
75 | logFile.flush()
76 |
77 | app = App(logFile, LIVE=self.LIVE)
78 | app.tick()
79 |
80 | if __name__ == "__main__":
81 | print 'Running the escalator app locally....'
82 | escApp = EscalatorApp()
83 | escApp.start()
84 | escApp.join()
85 |
--------------------------------------------------------------------------------
/hotCarApp.py:
--------------------------------------------------------------------------------
1 | """
2 | apps.hotCarApp
3 |
4 | Define the HotCarApp as a restartingGreenlet.
5 |
6 | This app will use the Twitter API to search for tweets about #wmata
7 | #hotcar's. These tweets and the hotcar data are stored in a database.
8 | Tweet acknowledgements are posted to the @MetroHotCars twitter account.
9 | """
10 |
11 | # TEST CODE
12 | if __name__ == "__main__":
13 | import test.setup
14 |
15 | import gevent
16 | import os
17 | import sys
18 | from datetime import datetime
19 |
20 | from dcmetrometrics.common.restartingGreenlet import RestartingGreenlet
21 | from dcmetrometrics.common import dbGlobals
22 | from dcmetrometrics.hotcars import hotCars
23 | from dcmetrometrics.common.globals import DATA_DIR, REPO_DIR, DATA_DIR
24 |
25 | OUTPUT_DIR = DATA_DIR
26 |
27 | ###############################################################
28 | # Log the HotCarApp App to a file.
29 | import logging
30 |
31 | LOG_FILE_NAME = os.path.join(DATA_DIR, 'HotCarApp.log')
32 | fh = logging.FileHandler(LOG_FILE_NAME)
33 | sh = logging.StreamHandler(sys.stderr)
34 |
35 | formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
36 | fh.setFormatter(formatter)
37 | sh.setFormatter(formatter)
38 |
39 | logger = logging.getLogger('HotCarApp')
40 | logger.addHandler(fh)
41 | logger.addHandler(sh)
42 | #################################################################
43 |
44 | class HotCarApp(RestartingGreenlet):
45 |
46 | def __init__(self, LIVE=False):
47 |
48 | dbGlobals.connect()
49 |
50 | RestartingGreenlet.__init__(self, LIVE=LIVE)
51 | self.SLEEP = 40 # Run every 10 seconds
52 | self.LIVE = LIVE
53 |
54 | # Run forever
55 | def _run(self):
56 |
57 | while True:
58 |
59 | try:
60 |
61 | hotCars.tick(tweetLive = self.LIVE)
62 |
63 | except Exception as e:
64 |
65 | import traceback
66 | tb = traceback.format_exc()
67 | logger.error("HotCarApp Caught Error! %s\nTraceback:\n%s"%(str(e), tb))
68 |
69 | gevent.sleep(self.SLEEP)
70 |
71 |
72 |
73 | if __name__ == '__main__':
74 | from time import sleep
75 | app = HotCarApp(LIVE = False)
76 | app.start()
77 | app.join()
78 |
79 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | bottle==0.12.5
2 | gevent==1.0.1
3 | gnureadline==6.3.3
4 | greenlet==0.4.5
5 | gunicorn==18.0
6 | httplib2==0.8
7 | ipython==2.3.1
8 | mongoengine==0.8.7
9 | numpy==1.9.1
10 | oauth2==1.5.211
11 | oauthlib==0.7.2
12 | pandas==0.15.2
13 | pymongo==2.6.3
14 | python-dateutil==1.5
15 | python-twitter==2.0
16 | pytz==2014.10
17 | requests==2.5.0
18 | requests-oauthlib==0.4.2
19 | simplejson==3.3.3
20 | wsgiref==0.1.2
21 |
--------------------------------------------------------------------------------
/run_app.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # Script for running the dcmetrometrics worker app on digital ocean
4 | ROOT=/home/lmendelo/dcmetrometrics
5 | source $ROOT/env.sh
6 | source $ROOT/python/virtenv/bin/activate
7 | cd $ROOT/repo
8 |
9 | # Run gunicorn
10 | python app.py
--------------------------------------------------------------------------------
/run_gunicorn.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # Script for running gunicorn server on digital ocean.
4 | ROOT=/home/lmendelo/dcmetrometrics
5 | source $ROOT/env.sh
6 | source $ROOT/python/virtenv/bin/activate
7 | cd $ROOT/repo
8 |
9 | # Run gunicorn
10 | gunicorn -w 4 -k gevent dcmetrometrics.web.server:app
--------------------------------------------------------------------------------
/serve_gunicorn.py:
--------------------------------------------------------------------------------
1 | """
2 | Expose the server app for gunicorn in a testing environment
3 | with the appropriate environmental variables.
4 |
5 | To run:
6 | gunicorn -w 4 -k gevent serve_gunicorn:app
7 | """
8 |
9 | # Local Testing
10 | import test.setup
11 | from gevent import monkey; monkey.patch_all()
12 | from dcmetrometrics.web.server import app
--------------------------------------------------------------------------------
/serverApp.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | """
3 | Run a local instance of the DCMetroMetrics server
4 | for local testing.
5 | """
6 |
7 | # Local Testing
8 | import test.setup
9 | from gevent import monkey; monkey.patch_all()
10 | from dcmetrometrics.web.server import Server
11 |
12 | print 'Running the server locally....'
13 | serverApp = Server()
14 | serverApp.start()
15 | serverApp.join()
16 |
17 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | from setuptools import setup
2 |
3 | packages = ['dcmetrometrics',
4 | 'dcmetrometrics.common',
5 | 'dcmetrometrics.eles',
6 | 'dcmetrometrics.hotcars',
7 | 'dcmetrometrics.keys',
8 | 'dcmetrometrics.test',
9 | 'dcmetrometrics.third_party',
10 | 'dcmetrometrics.web']
11 |
12 | setup(name='DC Metro Metrics',
13 | version='1.1',
14 | description='Collecting and sharing public data related to the DC WMATA Metrorail system.',
15 | author='Lee Mendelowitz',
16 | author_email='Lee.Mendelowitz@gmail.com',
17 | url='https://github.com/LeeMendelowitz/DCMetroMetrics',
18 | # Uncomment one or more lines below in the install_requires section
19 | # for the specific client drivers/modules your application needs.
20 | install_requires=[ 'greenlet',
21 | 'gevent',
22 | 'requests',
23 | 'python-dateutil==1.5',
24 | 'oauth2',
25 | 'pymongo',
26 | 'simplejson',
27 | 'httplib2',
28 | 'bottle',
29 | 'mongoengine'
30 | ],
31 | packages = []
32 | )
33 |
--------------------------------------------------------------------------------
/test/__init__.py:
--------------------------------------------------------------------------------
1 | from . import setup
2 | from .setup import connect
3 |
--------------------------------------------------------------------------------
/test/status_group.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import setup
3 |
4 | from dcmetrometrics.eles import models
5 | from dcmetrometrics.eles.StatusGroup import StatusGroup
6 | from dcmetrometrics.common.metroTimes import utcnow, nytz
7 | from datetime import timedelta, datetime
8 |
9 | class TestOutageDays(unittest.TestCase):
10 |
11 | def setUp(self):
12 |
13 | t1 = datetime(2015, 2, 16, 22, tzinfo = nytz)
14 |
15 | # Outage does not cover the 17th since resolved before system open
16 | t2 = datetime(2015, 2, 17, 3, tzinfo = nytz)
17 | t3 = datetime(2015, 2, 21, 4, tzinfo = nytz) # This covers 20 and 21
18 |
19 | s1 = models.UnitStatus(time = t1, end_time = t2, symptom_category = "BROKEN")
20 | s2 = models.UnitStatus(time = t2, end_time = t3, symptom_category = "ON" )
21 | s3 = models.UnitStatus(time = t3, end_time = None, symptom_category = "BROKEN")
22 |
23 | self.statuses = [s1, s2, s3]
24 | self.start_time = t1
25 |
26 | def test_end_time_1(self):
27 | end_time = datetime(2015, 2, 22, 10, tzinfo = nytz)
28 | sg = StatusGroup(self.statuses, self.start_time, end_time)
29 | bd = sg.break_days
30 | self.assertEqual(len(sg.break_days), 4)
31 |
32 | def test_end_time_2(self):
33 | end_time = datetime(2015, 2, 22, 4, tzinfo = nytz)
34 | sg = StatusGroup(self.statuses, self.start_time, end_time)
35 | bd = sg.break_days
36 | self.assertEqual(len(sg.break_days), 3)
37 |
38 | class TestTrimming(unittest.TestCase):
39 |
40 | def setUp(self):
41 |
42 | t1 = datetime(2015, 2, 16, 22, tzinfo = nytz)
43 |
44 | # Outage does not cover the 17th since resolved before system open
45 | t2 = datetime(2015, 2, 17, 3, tzinfo = nytz)
46 | t3 = datetime(2015, 2, 21, 4, tzinfo = nytz) # This covers 20 and 21
47 |
48 | s1 = models.UnitStatus(time = t1, end_time = t2, symptom_category = "BROKEN")
49 | s2 = models.UnitStatus(time = t2, end_time = t3, symptom_category = "ON" )
50 | s3 = models.UnitStatus(time = t3, end_time = None, symptom_category = "BROKEN")
51 |
52 | self.statuses = [s1, s2, s3]
53 | self.start_time = t1
54 |
55 | def test_trim_1(self):
56 | start_time = datetime(2015, 2, 17, 1, tzinfo = nytz)
57 | end_time = datetime(2015, 2, 25, 4, tzinfo = nytz)
58 | statuses = self.statuses
59 | sg = StatusGroup(statuses, start_time, end_time)
60 | self.assertEqual(sg.statuses_trimmed[0].time, start_time)
61 | self.assertEqual(sg.statuses_trimmed[-1].end_time, end_time)
62 |
63 | def test_trim_2(self):
64 | start_time = datetime(2015, 2, 17, 1, tzinfo = nytz)
65 | end_time = datetime(2015, 2, 21, 2, tzinfo = nytz)
66 | statuses = self.statuses
67 | sg = StatusGroup(statuses, start_time, end_time)
68 | self.assertEqual(sg.statuses_trimmed[0].time, start_time)
69 | self.assertEqual(sg.statuses_trimmed[-1].end_time, end_time)
70 |
71 |
72 |
73 | if __name__ == '__main__':
74 | unittest.main()
--------------------------------------------------------------------------------
/test/test_app.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | """
3 | Test the app.py in the repo directory
4 | This includes the hotcars app, and the escalator app
5 | """
6 |
7 | import os, sys, subprocess, imp
8 |
9 | # Setup environmental variables and fix the system path
10 | import setup
11 |
12 | from gevent import monkey; monkey.patch_all()
13 | appPath = os.path.join(setup.HOME_DIR, 'app.py')
14 | appModule = imp.load_source('app', appPath)
15 | appModule.run()
16 |
--------------------------------------------------------------------------------
/test/test_exportDB.py:
--------------------------------------------------------------------------------
1 | import setup
2 | import utils.exportDB
3 | utils.exportDB.run()
4 |
--------------------------------------------------------------------------------
/test/test_exportdata.py:
--------------------------------------------------------------------------------
1 | import setup
2 | import utils.exportData
3 | utils.exportData.run()
4 |
--------------------------------------------------------------------------------
/utils/__init__.py:
--------------------------------------------------------------------------------
1 | from .utils import isOpenshiftEnv, fixSysPath
2 |
--------------------------------------------------------------------------------
/utils/fixHotCarColors.py:
--------------------------------------------------------------------------------
1 | # Functions to help manually add a line color to hot car
2 | # reports which are missing a line color:
3 |
4 | # dumpCSV: Dump a CSV file with hot car reports which are missing line color
5 | # addColorsFromCSV: Read in colors from a csv file.
6 | # genUpdateScript: Generate a python script to update the MongoDB hotcars collection
7 | #####################################################
8 | #from . import test.test_setup
9 | from . import dbUtils
10 | from . import hotCars
11 |
12 | import pandas
13 | import bson
14 | from StringIO import StringIO
15 |
16 | db = dbUtils.getDB()
17 |
18 | def dumpCSV(fname = 'hotcars.colormissing.csv'):
19 | hotCarDict = hotCars.getAllHotCarReports(db)
20 | allReports = [v for vl in hotCarDict.values() for v in vl]
21 | missingColor = [h for h in allReports if h['color']=='NONE']
22 | print '%i reports are missing color'%len(missingColor)
23 |
24 | myFields = ['_id', 'text', 'color']
25 | data = []
26 | for h in missingColor:
27 | d = dict((k,h[k]) for k in myFields)
28 | d['_id'] = str(d['_id'])
29 | d['text'] = d['text'].encode('ascii', errors='ignore').replace(',',' ')
30 | data.append(d)
31 | dt = pandas.DataFrame(data)
32 | dt.to_csv(fname,index=False)
33 |
34 | def addColorsFromCSV(fname):
35 | dt = pandas.read_csv(fname)
36 | numRec = len(dt)
37 | colorInd = dt['color'] != 'NONE'
38 | colorDt = dt[colorInd]
39 | numColor = len(colorDt)
40 | print 'Found %i records out of %i with color'%(numColor, numRec)
41 | for recId, d in colorDt.iterrows():
42 | myId = d['_id']
43 | color = d['color']
44 | print myId, color
45 | myId = bson.objectid.ObjectId(myId)
46 | db.hotcars.update({"_id" : myId}, {"$set" : {"color" : color}})
47 |
48 | script =\
49 | """\
50 | #!/usr/bin/env python
51 | import bson, os, sys
52 |
53 | if "OPENSHIFT_MONGODB_DB_HOST" not in os.environ:
54 | print "Seems like this is not the OPENSHIFT environment. Importing test_setup."
55 | import test_setup
56 |
57 | import dbUtils
58 | db = dbUtils.getDB()
59 |
60 | {cmds}
61 |
62 | """
63 |
64 |
65 | def genUpdateScript(csvname, output='updateDBWithColors.py'):
66 | dt = pandas.read_csv(csvname)
67 | numRec = len(dt)
68 | colorInd = dt['color'] != 'NONE'
69 | colorDt = dt[colorInd]
70 | numColor = len(colorDt)
71 | print 'Found %i records out of %i with color'%(numColor, numRec)
72 | cmds = StringIO()
73 | for recId, d in colorDt.iterrows():
74 | myId = d['_id']
75 | color = d['color']
76 | print myId, color
77 | cmd = 'db.hotcars.update({{"_id" : bson.objectid.ObjectId("{0}")}}, {{"$set" : {{"color" : "{1}"}}}})'.format(myId,color)
78 | cmds.write(cmd + "\n")
79 | myScript = script.format(cmds=cmds.getvalue())
80 | cmds.close()
81 | fout = open(output, 'w')
82 | fout.write(myScript)
83 |
--------------------------------------------------------------------------------
/utils/generateAllWebPages.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | """
3 | Generate all web pages.
4 | This script works for both an OpenShift deployment
5 | and for local testing.
6 | """
7 |
8 | import sys, os
9 | import utils
10 |
11 | if not utils.isOpenshiftEnv():
12 | print 'This does not appear to be an Openshift Environment.'
13 | utils.fixSysPath()
14 | import test.setup # Fixes the system path so we can import dcmetrometrics.
15 | else:
16 | print 'This appears to be an Openshift Environment.'
17 |
18 | print '*'*50
19 |
20 | from dcmetrometrics.web import WebPageGenerator
21 | WebPageGenerator.updateAllPages()
22 |
23 |
24 |
--------------------------------------------------------------------------------
/utils/generate_site_map.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | """
3 | Generate a sitemap.xml file for dc metro metrics
4 | and a json file with an array of urls.
5 |
6 | The json file is used with grunt in order to crawl dcmetrometrics.com and regenerate
7 | the static version of the site.
8 | """
9 |
10 | from dcmetrometrics.eles import models as models_eles
11 | from dcmetrometrics.hotcars import models as models_hotcars
12 | import sys
13 |
14 | #######################
15 | # Set up logging
16 | import logging
17 | sh = logging.StreamHandler(sys.stderr)
18 | formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
19 | logger = logging.getLogger('SiteMapGenerator')
20 | logger.setLevel(logging.DEBUG)
21 | logger.addHandler(sh)
22 | #######################
23 |
24 | URL_ROOT = 'http://localhost:80'
25 |
26 | def unit_urls():
27 | logger.info("Generating Unit URLS...")
28 | units = models_eles.Unit.objects
29 | return ['{URL_ROOT}/unit/%s'%(unit.unit_id) for unit in units]
30 |
31 | def hotcar_urls():
32 | logger.info("Generating HotCar URLS...")
33 | hotcars = models_hotcars.HotCarReport.objects
34 | return list(set(['{URL_ROOT}/hotcars/detail/%s'%(h.car_number) for h in hotcars]))
35 |
36 | def station_urls():
37 | logger.info("Generating Station URLS...")
38 | stations = models_eles.Station.objects
39 | short_names = set(s.short_name for s in stations)
40 | return ['{URL_ROOT}/stations/detail/%s'%(s) for s in short_names]
41 |
42 | def main_urls():
43 | logger.info("Generating Main URLS...")
44 | pages = ['home', 'stations/list', 'about', 'outages', 'rankings', '']
45 | return ['{URL_ROOT}/%s'%(p) for p in pages]
46 |
47 | URLS = unit_urls() + hotcar_urls() + station_urls() + main_urls()
48 |
49 |
50 | def make_site_map(output_file_name, URL_ROOT = URL_ROOT):
51 | import xml.etree.ElementTree as ET
52 | urls = [u.format(URL_ROOT = URL_ROOT) for u in URLS]
53 | urlset_node = ET.Element('urlset')
54 | urlset_node.set('xmlns', 'http://www.sitemaps.org/schemas/sitemap/0.9')
55 | tree = ET.ElementTree(urlset_node)
56 |
57 | for url in urls:
58 | url_node = ET.SubElement(urlset_node, 'url')
59 | loc_node = ET.SubElement(url_node, 'loc')
60 | loc_node.text = url
61 | changefreq_node = ET.SubElement(url_node, 'changefreq')
62 | changefreq_node.text = 'daily'
63 |
64 | with open(output_file_name, 'w') as fout:
65 | tree.write(fout, encoding="utf-8", xml_declaration = True)
66 |
67 | def write_url_json(output_file_name, URL_ROOT = URL_ROOT):
68 | import json
69 | urls = [u.format(URL_ROOT = URL_ROOT) for u in URLS]
70 | with open(output_file_name, 'w') as fout:
71 | json.dump(urls, fout)
72 |
73 | if __name__ == "__main__":
74 | #make_site_map('sitemap.xml', URL_ROOT='http://www.dcmetrometrics.com')
75 | write_url_json('site_urls.json', URL_ROOT='')
76 |
77 |
78 |
79 |
80 |
81 |
--------------------------------------------------------------------------------
/utils/importData.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | """
3 | Downloads the latest DC Metro Metrics data from www.DCMetroMetrics.com
4 | Loads the data into the local MongoDB Database
5 | An instance of mongod must be running.
6 | """
7 |
8 | from pymongo import MongoClient
9 | import os, sys, shutil, glob, argparse
10 | from subprocess import call
11 |
12 | parser = argparse.ArgumentParser(description="Import DCMetroMetrics data into MongoDB")
13 | parser.add_argument('--db', default="MetroEscalators", help='The database to use.')
14 | parser.add_argument('--zip', default="./DCMetroMetricsData.zip", help='The zip file to use')
15 |
16 | c = MongoClient()
17 |
18 |
19 | def run():
20 | args = parser.parse_args()
21 |
22 | ZIP = args.zip
23 |
24 | if not os.path.exists(ZIP):
25 | raise RuntimeError("Could not find zip file: %s"%ZIP)
26 |
27 | UNZIPPED_DIR = os.path.abspath('./DCMetroMetricsData')
28 |
29 | # Download the latest data zip file
30 | # cmd = "curl http://www.dcmetrometrics.com/data/{ZIP} > {ZIP}".format(ZIP=ZIP)
31 | # call(cmd, shell=True)
32 |
33 | if os.path.exists(UNZIPPED_DIR):
34 | shutil.rmtree(UNZIPPED_DIR)
35 |
36 | # Unzip
37 | cmd = "unzip %s"%ZIP
38 | call(cmd, shell=True)
39 |
40 | db = c[args.db]
41 |
42 | jsons = glob.glob('%s/*.json'%UNZIPPED_DIR)
43 | print 'Json Directory: %s'%UNZIPPED_DIR
44 | print 'Found %i jsons'%len(jsons)
45 | for j in jsons:
46 | path, name = os.path.split(j)
47 | print ' - %s'%(name)
48 |
49 | for j in jsons:
50 | path, fname = os.path.split(j)
51 | base, ext = os.path.splitext(fname)
52 | collection = base
53 | print '*'*50
54 | print 'Dropping collection %s'%collection
55 | db[collection].drop()
56 | print 'Importing json %s into collection %s'%(j, collection)
57 | cmd = 'mongoimport --db {db} --collection {collection} {fname}'
58 | cmd = cmd.format(db = args.db, collection=collection, fname=j)
59 | call(cmd, shell=True)
60 |
61 | if __name__ == '__main__':
62 | run()
63 |
--------------------------------------------------------------------------------
/utils/migration.py:
--------------------------------------------------------------------------------
1 | """Migrate the database to the new version. 8/11/2014"""
2 |
3 | import test
4 | from dcmetrometrics.common import dbGlobals
5 | dbGlobals.connect()
6 |
7 | from utils import denormalize_db
8 | denormalize_db.update_symptom_codes()
9 | denormalize_db.update_unit_statuses()
10 | denormalize_db.denormalize_unit_statuses()
11 | denormalize_db.recompute_key_statuses()
12 |
13 |
14 | # Finally, we need to create the stations in the database
15 | denormalize_db.add_station_docs()
--------------------------------------------------------------------------------
/utils/migrations_hotcars.py:
--------------------------------------------------------------------------------
1 | """
2 | Script to perform database migrations for the ELES App.
3 |
4 | Script to denormalize the database by adding fields to the unit status records.
5 |
6 | This should be imported as a module and not run directly.
7 | """
8 |
9 | from . import utils
10 | utils.fixSysPath()
11 |
12 | import sys
13 | from datetime import datetime
14 |
15 | from dcmetrometrics.common.dbGlobals import G
16 | from dcmetrometrics.hotcars.models import (HotCarAppState, HotCarTweeter, HotCarTweet,
17 | HotCarReport, CarsForbiddenByMention, Temperature)
18 | from dcmetrometrics.hotcars.twitter_api import TwitterError, getTwitterAPI
19 | from datetime import timedelta
20 | import logging
21 |
22 | from dcmetrometrics.common.JSONifier import JSONWriter
23 | from dcmetrometrics.common.globals import WWW_DIR
24 |
25 | ##########################
26 | def update_twitter_handles():
27 | """
28 | For each twitter user, look up the user and the handle.
29 | Save the twitter user to the hotcars_tweeters collection.
30 | """
31 |
32 | user_ids = set(t.user_id for t in HotCarTweeter.objects)
33 | user_ids = list(user_ids)
34 |
35 | print "Have %i users"%len(user_ids)
36 |
37 | # Make requests in batch of 100
38 | def gen_batches(d):
39 | i = 0
40 | n = 100
41 | ret = d[i:i+n]
42 | while ret:
43 | yield ret
44 | i = i + n
45 | ret = d[i:i+n]
46 |
47 | T = getTwitterAPI()
48 |
49 | for i, user_id_batch in enumerate(gen_batches(user_ids)):
50 |
51 | print "Querying Twitter with batch %i"%i
52 |
53 | try:
54 |
55 | user_info = T.UsersLookup(user_id = user_id_batch)
56 |
57 | for user in user_info:
58 | user_id = user.id
59 | handle = user.screen_name
60 | HotCarTweeter.update(user_id, handle)
61 |
62 | except TwitterError as e:
63 | print("Caught twitter error:\n%s"%str(e))
64 |
65 | denormalize_reports()
66 |
67 | def denormalize_reports():
68 | """
69 | Denormalize all hot car report docs by setting the
70 | user_id field, text field, and handle field.
71 |
72 | """
73 | for doc in HotCarReport.iter_reports():
74 | doc.denormalize()
75 |
76 | def write_json():
77 | jwriter = JSONWriter(WWW_DIR)
78 | jwriter.write_hotcars()
79 | jwriter.write_hotcars_by_day()
--------------------------------------------------------------------------------
/utils/trimLogs.py:
--------------------------------------------------------------------------------
1 | import glob, sys, os
2 | import subprocess
3 | import shutil
4 |
5 | DATA_DIR = os.environ['OPENSHIFT_DATA_DIR']
6 | os.chdir(DATA_DIR)
7 |
8 | logFiles = glob.glob('*.log')
9 | logFiles = [os.path.abspath(l) for l in logFiles]
10 |
11 | print 'Found %i log files'%len(logFiles)
12 | print '\n'.join(logFiles)
13 |
14 | def trimLog(logFile, numLines = 200000):
15 | path, fn = os.path.split(logFile)
16 | bn, ext = os.path.splitext(fn)
17 | tempFile = '%s.temp'%fn
18 | tempFile = os.path.join(path, tempFile)
19 |
20 | # Execute tail
21 | cmd = ['tail', '-n', str(numLines), logFile]
22 | out = open(tempFile, 'w')
23 | subprocess.call(cmd, stdout = out)
24 | out.close()
25 |
26 | # Remove original log file
27 | os.remove(logFile)
28 |
29 | # Copy the shortened log file
30 | os.rename(tempFile, logFile)
31 |
32 | for logFile in logFiles:
33 | logFile = os.path.abspath(logFile)
34 | trimLog(logFile)
35 |
--------------------------------------------------------------------------------
/utils/utils.py:
--------------------------------------------------------------------------------
1 | import sys, os
2 |
3 | _this_file = os.path.abspath(__file__)
4 | _this_dir = os.path.split(_this_file)[0]
5 | ROOT_DIR = os.path.split(_this_dir)[0]
6 |
7 | def isOpenshiftEnv():
8 | """
9 | Return True if this appears to be an Openshift Environment.
10 | """
11 | MIN_ENV_COUNT = 30
12 | env_vars = os.environ.keys()
13 | isOpenshiftEnv = lambda s: 'OPENSHIFT' == s[:9]
14 | openshift_envs = [e for e in env_vars if isOpenshiftEnv(e)]
15 | return len(openshift_envs) >= MIN_ENV_COUNT
16 |
17 | def fixSysPath():
18 | if ROOT_DIR not in sys.path:
19 | sys.path = [ROOT_DIR] + sys.path
20 |
21 | fixSysPath()
22 |
--------------------------------------------------------------------------------
/webPageGeneratorApp.py:
--------------------------------------------------------------------------------
1 | """
2 | Run an instance of the WebPageGenerator restartingGreenlet
3 | for local testing.
4 | """
5 |
6 | import test.setup
7 | from dcmetrometrics.web.WebPageGenerator import WebPageGenerator
8 | print 'Running the webPageGenerator locally'
9 | app = WebPageGenerator()
10 | app.start()
11 | app.join()
12 |
--------------------------------------------------------------------------------
/wsgi/static/.gitignore:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LeeMendelowitz/DCMetroMetrics/73b28117fbdb35886cec0e05328a19416550c6a7/wsgi/static/.gitignore
--------------------------------------------------------------------------------
/wsgi/static/architect.logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LeeMendelowitz/DCMetroMetrics/73b28117fbdb35886cec0e05328a19416550c6a7/wsgi/static/architect.logo.png
--------------------------------------------------------------------------------
/wsgi/static/dailycaller.logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LeeMendelowitz/DCMetroMetrics/73b28117fbdb35886cec0e05328a19416550c6a7/wsgi/static/dailycaller.logo.png
--------------------------------------------------------------------------------
/wsgi/static/dcist.logo.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LeeMendelowitz/DCMetroMetrics/73b28117fbdb35886cec0e05328a19416550c6a7/wsgi/static/dcist.logo.gif
--------------------------------------------------------------------------------
/wsgi/static/hotair.logo.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LeeMendelowitz/DCMetroMetrics/73b28117fbdb35886cec0e05328a19416550c6a7/wsgi/static/hotair.logo.gif
--------------------------------------------------------------------------------
/wsgi/static/styles.css:
--------------------------------------------------------------------------------
1 | body {
2 | padding-top: 60px;
3 | padding-bottom: 40px;
4 | background-color:#D0D0D0;
5 | background: url('/static/wmata_ceiling_banner_web.jpg') no-repeat center center fixed;
6 | -webkit-background-size: cover;
7 | -moz-background-size: cover;
8 | -o-background-size: cover;
9 | background-size: cover;
10 | }
11 |
12 | @media (max-width:979px){
13 | body{
14 | padding-top:0px;
15 | padding-bottom:0px;
16 | }
17 | }
18 |
19 | .footer {
20 | margin-top: 10px;
21 | padding-top: 10px;
22 | background-color: white;
23 | border-radius: 6px 6px 6px 6px;
24 | }
25 |
26 | .navbar .nav > li > .dropdown-menu:after {
27 | border: none;
28 | }
29 |
30 | .googletable td{border: thin solid #dddddd;}
31 | .googletable td.success{background-color:#dff0d8;}
32 | .googletable td.error{background-color:#f2dede;}
33 | .googletable td.warning{background-color:#fcf8e3;}
34 | .googletable td.info{background-color:#d9edf7;}
35 |
36 |
37 | .main-content {
38 | padding: 25px;
39 | background-color: white;
40 | border-radius: 6px 6px 6px 6px;
41 | }
42 |
43 |
44 | .press-item .publisher {
45 | color: blue;
46 | }
47 |
48 | .updateTime
49 | {
50 | padding-top:20px;
51 | color:black
52 | font-size:small;
53 | text-align:right;
54 | }
55 |
56 | .redsquare
57 | {
58 | width:14px;
59 | height:14px;
60 | background-color:red;
61 | border:1px solid black;
62 | float:left;
63 | margin:5px;
64 | }
65 |
66 | .bluesquare
67 | {
68 | width:14px;
69 | height:14px;
70 | background-color:blue;
71 | border:1px solid black;
72 | float:left;
73 | margin:5px;
74 | }
75 | .yellowsquare
76 | {
77 | width:14px;
78 | height:14px;
79 | background-color:yellow;
80 | border:1px solid black;
81 | float:left;
82 | margin:5px;
83 | }
84 | .orangesquare
85 | {
86 | width:14px;
87 | height:14px;
88 | background-color:orange;
89 | border:1px solid black;
90 | float:left;
91 | margin:5px;
92 | }
93 | .greensquare
94 | {
95 | width:14px;
96 | height:14px;
97 | background-color:green;
98 | border:1px solid black;
99 | float:left;
100 | margin:5px;
101 | }
--------------------------------------------------------------------------------
/wsgi/static/unsuckdcmetro.logo.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LeeMendelowitz/DCMetroMetrics/73b28117fbdb35886cec0e05328a19416550c6a7/wsgi/static/unsuckdcmetro.logo.jpg
--------------------------------------------------------------------------------
/wsgi/static/wapo.logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LeeMendelowitz/DCMetroMetrics/73b28117fbdb35886cec0e05328a19416550c6a7/wsgi/static/wapo.logo.png
--------------------------------------------------------------------------------
/wsgi/static/wmata-gallery-place.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LeeMendelowitz/DCMetroMetrics/73b28117fbdb35886cec0e05328a19416550c6a7/wsgi/static/wmata-gallery-place.jpg
--------------------------------------------------------------------------------
/wsgi/static/wmata_ceiling_banner_blue.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LeeMendelowitz/DCMetroMetrics/73b28117fbdb35886cec0e05328a19416550c6a7/wsgi/static/wmata_ceiling_banner_blue.jpg
--------------------------------------------------------------------------------
/wsgi/static/wmata_ceiling_banner_web.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LeeMendelowitz/DCMetroMetrics/73b28117fbdb35886cec0e05328a19416550c6a7/wsgi/static/wmata_ceiling_banner_web.jpg
--------------------------------------------------------------------------------
/wsgi/static/wusa9logo.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/LeeMendelowitz/DCMetroMetrics/73b28117fbdb35886cec0e05328a19416550c6a7/wsgi/static/wusa9logo.gif
--------------------------------------------------------------------------------
/wsgi/views/DCMetroMetricRedesign/rankingsContent.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
Rankings
6 |
7 |
Explore which escalators are the most broken, and which stations have the
8 | best and worst performing escalators. Also explore historical daily counts
9 | of escalator breaks and inspections.
This website and its features are a work in progress.
20 | If you have any questions or comments, or would like to get involved,
21 | please contact me at info@dcmetrometrics.com.
70 |
71 |
72 |
73 |
74 | %rebase layout title='DC Metro Metrics: Nonoperational Escalators', description=''
75 |
--------------------------------------------------------------------------------
/wsgi/views/old/escalatorRankings.tpl:
--------------------------------------------------------------------------------
1 | %# Get rankings for escalators
2 | %from metroTimes import toLocalTime
3 |
4 | %description = 'Performance rankings of escalators in the WMATA Metrorail system.'
5 |
6 |
7 |
Rankings
8 |
9 |
Explore which escalators are the most broken, and which stations have the
10 | best and worst performing escalators. Also explore historical daily counts
11 | of escalator breaks and inspections.
This website and its features are a work in progress.
15 | If you have any questions or comments, or would like to get involved,
16 | please contact me at info@dcmetrometrics.com.