├── runtime.txt ├── Procfile ├── screenshot.png ├── requirements.txt ├── web ├── static │ ├── img │ │ ├── bus.png │ │ ├── busstop.png │ │ ├── bus-inbound.png │ │ └── bus-outbound.png │ ├── css │ │ └── images │ │ │ ├── ajax-loader.gif │ │ │ ├── icons-18-black.png │ │ │ ├── icons-18-white.png │ │ │ ├── icons-36-black.png │ │ │ └── icons-36-white.png │ └── js │ │ ├── underscore-min.js │ │ ├── backbone-min.js │ │ ├── busfinder.js │ │ └── json2.js ├── app.py ├── gtfs-realtime.proto └── gtfs_realtime_pb2.py ├── scripts ├── schedule.py ├── serverless.yml ├── bus.py ├── README.md ├── ftp.py ├── gtfs.py └── database.py ├── .gitignore └── README.md /runtime.txt: -------------------------------------------------------------------------------- 1 | python-2.7.14 2 | 3 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: python web/app.py 2 | clock: python scripts/schedule.py -------------------------------------------------------------------------------- /screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Code4HR/hrt-bus-api/HEAD/screenshot.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | APScheduler 2 | Flask 3 | Jinja2 4 | geopy 5 | protobuf 6 | pymongo 7 | pytz 8 | -------------------------------------------------------------------------------- /web/static/img/bus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Code4HR/hrt-bus-api/HEAD/web/static/img/bus.png -------------------------------------------------------------------------------- /web/static/img/busstop.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Code4HR/hrt-bus-api/HEAD/web/static/img/busstop.png -------------------------------------------------------------------------------- /web/static/img/bus-inbound.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Code4HR/hrt-bus-api/HEAD/web/static/img/bus-inbound.png -------------------------------------------------------------------------------- /web/static/img/bus-outbound.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Code4HR/hrt-bus-api/HEAD/web/static/img/bus-outbound.png -------------------------------------------------------------------------------- /web/static/css/images/ajax-loader.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Code4HR/hrt-bus-api/HEAD/web/static/css/images/ajax-loader.gif -------------------------------------------------------------------------------- /web/static/css/images/icons-18-black.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Code4HR/hrt-bus-api/HEAD/web/static/css/images/icons-18-black.png -------------------------------------------------------------------------------- /web/static/css/images/icons-18-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Code4HR/hrt-bus-api/HEAD/web/static/css/images/icons-18-white.png -------------------------------------------------------------------------------- /web/static/css/images/icons-36-black.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Code4HR/hrt-bus-api/HEAD/web/static/css/images/icons-36-black.png -------------------------------------------------------------------------------- /web/static/css/images/icons-36-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Code4HR/hrt-bus-api/HEAD/web/static/css/images/icons-36-white.png -------------------------------------------------------------------------------- /scripts/schedule.py: -------------------------------------------------------------------------------- 1 | import ftp 2 | import gtfs 3 | from apscheduler.schedulers.blocking import BlockingScheduler 4 | 5 | sched = BlockingScheduler() 6 | 7 | # Process FTP every minute 8 | @sched.scheduled_job('interval', minutes=1) 9 | def ftp_job(): 10 | print 'Running FTP Job' 11 | ftp.process({}, None) 12 | print 'FTP Job Complete' 13 | 14 | # Each day at 3 AM, load schedules for the next day 15 | @sched.scheduled_job('cron', hour=3) 16 | def gtfs_job(): 17 | print 'Running GTFS Job' 18 | gtfs.process({}, None) 19 | print 'GTFS Job Complete' 20 | 21 | sched.start() 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Build Folders (you can keep bin if you'd like, to store dlls and pdbs) 2 | bin 3 | obj 4 | 5 | # mstest test results 6 | TestResults 7 | 8 | *.suo 9 | *.user 10 | 11 | _ReSharper*/ 12 | packages 13 | 14 | .DS_Store 15 | 16 | venv 17 | *.pyc 18 | *.json 19 | 20 | 21 | .env 22 | 23 | web/debug.py 24 | 25 | # ide files 26 | .idea/* 27 | 28 | # Distribution / packaging 29 | .Python 30 | env/ 31 | build/ 32 | develop-eggs/ 33 | dist/ 34 | downloads/ 35 | eggs/ 36 | .eggs/ 37 | lib/ 38 | lib64/ 39 | parts/ 40 | sdist/ 41 | var/ 42 | *.egg-info/ 43 | .installed.cfg 44 | *.egg 45 | 46 | # Serverless directories 47 | .serverless 48 | 49 | .requirements -------------------------------------------------------------------------------- /scripts/serverless.yml: -------------------------------------------------------------------------------- 1 | # Welcome to Serverless! 2 | # 3 | # This file is the main config file for your service. 4 | # It's very minimal at this point and uses default values. 5 | # You can always add more config options for more control. 6 | # We've included some commented out config examples here. 7 | # Just uncomment any of them to get that config option. 8 | # 9 | # For full config options, check the docs: 10 | # docs.serverless.com 11 | # 12 | # Happy Coding! 13 | 14 | service: hrt 15 | 16 | provider: 17 | name: aws 18 | runtime: python2.7 19 | role: 20 | vpc: 21 | securityGroupIds: 22 | - 23 | subnetIds: 24 | - 25 | environment: 26 | MONGODB_URI: "mongodb://" 27 | 28 | functions: 29 | gtfs: 30 | handler: gtfs.process 31 | memorySize: 512 32 | timeout: 240 33 | events: 34 | - schedule: rate(1 day) 35 | ftp: 36 | handler: ftp.process 37 | memorySize: 128 38 | timeout: 30 39 | events: 40 | - schedule: rate(1 minute) 41 | 42 | plugins: 43 | - serverless-python-requirements 44 | -------------------------------------------------------------------------------- /scripts/bus.py: -------------------------------------------------------------------------------- 1 | import pytz 2 | from datetime import datetime, timedelta 3 | 4 | class Checkin: 5 | def __init__(self, data): 6 | parts = data.split(',') 7 | 8 | # checkin time 9 | local_tz = pytz.timezone('US/Eastern') 10 | 11 | utc = datetime.utcnow() 12 | utc = utc.replace(tzinfo=pytz.utc) 13 | local = utc.astimezone(local_tz) 14 | 15 | # handle checkins from both 12/31/N and 1/1/N+1 16 | year = local.year 17 | if parts[1].startswith('12/') and local.month == 1: 18 | year -= 1 19 | elif parts[1].startswith('1/') and local.month == 12: 20 | year += 1 21 | 22 | #print local, data, year 23 | 24 | naive = datetime.strptime('{} {}/{}'.format( 25 | parts[0], parts[1], str(year) 26 | ), '%H:%M:%S %m/%d/%Y') 27 | local_dt = local_tz.localize(naive, is_dst=False) 28 | self.time = local_dt.astimezone(pytz.utc) 29 | 30 | # bus id 31 | self.busId = int(parts[2]) 32 | 33 | # location 34 | if parts[4] == 'V': 35 | location = parts[3].split('/') 36 | lat = float(location[0][:2] + '.' + location[0][2:]) 37 | lon = float(location[1][:3] + '.' + location[1][3:]) 38 | self.location = [lat, lon] 39 | 40 | # adherence 41 | if parts[6] == 'V': 42 | self.adherence = int(parts[5]) 43 | 44 | if len(parts) == 10: 45 | self.routeShortName = int(parts[7]) 46 | self.direction = int(parts[8]) - 1 47 | self.stopId = parts[9].zfill(4) 48 | -------------------------------------------------------------------------------- /scripts/README.md: -------------------------------------------------------------------------------- 1 | # Scripts 2 | 3 | Both scripts require setting the following environment variable: 4 | 5 | * `MONGODB_URI` 6 | 7 | ## GTFS 8 | 9 | `python gtfs.py [daysRelativeToToday]` 10 | 11 | Downloads GTFS data and loads one day's schedule into the database. Realtime data from HRT is relative to schedule data, so this data is necessary to correct interpret realtime data. 12 | 13 | If `daysRelativeToToday` is not specified, it defaults to `0`, which means that the schedule for the current day is loaded. 14 | 15 | ## FTP 16 | 17 | `python ftp.py` 18 | 19 | Downloads realtime data from HRT, matches to scheduled data and stores results in database. 20 | 21 | # Deployment 22 | 23 | The current deployment strategy is to use a Heroku custom clock process. Below are details for deploying to AWS Lambda, where we used to run the scripts. 24 | 25 | # AWS Deployment (OLD) 26 | 27 | The scripts can be deployed to AWS Lambda using [Serverless](https://serverless.com) where they will run periodically. 28 | 29 | Set environment variables in `serverless.yml` 30 | 31 | Note that HRT's FTP server only allows whitelisted IPs, so you'll need to set up a VPC. [Here is one guide on how to do that](http://techblog.financialengines.com/2016/09/26/aws-lambdas-with-a-static-outgoing-ip/). 32 | 33 | Install Serverless 34 | `npm install -g serverless` 35 | 36 | Install python requirements plugin 37 | `npm install --save serverless-python-requirements` 38 | 39 | Configure AWS credentials 40 | 41 | Create and activate a virtual environment 42 | ``` 43 | virtualenv venv 44 | source venv/bin/activate 45 | ``` 46 | 47 | Deploy 48 | `serverless deploy -v` 49 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ARCHIVED 2 | HRT will begin supporting real time data and has stopped sharing their hack-y real-time feed. This project is no longer useful. 3 | 4 | # hrt-bus-api 5 | HRT Bus API publishes real time bus data from Hampton Roads Transit through an application programming interface for developers to make apps from it. 6 | 7 | HRT Bus API consists of Python scripts and a Flask app that transform, store, and expose HRT Bus data through a RESTful API. API's don't make for good demos, so we've created an HRT Bus Finder App. 8 | 9 | ## Try It 10 | * Web App: [hrtb.us](http://hrtb.us) 11 | * REST Api: [api.hrtb.us/api](http://api.hrtb.us/api/) 12 | 13 | ## Problem 14 | HRT exposes the real time location of their buses through a text file exposed through an FTP server. Unfortunately, this file gives us less than five minutes of data and most of the entries don't have a route number associated with them. Riders lookup bus information by route number, so without it, the data isn't very useful. 15 | 16 | ## Solution 17 | 18 | ### Transform and Store Data - Python Scripts 19 | * [Process GTFS](https://github.com/code4hr/hrt-bus-api/tree/master/scripts/gtfs.py) - Fetches the HRT GTFS package and stores the scheduled stop times for a single day in MongoDB. 20 | * [Process FTP](https://github.com/code4hr/hrt-bus-api/tree/master/scripts/ftp.py) - Fetches the HRT FTP file and stores the data in MongoDB. Also attempts to set route number when it's missing. 21 | 22 | ### Expose Data - Python Flask 23 | * Web App 24 | * [RESTful API](https://github.com/code4hr/hrt-bus-api/wiki/RESTful-API) 25 | 26 | ## Setup for Local Development 27 | 28 | 1. Install [Python 2](http://wiki.python.org/moin/BeginnersGuide/Download) 29 | 2. Install [virtualenv](https://pypi.python.org/pypi/virtualenv) 30 | 3. Clone this repo 31 | 4. Create a virtual environment in the top level directory of the repo 32 | 33 | ``` 34 | $ virtualenv venv --distribute 35 | ``` 36 | 37 | 5. Activate the environment 38 | 39 | ``` 40 | $ source venv/bin/activate 41 | ``` 42 | -OR- for Windows 43 | ``` 44 | $ venv\Scripts\activate.bat 45 | ``` 46 | 47 | 6. Install dependencies 48 | 49 | ``` 50 | $ pip install -r requirements.txt 51 | ``` 52 | 53 | ### Scripts 54 | If you would like to develop the scripts, you will need your own MongoDB instance. I recommend [mLab](https://mLab.com/). If you just want to work on the web app, feel free to skip the part about the scripts. Ask someone on the team for read-only access to our production MongoDB instance. 55 | 56 | ### Web App 57 | 58 | 1. Set MongoDB URI (substitue your own MongoDB instance if you have one) 59 | 60 | ``` 61 | $ export MONGO_URI=mongodb:// 62 | ``` 63 | -OR- for Windows 64 | ``` 65 | $ set MONGO_URI=mongodb:// 66 | ``` 67 | 2. Change to the web directory and run the flask app 68 | 69 | ``` 70 | $ cd web 71 | $ python app.py 72 | ``` 73 | 74 | 3. Browse to `http://0.0.0.0:5000/` 75 | 76 | ## Deployment 77 | 78 | We deploy to [Heroku](http://www.heroku.com/). Check this [wiki page](https://github.com/c4hrva/hrt-bus-api/wiki/Deploying-To-Heroku) 79 | 80 | ## We're Here to Help 81 | * Ben - [@OilyTheOtter](http://twitter.com/oilytheotter) 82 | * Blaine - [@wbprice](https://twitter.com/wbprice) 83 | * Stanley - [@StanZheng](https://twitter.com/StanZheng) 84 | -------------------------------------------------------------------------------- /scripts/ftp.py: -------------------------------------------------------------------------------- 1 | import os 2 | from ftplib import FTP 3 | from datetime import datetime, timedelta 4 | from bus import Checkin 5 | from database import HRTDatabase 6 | 7 | DATABASE = HRTDatabase() 8 | PROCESSOR = None 9 | LAST_REPEAT = None 10 | 11 | def process(event, context): 12 | global PROCESSOR 13 | PROCESSOR = Processor() 14 | print "Read {0} Bus Route Mappings".format(len(PROCESSOR.bus_route_mappings)) 15 | 16 | ftp = FTP('52.170.192.86') 17 | ftp.login() 18 | #print ftp.sendcmd('PASV') 19 | ftp.cwd('Anrd') 20 | ftp.retrlines('RETR hrtrtf.txt', processData) 21 | 22 | if PROCESSOR.last_checkins is not None: 23 | print "Latest Checkin Time From Previous Run: {0}".format(PROCESSOR.last_checkins["time"]) 24 | print "Latest Previously Processed Checkin Time From This Run: {0}".format(PROCESSOR.last_repeat) 25 | 26 | DATABASE.setBusRouteMappings(PROCESSOR.bus_route_mappings.values()) 27 | print "Inserted {0} Bus Route Mappings".format(len(PROCESSOR.bus_route_mappings)) 28 | 29 | DATABASE.updateCheckins(PROCESSOR.checkin_docs) 30 | print "Added {0} Checkins".format(len(PROCESSOR.checkin_docs)) 31 | 32 | DATABASE.updateRealTimeArrivals(PROCESSOR.schedule_changes) 33 | 34 | for key, value in PROCESSOR.stats.iteritems(): 35 | print "{0} {1}".format(key, value) 36 | 37 | class Processor: 38 | def __init__(self): 39 | self.bus_route_mappings = DATABASE.getBusRouteMappings() 40 | self.last_checkins = DATABASE.getLastCheckinSummary() 41 | self.checkin_docs = [] 42 | self.schedule_changes = {} 43 | self.last_repeat = None 44 | self.stats = { 45 | 'lines': 0, 46 | 'invalid': 0, 47 | 'processed': 0, 48 | 'hadRoute': 0, 49 | 'foundRoute': 0, 50 | 'foundTrip': 0, 51 | 'arriveTimesUpdated': 0 52 | } 53 | 54 | def processData(text): 55 | if not text.strip(): 56 | return 57 | 58 | PROCESSOR.stats['lines'] += 1 59 | 60 | try: 61 | checkin = Checkin(text) 62 | except ValueError: 63 | PROCESSOR.stats['invalid'] += 1 64 | return 65 | 66 | if checkinProcessed(checkin): 67 | PROCESSOR.last_repeat = checkin.time 68 | return 69 | 70 | PROCESSOR.stats['processed'] += 1 71 | 72 | if hasattr(checkin, 'routeShortName'): 73 | checkin.tripId = None 74 | checkin.blockId = None 75 | checkin.lastStopSequence = None 76 | checkin.lastStopSequenceOBA = None 77 | if hasattr(checkin, 'adherence'): 78 | scheduled_stop = DATABASE.getScheduledStop(checkin) 79 | if scheduled_stop is not None: 80 | PROCESSOR.stats['foundTrip'] += 1 81 | checkin.tripId = scheduled_stop['trip_id'] 82 | checkin.blockId = scheduled_stop['block_id'] 83 | checkin.lastStopSequence = scheduled_stop['stop_sequence'] 84 | checkin.lastStopSequenceOBA = scheduled_stop['stop_sequence_OBA'] 85 | checkin.scheduleMatch = True 86 | if checkin.tripId is None and checkin.busId in PROCESSOR.bus_route_mappings: 87 | checkin.tripId = PROCESSOR.bus_route_mappings[checkin.busId]['tripId'] 88 | checkin.blockId = PROCESSOR.bus_route_mappings[checkin.busId]['blockId'] 89 | checkin.lastStopSequence = PROCESSOR.bus_route_mappings[checkin.busId]['lastStopSequence'] 90 | checkin.lastStopSequenceOBA = PROCESSOR.bus_route_mappings[checkin.busId]['lastStopSequenceOBA'] 91 | PROCESSOR.bus_route_mappings[checkin.busId] = { 92 | 'busId': checkin.busId, 93 | 'routeShortName' : checkin.routeShortName, 94 | 'direction': checkin.direction, 95 | 'tripId': checkin.tripId, 96 | 'blockId': checkin.blockId, 97 | 'lastStopSequence': checkin.lastStopSequence, 98 | 'lastStopSequenceOBA': checkin.lastStopSequenceOBA, 99 | 'time': checkin.time 100 | } 101 | PROCESSOR.stats['hadRoute'] += 1 102 | elif checkin.busId in PROCESSOR.bus_route_mappings: 103 | checkin.routeShortName = PROCESSOR.bus_route_mappings[checkin.busId]['routeShortName'] 104 | checkin.direction = PROCESSOR.bus_route_mappings[checkin.busId]['direction'] 105 | checkin.tripId = PROCESSOR.bus_route_mappings[checkin.busId]['tripId'] 106 | checkin.blockId = PROCESSOR.bus_route_mappings[checkin.busId]['blockId'] 107 | checkin.lastStopSequence = PROCESSOR.bus_route_mappings[checkin.busId]['lastStopSequence'] 108 | checkin.lastStopSequenceOBA = PROCESSOR.bus_route_mappings[checkin.busId]['lastStopSequenceOBA'] 109 | PROCESSOR.stats['foundRoute'] += 1 110 | 111 | if hasattr(checkin, 'adherence') and hasattr(checkin, 'blockId'): 112 | collection, updates = DATABASE.getRealTimeArrivalUpdates(checkin) 113 | if collection not in PROCESSOR.schedule_changes: 114 | PROCESSOR.schedule_changes[collection] = [] 115 | PROCESSOR.schedule_changes[collection].extend(updates) 116 | PROCESSOR.stats['arriveTimesUpdated'] += len(updates) 117 | 118 | PROCESSOR.checkin_docs.append(checkin.__dict__) 119 | 120 | def checkinProcessed(checkin): 121 | if PROCESSOR.last_checkins is None: 122 | return False 123 | if checkin.time == PROCESSOR.last_checkins["time"]: 124 | return checkin.busId in PROCESSOR.last_checkins["busIds"] 125 | return checkin.time < PROCESSOR.last_checkins["time"] 126 | -------------------------------------------------------------------------------- /scripts/gtfs.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import pytz 4 | import sys 5 | from StringIO import StringIO 6 | from urllib import urlopen 7 | from zipfile import ZipFile 8 | from csv import DictReader 9 | from datetime import datetime, time, timedelta 10 | from database import HRTDatabase 11 | 12 | EST = pytz.timezone('US/Eastern') 13 | 14 | def process(event, context): 15 | print event 16 | 17 | now = datetime.utcnow().replace(tzinfo=pytz.utc).astimezone(EST) 18 | print 'Running at ' + str(now) 19 | 20 | database = HRTDatabase() 21 | if 'daysFromNow' not in event: 22 | database.removeOldGTFS(now) 23 | 24 | file_url = 'http://googletf.gohrt.com/google_transit.zip' 25 | zip_file = ZipFile(StringIO(urlopen(file_url).read())) 26 | 27 | days_from_now = 1 28 | if 'daysFromNow' in event: 29 | days_from_now = event['daysFromNow'] 30 | days = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'] 31 | cur_date = (now + timedelta(days=days_from_now)).date() 32 | midnight = datetime.combine(cur_date, time.min) 33 | cur_week_day = days[cur_date.weekday()] 34 | print cur_week_day + " " + str(cur_date) 35 | 36 | routes = [] 37 | route_map = {} 38 | route_errors = 0 39 | routes_reader = DictReader(open_from_zipfile(zip_file, "routes.txt")) 40 | for row in routes_reader: 41 | try: 42 | row['route_id'] = row['route_id'] 43 | row['route_short_name'] = int(row['route_short_name']) 44 | route_map[row['route_id']] = int(row['route_short_name']) 45 | routes.append(row) 46 | except ValueError: 47 | route_errors += 1 48 | pass 49 | print str(len(routes)) + " routes (" + str(route_errors) + " errors)" 50 | database.insertRoutes(routes, cur_date) 51 | 52 | active_service_ids = [] 53 | calendar = DictReader(open_from_zipfile(zip_file, "calendar.txt")) 54 | for row in calendar: 55 | start = datetime.strptime(row['start_date'], "%Y%m%d").date() 56 | end = datetime.strptime(row['end_date'], "%Y%m%d").date() 57 | if cur_date >= start and cur_date <= end and row[cur_week_day] == '1': 58 | active_service_ids.append(row['service_id']) 59 | print active_service_ids 60 | 61 | active_trips = {} 62 | trips = DictReader(open_from_zipfile(zip_file, "trips.txt")) 63 | for row in trips: 64 | if row['service_id'] in active_service_ids: 65 | active_trips[row['trip_id']] = row 66 | print str(len(active_trips)) + " active trips" 67 | 68 | active_stop_times = [] 69 | active_stop_time_errors = 0 70 | stop_times = DictReader(open_from_zipfile(zip_file, "stop_times.txt")) 71 | for row in stop_times: 72 | if row['trip_id'] in active_trips: 73 | try: 74 | trip = active_trips[row['trip_id']] 75 | row['route_id'] = trip['route_id'] 76 | row['route_short_name'] = route_map[trip['route_id']] 77 | row['direction_id'] = int(trip['direction_id']) 78 | row['block_id'] = trip['block_id'] 79 | row['stop_id'] = row['stop_id'] 80 | row['stop_sequence'] = int(row['stop_sequence']) 81 | 82 | arrive_time = row['arrival_time'].split(':') 83 | naive_arrive_time = midnight + timedelta( 84 | hours=int(arrive_time[0]), minutes=int(arrive_time[1]) 85 | ) 86 | local_arrive_time = EST.localize(naive_arrive_time, is_dst=False) 87 | row['arrival_time'] = local_arrive_time.astimezone(pytz.utc) 88 | 89 | depart_time = row['departure_time'].split(':') 90 | naive_dept_time = midnight + timedelta( 91 | hours=int(depart_time[0]), minutes=int(depart_time[1]) 92 | ) 93 | local_dept_time = EST.localize(naive_dept_time, is_dst=False) 94 | row['departure_time'] = local_dept_time.astimezone(pytz.utc) 95 | 96 | active_stop_times.append(row) 97 | except ValueError: 98 | active_stop_time_errors += 1 99 | pass 100 | print str(len(active_stop_times)) + " active stop times (" + str(active_stop_time_errors) + " errors)" 101 | 102 | database.insertGTFS(active_stop_times, cur_date) 103 | 104 | stops = [] 105 | stops_reader = DictReader(open_from_zipfile(zip_file, "stops.txt")) 106 | for row in stops_reader: 107 | try: 108 | stops.append({ 109 | 'stopId': row['stop_id'], 110 | 'stopName': row['stop_name'], 111 | 'location': [float(row['stop_lat']), float(row['stop_lon'])] 112 | }) 113 | except ValueError: 114 | pass 115 | print str(len(stops)) + " stops" 116 | database.insertStops(stops, cur_date) 117 | 118 | print "Generating Destinations Collection" 119 | destinations = [] 120 | for trip in database.getFinalStops(cur_date): 121 | destinations.append({ 122 | 'tripId': trip['_id'], 123 | 'stopName': database.getStopName(trip['stopId'], cur_date) 124 | }) 125 | print str(len(destinations)) + " destinations" 126 | database.insertDestinations(destinations, cur_date) 127 | 128 | print "Generating Indexes" 129 | database.generateIndicesForGTFS(cur_date) 130 | 131 | def open_from_zipfile(zip_file, filename): 132 | # Remove UTF-8 BOM (http://stackoverflow.com/a/18664752/438281) 133 | return StringIO( 134 | zip_file 135 | .open(filename) 136 | .read() 137 | .decode("utf-8-sig") 138 | .encode("utf-8") 139 | ) 140 | 141 | if __name__ == "__main__": 142 | days_from_now = 0 143 | if len(sys.argv) > 1: 144 | days_from_now = int(sys.argv[1]) 145 | process({'daysFromNow': days_from_now}, None) 146 | -------------------------------------------------------------------------------- /scripts/database.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytz 3 | from datetime import datetime, timedelta 4 | from pymongo import MongoClient, GEO2D, ASCENDING, UpdateOne 5 | 6 | class HRTDatabase: 7 | def __init__(self): 8 | self.database = MongoClient(os.environ['MONGODB_URI']).get_database(None) 9 | 10 | def removeOldGTFS(self, date): 11 | print "Removing Old GTFS Data" 12 | collection_prefix = self.genCollectionName('', date) 13 | for collection in self.database.collection_names(): 14 | if collection.find('_') != -1 and (not collection.endswith(collection_prefix)): 15 | self.database.drop_collection(collection) 16 | 17 | def insertGTFS(self, data, date): 18 | self.insertData(self.genCollectionName('gtfs_', date), data) 19 | 20 | def insertStops(self, data, date): 21 | collection_name = self.genCollectionName('stops_', date) 22 | self.insertData(collection_name, data) 23 | self.database[collection_name].ensure_index([('location', GEO2D)]) 24 | 25 | def insertRoutes(self, data, date): 26 | self.insertData(self.genCollectionName('routes_', date), data) 27 | 28 | def insertDestinations(self, data, date): 29 | self.insertData(self.genCollectionName('destinations_', date), data) 30 | 31 | def getStopName(self, stop_id, date): 32 | collection_name = self.genCollectionName('stops_', date) 33 | stop = self.database[collection_name].find_one({"stopId": stop_id}) 34 | return stop['stopName'] 35 | 36 | def getFinalStops(self, date): 37 | collection_name = self.genCollectionName('gtfs_', date) 38 | final_stops = self.database[collection_name].aggregate([ 39 | {"$group": { 40 | "_id": "$trip_id", 41 | "stopId": {"$last": "$stop_id"}, 42 | "sequence": {"$last": "$stop_sequence"} 43 | }} 44 | ]) 45 | return final_stops 46 | 47 | def generateIndicesForGTFS(self, date): 48 | collection_name = self.genCollectionName('gtfs_', date) 49 | self.database[collection_name].create_index([ 50 | ("block_id", ASCENDING) 51 | ], background=True) 52 | self.database[collection_name].create_index([ 53 | ("block_id", ASCENDING), 54 | ("arrival_time", ASCENDING) 55 | ], background=True) 56 | self.database[collection_name].create_index([ 57 | ("block_id", ASCENDING), 58 | ("actual_arrival_time", ASCENDING) 59 | ], background=True) 60 | self.database[collection_name].create_index([ 61 | ("stop_id", ASCENDING), 62 | ("arrival_time", ASCENDING), 63 | ("actual_arrival_time", ASCENDING) 64 | ], background=True) 65 | self.database[collection_name].create_index([ 66 | ("route_short_name", ASCENDING), 67 | ("stop_id", ASCENDING), 68 | ("direction_id", ASCENDING), 69 | ("arrival_time", ASCENDING) 70 | ], background=True) 71 | self.database[collection_name].create_index([ 72 | ("route_short_name", ASCENDING), 73 | ("stop_id", ASCENDING), 74 | ("direction_id", ASCENDING), 75 | ("departure_time", ASCENDING) 76 | ], background=True) 77 | 78 | def genCollectionName(self, prefix, date): 79 | return prefix + date.strftime('%Y%m%d') 80 | 81 | def insertData(self, collection_name, data): 82 | if len(data) > 0: 83 | self.database[collection_name].remove() 84 | self.database[collection_name].insert_many(data) 85 | 86 | 87 | # get bus route mappings that are not more than 30 minutes old 88 | def getBusRouteMappings(self): 89 | mappings = {} 90 | for mapping in self.database['busRouteMappings'].find(): 91 | if mapping['time'] > datetime.utcnow() + timedelta(minutes=-30): 92 | mappings[mapping['busId']] = mapping 93 | return mappings 94 | 95 | def setBusRouteMappings(self, mappings): 96 | self.database['busRouteMappings'].remove() 97 | if len(mappings) > 0: 98 | self.database['busRouteMappings'].insert(mappings) 99 | 100 | # return the last time to the minute that a bus checked in and 101 | # a list of all buses that checked in during that minute 102 | def getLastCheckinSummary(self): 103 | if self.database['checkins'].find().count() > 0: 104 | last_time = self.database['checkins'].find().sort("$natural", -1)[0]["time"] 105 | last_buses = self.database['checkins'].find({"time" : last_time}).distinct("busId") 106 | return {"time": last_time.replace(tzinfo=pytz.UTC), "busIds": last_buses} 107 | return None 108 | 109 | def updateCheckins(self, checkins): 110 | # purge checkins that are more than 2 hours 111 | self.database['checkins'].remove({"time": {"$lt": datetime.utcnow() + timedelta(hours=-2)}}) 112 | if len(checkins) > 0: 113 | self.database['checkins'].insert(checkins) 114 | 115 | def getRealTimeArrivalUpdates(self, checkin): 116 | checkin_local_time = checkin.time + timedelta(hours=-5) 117 | collection_name = 'gtfs_' + checkin_local_time.strftime('%Y%m%d') 118 | stop_times = self.database[collection_name].find({ 119 | 'block_id': checkin.blockId, 120 | '$or': [ 121 | {'arrival_time': { 122 | '$gte': datetime.utcnow() + timedelta(minutes=-5-checkin.adherence), 123 | '$lte': datetime.utcnow() + timedelta(minutes=30-checkin.adherence) 124 | }}, 125 | {'actual_arrival_time': { 126 | '$gte': datetime.utcnow() + timedelta(minutes=-5), 127 | '$lte': datetime.utcnow() + timedelta(minutes=30) 128 | }} 129 | ] 130 | }, {'arrival_time': 1, 'actual_arrival_time': 1}) 131 | updates = [] 132 | for stoptime in stop_times: 133 | new_arrival_time = stoptime['arrival_time'] - timedelta(minutes=checkin.adherence) 134 | if 'actual_arrival_time' not in stoptime or new_arrival_time != stoptime['actual_arrival_time']: 135 | updates.append(UpdateOne( 136 | {'_id': stoptime['_id']}, 137 | {'$set': {'actual_arrival_time': new_arrival_time}} 138 | )) 139 | return (collection_name, updates) 140 | 141 | def updateRealTimeArrivals(self, updates): 142 | for collection_name in updates: 143 | if updates[collection_name]: 144 | result = self.database[collection_name].bulk_write(updates[collection_name]) 145 | #print result.bulk_api_result 146 | 147 | def getScheduledStop(self, checkin): 148 | checkin_local_time = checkin.time + timedelta(hours=-5) 149 | collection_name = 'gtfs_' + checkin_local_time.strftime('%Y%m%d') 150 | scheduled_stop = self.database[collection_name].find_one({ 151 | "route_short_name" : checkin.routeShortName, 152 | "stop_id": checkin.stopId, 153 | "direction_id": {"$ne": checkin.direction}, 154 | "$or": [ 155 | {"arrival_time": { 156 | "$gte": checkin.time + timedelta(minutes=checkin.adherence - 2), 157 | "$lte": checkin.time + timedelta(minutes=checkin.adherence + 2) 158 | }}, 159 | {"departure_time": { 160 | "$gte": checkin.time + timedelta(minutes=checkin.adherence - 2), 161 | "$lte": checkin.time + timedelta(minutes=checkin.adherence + 2) 162 | }} 163 | ] 164 | }) 165 | if scheduled_stop is None: 166 | print "No scheduled stop found for the following checkin in {0}".format(collection_name) 167 | print checkin.__dict__ 168 | return None 169 | 170 | # get the stop sequence that OneBusAway uses 171 | scheduled_stop['stop_sequence_OBA'] = self.database[collection_name].find({ 172 | "trip_id": scheduled_stop["trip_id"], 173 | "stop_sequence": {"$lt": scheduled_stop["stop_sequence"]} 174 | }).count() 175 | 176 | return scheduled_stop 177 | -------------------------------------------------------------------------------- /web/static/js/underscore-min.js: -------------------------------------------------------------------------------- 1 | (function(){var n=this,t=n._,r={},e=Array.prototype,u=Object.prototype,i=Function.prototype,a=e.push,o=e.slice,c=e.concat,l=u.toString,f=u.hasOwnProperty,s=e.forEach,p=e.map,h=e.reduce,v=e.reduceRight,d=e.filter,g=e.every,m=e.some,y=e.indexOf,b=e.lastIndexOf,x=Array.isArray,_=Object.keys,j=i.bind,w=function(n){return n instanceof w?n:this instanceof w?(this._wrapped=n,void 0):new w(n)};"undefined"!=typeof exports?("undefined"!=typeof module&&module.exports&&(exports=module.exports=w),exports._=w):n._=w,w.VERSION="1.4.4";var A=w.each=w.forEach=function(n,t,e){if(null!=n)if(s&&n.forEach===s)n.forEach(t,e);else if(n.length===+n.length){for(var u=0,i=n.length;i>u;u++)if(t.call(e,n[u],u,n)===r)return}else for(var a in n)if(w.has(n,a)&&t.call(e,n[a],a,n)===r)return};w.map=w.collect=function(n,t,r){var e=[];return null==n?e:p&&n.map===p?n.map(t,r):(A(n,function(n,u,i){e[e.length]=t.call(r,n,u,i)}),e)};var O="Reduce of empty array with no initial value";w.reduce=w.foldl=w.inject=function(n,t,r,e){var u=arguments.length>2;if(null==n&&(n=[]),h&&n.reduce===h)return e&&(t=w.bind(t,e)),u?n.reduce(t,r):n.reduce(t);if(A(n,function(n,i,a){u?r=t.call(e,r,n,i,a):(r=n,u=!0)}),!u)throw new TypeError(O);return r},w.reduceRight=w.foldr=function(n,t,r,e){var u=arguments.length>2;if(null==n&&(n=[]),v&&n.reduceRight===v)return e&&(t=w.bind(t,e)),u?n.reduceRight(t,r):n.reduceRight(t);var i=n.length;if(i!==+i){var a=w.keys(n);i=a.length}if(A(n,function(o,c,l){c=a?a[--i]:--i,u?r=t.call(e,r,n[c],c,l):(r=n[c],u=!0)}),!u)throw new TypeError(O);return r},w.find=w.detect=function(n,t,r){var e;return E(n,function(n,u,i){return t.call(r,n,u,i)?(e=n,!0):void 0}),e},w.filter=w.select=function(n,t,r){var e=[];return null==n?e:d&&n.filter===d?n.filter(t,r):(A(n,function(n,u,i){t.call(r,n,u,i)&&(e[e.length]=n)}),e)},w.reject=function(n,t,r){return w.filter(n,function(n,e,u){return!t.call(r,n,e,u)},r)},w.every=w.all=function(n,t,e){t||(t=w.identity);var u=!0;return null==n?u:g&&n.every===g?n.every(t,e):(A(n,function(n,i,a){return(u=u&&t.call(e,n,i,a))?void 0:r}),!!u)};var E=w.some=w.any=function(n,t,e){t||(t=w.identity);var u=!1;return null==n?u:m&&n.some===m?n.some(t,e):(A(n,function(n,i,a){return u||(u=t.call(e,n,i,a))?r:void 0}),!!u)};w.contains=w.include=function(n,t){return null==n?!1:y&&n.indexOf===y?n.indexOf(t)!=-1:E(n,function(n){return n===t})},w.invoke=function(n,t){var r=o.call(arguments,2),e=w.isFunction(t);return w.map(n,function(n){return(e?t:n[t]).apply(n,r)})},w.pluck=function(n,t){return w.map(n,function(n){return n[t]})},w.where=function(n,t,r){return w.isEmpty(t)?r?null:[]:w[r?"find":"filter"](n,function(n){for(var r in t)if(t[r]!==n[r])return!1;return!0})},w.findWhere=function(n,t){return w.where(n,t,!0)},w.max=function(n,t,r){if(!t&&w.isArray(n)&&n[0]===+n[0]&&65535>n.length)return Math.max.apply(Math,n);if(!t&&w.isEmpty(n))return-1/0;var e={computed:-1/0,value:-1/0};return A(n,function(n,u,i){var a=t?t.call(r,n,u,i):n;a>=e.computed&&(e={value:n,computed:a})}),e.value},w.min=function(n,t,r){if(!t&&w.isArray(n)&&n[0]===+n[0]&&65535>n.length)return Math.min.apply(Math,n);if(!t&&w.isEmpty(n))return 1/0;var e={computed:1/0,value:1/0};return A(n,function(n,u,i){var a=t?t.call(r,n,u,i):n;e.computed>a&&(e={value:n,computed:a})}),e.value},w.shuffle=function(n){var t,r=0,e=[];return A(n,function(n){t=w.random(r++),e[r-1]=e[t],e[t]=n}),e};var k=function(n){return w.isFunction(n)?n:function(t){return t[n]}};w.sortBy=function(n,t,r){var e=k(t);return w.pluck(w.map(n,function(n,t,u){return{value:n,index:t,criteria:e.call(r,n,t,u)}}).sort(function(n,t){var r=n.criteria,e=t.criteria;if(r!==e){if(r>e||r===void 0)return 1;if(e>r||e===void 0)return-1}return n.indexi;){var o=i+a>>>1;u>r.call(e,n[o])?i=o+1:a=o}return i},w.toArray=function(n){return n?w.isArray(n)?o.call(n):n.length===+n.length?w.map(n,w.identity):w.values(n):[]},w.size=function(n){return null==n?0:n.length===+n.length?n.length:w.keys(n).length},w.first=w.head=w.take=function(n,t,r){return null==n?void 0:null==t||r?n[0]:o.call(n,0,t)},w.initial=function(n,t,r){return o.call(n,0,n.length-(null==t||r?1:t))},w.last=function(n,t,r){return null==n?void 0:null==t||r?n[n.length-1]:o.call(n,Math.max(n.length-t,0))},w.rest=w.tail=w.drop=function(n,t,r){return o.call(n,null==t||r?1:t)},w.compact=function(n){return w.filter(n,w.identity)};var R=function(n,t,r){return A(n,function(n){w.isArray(n)?t?a.apply(r,n):R(n,t,r):r.push(n)}),r};w.flatten=function(n,t){return R(n,t,[])},w.without=function(n){return w.difference(n,o.call(arguments,1))},w.uniq=w.unique=function(n,t,r,e){w.isFunction(t)&&(e=r,r=t,t=!1);var u=r?w.map(n,r,e):n,i=[],a=[];return A(u,function(r,e){(t?e&&a[a.length-1]===r:w.contains(a,r))||(a.push(r),i.push(n[e]))}),i},w.union=function(){return w.uniq(c.apply(e,arguments))},w.intersection=function(n){var t=o.call(arguments,1);return w.filter(w.uniq(n),function(n){return w.every(t,function(t){return w.indexOf(t,n)>=0})})},w.difference=function(n){var t=c.apply(e,o.call(arguments,1));return w.filter(n,function(n){return!w.contains(t,n)})},w.zip=function(){for(var n=o.call(arguments),t=w.max(w.pluck(n,"length")),r=Array(t),e=0;t>e;e++)r[e]=w.pluck(n,""+e);return r},w.object=function(n,t){if(null==n)return{};for(var r={},e=0,u=n.length;u>e;e++)t?r[n[e]]=t[e]:r[n[e][0]]=n[e][1];return r},w.indexOf=function(n,t,r){if(null==n)return-1;var e=0,u=n.length;if(r){if("number"!=typeof r)return e=w.sortedIndex(n,t),n[e]===t?e:-1;e=0>r?Math.max(0,u+r):r}if(y&&n.indexOf===y)return n.indexOf(t,r);for(;u>e;e++)if(n[e]===t)return e;return-1},w.lastIndexOf=function(n,t,r){if(null==n)return-1;var e=null!=r;if(b&&n.lastIndexOf===b)return e?n.lastIndexOf(t,r):n.lastIndexOf(t);for(var u=e?r:n.length;u--;)if(n[u]===t)return u;return-1},w.range=function(n,t,r){1>=arguments.length&&(t=n||0,n=0),r=arguments[2]||1;for(var e=Math.max(Math.ceil((t-n)/r),0),u=0,i=Array(e);e>u;)i[u++]=n,n+=r;return i},w.bind=function(n,t){if(n.bind===j&&j)return j.apply(n,o.call(arguments,1));var r=o.call(arguments,2);return function(){return n.apply(t,r.concat(o.call(arguments)))}},w.partial=function(n){var t=o.call(arguments,1);return function(){return n.apply(this,t.concat(o.call(arguments)))}},w.bindAll=function(n){var t=o.call(arguments,1);return 0===t.length&&(t=w.functions(n)),A(t,function(t){n[t]=w.bind(n[t],n)}),n},w.memoize=function(n,t){var r={};return t||(t=w.identity),function(){var e=t.apply(this,arguments);return w.has(r,e)?r[e]:r[e]=n.apply(this,arguments)}},w.delay=function(n,t){var r=o.call(arguments,2);return setTimeout(function(){return n.apply(null,r)},t)},w.defer=function(n){return w.delay.apply(w,[n,1].concat(o.call(arguments,1)))},w.throttle=function(n,t){var r,e,u,i,a=0,o=function(){a=new Date,u=null,i=n.apply(r,e)};return function(){var c=new Date,l=t-(c-a);return r=this,e=arguments,0>=l?(clearTimeout(u),u=null,a=c,i=n.apply(r,e)):u||(u=setTimeout(o,l)),i}},w.debounce=function(n,t,r){var e,u;return function(){var i=this,a=arguments,o=function(){e=null,r||(u=n.apply(i,a))},c=r&&!e;return clearTimeout(e),e=setTimeout(o,t),c&&(u=n.apply(i,a)),u}},w.once=function(n){var t,r=!1;return function(){return r?t:(r=!0,t=n.apply(this,arguments),n=null,t)}},w.wrap=function(n,t){return function(){var r=[n];return a.apply(r,arguments),t.apply(this,r)}},w.compose=function(){var n=arguments;return function(){for(var t=arguments,r=n.length-1;r>=0;r--)t=[n[r].apply(this,t)];return t[0]}},w.after=function(n,t){return 0>=n?t():function(){return 1>--n?t.apply(this,arguments):void 0}},w.keys=_||function(n){if(n!==Object(n))throw new TypeError("Invalid object");var t=[];for(var r in n)w.has(n,r)&&(t[t.length]=r);return t},w.values=function(n){var t=[];for(var r in n)w.has(n,r)&&t.push(n[r]);return t},w.pairs=function(n){var t=[];for(var r in n)w.has(n,r)&&t.push([r,n[r]]);return t},w.invert=function(n){var t={};for(var r in n)w.has(n,r)&&(t[n[r]]=r);return t},w.functions=w.methods=function(n){var t=[];for(var r in n)w.isFunction(n[r])&&t.push(r);return t.sort()},w.extend=function(n){return A(o.call(arguments,1),function(t){if(t)for(var r in t)n[r]=t[r]}),n},w.pick=function(n){var t={},r=c.apply(e,o.call(arguments,1));return A(r,function(r){r in n&&(t[r]=n[r])}),t},w.omit=function(n){var t={},r=c.apply(e,o.call(arguments,1));for(var u in n)w.contains(r,u)||(t[u]=n[u]);return t},w.defaults=function(n){return A(o.call(arguments,1),function(t){if(t)for(var r in t)null==n[r]&&(n[r]=t[r])}),n},w.clone=function(n){return w.isObject(n)?w.isArray(n)?n.slice():w.extend({},n):n},w.tap=function(n,t){return t(n),n};var I=function(n,t,r,e){if(n===t)return 0!==n||1/n==1/t;if(null==n||null==t)return n===t;n instanceof w&&(n=n._wrapped),t instanceof w&&(t=t._wrapped);var u=l.call(n);if(u!=l.call(t))return!1;switch(u){case"[object String]":return n==t+"";case"[object Number]":return n!=+n?t!=+t:0==n?1/n==1/t:n==+t;case"[object Date]":case"[object Boolean]":return+n==+t;case"[object RegExp]":return n.source==t.source&&n.global==t.global&&n.multiline==t.multiline&&n.ignoreCase==t.ignoreCase}if("object"!=typeof n||"object"!=typeof t)return!1;for(var i=r.length;i--;)if(r[i]==n)return e[i]==t;r.push(n),e.push(t);var a=0,o=!0;if("[object Array]"==u){if(a=n.length,o=a==t.length)for(;a--&&(o=I(n[a],t[a],r,e)););}else{var c=n.constructor,f=t.constructor;if(c!==f&&!(w.isFunction(c)&&c instanceof c&&w.isFunction(f)&&f instanceof f))return!1;for(var s in n)if(w.has(n,s)&&(a++,!(o=w.has(t,s)&&I(n[s],t[s],r,e))))break;if(o){for(s in t)if(w.has(t,s)&&!a--)break;o=!a}}return r.pop(),e.pop(),o};w.isEqual=function(n,t){return I(n,t,[],[])},w.isEmpty=function(n){if(null==n)return!0;if(w.isArray(n)||w.isString(n))return 0===n.length;for(var t in n)if(w.has(n,t))return!1;return!0},w.isElement=function(n){return!(!n||1!==n.nodeType)},w.isArray=x||function(n){return"[object Array]"==l.call(n)},w.isObject=function(n){return n===Object(n)},A(["Arguments","Function","String","Number","Date","RegExp"],function(n){w["is"+n]=function(t){return l.call(t)=="[object "+n+"]"}}),w.isArguments(arguments)||(w.isArguments=function(n){return!(!n||!w.has(n,"callee"))}),"function"!=typeof/./&&(w.isFunction=function(n){return"function"==typeof n}),w.isFinite=function(n){return isFinite(n)&&!isNaN(parseFloat(n))},w.isNaN=function(n){return w.isNumber(n)&&n!=+n},w.isBoolean=function(n){return n===!0||n===!1||"[object Boolean]"==l.call(n)},w.isNull=function(n){return null===n},w.isUndefined=function(n){return n===void 0},w.has=function(n,t){return f.call(n,t)},w.noConflict=function(){return n._=t,this},w.identity=function(n){return n},w.times=function(n,t,r){for(var e=Array(n),u=0;n>u;u++)e[u]=t.call(r,u);return e},w.random=function(n,t){return null==t&&(t=n,n=0),n+Math.floor(Math.random()*(t-n+1))};var M={escape:{"&":"&","<":"<",">":">",'"':""","'":"'","/":"/"}};M.unescape=w.invert(M.escape);var S={escape:RegExp("["+w.keys(M.escape).join("")+"]","g"),unescape:RegExp("("+w.keys(M.unescape).join("|")+")","g")};w.each(["escape","unescape"],function(n){w[n]=function(t){return null==t?"":(""+t).replace(S[n],function(t){return M[n][t]})}}),w.result=function(n,t){if(null==n)return null;var r=n[t];return w.isFunction(r)?r.call(n):r},w.mixin=function(n){A(w.functions(n),function(t){var r=w[t]=n[t];w.prototype[t]=function(){var n=[this._wrapped];return a.apply(n,arguments),D.call(this,r.apply(w,n))}})};var N=0;w.uniqueId=function(n){var t=++N+"";return n?n+t:t},w.templateSettings={evaluate:/<%([\s\S]+?)%>/g,interpolate:/<%=([\s\S]+?)%>/g,escape:/<%-([\s\S]+?)%>/g};var T=/(.)^/,q={"'":"'","\\":"\\","\r":"r","\n":"n"," ":"t","\u2028":"u2028","\u2029":"u2029"},B=/\\|'|\r|\n|\t|\u2028|\u2029/g;w.template=function(n,t,r){var e;r=w.defaults({},r,w.templateSettings);var u=RegExp([(r.escape||T).source,(r.interpolate||T).source,(r.evaluate||T).source].join("|")+"|$","g"),i=0,a="__p+='";n.replace(u,function(t,r,e,u,o){return a+=n.slice(i,o).replace(B,function(n){return"\\"+q[n]}),r&&(a+="'+\n((__t=("+r+"))==null?'':_.escape(__t))+\n'"),e&&(a+="'+\n((__t=("+e+"))==null?'':__t)+\n'"),u&&(a+="';\n"+u+"\n__p+='"),i=o+t.length,t}),a+="';\n",r.variable||(a="with(obj||{}){\n"+a+"}\n"),a="var __t,__p='',__j=Array.prototype.join,"+"print=function(){__p+=__j.call(arguments,'');};\n"+a+"return __p;\n";try{e=Function(r.variable||"obj","_",a)}catch(o){throw o.source=a,o}if(t)return e(t,w);var c=function(n){return e.call(this,n,w)};return c.source="function("+(r.variable||"obj")+"){\n"+a+"}",c},w.chain=function(n){return w(n).chain()};var D=function(n){return this._chain?w(n).chain():n};w.mixin(w),A(["pop","push","reverse","shift","sort","splice","unshift"],function(n){var t=e[n];w.prototype[n]=function(){var r=this._wrapped;return t.apply(r,arguments),"shift"!=n&&"splice"!=n||0!==r.length||delete r[0],D.call(this,r)}}),A(["concat","join","slice"],function(n){var t=e[n];w.prototype[n]=function(){return D.call(this,t.apply(this._wrapped,arguments))}}),w.extend(w.prototype,{chain:function(){return this._chain=!0,this},value:function(){return this._wrapped}})}).call(this); -------------------------------------------------------------------------------- /web/app.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | from flask import Flask, Response, render_template, redirect, url_for, request, current_app 3 | from functools import wraps 4 | from geopy import geocoders 5 | from google.protobuf import text_format 6 | import json 7 | import os 8 | import pymongo 9 | from bson import json_util 10 | import gtfs_realtime_pb2 11 | 12 | 13 | app = Flask(__name__) 14 | dthandler = lambda obj: obj.isoformat() if isinstance(obj, datetime) else None 15 | 16 | 17 | db = None 18 | curDateTime = None 19 | collectionPrefix = None 20 | 21 | 22 | def support_jsonp(f): 23 | """Wraps JSONified output for JSONP""" 24 | @wraps(f) 25 | def decorated_function(*args, **kwargs): 26 | callback = request.args.get('callback', False) 27 | if callback: 28 | content = str(callback) + '(' + str(f(*args, **kwargs)) + ')' 29 | return current_app.response_class(content, mimetype='application/json') 30 | else: 31 | return Response(f(*args, **kwargs), mimetype='application/json') 32 | return decorated_function 33 | 34 | 35 | @app.before_request 36 | def beforeRequest(): 37 | global db 38 | global curDateTime 39 | global collectionPrefix 40 | 41 | db = pymongo.MongoClient(os.environ['MONGODB_URI']).get_database(None) 42 | curDateTime = datetime.utcnow() + timedelta(hours=-5) 43 | collectionPrefix = curDateTime.strftime('%Y%m%d') 44 | 45 | 46 | @app.route('/') 47 | def index(): 48 | return redirect(url_for('getApiInfo')) 49 | 50 | 51 | @app.route('/gtfs/trip_update/') 52 | def tripUpdate(): 53 | # PROTOCAL BUFFER!!! 54 | # https://developers.google.com/protocol-buffers/docs/pythontutorial 55 | 56 | # Create feed 57 | feed = gtfs_realtime_pb2.FeedMessage() 58 | 59 | # header 60 | feed.header.gtfs_realtime_version = '1.0' 61 | feed.header.timestamp = long( 62 | (datetime.utcnow() - datetime(1970, 1, 1)).total_seconds()) 63 | 64 | # create an entity for each active trip id 65 | activeTrips = db['checkins'].aggregate([{"$match": 66 | {"tripId": {'$ne': None}, 67 | "adherence": {'$exists': True}, 68 | "lastStopSequence": {'$exists': True}, 69 | "lastStopSequenceOBA": {'$exists': True}}}, 70 | {"$sort": {"time": 1}}, 71 | {"$group": 72 | {"_id": {"trip": "$tripId", "seq": "$lastStopSequence"}, 73 | "time": {"$last": "$time"}, 74 | "bus": {"$last": "$busId"}, 75 | "adherence": {"$last": "$adherence"}, 76 | "seqOBA": {"$last": "$lastStopSequenceOBA"}}}, 77 | {"$sort": {"_id.seq": 1}}, 78 | {"$group": 79 | {"_id": {"trip": "$_id.trip"}, 80 | "time": {"$last": "$time"}, 81 | "bus": {"$last": "$bus"}, 82 | "timeChecks": 83 | {"$push": 84 | {"seq": "$_id.seq", 85 | "time": "$time", 86 | "adherence": "$adherence", 87 | "seqOBA": "$seqOBA"}}}}]) 88 | # return json.dumps(activeTrips, default=dthandler) 89 | 90 | for trip in activeTrips['result']: 91 | # add the trip entity 92 | entity = feed.entity.add() 93 | entity.id = 'trip' + trip['_id']['trip'] 94 | entity.trip_update.trip.trip_id = trip['_id']['trip'] 95 | entity.trip_update.vehicle.id = str(trip['bus']) 96 | entity.trip_update.vehicle.label = str(trip['bus']) 97 | entity.trip_update.timestamp = long( 98 | (trip['time'] - datetime(1970, 1, 1)).total_seconds()) 99 | 100 | # add the stop time updates 101 | for update in trip['timeChecks']: 102 | stopTime = entity.trip_update.stop_time_update.add() 103 | stopTime.stop_sequence = update['seq'] if not request.args.get('oba') else update[ 104 | 'seqOBA'] 105 | stopTime.arrival.delay = update[ 106 | 'adherence'] * -60 # convert minutes to seconds 107 | 108 | if request.args.get('debug'): 109 | return text_format.MessageToString(feed) 110 | return feed.SerializeToString() 111 | 112 | 113 | @app.route('/gtfs/vehicle_position/') 114 | def vehiclePosition(): 115 | # PROTOCAL BUFFER!!! 116 | # https://developers.google.com/protocol-buffers/docs/pythontutorial 117 | 118 | # Create feed 119 | feed = gtfs_realtime_pb2.FeedMessage() 120 | 121 | # header 122 | feed.header.gtfs_realtime_version = '1.0' 123 | feed.header.timestamp = long( 124 | (datetime.utcnow() - datetime(1970, 1, 1)).total_seconds()) 125 | 126 | # create an entity for each active trip id 127 | lastBusLocations = db['checkins'].aggregate([{"$match": 128 | {"tripId": {'$ne': None}, 129 | "location": {'$exists': True}}}, 130 | {"$group": 131 | {"_id": {"bus": "$busId"}, 132 | "trip": {"$last": "$tripId"}, 133 | "time": {"$last": "$time"}, 134 | "location": {"$last": "$location"}}}]) 135 | # return json.dumps(lastLocations, default=dthandler) 136 | 137 | for bus in lastBusLocations['result']: 138 | # add the trip entity 139 | entity = feed.entity.add() 140 | entity.id = 'bus' + str(bus['_id']['bus']) 141 | entity.vehicle.trip.trip_id = bus['trip'] 142 | entity.vehicle.vehicle.id = str(bus['_id']['bus']) 143 | entity.vehicle.vehicle.label = str(bus['_id']['bus']) 144 | entity.vehicle.position.latitude = float(bus['location'][0]) 145 | entity.vehicle.position.longitude = float(bus['location'][1]) 146 | entity.vehicle.timestamp = long( 147 | (bus['time'] - datetime(1970, 1, 1)).total_seconds()) 148 | 149 | if request.args.get('debug'): 150 | return text_format.MessageToString(feed) 151 | return feed.SerializeToString() 152 | 153 | 154 | @app.route('/api/') 155 | @support_jsonp 156 | def getApiInfo(): 157 | return json.dumps({'version': '0.10', 'dbHost': db.client.address, 'curDateTime': curDateTime, 'collectionPrefix': collectionPrefix}, default=dthandler) 158 | 159 | 160 | @app.route('/api/routes/active/') 161 | @support_jsonp 162 | def getActiveRoutes(): 163 | # List the routes from the checkins 164 | activeRoutes = db['checkins'].find( 165 | {'location': {'$exists': True}}).distinct('routeShortName') 166 | 167 | # Get details about those routes from the GTFS data 168 | activeRoutesWithDetails = db['routes_' + collectionPrefix].find( 169 | {'route_short_name': {'$in': activeRoutes}}, {'_id': 0}).sort('route_short_name') 170 | return json.dumps(list(activeRoutesWithDetails)) 171 | 172 | 173 | @app.route('/api/buses/on_route//') 174 | @support_jsonp 175 | def getBusesOnRoute(routeShortName): 176 | # Get all checkins for the route, only keep the last one for each bus 177 | checkins = {} 178 | for checkin in db['checkins'].find({'routeShortName': routeShortName, 'location': {'$exists': True}}, {'_id': 0}).sort('time'): 179 | checkins[checkin['busId']] = checkin 180 | return json.dumps(checkins.values(), default=dthandler) 181 | 182 | 183 | @app.route('/api/buses/routes') 184 | @app.route('/api/buses/routes//') 185 | @support_jsonp 186 | def getBusesByRoute(routeShortNames=None): 187 | match = {'location': {'$exists': True}} 188 | if routeShortNames is not None: 189 | ids = map(int, filter(None, routeShortNames.split('/'))) 190 | match['routeShortName'] = {'$in': ids} 191 | 192 | cursor = db['checkins'].find(match).sort('time') 193 | checkins = {} 194 | for c in cursor: 195 | c['_id'] = str(c['_id']) 196 | checkins[c['busId']] = c 197 | 198 | return json.dumps(checkins.values(), default=dthandler) 199 | 200 | 201 | @app.route('/api/buses/history//') 202 | @support_jsonp 203 | def getBusHistory(busId): 204 | # Get all checkins for a bus 205 | checkins = db['checkins'].find({'busId': busId, 'location': {'$exists': True}}, { 206 | '_id': 0, 'tripId': 0}).sort('time', pymongo.DESCENDING) 207 | return json.dumps(list(checkins), default=dthandler) 208 | 209 | 210 | @app.route('/api/stops/near/intersection///') 211 | def getStopsNearIntersection(city, intersection): 212 | place, (lat, lng) = geocoders.googlev3.GoogleV3().geocode( 213 | "{0}, {1}, VA".format(intersection, city), exactly_one=False)[0] 214 | return getStopsNear(lat, lng) 215 | 216 | 217 | @app.route('/api/stops/near///') 218 | @support_jsonp 219 | def getStopsNear(lat, lng): 220 | stops = db['stops_' + collectionPrefix].find( 221 | {"location": {"$near": [float(lat), float(lng)]}}).limit(6) 222 | stops = list(stops) 223 | for stop in stops: 224 | stop['_id'] = str(stop['_id']) 225 | return json.dumps(stops) 226 | 227 | 228 | @app.route('/api/stops/id//') 229 | @support_jsonp 230 | def getStopsById(stopIds): 231 | ids = stopIds.split('/') 232 | stops = db['stops_' + collectionPrefix].find({'stopId': {'$in': ids}}) 233 | stops = list(stops) 234 | for stop in stops: 235 | stop['_id'] = str(stop['_id']) 236 | return json.dumps(stops) 237 | 238 | 239 | @app.route('/api/stop_times///') 240 | @support_jsonp 241 | def getNextBus(routeShortName, stopId): 242 | scheduledStops = db['gtfs_' + collectionPrefix].find({'route_short_name': routeShortName, 'stop_id': stopId, 'arrival_time': { 243 | '$gte': datetime.utcnow()}}).sort('arrival_time').limit(3) 244 | lastStop = db['gtfs_' + collectionPrefix].find({'route_short_name': routeShortName, 'stop_id': stopId, 'arrival_time': { 245 | '$lt': datetime.utcnow()}}).sort('arrival_time', pymongo.DESCENDING).limit(1) 246 | data = list(lastStop) 247 | data += list(scheduledStops) 248 | for stop in data: 249 | stop['all_trip_ids'] = list( 250 | db['gtfs_' + collectionPrefix].find({'block_id': stop['block_id']}).distinct('trip_id')) 251 | checkins = db['checkins'].find( 252 | {'tripId': {'$in': stop['all_trip_ids']}}).sort('time', pymongo.DESCENDING) 253 | for checkin in checkins: 254 | try: 255 | stop['adherence'] = checkin['adherence'] 256 | stop['busId'] = checkin['busId'] 257 | break 258 | except KeyError: 259 | pass 260 | return json.dumps(data, default=dthandler) 261 | 262 | 263 | @app.route('/api/stop_times//') 264 | @support_jsonp 265 | def getBusesAtStop(stopId): 266 | return json.dumps(find_buses_at_stop(stopId), default=dthandler) 267 | 268 | 269 | def find_buses_at_stop(stopId): 270 | print(stopId) 271 | scheduledStops = list(db['gtfs_' + collectionPrefix].find({'stop_id': stopId, 272 | '$or': [ 273 | {'arrival_time': {'$gte': datetime.utcnow() + timedelta(minutes=-5), 274 | '$lte': datetime.utcnow() + timedelta(minutes=30)}}, 275 | {'actual_arrival_time': {'$gte': datetime.utcnow() + timedelta(minutes=-5), 276 | '$lte': datetime.utcnow() + timedelta(minutes=30)}}] 277 | }).sort('arrival_time')) 278 | for stop in scheduledStops: 279 | stop['destination'] = db['destinations_' + 280 | collectionPrefix].find_one({'tripId': stop['trip_id']})['stopName'] 281 | stop['all_trip_ids'] = list( 282 | db['gtfs_' + collectionPrefix].find({'block_id': stop['block_id']}).distinct('trip_id')) 283 | stop['_id'] = str(stop['_id']) 284 | 285 | routeDetails = db[ 286 | 'routes_' + collectionPrefix].find_one({'route_id': stop['route_id']}) 287 | stop['routeLongName'] = routeDetails['route_long_name'] 288 | stop['routeShortName'] = routeDetails['route_short_name'] 289 | stop['routeDescription'] = routeDetails['route_desc'] 290 | stop['routeType'] = routeDetails['route_type'] 291 | 292 | checkins = db['checkins'].find( 293 | {'tripId': {'$in': stop['all_trip_ids']}}).sort('time', pymongo.DESCENDING) 294 | for checkin in checkins: 295 | try: 296 | stop['busAdherence'] = checkin['adherence'] 297 | stop['busId'] = checkin['busId'] 298 | stop['busPosition'] = checkin['location'] 299 | stop['busCheckinTime'] = checkin['time'] 300 | break 301 | except KeyError: 302 | pass 303 | stop.pop('all_trip_ids') 304 | return scheduledStops 305 | 306 | 307 | @app.route('/api/v2/stops/near///') 308 | @support_jsonp 309 | def get_stops_near(lat=None, lng=None, cap=6): 310 | """ 311 | @lat - Lattitude Value (36.850769) 312 | @lng - Longitude Value fmt (-76.285873) 313 | @cap - Cap = number of values to return 314 | """ 315 | stops = db['stops_' + collectionPrefix].find( 316 | {"location": {"$near": [float(lat), float(lng)]}}).limit(cap) 317 | stops = list(stops) 318 | for stop in stops: 319 | stop['_id'] = str(stop['_id']) 320 | stop['buses'] = find_buses_at_stop(stop.get('stopId')) 321 | 322 | return json.dumps(stops, default=json_util.default) 323 | 324 | 325 | @app.route('/api/v2/stops') 326 | @support_jsonp 327 | def get_buses_at_stop(): 328 | """ 329 | Similar to the stop-times function but for the v2 api 330 | """ 331 | # Unique Sets Please 332 | stop_ids = set(request.args.get("id").split(',')) 333 | return json.dumps([find_buses_at_stop(stop_id) for stop_id in stop_ids], default=dthandler) 334 | 335 | if __name__ == '__main__': 336 | # Bind to PORT if defined, otherwise default to 5000. 337 | port = int(os.environ.get('PORT', 5000)) 338 | app.run(host='0.0.0.0', port=port, debug=True) 339 | -------------------------------------------------------------------------------- /web/static/js/backbone-min.js: -------------------------------------------------------------------------------- 1 | // Backbone.js 0.9.10 2 | 3 | // (c) 2010-2012 Jeremy Ashkenas, DocumentCloud Inc. 4 | // Backbone may be freely distributed under the MIT license. 5 | // For all details and documentation: 6 | // http://backbonejs.org 7 | (function(){var n=this,B=n.Backbone,h=[],C=h.push,u=h.slice,D=h.splice,g;g="undefined"!==typeof exports?exports:n.Backbone={};g.VERSION="0.9.10";var f=n._;!f&&"undefined"!==typeof require&&(f=require("underscore"));g.$=n.jQuery||n.Zepto||n.ender;g.noConflict=function(){n.Backbone=B;return this};g.emulateHTTP=!1;g.emulateJSON=!1;var v=/\s+/,q=function(a,b,c,d){if(!c)return!0;if("object"===typeof c)for(var e in c)a[b].apply(a,[e,c[e]].concat(d));else if(v.test(c)){c=c.split(v);e=0;for(var f=c.length;e< 8 | f;e++)a[b].apply(a,[c[e]].concat(d))}else return!0},w=function(a,b){var c,d=-1,e=a.length;switch(b.length){case 0:for(;++d=b);this.root=("/"+this.root+"/").replace(I,"/");b&&this._wantsHashChange&&(this.iframe=g.$('