├── .python-version ├── lambda ├── update_web_site │ ├── setup.cfg │ ├── requirements.txt │ ├── dev-requirements.txt │ ├── Makefile │ └── imtf.py └── retrieve_latest_status │ └── main.py ├── s3_website.yml.example ├── website └── error.html ├── README.md ├── LICENSE └── .gitignore /.python-version: -------------------------------------------------------------------------------- 1 | 3.7.1 2 | -------------------------------------------------------------------------------- /lambda/update_web_site/setup.cfg: -------------------------------------------------------------------------------- 1 | [install] 2 | prefix= 3 | -------------------------------------------------------------------------------- /lambda/update_web_site/requirements.txt: -------------------------------------------------------------------------------- 1 | beautifulsoup4==4.4.1 2 | pytz==2017.3 3 | requests==2.9.1 4 | tabulate==0.8.2 5 | -------------------------------------------------------------------------------- /s3_website.yml.example: -------------------------------------------------------------------------------- 1 | s3_id: YOUR_AWS_S3_ACCESS_KEY_ID 2 | s3_secret: YOUR_AWS_S3_SECRET_ACCESS_KEY 3 | s3_bucket: your.blog.bucket.com 4 | 5 | site: website 6 | # the index.html is generated by a lambda function! 7 | ignore_on_server: index.html 8 | # index_document: index.html 9 | 10 | error_document: error.html 11 | -------------------------------------------------------------------------------- /lambda/update_web_site/dev-requirements.txt: -------------------------------------------------------------------------------- 1 | appnope==0.1.0 2 | backcall==0.1.0 3 | beautifulsoup4==4.4.1 4 | boto3==1.9.71 5 | botocore==1.12.71 6 | decorator==4.3.0 7 | docutils==0.14 8 | ipython==7.2.0 9 | ipython-genutils==0.2.0 10 | jedi==0.13.2 11 | jmespath==0.9.3 12 | parso==0.3.1 13 | pexpect==4.6.0 14 | pickleshare==0.7.5 15 | prompt-toolkit==2.0.7 16 | ptyprocess==0.6.0 17 | Pygments==2.3.1 18 | python-dateutil==2.7.5 19 | pytz==2017.3 20 | requests==2.9.1 21 | s3transfer==0.1.13 22 | six==1.12.0 23 | tabulate==0.8.2 24 | traitlets==4.3.2 25 | urllib3==1.24.1 26 | wcwidth==0.1.7 27 | -------------------------------------------------------------------------------- /website/error.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 |

 7 |    ┌───────┐
 8 |    | error |
 9 |    └───────┘
10 | (\__/) ||
11 | (•ㅅ•) ||
12 | / 　 づ
13 |

14 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Is My Train Fucked? 2 | 3 | A tiny web app that answers the question "Is your train fucked?" 4 | 5 | Hint: it probably is. 6 | 7 | ## Technology 8 | 9 | IMTF is written in python and intended to be an 10 | [AWS Lambda](https://aws.amazon.com/lambda/) microservice. 11 | [Apex](http://apex.run) is used for function uploading, version management, and 12 | testing. 13 | 14 | ## Data Flow 15 | 16 | 1. MTA information is queried every minute 17 | 18 | a. New data is written to S3 for record-keeping purposes (perhaps querying w/ Atlas in the future?) 19 | 20 | b. New data is written to index.html file and uploaded to S3 hosted website 21 | 22 | Pretty simple! 23 | 24 | ## "API" 25 | 26 | Right now, you can hit an "API" and see the latest train status in a really 27 | crappy format. I'm working on fixing it soon, so stay tuned! 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Chris Becker 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /lambda/update_web_site/Makefile: -------------------------------------------------------------------------------- 1 | REGION = us-east-1 2 | 3 | .PHONY: clean deploy 4 | 5 | venv: venv/bin/activate 6 | venv/bin/activate: dev-requirements.txt 7 | test -d venv || python3 -m venv venv 8 | venv/bin/pip3 install -U pip 9 | venv/bin/pip3 install -Ur dev-requirements.txt 10 | touch venv/bin/activate 11 | 12 | clean: cleandist 13 | rm -rf venv 14 | 15 | cleandist: 16 | rm -rf build 17 | 18 | dist: cleandist build/deploy.zip 19 | 20 | build/deploy.zip: imtf.py 21 | mkdir -p build/ 22 | zip -r build/deploy.zip . \ 23 | -x "*.DS_Store*" \ 24 | "build" \ 25 | "*requirements.txt" \ 26 | ".gitignore" \ 27 | "Makefile" \ 28 | "build*" \ 29 | "setup.cfg" \ 30 | "venv*" 31 | python3 -m venv build/venv 32 | source build/venv/bin/activate; \ 33 | pip3 install -r requirements.txt; \ 34 | cp -r $$VIRTUAL_ENV/lib/python3.7/site-packages/ build/site-packages 35 | cd build/site-packages; zip -g -r ../deploy.zip . -x "*__pycache__*" 36 | 37 | 38 | deploy: dist 39 | aws lambda update-function-code \ 40 | --region $(REGION) \ 41 | --function-name ismytrainfucked_update_site \ 42 | --zip-file fileb://build/deploy.zip \ 43 | --publish 44 | -------------------------------------------------------------------------------- /lambda/retrieve_latest_status/main.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import datetime 3 | 4 | TRAIN_LINES = [ 5 | '123', 6 | '456', 7 | '7', 8 | 'ACE', 9 | 'BDFM', 10 | 'G', 11 | 'JZ', 12 | 'L', 13 | 'NQR', 14 | 'S', 15 | 'SIR', 16 | ] 17 | 18 | 19 | def _normalize_dynamo_response(response): 20 | """ 21 | Flatten DynamoDB's response from a list of dicts to a structure that's 22 | easier to work with: 23 | ``` 24 | { 'TRAIN_LINE': 25 | { 'date': $DATE, 26 | 'status_title': $STATUS_TITLE, 27 | 'is_it_fucked': $IS_IT_FUCKED } 28 | } 29 | ``` 30 | 31 | Which makes querying in python easier: 32 | ``` 33 | response.get('NQR').get('is_it_fucked') 34 | ``` 35 | """ 36 | normalized_dict = {} 37 | for item in response: 38 | line = item.get('line') 39 | del item['line'] 40 | normalized_dict[line] = item 41 | return normalized_dict 42 | 43 | 44 | def handle(event, context): 45 | print "Fetching status from IMTF API..." 46 | 47 | dynamo = boto3.resource('dynamodb') 48 | table = dynamo.Table('imtf-test') 49 | 50 | response = table.meta.client.batch_get_item( 51 | RequestItems={ 52 | 'imtf-test': { 53 | 'Keys': [{'line': line, 'date': 'latest'} for line in TRAIN_LINES] 54 | } 55 | } 56 | ) 57 | latest = _normalize_dynamo_response(response.get('Responses').get('imtf-test')) 58 | return latest 59 | 60 | 61 | if __name__ == '__main__': 62 | print(handle('', '')) 63 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | .hypothesis/ 50 | .pytest_cache/ 51 | 52 | # Translations 53 | *.mo 54 | *.pot 55 | 56 | # Django stuff: 57 | *.log 58 | local_settings.py 59 | db.sqlite3 60 | 61 | # Flask stuff: 62 | instance/ 63 | .webassets-cache 64 | 65 | # Scrapy stuff: 66 | .scrapy 67 | 68 | # Sphinx documentation 69 | docs/_build/ 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # Jupyter Notebook 75 | .ipynb_checkpoints 76 | 77 | # IPython 78 | profile_default/ 79 | ipython_config.py 80 | 81 | # celery beat schedule file 82 | celerybeat-schedule 83 | 84 | # SageMath parsed files 85 | *.sage.py 86 | 87 | # Environments 88 | .env 89 | .venv 90 | env/ 91 | venv/ 92 | ENV/ 93 | env.bak/ 94 | venv.bak/ 95 | 96 | # Spyder project settings 97 | .spyderproject 98 | .spyproject 99 | 100 | # Rope project settings 101 | .ropeproject 102 | 103 | # mkdocs documentation 104 | /site 105 | 106 | # mypy 107 | .mypy_cache/ 108 | .dmypy.json 109 | dmypy.json 110 | 111 | # Pyre type checker 112 | .pyre/ 113 | -------------------------------------------------------------------------------- /lambda/update_web_site/imtf.py: -------------------------------------------------------------------------------- 1 | """ 2 | Query MTA API, and write data and HTML website to S3 3 | """ 4 | from bs4 import BeautifulSoup as Soup 5 | from pytz import timezone 6 | from tabulate import tabulate 7 | 8 | import boto3 9 | import datetime 10 | import json 11 | import requests 12 | import time 13 | 14 | 15 | # NYC is only in one time zone 16 | TIMEZONE = timezone('US/Eastern') 17 | 18 | HTML = ''' 19 | 20 | 21 | 22 | 23 | 24 | 25 | {0} 26 |
27 | last updated: {1} 28 |
29 | source code on github 30 | {2} 31 | 32 | 33 | ''' 34 | 35 | GA = ''' 36 | 45 | ''' 46 | 47 | STATUS_PAGE_HTML = ''' 48 | 49 | 50 | 51 | 52 | 53 | 54 | {0} 55 |
56 |
57 | {1} 58 | {2} 59 | 60 | 61 | ''' 62 | 63 | 64 | def _log_print(message): 65 | print(json.dumps({'message': message})) 66 | 67 | 68 | def _put_item_with_latest_and_timestamp(dynamo_resource, item): 69 | """ 70 | Put an item into DynamoDB with the sort key set as `latest` and the actual 71 | timestamp. 72 | 73 | This function is DISABLED rn because idk how I want to store this data yet. 74 | """ 75 | for timestamp in ('latest', datetime.datetime.now(TIMEZONE).strftime("%Y-%m-%d-%H-%M")): 76 | item['date'] = timestamp 77 | dynamo_resource.put_item(Item=item) 78 | 79 | 80 | def handle(_, __): 81 | dynamodb = boto3.resource('dynamodb') 82 | imtf = dynamodb.Table('imtf-test') 83 | s3 = boto3.resource('s3') 84 | 85 | # this is gross, but I don't know a better way to get the nice dotted lines 86 | # in the headers right now 87 | data = [[ 88 | '------------- ', 89 | '------------ ', 90 | '--------------- ', 91 | ]] 92 | r = requests.get('http://web.mta.info/status/serviceStatus.txt') 93 | _log_print("Fetched status from MTA") 94 | soup = Soup(r.text, "html.parser") 95 | subway = soup.find("subway") 96 | for status in subway.findAll("line"): 97 | line = status.contents[1].contents[0] 98 | status_title = status.contents[3].contents[0] 99 | try: 100 | status_description = status.contents[5].contents[0].strip() 101 | except IndexError: 102 | _log_print(f"{line} doesn't have a status, assuming GOOD SERVICE") 103 | status_description = 'GOOD SERVICE' 104 | is_it_fucked = "YUP" if status_title != "GOOD SERVICE" else "NOPE" 105 | 106 | # TODO: figure out how I want to store data, and do this the right 107 | # way... 108 | # _put_item_with_latest_and_timestamp(imtf, { 109 | # "line": line, 110 | # "status_title": status_title, 111 | # # "status_description": status_description, 112 | # "is_it_fucked": is_it_fucked, 113 | # }) 114 | 115 | s3.Object('www.ismytrainfucked.com', f"{line}.html").put( 116 | Body=STATUS_PAGE_HTML.format(line, status_description, GA), 117 | ContentType="text/html" 118 | ) 119 | _log_print(f"Wrote {line}.html file to website bucket") 120 | 121 | data.append([ 122 | f"{line}", 123 | status_title, 124 | is_it_fucked 125 | ]) 126 | 127 | html_table = tabulate(data, headers=["Subway Line", "Status", "Is it fucked?"], tablefmt="html") 128 | last_updated = datetime.datetime.now(TIMEZONE).strftime("%A %B %d %I:%M %p") 129 | s3.Object('www.ismytrainfucked.com', 'index.html').put( 130 | Body=HTML.format(html_table, last_updated, GA), 131 | ContentType="text/html" 132 | ) 133 | _log_print("Wrote index file to website bucket") 134 | return {'message': "ALL GOOD, YO!"} 135 | 136 | 137 | if __name__ == '__main__': 138 | handle('', '') 139 | --------------------------------------------------------------------------------