├── datapipe ├── __init__.py ├── importer.py └── transformer.py ├── correlation_trading ├── __init__.py ├── transformers.py └── trading_pipe.py ├── docker-ci-demo ├── app │ ├── tests │ │ ├── __init__.py │ │ └── test_routes.py │ ├── templates │ │ ├── index.html │ │ ├── new.html │ │ └── base.html │ ├── models.py │ ├── forms.py │ ├── __init__.py │ ├── configuration.py │ └── views.py ├── Dockerfile ├── Dockerfile.test ├── run_test_mode.py ├── run.py ├── requirements.txt ├── .gitignore ├── jenkins-pipeline.groovy └── docker-compose-ci-test.yaml ├── citibikekafkastreams ├── .gitignore ├── src │ ├── main │ │ ├── resources │ │ │ └── avro │ │ │ │ └── com │ │ │ │ └── cloudboxlabs │ │ │ │ └── streams │ │ │ │ └── citi-bike-streams.avsc │ │ └── java │ │ │ └── com │ │ │ └── cloudboxlabs │ │ │ ├── CitiBikeStationInfoAPI.java │ │ │ ├── CitiBikeStationStatusAPI.java │ │ │ ├── LowAvailability.java │ │ │ └── TurnoverRatio.java │ └── assembly │ │ ├── standalone.xml │ │ ├── development.xml │ │ └── package.xml └── pom.xml ├── .gitignore ├── .DS_Store ├── README.md ├── rpc ├── interstellar_pb2.pyc ├── interstellar_pb2_grpc.pyc ├── protos │ └── interstellar.proto ├── interstellar_client.py ├── interstellar_server.py ├── interstellar_pb2_grpc.py └── interstellar_pb2.py ├── streaming ├── gtfs_realtime_pb2.pyc ├── nyct_subway_pb2_grpc.py ├── gtfs_realtime_pb2_grpc.py ├── producer.py ├── consumer.py ├── protos │ ├── nyct-subway.proto │ └── gtfs-realtime.proto ├── nyct_subway_pb2.py └── static │ └── mta_stations.csv └── distributed-logging ├── fluentd ├── .DS_Store └── conf │ └── fluent.conf ├── connector_jars ├── gson-2.4.jar ├── guava-18.0.jar ├── jest-2.0.0.jar ├── httpclient-4.5.1.jar ├── httpcore-4.4.4.jar ├── commons-codec-1.9.jar ├── commons-lang3-3.4.jar ├── httpcore-nio-4.4.4.jar ├── jest-common-2.0.0.jar ├── slf4j-simple-1.7.5.jar ├── commons-logging-1.2.jar ├── httpasyncclient-4.1.1.jar └── kafka-connect-elasticsearch-3.2.0-SNAPSHOT.jar ├── Dockerfile-fluentd ├── connector_conf ├── connector_console.json ├── connector_elasticsearch.json └── connector_s3.json ├── Dockerfile-kafka-connect └── docker-compose-dist-logging.yaml /datapipe/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /correlation_trading/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docker-ci-demo/app/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /citibikekafkastreams/.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | *.iml 3 | target/ 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.idea/* 2 | .mta_api_key 3 | *.pytest_cache/* 4 | *.pyc 5 | -------------------------------------------------------------------------------- /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudboxlabs/blog-code/HEAD/.DS_Store -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # blog-code 2 | Code for public blog http://cloudboxlabs.com/blog 3 | -------------------------------------------------------------------------------- /rpc/interstellar_pb2.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudboxlabs/blog-code/HEAD/rpc/interstellar_pb2.pyc -------------------------------------------------------------------------------- /rpc/interstellar_pb2_grpc.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudboxlabs/blog-code/HEAD/rpc/interstellar_pb2_grpc.pyc -------------------------------------------------------------------------------- /streaming/gtfs_realtime_pb2.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudboxlabs/blog-code/HEAD/streaming/gtfs_realtime_pb2.pyc -------------------------------------------------------------------------------- /streaming/nyct_subway_pb2_grpc.py: -------------------------------------------------------------------------------- 1 | # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! 2 | import grpc 3 | 4 | -------------------------------------------------------------------------------- /distributed-logging/fluentd/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudboxlabs/blog-code/HEAD/distributed-logging/fluentd/.DS_Store -------------------------------------------------------------------------------- /streaming/gtfs_realtime_pb2_grpc.py: -------------------------------------------------------------------------------- 1 | # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! 2 | import grpc 3 | 4 | -------------------------------------------------------------------------------- /distributed-logging/connector_jars/gson-2.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudboxlabs/blog-code/HEAD/distributed-logging/connector_jars/gson-2.4.jar -------------------------------------------------------------------------------- /distributed-logging/connector_jars/guava-18.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudboxlabs/blog-code/HEAD/distributed-logging/connector_jars/guava-18.0.jar -------------------------------------------------------------------------------- /distributed-logging/connector_jars/jest-2.0.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudboxlabs/blog-code/HEAD/distributed-logging/connector_jars/jest-2.0.0.jar -------------------------------------------------------------------------------- /distributed-logging/connector_jars/httpclient-4.5.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudboxlabs/blog-code/HEAD/distributed-logging/connector_jars/httpclient-4.5.1.jar -------------------------------------------------------------------------------- /distributed-logging/connector_jars/httpcore-4.4.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudboxlabs/blog-code/HEAD/distributed-logging/connector_jars/httpcore-4.4.4.jar -------------------------------------------------------------------------------- /distributed-logging/connector_jars/commons-codec-1.9.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudboxlabs/blog-code/HEAD/distributed-logging/connector_jars/commons-codec-1.9.jar -------------------------------------------------------------------------------- /distributed-logging/connector_jars/commons-lang3-3.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudboxlabs/blog-code/HEAD/distributed-logging/connector_jars/commons-lang3-3.4.jar -------------------------------------------------------------------------------- /distributed-logging/connector_jars/httpcore-nio-4.4.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudboxlabs/blog-code/HEAD/distributed-logging/connector_jars/httpcore-nio-4.4.4.jar -------------------------------------------------------------------------------- /distributed-logging/connector_jars/jest-common-2.0.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudboxlabs/blog-code/HEAD/distributed-logging/connector_jars/jest-common-2.0.0.jar -------------------------------------------------------------------------------- /distributed-logging/connector_jars/slf4j-simple-1.7.5.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudboxlabs/blog-code/HEAD/distributed-logging/connector_jars/slf4j-simple-1.7.5.jar -------------------------------------------------------------------------------- /distributed-logging/connector_jars/commons-logging-1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudboxlabs/blog-code/HEAD/distributed-logging/connector_jars/commons-logging-1.2.jar -------------------------------------------------------------------------------- /distributed-logging/connector_jars/httpasyncclient-4.1.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudboxlabs/blog-code/HEAD/distributed-logging/connector_jars/httpasyncclient-4.1.1.jar -------------------------------------------------------------------------------- /docker-ci-demo/app/templates/index.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {%block title%}Index!{%endblock%} 4 | 5 | {%block main_content%} 6 | 7 |

Index

8 | 9 | {%endblock%} 10 | -------------------------------------------------------------------------------- /distributed-logging/Dockerfile-fluentd: -------------------------------------------------------------------------------- 1 | FROM fluent/fluentd:v0.12-debian 2 | 3 | ENV FLUENT_UID=0 4 | RUN mkdir /buffer 5 | RUN ["gem", "install", "fluent-plugin-kafka", "--no-rdoc", "--no-ri", "--version", "0.7.9"] 6 | -------------------------------------------------------------------------------- /distributed-logging/connector_jars/kafka-connect-elasticsearch-3.2.0-SNAPSHOT.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudboxlabs/blog-code/HEAD/distributed-logging/connector_jars/kafka-connect-elasticsearch-3.2.0-SNAPSHOT.jar -------------------------------------------------------------------------------- /docker-ci-demo/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:2.7 2 | 3 | # Install packages 4 | RUN set -ex; \ 5 | apt-get update; \ 6 | apt-get -y -qq install postgresql 7 | 8 | ADD . /app 9 | WORKDIR /app 10 | RUN pip install -r requirements.txt 11 | -------------------------------------------------------------------------------- /docker-ci-demo/Dockerfile.test: -------------------------------------------------------------------------------- 1 | FROM python:2.7 2 | 3 | # Install packages 4 | RUN set -ex; \ 5 | apt-get update; \ 6 | apt-get -y -qq install postgresql redis-tools 7 | 8 | ADD . /app 9 | WORKDIR /app 10 | 11 | RUN pip install -r requirements.txt 12 | 13 | -------------------------------------------------------------------------------- /distributed-logging/connector_conf/connector_console.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"local-console-sink", 3 | "config":{ 4 | "connector.class":"org.apache.kafka.connect.file.FileStreamSinkConnector", 5 | "tasks.max":"1", 6 | "topics":"log-messages" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /docker-ci-demo/run_test_mode.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from app import app, db 4 | from app.models import Post 5 | 6 | 7 | if __name__ == "__main__": 8 | # wait for postgres to be ready 9 | time.sleep(5) 10 | db.create_all() 11 | 12 | app.run(host='0.0.0.0', port=5000) 13 | -------------------------------------------------------------------------------- /docker-ci-demo/app/models.py: -------------------------------------------------------------------------------- 1 | from app import db 2 | 3 | 4 | class Post(db.Model): 5 | id = db.Column(db.Integer, primary_key=True) 6 | title = db.Column(db.String(250)) 7 | body = db.Column(db.Text) 8 | date = db.Column(db.DateTime) 9 | author = db.Column(db.String(50)) 10 | -------------------------------------------------------------------------------- /docker-ci-demo/run.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | """ 3 | Python Aplication Template 4 | Licence: GPLv3 5 | """ 6 | 7 | from flask_sqlalchemy import SQLAlchemy 8 | 9 | from app import app 10 | 11 | 12 | db = SQLAlchemy(app) 13 | 14 | 15 | if __name__ == "__main__": 16 | app.run(host='0.0.0.0', port=5000) 17 | -------------------------------------------------------------------------------- /docker-ci-demo/app/forms.py: -------------------------------------------------------------------------------- 1 | from flask.ext.wtf import Form, TextField, TextAreaField, DateTimeField, PasswordField 2 | from flask.ext.wtf import Required 3 | 4 | 5 | class BlogPostForm(Form): 6 | title = TextField(u'Title', validators=[Required()]) 7 | body = TextAreaField(u'Content') 8 | date = DateTimeField(u'Date', format='%Y-%m-%d') 9 | author = TextField(u'Author', validators=[Required()]) 10 | 11 | -------------------------------------------------------------------------------- /docker-ci-demo/requirements.txt: -------------------------------------------------------------------------------- 1 | Flask==0.9 2 | Flask-Admin==1.0.4 3 | Flask-Cache==0.10.1 4 | Flask-Login==0.1.3 5 | Flask-SQLAlchemy==0.16 6 | Flask-WTF==0.8.2 7 | Jinja2==2.6 8 | Markdown==2.2.1 9 | PyYAML==3.10 10 | SQLAlchemy==0.8.0b2 11 | WTForms==1.0.3 12 | Werkzeug==0.8.3 13 | argparse==1.2.1 14 | blinker==1.2 15 | python-dateutil==1.5 16 | psycopg2==2.7 17 | redis==2.10 18 | requests==2.19.1 19 | pytest 20 | wsgiref==0.1.2 21 | 22 | -------------------------------------------------------------------------------- /distributed-logging/connector_conf/connector_elasticsearch.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"elasticsearch-sink", 3 | "config":{ 4 | "connector.class":"io.confluent.connect.elasticsearch.ElasticsearchSinkConnector", 5 | "tasks.max":"1", 6 | "topics":"log-messages", 7 | "key.ignore":"true", 8 | "schema.ignore": "true", 9 | "connection.url": "http://elasticsearch:9200", 10 | "type.name": "kafka-connect" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /distributed-logging/Dockerfile-kafka-connect: -------------------------------------------------------------------------------- 1 | FROM confluentinc/cp-kafka-connect:4.0.0 2 | 3 | RUN apt-get update && apt-get install -y vim 4 | 5 | RUN mkdir -p /opt 6 | RUN mkdir -p /opt/connectors 7 | ADD connector_jars /opt/connectors/connector_jars 8 | WORKDIR /opt 9 | 10 | ADD connector_conf/ /opt/connector_conf 11 | RUN wget https://github.com/confluentinc/kafka-connect-elasticsearch/archive/v3.3.3-rc1.tar.gz; \ 12 | tar xzf v3.3.3-rc1.tar.gz -C /opt/connectors 13 | -------------------------------------------------------------------------------- /docker-ci-demo/app/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | """ 3 | Python Aplication Template 4 | Licence: GPLv3 5 | """ 6 | import os 7 | 8 | from flask import Flask 9 | from flask.ext.sqlalchemy import SQLAlchemy 10 | from flask.ext.login import LoginManager 11 | 12 | from app.configuration import config 13 | 14 | app = Flask(__name__) 15 | 16 | env = os.environ.get("FLASK_ENV", "dev") 17 | app.config.from_object(config[env]) 18 | 19 | db = SQLAlchemy(app) 20 | 21 | lm = LoginManager() 22 | lm.setup_app(app) 23 | lm.login_view = 'login' 24 | 25 | from app import views, models 26 | -------------------------------------------------------------------------------- /distributed-logging/fluentd/conf/fluent.conf: -------------------------------------------------------------------------------- 1 | 2 | @type forward 3 | port 24224 4 | bind 0.0.0.0 5 | 6 | 7 | @type kafka_buffered 8 | 9 | # list of seed brokers 10 | brokers kafka:9092 11 | 12 | # buffer settings 13 | buffer_type file 14 | buffer_path /buffer/td 15 | flush_interval 3s 16 | 17 | # topic settings 18 | default_topic log-messages 19 | 20 | # data type settings 21 | output_data_type json 22 | compression_codec gzip 23 | 24 | # producer settings 25 | max_send_retries 1 26 | required_acks -1 27 | 28 | -------------------------------------------------------------------------------- /docker-ci-demo/.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | #virtualenv 7 | env 8 | 9 | # Packages 10 | *.egg 11 | *.egg-info 12 | dist 13 | build 14 | eggs 15 | parts 16 | bin 17 | var 18 | sdist 19 | develop-eggs 20 | .installed.cfg 21 | lib 22 | lib64 23 | 24 | # Installer logs 25 | pip-log.txt 26 | 27 | # Unit test / coverage reports 28 | .coverage 29 | .tox 30 | nosetests.xml 31 | 32 | # Translations 33 | *.mo 34 | 35 | # Mr Developer 36 | .mr.developer.cfg 37 | .project 38 | .pydevproject 39 | 40 | # Nilton Comandos 41 | comandos_git.sh 42 | .directory 43 | 44 | .idea/ 45 | .pytest_cache 46 | venv 47 | -------------------------------------------------------------------------------- /docker-ci-demo/app/configuration.py: -------------------------------------------------------------------------------- 1 | 2 | class Config(object): 3 | """ 4 | Configuration base, for all environments. 5 | """ 6 | DEBUG = False 7 | TESTING = False 8 | SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://postgres@localhost:5432/postgres' 9 | BOOTSTRAP_FONTAWESOME = True 10 | SECRET_KEY = "MINHACHAVESECRETA" 11 | CSRF_ENABLED = True 12 | 13 | 14 | class ProductionConfig(Config): 15 | DATABASE_URI = 'mysql://user@localhost/foo' 16 | 17 | 18 | class DevelopmentConfig(Config): 19 | DEBUG = True 20 | 21 | 22 | class TestingConfig(Config): 23 | SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://postgres@postgres:5432/postgres' 24 | TESTING = True 25 | CSRF_ENABLED = False 26 | 27 | 28 | config = {"dev": DevelopmentConfig, "prod": ProductionConfig, "docker": TestingConfig} -------------------------------------------------------------------------------- /docker-ci-demo/app/templates/new.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {%block title%}New Post{%endblock%} 4 | 5 | {%block main_content%} 6 | 7 |

Create new blog post

8 |
9 |
10 | 11 |
12 | {{ form.hidden_tag() }} 13 | {{ form.title }}
14 | {{ form.body }} 15 | {{ form.date }} 16 | {{ form.author }} 17 | 18 |
19 | 20 | 21 |
22 |
23 | 24 |
25 |
26 | 27 | {%endblock%} 28 | -------------------------------------------------------------------------------- /docker-ci-demo/jenkins-pipeline.groovy: -------------------------------------------------------------------------------- 1 | node { 2 | try { 3 | stage('Clone') { 4 | checkout([ 5 | $class: 'GitSCM', 6 | branch: 'master', 7 | userRemoteConfigs: [[ 8 | credentialsId: 'ebc0533e-db18-45d5-b041-4ca58fb25b15', 9 | url: 'https://github.com/cloudboxlabs/blog-code.git' 10 | ]] 11 | ]) 12 | } 13 | stage('Integration Test') { 14 | sh "/usr/local/bin/docker-compose -f docker-ci-demo/docker-compose-ci-test.yaml up -d" 15 | sh "/usr/bin/docker wait docker-ci-demo_integration_test_1" 16 | } 17 | stage('Deploy') { 18 | sh "cd docker-ci-demo && /usr/bin/docker build ." 19 | } 20 | } catch (e) { 21 | currentBuild.result = 'FAILURE' 22 | throw e 23 | } finally { 24 | sh "/usr/local/bin/docker-compose -f docker-ci-demo/docker-compose-ci-test.yaml down" 25 | } 26 | } -------------------------------------------------------------------------------- /distributed-logging/connector_conf/connector_s3.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "s3-sink", 3 | "config": { 4 | "connector.class": "io.confluent.connect.s3.S3SinkConnector", 5 | "tasks.max": 1, 6 | "topics": "log-messages", 7 | "s3.region": "us-east-1", 8 | "s3.bucket.name": "distributed-logging", 9 | "s3.part.size": 5242880, 10 | "flush.size": 10000, 11 | "storage.class": "io.confluent.connect.s3.storage.S3Storage", 12 | "format.class": "io.confluent.connect.s3.format.json.JsonFormat", 13 | "schema.generator.class": "io.confluent.connect.storage.hive.schema.DefaultSchemaGenerator", 14 | "partitioner.class": "io.confluent.connect.storage.partitioner.TimeBasedPartitioner", 15 | "schema.compatibility": "NONE", 16 | "partition.duration.ms": 10000, 17 | "path.format": "YYYY/M/d/h", 18 | "locale": "US", 19 | "timezone": "UTC", 20 | "rotate.schedule.interval.ms": 60000 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /docker-ci-demo/docker-compose-ci-test.yaml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | integration_test: 4 | build: 5 | context: ./ 6 | dockerfile: Dockerfile.test 7 | volumes: 8 | - .:/app 9 | depends_on: 10 | - web_app 11 | links: 12 | - web_app 13 | environment: 14 | - FLASK_ENV=docker 15 | command: ["bash", "-c", "sleep 10 && py.test"] 16 | web_app: 17 | build: 18 | context: ./ 19 | dockerfile: Dockerfile 20 | volumes: 21 | - .:/app 22 | ports: 23 | - "5000" 24 | depends_on: 25 | - redis 26 | - postgres 27 | links: 28 | - redis 29 | - postgres 30 | environment: 31 | - FLASK_ENV=docker 32 | - REDIS_HOST=redis 33 | command: ["python", "run_test_mode.py"] 34 | redis: 35 | image: redis:latest 36 | ports: 37 | - "6379" 38 | postgres: 39 | image: postgres:latest 40 | ports: 41 | - "5432" 42 | -------------------------------------------------------------------------------- /rpc/protos/interstellar.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package interstellar; 4 | 5 | service InterstellarCommunication { 6 | // Say hello and receive a hello back 7 | rpc SayHello (HelloRequest) returns (HelloResponse) {} 8 | // Get a stream of words from Mars 9 | rpc GetMessageFromMars(MessageRequest) returns (stream StreamMessageFromMars) {} 10 | // Send a stream of words from earth and get a quick reply back from Mars 11 | rpc SendMessageFromEarth(stream StreamMessageFromEarth) returns (ReplyFromMars) {} 12 | // Send a stream of words from earth and get a stream of words back from Mars 13 | rpc SendAndReceiveMessage(stream StreamMessageFromEarth) returns (stream StreamMessageFromMars) {} 14 | } 15 | 16 | message HelloRequest { 17 | string hello_from_earth = 1; 18 | } 19 | 20 | message HelloResponse { 21 | string hello_from_mars = 1; 22 | } 23 | 24 | message MessageRequest { 25 | string request = 1; 26 | } 27 | 28 | message StreamMessageFromEarth { 29 | string message = 1; 30 | } 31 | 32 | message ReplyFromMars { 33 | string reply = 1; 34 | } 35 | 36 | message StreamMessageFromMars { 37 | string message = 1; 38 | } 39 | -------------------------------------------------------------------------------- /citibikekafkastreams/src/main/resources/avro/com/cloudboxlabs/streams/citi-bike-streams.avsc: -------------------------------------------------------------------------------- 1 | [ 2 | {"namespace": "com.cloudboxlabs.streams.avro", 3 | "type": "record", 4 | "name": "StationInformation", 5 | "fields": [ 6 | {"name": "station_id", "type": "string"}, 7 | {"name": "name", "type": "string"}, 8 | {"name": "latitude", "type": "float"}, 9 | {"name": "longitude", "type": "float"}, 10 | {"name": "capacity", "type": "int"} 11 | ] 12 | }, 13 | 14 | {"namespace": "com.cloudboxlabs.streams.avro", 15 | "type": "record", 16 | "name": "StationStatus", 17 | "fields": [ 18 | {"name": "station_id", "type": "string"}, 19 | {"name": "num_bikes_available", "type": "int"}, 20 | {"name": "num_ebikes_available", "type": "int"}, 21 | {"name": "num_bikes_disabled", "type": "int"}, 22 | {"name": "num_docks_available", "type": "int"}, 23 | {"name": "num_docks_disabled", "type": "int"}, 24 | {"name": "is_installed", "type": "int"}, 25 | {"name": "is_renting", "type": "int"}, 26 | {"name": "is_returning", "type": "int"} 27 | ] 28 | } 29 | ] 30 | 31 | 32 | -------------------------------------------------------------------------------- /rpc/interstellar_client.py: -------------------------------------------------------------------------------- 1 | import grpc 2 | 3 | import interstellar_pb2 as pb2 4 | import interstellar_pb2_grpc 5 | 6 | 7 | def get_message_iterator(): 8 | earth_message = 'Live on Earth in fantastic!' 9 | 10 | for word in earth_message.split(' '): 11 | yield pb2.StreamMessageFromEarth(message=word) 12 | 13 | 14 | def run(): 15 | channel = grpc.insecure_channel('localhost:50051') 16 | stub = interstellar_pb2_grpc.InterstellarCommunicationStub(channel) 17 | 18 | # say hello 19 | print('Calling SayHello...') 20 | print(stub.SayHello(pb2.HelloRequest(hello_from_earth='Hello!'))) 21 | 22 | # get a message from Mars 23 | print('Calling GetMessageFromMars...') 24 | for msg in stub.GetMessageFromMars(pb2.MessageRequest(request='Requesting Mars message')): 25 | print(msg) 26 | 27 | # send message from Earth to Mars asynchronously 28 | print('Calling SendMessageFromEarth...') 29 | print(stub.SendMessageFromEarth(get_message_iterator())) 30 | 31 | # send and receive messages between Earth and Mars asynchronously 32 | print('Calling SendAndReceiveMessage...') 33 | for msg in stub.SendAndReceiveMessage(get_message_iterator()): 34 | print(msg) 35 | 36 | 37 | if __name__ == '__main__': 38 | run() 39 | 40 | -------------------------------------------------------------------------------- /docker-ci-demo/app/views.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from flask import render_template, request 4 | import redis 5 | 6 | from app import app, db 7 | from app.models import Post 8 | from forms import BlogPostForm 9 | 10 | 11 | redis_client = redis.StrictRedis(host=os.getenv('REDIS_HOST'), port=6379) 12 | 13 | 14 | @app.route('/') 15 | def index(): 16 | return render_template('index.html') 17 | 18 | 19 | @app.route('/new/') 20 | def new(): 21 | form = BlogPostForm() 22 | return render_template('new.html', form=form) 23 | 24 | 25 | @app.route('/save/', methods=['GET', 'POST']) 26 | def save(): 27 | form = BlogPostForm() 28 | if form.validate_on_submit(): 29 | if request.form['action'] == 'draft': 30 | print('Saving to redis') 31 | redis_client.set(form.title.data, form.body.data) 32 | else: 33 | print('Saving to postgres') 34 | model = Post() 35 | model.title = form.title.data 36 | model.body = form.body.data 37 | model.date = form.date.data 38 | model.author = form.author.data 39 | db.session.add(model) 40 | db.session.commit() 41 | return render_template('new.html', form=form) 42 | 43 | 44 | @app.route('/view//') 45 | def view(id): 46 | return render_template('view.html') 47 | 48 | 49 | -------------------------------------------------------------------------------- /streaming/producer.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import requests 4 | from google.protobuf.json_format import MessageToJson 5 | from confluent_kafka import Producer 6 | 7 | import gtfs_realtime_pb2 8 | 9 | 10 | class MTARealTime(object): 11 | 12 | def __init__(self): 13 | with open('.mta_api_key', 'r') as key_in: 14 | self.api_key = key_in.read().strip() 15 | 16 | self.mta_api_url = 'http://datamine.mta.info/mta_esi.php?key={}&feed_id=1'.format( 17 | self.api_key) 18 | self.kafka_topic = 'test' 19 | self.kafka_producer = Producer({'bootstrap.servers': 'localhost:9092'}) 20 | 21 | def produce_trip_updates(self): 22 | feed = gtfs_realtime_pb2.FeedMessage() 23 | response = requests.get(self.mta_api_url) 24 | feed.ParseFromString(response.content) 25 | 26 | for entity in feed.entity: 27 | if entity.HasField('trip_update'): 28 | update_json = MessageToJson(entity.trip_update) 29 | self.kafka_producer.produce( 30 | self.kafka_topic, update_json.encode('utf-8')) 31 | 32 | self.kafka_producer.flush() 33 | 34 | def run(self): 35 | while True: 36 | self.produce_trip_updates() 37 | time.sleep(30) 38 | 39 | 40 | if __name__ == '__main__': 41 | MTARealTime().run() 42 | 43 | -------------------------------------------------------------------------------- /datapipe/importer.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | from datetime import datetime 4 | 5 | import boto3 6 | import requests 7 | 8 | 9 | EXTERNAL_API_URL = 'https://jsonplaceholder.typicode.com/posts' 10 | BUCKET_NAME = 'cloudboxlabs' 11 | QUEUE_NAME = 'cloudboxlabs_datapipe_tutorial' 12 | 13 | 14 | class Importer(object): 15 | def __init__(self): 16 | self.s3 = boto3.resource('s3') 17 | sqs = boto3.resource('sqs') 18 | self.queue = sqs.get_queue_by_name(QueueName=QUEUE_NAME) 19 | 20 | def run(self): 21 | response = requests.get(EXTERNAL_API_URL) 22 | 23 | file_name = 'posts_{}.json'.format(datetime.strftime(datetime.now(), '%Y%m%d%H%M%S')) 24 | with open(file_name, 'w') as file_obj: 25 | file_obj.write(json.dumps(response.json())) 26 | 27 | logging.info('Received API response') 28 | 29 | with open(file_name, 'r') as file_obj: 30 | self.s3.Bucket(BUCKET_NAME).put_object(Key=file_name, Body=file_obj) 31 | 32 | logging.info('Put json into S3') 33 | 34 | self.queue.send_message(MessageBody='post', MessageAttributes={ 35 | 's3_path': { 36 | 'StringValue': 's3://{}/{}'.format(BUCKET_NAME, file_name), 37 | 'DataType': 'String' 38 | } 39 | }) 40 | 41 | 42 | if __name__ == '__main__': 43 | Importer().run() 44 | 45 | -------------------------------------------------------------------------------- /docker-ci-demo/app/templates/base.html: -------------------------------------------------------------------------------- 1 | {%block title%}Base{%endblock%} 2 | 3 | {%block body_content%} 4 | 5 |
6 |
7 |
8 | 9 |
10 |
11 | MySiteName 12 | 17 |
18 |
19 | 20 |
21 | 22 | 23 |
24 | 25 |
26 |
27 | 28 | {% with messages = get_flashed_messages() %} 29 | {% if messages %} 30 |
31 |
32 |
    33 | {% for message in messages %} 34 |
  • {{ message }}
    • 35 | {% endfor %} 36 |
37 |
38 |
39 | {% endif %} 40 | {% endwith %} 41 | 42 | {%block main_content%}{%endblock%} 43 | 44 |
45 | 46 | {%endblock%} 47 | -------------------------------------------------------------------------------- /datapipe/transformer.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import boto3 4 | import psycopg2 5 | 6 | BUCKET_NAME = 'cloudboxlabs' 7 | QUEUE_NAME = 'cloudboxlabs_datapipe_tutorial' 8 | 9 | 10 | class Transformer(object): 11 | def __init__(self): 12 | self.s3 = boto3.client('s3') 13 | sqs = boto3.resource('sqs') 14 | self.queue = sqs.get_queue_by_name(QueueName=QUEUE_NAME) 15 | 16 | self.conn = psycopg2.connect('connection string for postgres RDS') 17 | self.cursor = self.conn.cursor() 18 | 19 | def run(self): 20 | for message in self.queue.receive_messages(MessageAttributeNames=['s3_path']): 21 | if message.message_attributes: 22 | s3_path = message.message_attributes.get('s3_path').get('StringValue') 23 | response = self.s3.get_object(Bucket=BUCKET_NAME, 24 | Key=s3_path.split('/')[-1]) 25 | 26 | data = json.loads(response['Body'].read()) 27 | 28 | for post in data: 29 | self.cursor.execute( 30 | 'insert into posts (id, userId, title, body) ' 31 | 'values (%s, %s, %s, %s)', 32 | (post['id'], post['userId'], post['title'], post['body'])) 33 | 34 | message.delete() 35 | self.conn.commit() 36 | 37 | 38 | if __name__ == '__main__': 39 | Transformer().run() 40 | -------------------------------------------------------------------------------- /docker-ci-demo/app/tests/test_routes.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import redis 4 | import requests 5 | 6 | from app import db 7 | from app.models import Post 8 | 9 | 10 | class TestRoutes(unittest.TestCase): 11 | 12 | def setUp(self): 13 | self.redis = redis.StrictRedis(host='redis', port=6379, db=0) 14 | 15 | def test_save_post_draft(self): 16 | response = requests.post('http://web_app:5000/save/', data={ 17 | 'title': 'Test integration draft title', 18 | 'body': 'Test integration draft body', 19 | 'date': '2018-07-01', 20 | 'author': 'test_user', 21 | 'action': 'draft' 22 | }) 23 | 24 | self.assertEqual(response.status_code, 200) 25 | redis_value = self.redis.get('Test integration draft title') 26 | self.assertEqual(redis_value, 'Test integration draft body') 27 | 28 | def test_save_post(self): 29 | response = requests.post('http://web_app:5000/save/', data={ 30 | 'title': 'Test integration final title', 31 | 'body': 'Test integration final body', 32 | 'date': '2018-07-01', 33 | 'author': 'test_user', 34 | 'action': 'save' 35 | }) 36 | 37 | self.assertEqual(response.status_code, 200) 38 | 39 | posts = db.session.query(Post).all() 40 | self.assertEqual(len(posts), 1) 41 | self.assertEqual(posts[0].title, 'Test integration final title') 42 | self.assertEqual(posts[0].body, 'Test integration final body') 43 | -------------------------------------------------------------------------------- /correlation_trading/transformers.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import itertools 3 | 4 | import apache_beam as beam 5 | import pandas as pd 6 | 7 | 8 | TICKER_LIST = ['goog', 'aapl', 'fb'] 9 | PAIRS = list(itertools.combinations(TICKER_LIST, 2)) 10 | 11 | 12 | class CorrelationPairDoFn(beam.DoFn): 13 | """Parse each line of input text into words.""" 14 | 15 | def __init__(self, ticker): 16 | super(CorrelationPairDoFn, self).__init__() 17 | self.ticker = ticker 18 | 19 | def process(self, element, *args, **kwargs): 20 | fields = element.split(',') 21 | dt, price = tuple(fields)[:2] 22 | 23 | for pair in PAIRS: 24 | if pair[0] == self.ticker: 25 | yield pair, (dt, price) 26 | elif pair[1] == self.ticker: 27 | yield pair, (dt, price) 28 | 29 | 30 | class AddTimestampDoFn(beam.DoFn): 31 | 32 | def process(self, element, *args, **kwargs): 33 | trade_date = element.split(',')[0] 34 | unix_timestamp = int(datetime.datetime.strptime(trade_date, '%Y/%m/%d').strftime("%s")) 35 | yield beam.window.TimestampedValue(element, unix_timestamp) 36 | 37 | 38 | def calculate_correlation_pair(element): 39 | pair, price_dict = element 40 | 41 | prices_1, prices_2 = price_dict[pair[0]], price_dict[pair[1]] 42 | 43 | if prices_1 and prices_2: 44 | ind, vals = zip(*prices_1) 45 | ser1 = pd.Series([float(v) for v in vals], 46 | index=[pd.Timestamp(i) for i in ind]) 47 | ind, vals = zip(*prices_2) 48 | ser2 = pd.Series([float(v) for v in vals], 49 | index=[pd.Timestamp(i) for i in ind]) 50 | 51 | return pair, ser1.corr(ser2) 52 | else: 53 | return pair, 0 54 | -------------------------------------------------------------------------------- /citibikekafkastreams/src/assembly/standalone.xml: -------------------------------------------------------------------------------- 1 | 5 | 6 | standalone 7 | 8 | jar 9 | 10 | false 11 | 12 | 13 | ${project.parent.basedir} 14 | / 15 | 16 | README* 17 | COPYRIGHT* 18 | 19 | 20 | 21 | 22 | 23 | / 24 | true 25 | true 26 | runtime 27 | 28 | ${project.groupId}:${project.artifactId} 29 | 30 | 31 | 32 | 33 | / 34 | false 35 | true 36 | runtime 37 | 38 | 39 | log4j.properties 40 | 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /citibikekafkastreams/src/assembly/development.xml: -------------------------------------------------------------------------------- 1 | 5 | 7 | development 8 | 9 | dir 10 | 11 | false 12 | 13 | 14 | ${project.parent.basedir} 15 | share/doc/${project.artifactId}/ 16 | 17 | README* 18 | LICENSE* 19 | NOTICE* 20 | 21 | 22 | 23 | 24 | 25 | share/java/${project.artifactId} 26 | true 27 | 32 | true 33 | 34 | org.slf4j:slf4j-log4j12 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /rpc/interstellar_server.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import grpc 4 | from concurrent import futures 5 | 6 | import interstellar_pb2 as pb2 7 | import interstellar_pb2_grpc 8 | 9 | 10 | class InterstellarServer(interstellar_pb2_grpc.InterstellarCommunicationServicer): 11 | 12 | def __init__(self): 13 | self.earth_message_history = [] 14 | 15 | def SayHello(self, request, context): 16 | """ 17 | Receive a hello request from Earth and returns a hello from Mars 18 | """ 19 | self.earth_message_history.append(request.hello_from_earth) 20 | return pb2.HelloResponse(hello_from_mars='Hello from Mars!') 21 | 22 | def GetMessageFromMars(self, request, context): 23 | """ 24 | Receive a message request from Earth and returns a iterator of words from Mars 25 | """ 26 | self.earth_message_history.append(request.request) 27 | for word in 'Message from Mars!'.split(' '): 28 | yield pb2.StreamMessageFromMars(message=word) 29 | 30 | def SendMessageFromEarth(self, request_iterator, context): 31 | """ 32 | Receive an iterator of requests with words from Earth and returns a simple acknowledgement 33 | """ 34 | for earth_request in request_iterator: 35 | self.earth_message_history.append(earth_request.message) 36 | 37 | return pb2.ReplyFromMars(reply='Copy!') 38 | 39 | def SendAndReceiveMessage(self, request_iterator, context): 40 | """ 41 | Receive an iterator of requests with words from Earth and 42 | returns an iterator of words from Mars 43 | """ 44 | for earth_request in request_iterator: 45 | self.earth_message_history.append(earth_request.message) 46 | 47 | for word in 'Live on Mars is great!'.split(' '): 48 | yield pb2.StreamMessageFromMars(message=word) 49 | 50 | 51 | def serve(): 52 | server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) 53 | interstellar_pb2_grpc.add_InterstellarCommunicationServicer_to_server( 54 | InterstellarServer(), server) 55 | server.add_insecure_port('[::]:50051') 56 | server.start() 57 | try: 58 | while True: 59 | time.sleep(60 * 60 * 24) 60 | except KeyboardInterrupt: 61 | server.stop(0) 62 | 63 | 64 | if __name__ == '__main__': 65 | serve() 66 | -------------------------------------------------------------------------------- /citibikekafkastreams/src/assembly/package.xml: -------------------------------------------------------------------------------- 1 | 5 | 8 | package 9 | 10 | dir 11 | 12 | false 13 | 14 | 15 | ${project.parent.basedir} 16 | share/doc/${project.artifactId}/ 17 | 18 | version.txt 19 | COPYRIGHT* 20 | 21 | 22 | 23 | ${project.parent.basedir} 24 | 25 | 26 | bin/* 27 | 28 | 29 | 30 | ${project.parent.basedir}/config 31 | etc/${project.artifactId} 32 | 33 | * 34 | 35 | 36 | 37 | 38 | 39 | share/java/${project.artifactId} 40 | true 41 | 46 | true 47 | 48 | io.confluent:common-* 49 | 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /distributed-logging/docker-compose-dist-logging.yaml: -------------------------------------------------------------------------------- 1 | version: '2.1' 2 | services: 3 | web: 4 | image: httpd 5 | ports: 6 | - "8080:80" 7 | links: 8 | - fluentd 9 | logging: 10 | driver: "fluentd" 11 | options: 12 | fluentd-address: localhost:24224 13 | tag: httpd.access 14 | fluentd: 15 | build: 16 | context: ./ 17 | dockerfile: Dockerfile-fluentd 18 | volumes: 19 | - ./fluentd/conf:/fluentd/etc 20 | depends_on: 21 | - kafka 22 | links: 23 | - kafka 24 | ports: 25 | - "24224:24224" 26 | - "24224:24224/udp" 27 | zookeeper: 28 | image: wurstmeister/zookeeper 29 | ports: 30 | - "2181:2181" 31 | kafka: 32 | image: wurstmeister/kafka 33 | hostname: kafka 34 | depends_on: 35 | - zookeeper 36 | ports: 37 | - "9092:9092" 38 | environment: 39 | KAFKA_BROKER_ID: 1 40 | KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 41 | KAFKA_CREATE_TOPICS: log-messages:1:1,connect-config:1:1:compact,connect-offset:1:1:compact,connect-status:1:1:compact 42 | KAFKA_LISTENERS: PLAINTEXT://kafka:9092 43 | kafka_connect: 44 | build: 45 | context: ./ 46 | dockerfile: Dockerfile-kafka-connect 47 | depends_on: 48 | - kafka 49 | - elasticsearch 50 | links: 51 | - kafka 52 | - elasticsearch 53 | environment: 54 | CONNECT_BOOTSTRAP_SERVERS: kafka:9092 55 | CONNECT_GROUP_ID: 1 56 | CONNECT_CONFIG_STORAGE_TOPIC: connect-config 57 | CONNECT_OFFSET_STORAGE_TOPIC: connect-offset 58 | CONNECT_STATUS_STORAGE_TOPIC: connect-status 59 | CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1 60 | CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1 61 | CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1 62 | CONNECT_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter 63 | CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter 64 | CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "false" 65 | CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "false" 66 | CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter 67 | CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter 68 | CONNECT_REST_ADVERTISED_HOST_NAME: localhost 69 | CONNECT_PLUGIN_PATH: "/usr/share/java,/opt/connectors" 70 | elasticsearch: 71 | image: elasticsearch 72 | expose: 73 | - 9200 74 | ports: 75 | - "9200:9200" 76 | kibana: 77 | image: kibana 78 | links: 79 | - "elasticsearch" 80 | ports: 81 | - "5601:5601" 82 | -------------------------------------------------------------------------------- /streaming/consumer.py: -------------------------------------------------------------------------------- 1 | import csv 2 | from collections import defaultdict 3 | import json 4 | 5 | import arrow 6 | from confluent_kafka import Consumer, KafkaError 7 | 8 | 9 | class MTATrainTracker(object): 10 | 11 | def __init__(self): 12 | self.kafka_consumer = Consumer({ 13 | 'bootstrap.servers': 'localhost:9092', 14 | 'group.id': 'test_consumer_group', 15 | 'default.topic.config': { 16 | 'auto.offset.reset': 'smallest' 17 | } 18 | }) 19 | self.kafka_topic = 'test' 20 | 21 | # subway line number -> (stop_id, direction) -> next arrival time 22 | self.arrival_times = defaultdict(lambda: defaultdict(lambda: -1)) 23 | 24 | self.stations = {} 25 | with open('static/mta_stations.csv') as csvf: 26 | reader = csv.DictReader(csvf) 27 | for row in reader: 28 | self.stations[row['GTFS Stop ID']] = row['Stop Name'] 29 | 30 | def process_message(self, message): 31 | trip_update = json.loads(message) 32 | 33 | trip_header = trip_update.get('trip') 34 | if not trip_header: 35 | return 36 | 37 | route_id = trip_header['routeId'] 38 | stop_time_updates = trip_update.get('stopTimeUpdate') 39 | if not stop_time_updates: 40 | return 41 | 42 | for update in stop_time_updates: 43 | if 'arrival' not in update or 'stopId' not in update: 44 | continue 45 | 46 | stop_id, direction = update['stopId'][0:3], update['stopId'][3:] 47 | new_arrival_ts = int(update['arrival']['time']) 48 | 49 | next_arrival_ts = self.arrival_times[route_id][(stop_id, direction)] 50 | now = arrow.now(tz='US/Eastern') 51 | 52 | if new_arrival_ts >= now.timestamp and \ 53 | (next_arrival_ts == -1 or new_arrival_ts < next_arrival_ts): 54 | self.arrival_times[route_id][(stop_id, direction)] = new_arrival_ts 55 | 56 | # convert time delta to minutes 57 | time_delta = arrow.get(new_arrival_ts) - now 58 | minutes = divmod(divmod(time_delta.seconds, 3600)[1], 60)[0] 59 | print('Next {} bound {} train will arrive at station {} in {} minutes'.format( 60 | direction, route_id, self.stations[stop_id], minutes)) 61 | 62 | def run(self): 63 | self.kafka_consumer.subscribe([self.kafka_topic]) 64 | 65 | while True: 66 | msg = self.kafka_consumer.poll(1.0) 67 | 68 | if msg is None or not msg.value(): 69 | continue 70 | if msg.error() and msg.error().code() != KafkaError._PARTITION_EOF: 71 | raise ValueError('Kafka consumer exception: {}'.format(msg.error())) 72 | 73 | msg = msg.value() 74 | self.process_message(msg.decode('utf-8')) 75 | 76 | 77 | if __name__ == '__main__': 78 | MTATrainTracker().run() 79 | -------------------------------------------------------------------------------- /citibikekafkastreams/src/main/java/com/cloudboxlabs/CitiBikeStationInfoAPI.java: -------------------------------------------------------------------------------- 1 | package com.cloudboxlabs; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import com.google.api.client.http.*; 5 | import com.google.api.client.http.javanet.NetHttpTransport; 6 | import com.google.api.client.json.JsonFactory; 7 | import com.google.api.client.json.JsonObjectParser; 8 | import com.google.api.client.json.jackson2.JacksonFactory; 9 | import com.google.api.client.util.Key; 10 | import org.apache.kafka.clients.producer.KafkaProducer; 11 | import org.apache.kafka.clients.producer.Producer; 12 | import org.apache.kafka.clients.producer.ProducerRecord; 13 | 14 | import java.util.List; 15 | import java.util.Properties; 16 | 17 | 18 | public class CitiBikeStationInfoAPI { 19 | private static final String stationInformationUrl = 20 | "https://gbfs.citibikenyc.com/gbfs/en/station_information.json"; 21 | public static final String topicName = "station_information"; 22 | 23 | static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport(); 24 | static final JsonFactory JSON_FACTORY = new JacksonFactory(); 25 | 26 | /** URL for Citi bike API. */ 27 | public static class CitiBikeURL extends GenericUrl { 28 | 29 | public CitiBikeURL(String encodedUrl) { 30 | super(encodedUrl); 31 | } 32 | 33 | @Key 34 | public String fields; 35 | } 36 | 37 | /** Represents a StationInfo feed. */ 38 | public static class StationInfoFeed { 39 | @Key("ttl") 40 | public int ttl; 41 | 42 | @Key("data") 43 | public StationInfoData data; 44 | } 45 | 46 | /** Represents a data object in response */ 47 | public static class StationInfoData { 48 | @Key 49 | public List stations; 50 | } 51 | 52 | /** Represents a station info object */ 53 | public static class StationInfo { 54 | @Key("station_id") 55 | public String stationId; 56 | 57 | @Key 58 | public String name; 59 | 60 | @Key("lat") 61 | public float latitude; 62 | 63 | @Key("lon") 64 | public float longitude; 65 | 66 | @Key 67 | public int capacity; 68 | } 69 | 70 | public static void main(final String[] args) throws Exception { 71 | HttpRequestFactory requestFactory = 72 | HTTP_TRANSPORT.createRequestFactory(new HttpRequestInitializer() { 73 | @Override 74 | public void initialize(HttpRequest request) { 75 | request.setParser(new JsonObjectParser(JSON_FACTORY)); 76 | } 77 | }); 78 | CitiBikeURL url = new CitiBikeURL(stationInformationUrl); 79 | HttpRequest request = requestFactory.buildGetRequest(url); 80 | StationInfoFeed stationInfo = request.execute().parseAs(StationInfoFeed.class); 81 | if (stationInfo.data.stations.isEmpty()) { 82 | System.out.println("No station found."); 83 | } else { 84 | System.out.println(stationInfo.data.stations.size()); 85 | } 86 | 87 | // public station status stream to Kafka 88 | Properties props = new Properties(); 89 | props.put("bootstrap.servers", "localhost:9092"); 90 | props.put("acks", "all"); 91 | props.put("retries", 0); 92 | props.put("key.serializer", org.apache.kafka.common.serialization.StringSerializer.class); 93 | props.put("value.serializer", org.apache.kafka.common.serialization.StringSerializer.class); 94 | props.put("schema.registry.url", "http://localhost:8081"); 95 | 96 | Producer producer = new KafkaProducer<>(props); 97 | 98 | // send events to kafka 99 | ObjectMapper mapper = new ObjectMapper(); 100 | stationInfo.data.stations.forEach(st -> { 101 | try { 102 | String value = mapper.writeValueAsString(st); 103 | producer.send(new ProducerRecord<>(topicName, st.stationId, value)); 104 | } catch (Exception e) { 105 | System.out.println("Failed to serialize"); 106 | throw new RuntimeException(e.getMessage()); 107 | } 108 | }); 109 | 110 | producer.flush(); 111 | producer.close(); 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /rpc/interstellar_pb2_grpc.py: -------------------------------------------------------------------------------- 1 | # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! 2 | import grpc 3 | 4 | import interstellar_pb2 as interstellar__pb2 5 | 6 | 7 | class InterstellarCommunicationStub(object): 8 | # missing associated documentation comment in .proto file 9 | pass 10 | 11 | def __init__(self, channel): 12 | """Constructor. 13 | 14 | Args: 15 | channel: A grpc.Channel. 16 | """ 17 | self.SayHello = channel.unary_unary( 18 | '/interstellar.InterstellarCommunication/SayHello', 19 | request_serializer=interstellar__pb2.HelloRequest.SerializeToString, 20 | response_deserializer=interstellar__pb2.HelloResponse.FromString, 21 | ) 22 | self.GetMessageFromMars = channel.unary_stream( 23 | '/interstellar.InterstellarCommunication/GetMessageFromMars', 24 | request_serializer=interstellar__pb2.MessageRequest.SerializeToString, 25 | response_deserializer=interstellar__pb2.StreamMessageFromMars.FromString, 26 | ) 27 | self.SendMessageFromEarth = channel.stream_unary( 28 | '/interstellar.InterstellarCommunication/SendMessageFromEarth', 29 | request_serializer=interstellar__pb2.StreamMessageFromEarth.SerializeToString, 30 | response_deserializer=interstellar__pb2.ReplyFromMars.FromString, 31 | ) 32 | self.SendAndReceiveMessage = channel.stream_stream( 33 | '/interstellar.InterstellarCommunication/SendAndReceiveMessage', 34 | request_serializer=interstellar__pb2.StreamMessageFromEarth.SerializeToString, 35 | response_deserializer=interstellar__pb2.StreamMessageFromMars.FromString, 36 | ) 37 | 38 | 39 | class InterstellarCommunicationServicer(object): 40 | # missing associated documentation comment in .proto file 41 | pass 42 | 43 | def SayHello(self, request, context): 44 | """Say hello and receive a hello back 45 | """ 46 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 47 | context.set_details('Method not implemented!') 48 | raise NotImplementedError('Method not implemented!') 49 | 50 | def GetMessageFromMars(self, request, context): 51 | """Get a stream of words from Mars 52 | """ 53 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 54 | context.set_details('Method not implemented!') 55 | raise NotImplementedError('Method not implemented!') 56 | 57 | def SendMessageFromEarth(self, request_iterator, context): 58 | """Send a stream of words from earth and get a quick reply back from Mars 59 | """ 60 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 61 | context.set_details('Method not implemented!') 62 | raise NotImplementedError('Method not implemented!') 63 | 64 | def SendAndReceiveMessage(self, request_iterator, context): 65 | """Send a stream of words from earth and get a stream of words back from Mars 66 | """ 67 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 68 | context.set_details('Method not implemented!') 69 | raise NotImplementedError('Method not implemented!') 70 | 71 | 72 | def add_InterstellarCommunicationServicer_to_server(servicer, server): 73 | rpc_method_handlers = { 74 | 'SayHello': grpc.unary_unary_rpc_method_handler( 75 | servicer.SayHello, 76 | request_deserializer=interstellar__pb2.HelloRequest.FromString, 77 | response_serializer=interstellar__pb2.HelloResponse.SerializeToString, 78 | ), 79 | 'GetMessageFromMars': grpc.unary_stream_rpc_method_handler( 80 | servicer.GetMessageFromMars, 81 | request_deserializer=interstellar__pb2.MessageRequest.FromString, 82 | response_serializer=interstellar__pb2.StreamMessageFromMars.SerializeToString, 83 | ), 84 | 'SendMessageFromEarth': grpc.stream_unary_rpc_method_handler( 85 | servicer.SendMessageFromEarth, 86 | request_deserializer=interstellar__pb2.StreamMessageFromEarth.FromString, 87 | response_serializer=interstellar__pb2.ReplyFromMars.SerializeToString, 88 | ), 89 | 'SendAndReceiveMessage': grpc.stream_stream_rpc_method_handler( 90 | servicer.SendAndReceiveMessage, 91 | request_deserializer=interstellar__pb2.StreamMessageFromEarth.FromString, 92 | response_serializer=interstellar__pb2.StreamMessageFromMars.SerializeToString, 93 | ), 94 | } 95 | generic_handler = grpc.method_handlers_generic_handler( 96 | 'interstellar.InterstellarCommunication', rpc_method_handlers) 97 | server.add_generic_rpc_handlers((generic_handler,)) 98 | -------------------------------------------------------------------------------- /citibikekafkastreams/src/main/java/com/cloudboxlabs/CitiBikeStationStatusAPI.java: -------------------------------------------------------------------------------- 1 | package com.cloudboxlabs; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import com.google.api.client.http.*; 5 | import com.google.api.client.http.javanet.NetHttpTransport; 6 | import com.google.api.client.json.JsonFactory; 7 | import com.google.api.client.json.JsonObjectParser; 8 | import com.google.api.client.json.jackson2.JacksonFactory; 9 | import com.google.api.client.util.Key; 10 | import org.apache.kafka.clients.producer.KafkaProducer; 11 | import org.apache.kafka.clients.producer.Producer; 12 | import org.apache.kafka.clients.producer.ProducerRecord; 13 | 14 | import java.util.List; 15 | import java.util.Properties; 16 | 17 | public class CitiBikeStationStatusAPI { 18 | private static final String stationInformationUrl = 19 | "https://gbfs.citibikenyc.com/gbfs/en/station_status.json"; 20 | public static final String topicName = "station_status"; 21 | 22 | static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport(); 23 | static final JsonFactory JSON_FACTORY = new JacksonFactory(); 24 | 25 | /** URL for Citi bike API. */ 26 | public static class CitiBikeURL extends GenericUrl { 27 | 28 | public CitiBikeURL(String encodedUrl) { 29 | super(encodedUrl); 30 | } 31 | 32 | @Key 33 | public String fields; 34 | } 35 | 36 | /** Represents a StationStatus feed. */ 37 | public static class StationStatusFeed { 38 | @Key("ttl") 39 | public int ttl; 40 | 41 | @Key("data") 42 | public StationInfoData data; 43 | } 44 | 45 | /** Represents a data object in response */ 46 | public static class StationInfoData { 47 | @Key 48 | public List stations; 49 | } 50 | 51 | /** Represents a station status object */ 52 | public static class StationStatus { 53 | @Key("station_id") 54 | public String stationId; 55 | 56 | @Key 57 | public int num_bikes_available; 58 | 59 | @Key 60 | public int num_bikes_disabled; 61 | 62 | @Key 63 | public int num_docks_available; 64 | 65 | @Key 66 | public int num_docks_disabled; 67 | 68 | @Key 69 | public int is_installed; 70 | 71 | @Key 72 | public int is_renting; 73 | 74 | @Key 75 | public int is_returning; 76 | } 77 | 78 | 79 | public static void main(final String[] args) throws Exception { 80 | // http GET citi bike station status 81 | HttpRequestFactory requestFactory = 82 | HTTP_TRANSPORT.createRequestFactory(new HttpRequestInitializer() { 83 | @Override 84 | public void initialize(HttpRequest request) { 85 | request.setParser(new JsonObjectParser(JSON_FACTORY)); 86 | } 87 | }); 88 | CitiBikeURL url = new CitiBikeURL(stationInformationUrl); 89 | HttpRequest request = requestFactory.buildGetRequest(url); 90 | StationStatusFeed stationStatus = request.execute().parseAs(StationStatusFeed.class); 91 | if (stationStatus.data.stations.isEmpty()) { 92 | System.out.println("No station found."); 93 | } else { 94 | System.out.println(stationStatus.data.stations.size()); 95 | } 96 | 97 | // public station status stream to Kafka 98 | Properties props = new Properties(); 99 | props.put("bootstrap.servers", "localhost:9092"); 100 | props.put("acks", "all"); 101 | props.put("retries", 0); 102 | props.put("key.serializer", org.apache.kafka.common.serialization.StringSerializer.class); 103 | props.put("value.serializer", org.apache.kafka.common.serialization.StringSerializer.class); 104 | props.put("schema.registry.url", "http://localhost:8081"); 105 | 106 | Producer producer = new KafkaProducer<>(props); 107 | 108 | // send events to kafka 109 | ObjectMapper mapper = new ObjectMapper(); 110 | stationStatus.data.stations.forEach(st -> { 111 | try { 112 | String value = mapper.writeValueAsString(st); 113 | producer.send(new ProducerRecord<>(topicName, st.stationId, value)); 114 | } catch (Exception e) { 115 | System.out.println("Failed to serialize"); 116 | throw new RuntimeException(e.getMessage()); 117 | } 118 | }); 119 | 120 | producer.flush(); 121 | producer.close(); 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /correlation_trading/trading_pipe.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | A Dataflow pipeline that streams ingest real time tick prices for a set of stock tickers 4 | and calculate pair-wise correlations in a 10 minutes sliding window. If the correlation 5 | breaks certain threshold, stream output a trading signal to open a long/short position on the pair. 6 | 7 | The underlying assumption of correlation trading is that similar stocks moves in tandem. 8 | In statistical terms, their correlation should revert to mean. 9 | We can pair trade them if the sliding window correlation breaks out of a threshold with the 10 | assumption that we can close position once correlation reverts to mean. 11 | 12 | """ 13 | 14 | from __future__ import absolute_import 15 | 16 | import argparse 17 | import itertools 18 | import logging 19 | 20 | import apache_beam as beam 21 | from apache_beam.io import ReadFromText, WriteToText 22 | from apache_beam.options.pipeline_options import PipelineOptions, SetupOptions, StandardOptions 23 | from apache_beam.transforms import window 24 | import six 25 | 26 | from transformers import calculate_correlation_pair, CorrelationPairDoFn, AddTimestampDoFn 27 | 28 | 29 | SECONDS_IN_1_DAY = 3600 * 24 30 | CORRELATION_THRESHOLD = -0.75 31 | TICKER_LIST = ['goog', 'aapl', 'fb'] 32 | PAIRS = list(itertools.combinations(TICKER_LIST, 2)) 33 | 34 | 35 | def run(argv=None): 36 | parser = argparse.ArgumentParser() 37 | parser.add_argument( 38 | '--input_mode', 39 | default='file', 40 | help='Streaming input or file based batch input') 41 | 42 | for ticker in TICKER_LIST: 43 | parser.add_argument( 44 | '--input_{}'.format(ticker), 45 | default='{}_hist.csv'.format(ticker), 46 | help='Cloud Pub/Sub topic of tick market data for a stock, fall back to flat csv') 47 | 48 | parser.add_argument('--output_topic', 49 | default='/tmp/trading_signals.txt', 50 | help='Topic of output trading signals in Google Cloud Pub/Sub') 51 | 52 | known_args, pipeline_args = parser.parse_known_args(argv) 53 | # We use the save_main_session option because one or more DoFn's in this 54 | # workflow rely on global context (e.g., a module imported at module level). 55 | pipeline_options = PipelineOptions(pipeline_args) 56 | pipeline_options.view_as(SetupOptions).save_main_session = True 57 | 58 | if known_args.input_mode == 'stream': 59 | pipeline_options.view_as(StandardOptions).streaming = True 60 | 61 | with beam.Pipeline(options=pipeline_options) as p: 62 | 63 | # Read input 64 | input_stage = {} 65 | for ticker in TICKER_LIST: 66 | if known_args.input_mode == 'streaming': 67 | input_ticks = (p | beam.io.ReadFromPubSub(topic=known_args.input_topic) 68 | .with_output_types(six.binary_type)) 69 | else: 70 | input_ticks = (p | 'Read: %s' % ticker >> ReadFromText( 71 | getattr(known_args, 'input_%s' % ticker))) 72 | 73 | input_stage[ticker] = (input_ticks 74 | | 'decode: %s' % ticker >> beam.Map(lambda x: x.decode('utf-8')) 75 | | 'Filter: %s' % ticker >> beam.Filter( 76 | lambda row: row.split(',')[0] != 'date') 77 | | 'Add Timestamp: %s' % ticker >> beam.ParDo(AddTimestampDoFn()) 78 | | 'Window: %s' % ticker >> beam.WindowInto( 79 | window.SlidingWindows(size=SECONDS_IN_1_DAY * 10, period=SECONDS_IN_1_DAY)) 80 | | 'Pair: %s' % ticker >> beam.ParDo(CorrelationPairDoFn(ticker))) 81 | 82 | # Group together all entries under the same ticker 83 | grouped = input_stage | 'group_by_name' >> beam.CoGroupByKey() 84 | 85 | correlations = (grouped 86 | | 'Calculate pair correlation' >> beam.Map(calculate_correlation_pair)) 87 | 88 | if known_args.input_mode == 'stream': 89 | trading_signals = (correlations | 'Filter correlation threshold' >> beam.Filter( 90 | lambda x: x[1] < CORRELATION_THRESHOLD) 91 | .with_output_types(six.binary_type)) 92 | # pylint: disable=expression-not-assigned 93 | trading_signals | beam.io.WriteToPubSub(known_args.output_topic) 94 | else: 95 | trading_signals = (correlations | 'Filter correlation threshold' >> beam.Filter( 96 | lambda x: x[1] < CORRELATION_THRESHOLD)) 97 | # pylint: disable=expression-not-assigned 98 | trading_signals | 'WriteOutput' >> WriteToText(known_args.output_topic) 99 | 100 | 101 | if __name__ == '__main__': 102 | logging.getLogger().setLevel(logging.INFO) 103 | run() 104 | -------------------------------------------------------------------------------- /citibikekafkastreams/src/main/java/com/cloudboxlabs/LowAvailability.java: -------------------------------------------------------------------------------- 1 | package com.cloudboxlabs; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import org.apache.kafka.clients.consumer.ConsumerConfig; 5 | import org.apache.kafka.common.serialization.Serdes; 6 | import org.apache.kafka.streams.KafkaStreams; 7 | import org.apache.kafka.streams.KeyValue; 8 | import org.apache.kafka.streams.StreamsBuilder; 9 | import org.apache.kafka.streams.StreamsConfig; 10 | import org.apache.kafka.streams.kstream.Consumed; 11 | import org.apache.kafka.streams.kstream.KStream; 12 | import org.apache.kafka.streams.kstream.KTable; 13 | import org.apache.kafka.streams.kstream.Produced; 14 | 15 | import java.util.Properties; 16 | 17 | 18 | public class LowAvailability { 19 | static final String LOW_BIKE_TOPIC = "low-bike-availability"; 20 | 21 | 22 | public static void main(String[] args) { 23 | final KafkaStreams streams = createStreams("localhost:9092", "/tmp/kafka-streams"); 24 | streams.cleanUp(); 25 | 26 | streams.start(); 27 | 28 | // Add shutdown hook to respond to SIGTERM and gracefully close Kafka Streams 29 | Runtime.getRuntime().addShutdownHook(new Thread(streams::close)); 30 | } 31 | 32 | private static KafkaStreams createStreams(final String bootstrapServers, 33 | final String stateDir) { 34 | 35 | final Properties streamsConfiguration = new Properties(); 36 | // unique app id on kafka cluster 37 | streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, "citi-bike-low-availability"); 38 | streamsConfiguration.put(StreamsConfig.CLIENT_ID_CONFIG, "citi-bike-low-availability-client"); 39 | // kafka broker address 40 | streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); 41 | // local state store 42 | streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, stateDir); 43 | // consumer from the beginning of the topic or last offset 44 | streamsConfiguration.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); 45 | // override default serdes 46 | streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName()); 47 | streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName()); 48 | 49 | StreamsBuilder builder = new StreamsBuilder(); 50 | 51 | // Get the stream of station statuses 52 | ObjectMapper mapper = new ObjectMapper(); 53 | KStream statusStream = builder.stream( 54 | CitiBikeStationStatusAPI.topicName, 55 | Consumed.with(Serdes.String(), Serdes.String())) 56 | .map((station_id, v) -> { 57 | try { 58 | CitiBikeStationStatusAPI.StationStatus status = mapper.readValue( 59 | v, CitiBikeStationStatusAPI.StationStatus.class); 60 | return new KeyValue<>(station_id, Integer.toString(status.num_bikes_available)); 61 | } catch (Exception e) { 62 | throw new RuntimeException("Deserialize error" + e); 63 | } 64 | }); 65 | 66 | // Build KTable of station information 67 | KTable stationInfo = builder.table(CitiBikeStationInfoAPI.topicName); 68 | KTable stationInfoTable = stationInfo 69 | .mapValues(v -> { 70 | try { 71 | CitiBikeStationInfoAPI.StationInfo info = mapper.readValue( 72 | v, CitiBikeStationInfoAPI.StationInfo.class); 73 | return info; 74 | } catch (Exception e) { 75 | throw new RuntimeException("Deserialize error" + e); 76 | } 77 | }); 78 | 79 | // stations with bike availability ratio (num_bikes_avail / capacity) < threshold 80 | KStream outputStream = statusStream 81 | .leftJoin(stationInfoTable, (num_bikes, info) -> { 82 | return new BikeStats(Integer.parseInt(num_bikes), info.capacity, 83 | info.latitude, info.longitude); 84 | }) 85 | .filter((k, stats) -> stats.availabilityRatio < 0.1) 86 | .map((k, stats) -> new KeyValue<>(k, "station_id: " + k + 87 | ", longitude " + stats.longitude + 88 | ", latitude " + stats.latitude + 89 | ", bikes: " + stats.numBikesAvailable + 90 | ", capacity: " + stats.stationCapacity + 91 | ", ratio: " + String.format("%.2f", stats.availabilityRatio * 100) + "%")); 92 | 93 | // output to kafka topic 94 | outputStream 95 | .to(LOW_BIKE_TOPIC, Produced.with(Serdes.String(), Serdes.String())); 96 | 97 | 98 | return new KafkaStreams(builder.build(), streamsConfiguration); 99 | } 100 | 101 | private static class BikeStats { 102 | public final Integer numBikesAvailable; 103 | public final Integer stationCapacity; 104 | public final Float availabilityRatio; 105 | public final Float latitude; 106 | public final Float longitude; 107 | 108 | public BikeStats(Integer bikesAvailable, Integer capacity, Float lat, Float lng) { 109 | numBikesAvailable = bikesAvailable; 110 | stationCapacity = capacity; 111 | availabilityRatio = capacity == null ? 1 : bikesAvailable / (float) stationCapacity; 112 | latitude = lat; 113 | longitude = lng; 114 | } 115 | } 116 | 117 | } 118 | -------------------------------------------------------------------------------- /streaming/protos/nyct-subway.proto: -------------------------------------------------------------------------------- 1 | // 2 | // NYCT Subway extensions for the GTFS-realtime protocol. 3 | // 4 | option java_package = "com.google.transit.realtime"; 5 | 6 | import "gtfs-realtime.proto"; 7 | 8 | message TripReplacementPeriod { 9 | // The replacement period is for this route 10 | optional string route_id = 1; 11 | // The start time is omitted, the end time is currently now + 30 minutes for 12 | // all routes of the A division 13 | optional transit_realtime.TimeRange replacement_period = 2; 14 | } 15 | 16 | // NYCT Subway extensions for the feed header 17 | message NyctFeedHeader { 18 | // Version of the NYCT Subway extensions 19 | // The current version is 1.0 20 | required string nyct_subway_version = 1; 21 | // For the NYCT Subway, the GTFS-realtime feed replaces any scheduled 22 | // trip within the trip_replacement_period. 23 | // This feed is a full dataset, it contains all trips starting 24 | // in the trip_replacement_period. If a trip from the static GTFS is not 25 | // found in the GTFS-realtime feed, it should be considered as cancelled. 26 | // The replacement period can be different for each route, so here is 27 | // a list of the routes where the trips in the feed replace all 28 | // scheduled trips within the replacement period. 29 | repeated TripReplacementPeriod trip_replacement_period = 2; 30 | } 31 | 32 | extend transit_realtime.FeedHeader { 33 | optional NyctFeedHeader nyct_feed_header = 1001; 34 | } 35 | 36 | // NYCT Subway extensions for the trip descriptor 37 | message NyctTripDescriptor { 38 | // The nyct_train_id is meant for internal use only. It provides an 39 | // easy way to associated GTFS-realtime trip identifiers with NYCT rail 40 | // operations identifier 41 | // 42 | // The ATS office system assigns unique train identification (Train ID) to 43 | // each train operating within or ready to enter the mainline of the 44 | // monitored territory. An example of this is 06 0123+ PEL/BBR and is decoded 45 | // as follows: 46 | // 47 | // The first character represents the trip type designator. 0 identifies a 48 | // scheduled revenue trip. Other revenue trip values that are a result of a 49 | // change to the base schedule include; [= reroute], [/ skip stop], [$ turn 50 | // train] also known as shortly lined service. 51 | // 52 | // The second character 6 represents the trip line i.e. number 6 train The 53 | // third set of characters identify the decoded origin time. The last 54 | // character may be blank "on the whole minute" or + "30 seconds" 55 | // 56 | // Note: Origin times will not change when there is a trip type change. This 57 | // is followed by a three character "Origin Location" / "Destination 58 | // Location" 59 | optional string train_id = 1; 60 | 61 | // This trip has been assigned to a physical train. If true, this trip is 62 | // already underway or most likely will depart shortly. 63 | // 64 | // Train Assignment is a function of the Automatic Train Supervision (ATS) 65 | // office system used by NYCT Rail Operations to monitor and track train 66 | // movements. ATS provides the ability to "assign" the nyct_train_id 67 | // attribute when a physical train is at its origin terminal. These assigned 68 | // trips have the is_assigned field set in the TripDescriptor. 69 | // 70 | // When a train is at a terminal but has not been given a work program it is 71 | // declared unassigned and is tagged as such. Unassigned trains can be moved 72 | // to a storage location or assigned a nyct_train_id when a determination for 73 | // service is made. 74 | optional bool is_assigned = 2; 75 | 76 | // The direction the train is moving. 77 | enum Direction { 78 | NORTH = 1; 79 | EAST = 2; 80 | SOUTH = 3; 81 | WEST = 4; 82 | } 83 | // Uptown and Bronx-bound trains are moving NORTH. 84 | // Times Square Shuttle to Grand Central is also northbound. 85 | // 86 | // Downtown and Brooklyn-bound trains are moving SOUTH. 87 | // Times Square Shuttle to Times Square is also southbound. 88 | // 89 | // EAST and WEST are not used currently. 90 | optional Direction direction = 3; 91 | } 92 | 93 | extend transit_realtime.TripDescriptor { 94 | optional NyctTripDescriptor nyct_trip_descriptor = 1001; 95 | } 96 | 97 | // NYCT Subway extensions for the stop time update 98 | message NyctStopTimeUpdate { 99 | // Provides the planned station arrival track. The following is the Manhattan 100 | // track configurations: 101 | // 1: southbound local 102 | // 2: southbound express 103 | // 3: northbound express 104 | // 4: northbound local 105 | // 106 | // In the Bronx (except Dyre Ave line) 107 | // M: bi-directional express (in the AM express to Manhattan, in the PM 108 | // express away). 109 | // 110 | // The Dyre Ave line is configured: 111 | // 1: southbound 112 | // 2: northbound 113 | // 3: bi-directional 114 | optional string scheduled_track = 1; 115 | 116 | // This is the actual track that the train is operating on and can be used to 117 | // determine if a train is operating according to its current schedule 118 | // (plan). 119 | // 120 | // The actual track is known only shortly before the train reaches a station, 121 | // typically not before it leaves the previous station. Therefore, the NYCT 122 | // feed sets this field only for the first station of the remaining trip. 123 | // 124 | // Different actual and scheduled track is the result of manually rerouting a 125 | // train off it scheduled path. When this occurs, prediction data may become 126 | // unreliable since the train is no longer operating in accordance to its 127 | // schedule. The rules engine for the 'countdown' clocks will remove this 128 | // train from all schedule stations. 129 | optional string actual_track = 2; 130 | } 131 | 132 | extend transit_realtime.TripUpdate.StopTimeUpdate { 133 | optional NyctStopTimeUpdate nyct_stop_time_update = 1001; 134 | } -------------------------------------------------------------------------------- /citibikekafkastreams/src/main/java/com/cloudboxlabs/TurnoverRatio.java: -------------------------------------------------------------------------------- 1 | package com.cloudboxlabs; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import org.apache.kafka.clients.consumer.ConsumerConfig; 5 | import org.apache.kafka.common.serialization.Serdes; 6 | import org.apache.kafka.streams.KafkaStreams; 7 | import org.apache.kafka.streams.KeyValue; 8 | import org.apache.kafka.streams.StreamsBuilder; 9 | import org.apache.kafka.streams.StreamsConfig; 10 | import org.apache.kafka.streams.kstream.*; 11 | 12 | import java.util.Comparator; 13 | import java.util.Properties; 14 | import java.util.concurrent.TimeUnit; 15 | 16 | 17 | public class TurnoverRatio { 18 | static final String HIGH_TURNOVER_TOPIC = "top-N-high-turnover"; 19 | 20 | 21 | public static void main(String[] args) { 22 | final KafkaStreams streams = createStreams("localhost:9092", "/tmp/kafka-streams-1"); 23 | streams.cleanUp(); 24 | 25 | streams.start(); 26 | 27 | // Add shutdown hook to respond to SIGTERM and gracefully close Kafka Streams 28 | Runtime.getRuntime().addShutdownHook(new Thread(streams::close)); 29 | } 30 | 31 | private static KafkaStreams createStreams(final String bootstrapServers, 32 | final String stateDir) { 33 | 34 | final Comparator comparator = 35 | (o1, o2) -> o2 - o1; 36 | 37 | final Properties streamsConfiguration = new Properties(); 38 | // unique app id on kafka cluster 39 | streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, "citi-bike-turnover"); 40 | streamsConfiguration.put(StreamsConfig.CLIENT_ID_CONFIG, "citi-bike-turnover-client"); 41 | // kafka broker address 42 | streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); 43 | // local state store 44 | streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, stateDir); 45 | // consumer from the beginning of the topic or last offset 46 | streamsConfiguration.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); 47 | // override default serdes 48 | streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName()); 49 | streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName()); 50 | 51 | StreamsBuilder builder = new StreamsBuilder(); 52 | 53 | // Get the stream of station statuses 54 | ObjectMapper mapper = new ObjectMapper(); 55 | KStream statusStream = builder.stream( 56 | CitiBikeStationStatusAPI.topicName, 57 | Consumed.with(Serdes.String(), Serdes.String())) 58 | .map((station_id, v) -> { 59 | try { 60 | CitiBikeStationStatusAPI.StationStatus status = mapper.readValue( 61 | v, CitiBikeStationStatusAPI.StationStatus.class); 62 | return new KeyValue<>(station_id, Integer.toString(status.num_bikes_available)); 63 | } catch (Exception e) { 64 | throw new RuntimeException("Deserialize error" + e); 65 | } 66 | }); 67 | 68 | // Build KTable of station information 69 | KTable stationInfo = builder.table(CitiBikeStationInfoAPI.topicName); 70 | KTable stationInfoTable = stationInfo 71 | .mapValues(v -> { 72 | try { 73 | CitiBikeStationInfoAPI.StationInfo info = mapper.readValue( 74 | v, CitiBikeStationInfoAPI.StationInfo.class); 75 | return info; 76 | } catch (Exception e) { 77 | throw new RuntimeException("Deserialize error" + e); 78 | } 79 | }); 80 | 81 | // aggregate hourly over the net change in number of bikes 82 | final KTable, String> netDeltaTable = statusStream 83 | .groupByKey() 84 | .windowedBy(TimeWindows.of(TimeUnit.MINUTES.toMillis(60))) 85 | .aggregate( 86 | // the initializer 87 | () -> "0,0", 88 | 89 | // the "add" aggregator 90 | (stationId, record, accum) -> { 91 | Integer newValue = Integer.parseInt(record); 92 | Integer netDelta = Integer.parseInt(accum.split(",")[0]); 93 | Integer lastValue = Integer.parseInt(accum.split(",")[1]); 94 | Integer delta = Math.abs(newValue - lastValue); 95 | return (netDelta + delta) + "," + newValue; 96 | } 97 | ); 98 | 99 | // join station information table to get station capacity and calculate turnover ratio 100 | final KStream ratio = netDeltaTable 101 | .toStream() 102 | .map((windowedKey, v) -> new KeyValue<>(windowedKey.key(), v)) 103 | .join(stationInfoTable, (delta, info) -> delta + "," + info.capacity) 104 | .map((k, v) -> { 105 | Integer delta = Integer.parseInt(v.split(",")[0]); 106 | Integer capacity = Integer.parseInt(v.split(",")[2]); 107 | 108 | if (capacity == 0) { 109 | return new KeyValue<>(k, "NA"); 110 | } 111 | Double turnover = delta / (double)capacity * 100.0; 112 | return new KeyValue<>(k, String.format("%.2f", turnover)); 113 | }); 114 | 115 | ratio 116 | .map((k, v) -> new KeyValue<>(k, "station_id: " + k + ", turnover: " + v + "%")) 117 | .to(HIGH_TURNOVER_TOPIC, Produced.with(Serdes.String(), Serdes.String())); 118 | 119 | 120 | return new KafkaStreams(builder.build(), streamsConfiguration); 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /rpc/interstellar_pb2.py: -------------------------------------------------------------------------------- 1 | # Generated by the protocol buffer compiler. DO NOT EDIT! 2 | # source: interstellar.proto 3 | 4 | import sys 5 | _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) 6 | from google.protobuf import descriptor as _descriptor 7 | from google.protobuf import message as _message 8 | from google.protobuf import reflection as _reflection 9 | from google.protobuf import symbol_database as _symbol_database 10 | from google.protobuf import descriptor_pb2 11 | # @@protoc_insertion_point(imports) 12 | 13 | _sym_db = _symbol_database.Default() 14 | 15 | 16 | 17 | 18 | DESCRIPTOR = _descriptor.FileDescriptor( 19 | name='interstellar.proto', 20 | package='interstellar', 21 | syntax='proto3', 22 | serialized_pb=_b('\n\x12interstellar.proto\x12\x0cinterstellar\"(\n\x0cHelloRequest\x12\x18\n\x10hello_from_earth\x18\x01 \x01(\t\"(\n\rHelloResponse\x12\x17\n\x0fhello_from_mars\x18\x01 \x01(\t\"!\n\x0eMessageRequest\x12\x0f\n\x07request\x18\x01 \x01(\t\")\n\x16StreamMessageFromEarth\x12\x0f\n\x07message\x18\x01 \x01(\t\"\x1e\n\rReplyFromMars\x12\r\n\x05reply\x18\x01 \x01(\t\"(\n\x15StreamMessageFromMars\x12\x0f\n\x07message\x18\x01 \x01(\t2\x88\x03\n\x19InterstellarCommunication\x12\x45\n\x08SayHello\x12\x1a.interstellar.HelloRequest\x1a\x1b.interstellar.HelloResponse\"\x00\x12[\n\x12GetMessageFromMars\x12\x1c.interstellar.MessageRequest\x1a#.interstellar.StreamMessageFromMars\"\x00\x30\x01\x12]\n\x14SendMessageFromEarth\x12$.interstellar.StreamMessageFromEarth\x1a\x1b.interstellar.ReplyFromMars\"\x00(\x01\x12h\n\x15SendAndReceiveMessage\x12$.interstellar.StreamMessageFromEarth\x1a#.interstellar.StreamMessageFromMars\"\x00(\x01\x30\x01\x62\x06proto3') 23 | ) 24 | 25 | 26 | 27 | 28 | _HELLOREQUEST = _descriptor.Descriptor( 29 | name='HelloRequest', 30 | full_name='interstellar.HelloRequest', 31 | filename=None, 32 | file=DESCRIPTOR, 33 | containing_type=None, 34 | fields=[ 35 | _descriptor.FieldDescriptor( 36 | name='hello_from_earth', full_name='interstellar.HelloRequest.hello_from_earth', index=0, 37 | number=1, type=9, cpp_type=9, label=1, 38 | has_default_value=False, default_value=_b("").decode('utf-8'), 39 | message_type=None, enum_type=None, containing_type=None, 40 | is_extension=False, extension_scope=None, 41 | options=None, file=DESCRIPTOR), 42 | ], 43 | extensions=[ 44 | ], 45 | nested_types=[], 46 | enum_types=[ 47 | ], 48 | options=None, 49 | is_extendable=False, 50 | syntax='proto3', 51 | extension_ranges=[], 52 | oneofs=[ 53 | ], 54 | serialized_start=36, 55 | serialized_end=76, 56 | ) 57 | 58 | 59 | _HELLORESPONSE = _descriptor.Descriptor( 60 | name='HelloResponse', 61 | full_name='interstellar.HelloResponse', 62 | filename=None, 63 | file=DESCRIPTOR, 64 | containing_type=None, 65 | fields=[ 66 | _descriptor.FieldDescriptor( 67 | name='hello_from_mars', full_name='interstellar.HelloResponse.hello_from_mars', index=0, 68 | number=1, type=9, cpp_type=9, label=1, 69 | has_default_value=False, default_value=_b("").decode('utf-8'), 70 | message_type=None, enum_type=None, containing_type=None, 71 | is_extension=False, extension_scope=None, 72 | options=None, file=DESCRIPTOR), 73 | ], 74 | extensions=[ 75 | ], 76 | nested_types=[], 77 | enum_types=[ 78 | ], 79 | options=None, 80 | is_extendable=False, 81 | syntax='proto3', 82 | extension_ranges=[], 83 | oneofs=[ 84 | ], 85 | serialized_start=78, 86 | serialized_end=118, 87 | ) 88 | 89 | 90 | _MESSAGEREQUEST = _descriptor.Descriptor( 91 | name='MessageRequest', 92 | full_name='interstellar.MessageRequest', 93 | filename=None, 94 | file=DESCRIPTOR, 95 | containing_type=None, 96 | fields=[ 97 | _descriptor.FieldDescriptor( 98 | name='request', full_name='interstellar.MessageRequest.request', index=0, 99 | number=1, type=9, cpp_type=9, label=1, 100 | has_default_value=False, default_value=_b("").decode('utf-8'), 101 | message_type=None, enum_type=None, containing_type=None, 102 | is_extension=False, extension_scope=None, 103 | options=None, file=DESCRIPTOR), 104 | ], 105 | extensions=[ 106 | ], 107 | nested_types=[], 108 | enum_types=[ 109 | ], 110 | options=None, 111 | is_extendable=False, 112 | syntax='proto3', 113 | extension_ranges=[], 114 | oneofs=[ 115 | ], 116 | serialized_start=120, 117 | serialized_end=153, 118 | ) 119 | 120 | 121 | _STREAMMESSAGEFROMEARTH = _descriptor.Descriptor( 122 | name='StreamMessageFromEarth', 123 | full_name='interstellar.StreamMessageFromEarth', 124 | filename=None, 125 | file=DESCRIPTOR, 126 | containing_type=None, 127 | fields=[ 128 | _descriptor.FieldDescriptor( 129 | name='message', full_name='interstellar.StreamMessageFromEarth.message', index=0, 130 | number=1, type=9, cpp_type=9, label=1, 131 | has_default_value=False, default_value=_b("").decode('utf-8'), 132 | message_type=None, enum_type=None, containing_type=None, 133 | is_extension=False, extension_scope=None, 134 | options=None, file=DESCRIPTOR), 135 | ], 136 | extensions=[ 137 | ], 138 | nested_types=[], 139 | enum_types=[ 140 | ], 141 | options=None, 142 | is_extendable=False, 143 | syntax='proto3', 144 | extension_ranges=[], 145 | oneofs=[ 146 | ], 147 | serialized_start=155, 148 | serialized_end=196, 149 | ) 150 | 151 | 152 | _REPLYFROMMARS = _descriptor.Descriptor( 153 | name='ReplyFromMars', 154 | full_name='interstellar.ReplyFromMars', 155 | filename=None, 156 | file=DESCRIPTOR, 157 | containing_type=None, 158 | fields=[ 159 | _descriptor.FieldDescriptor( 160 | name='reply', full_name='interstellar.ReplyFromMars.reply', index=0, 161 | number=1, type=9, cpp_type=9, label=1, 162 | has_default_value=False, default_value=_b("").decode('utf-8'), 163 | message_type=None, enum_type=None, containing_type=None, 164 | is_extension=False, extension_scope=None, 165 | options=None, file=DESCRIPTOR), 166 | ], 167 | extensions=[ 168 | ], 169 | nested_types=[], 170 | enum_types=[ 171 | ], 172 | options=None, 173 | is_extendable=False, 174 | syntax='proto3', 175 | extension_ranges=[], 176 | oneofs=[ 177 | ], 178 | serialized_start=198, 179 | serialized_end=228, 180 | ) 181 | 182 | 183 | _STREAMMESSAGEFROMMARS = _descriptor.Descriptor( 184 | name='StreamMessageFromMars', 185 | full_name='interstellar.StreamMessageFromMars', 186 | filename=None, 187 | file=DESCRIPTOR, 188 | containing_type=None, 189 | fields=[ 190 | _descriptor.FieldDescriptor( 191 | name='message', full_name='interstellar.StreamMessageFromMars.message', index=0, 192 | number=1, type=9, cpp_type=9, label=1, 193 | has_default_value=False, default_value=_b("").decode('utf-8'), 194 | message_type=None, enum_type=None, containing_type=None, 195 | is_extension=False, extension_scope=None, 196 | options=None, file=DESCRIPTOR), 197 | ], 198 | extensions=[ 199 | ], 200 | nested_types=[], 201 | enum_types=[ 202 | ], 203 | options=None, 204 | is_extendable=False, 205 | syntax='proto3', 206 | extension_ranges=[], 207 | oneofs=[ 208 | ], 209 | serialized_start=230, 210 | serialized_end=270, 211 | ) 212 | 213 | DESCRIPTOR.message_types_by_name['HelloRequest'] = _HELLOREQUEST 214 | DESCRIPTOR.message_types_by_name['HelloResponse'] = _HELLORESPONSE 215 | DESCRIPTOR.message_types_by_name['MessageRequest'] = _MESSAGEREQUEST 216 | DESCRIPTOR.message_types_by_name['StreamMessageFromEarth'] = _STREAMMESSAGEFROMEARTH 217 | DESCRIPTOR.message_types_by_name['ReplyFromMars'] = _REPLYFROMMARS 218 | DESCRIPTOR.message_types_by_name['StreamMessageFromMars'] = _STREAMMESSAGEFROMMARS 219 | _sym_db.RegisterFileDescriptor(DESCRIPTOR) 220 | 221 | HelloRequest = _reflection.GeneratedProtocolMessageType('HelloRequest', (_message.Message,), dict( 222 | DESCRIPTOR = _HELLOREQUEST, 223 | __module__ = 'interstellar_pb2' 224 | # @@protoc_insertion_point(class_scope:interstellar.HelloRequest) 225 | )) 226 | _sym_db.RegisterMessage(HelloRequest) 227 | 228 | HelloResponse = _reflection.GeneratedProtocolMessageType('HelloResponse', (_message.Message,), dict( 229 | DESCRIPTOR = _HELLORESPONSE, 230 | __module__ = 'interstellar_pb2' 231 | # @@protoc_insertion_point(class_scope:interstellar.HelloResponse) 232 | )) 233 | _sym_db.RegisterMessage(HelloResponse) 234 | 235 | MessageRequest = _reflection.GeneratedProtocolMessageType('MessageRequest', (_message.Message,), dict( 236 | DESCRIPTOR = _MESSAGEREQUEST, 237 | __module__ = 'interstellar_pb2' 238 | # @@protoc_insertion_point(class_scope:interstellar.MessageRequest) 239 | )) 240 | _sym_db.RegisterMessage(MessageRequest) 241 | 242 | StreamMessageFromEarth = _reflection.GeneratedProtocolMessageType('StreamMessageFromEarth', (_message.Message,), dict( 243 | DESCRIPTOR = _STREAMMESSAGEFROMEARTH, 244 | __module__ = 'interstellar_pb2' 245 | # @@protoc_insertion_point(class_scope:interstellar.StreamMessageFromEarth) 246 | )) 247 | _sym_db.RegisterMessage(StreamMessageFromEarth) 248 | 249 | ReplyFromMars = _reflection.GeneratedProtocolMessageType('ReplyFromMars', (_message.Message,), dict( 250 | DESCRIPTOR = _REPLYFROMMARS, 251 | __module__ = 'interstellar_pb2' 252 | # @@protoc_insertion_point(class_scope:interstellar.ReplyFromMars) 253 | )) 254 | _sym_db.RegisterMessage(ReplyFromMars) 255 | 256 | StreamMessageFromMars = _reflection.GeneratedProtocolMessageType('StreamMessageFromMars', (_message.Message,), dict( 257 | DESCRIPTOR = _STREAMMESSAGEFROMMARS, 258 | __module__ = 'interstellar_pb2' 259 | # @@protoc_insertion_point(class_scope:interstellar.StreamMessageFromMars) 260 | )) 261 | _sym_db.RegisterMessage(StreamMessageFromMars) 262 | 263 | 264 | 265 | _INTERSTELLARCOMMUNICATION = _descriptor.ServiceDescriptor( 266 | name='InterstellarCommunication', 267 | full_name='interstellar.InterstellarCommunication', 268 | file=DESCRIPTOR, 269 | index=0, 270 | options=None, 271 | serialized_start=273, 272 | serialized_end=665, 273 | methods=[ 274 | _descriptor.MethodDescriptor( 275 | name='SayHello', 276 | full_name='interstellar.InterstellarCommunication.SayHello', 277 | index=0, 278 | containing_service=None, 279 | input_type=_HELLOREQUEST, 280 | output_type=_HELLORESPONSE, 281 | options=None, 282 | ), 283 | _descriptor.MethodDescriptor( 284 | name='GetMessageFromMars', 285 | full_name='interstellar.InterstellarCommunication.GetMessageFromMars', 286 | index=1, 287 | containing_service=None, 288 | input_type=_MESSAGEREQUEST, 289 | output_type=_STREAMMESSAGEFROMMARS, 290 | options=None, 291 | ), 292 | _descriptor.MethodDescriptor( 293 | name='SendMessageFromEarth', 294 | full_name='interstellar.InterstellarCommunication.SendMessageFromEarth', 295 | index=2, 296 | containing_service=None, 297 | input_type=_STREAMMESSAGEFROMEARTH, 298 | output_type=_REPLYFROMMARS, 299 | options=None, 300 | ), 301 | _descriptor.MethodDescriptor( 302 | name='SendAndReceiveMessage', 303 | full_name='interstellar.InterstellarCommunication.SendAndReceiveMessage', 304 | index=3, 305 | containing_service=None, 306 | input_type=_STREAMMESSAGEFROMEARTH, 307 | output_type=_STREAMMESSAGEFROMMARS, 308 | options=None, 309 | ), 310 | ]) 311 | _sym_db.RegisterServiceDescriptor(_INTERSTELLARCOMMUNICATION) 312 | 313 | DESCRIPTOR.services_by_name['InterstellarCommunication'] = _INTERSTELLARCOMMUNICATION 314 | 315 | # @@protoc_insertion_point(module_scope) 316 | -------------------------------------------------------------------------------- /streaming/nyct_subway_pb2.py: -------------------------------------------------------------------------------- 1 | # Generated by the protocol buffer compiler. DO NOT EDIT! 2 | # source: nyct-subway.proto 3 | 4 | import sys 5 | _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) 6 | from google.protobuf import descriptor as _descriptor 7 | from google.protobuf import message as _message 8 | from google.protobuf import reflection as _reflection 9 | from google.protobuf import symbol_database as _symbol_database 10 | from google.protobuf import descriptor_pb2 11 | # @@protoc_insertion_point(imports) 12 | 13 | _sym_db = _symbol_database.Default() 14 | 15 | 16 | import gtfs_realtime_pb2 as gtfs__realtime__pb2 17 | 18 | 19 | DESCRIPTOR = _descriptor.FileDescriptor( 20 | name='nyct-subway.proto', 21 | package='', 22 | syntax='proto2', 23 | serialized_pb=_b('\n\x11nyct-subway.proto\x1a\x13gtfs-realtime.proto\"b\n\x15TripReplacementPeriod\x12\x10\n\x08route_id\x18\x01 \x01(\t\x12\x37\n\x12replacement_period\x18\x02 \x01(\x0b\x32\x1b.transit_realtime.TimeRange\"f\n\x0eNyctFeedHeader\x12\x1b\n\x13nyct_subway_version\x18\x01 \x02(\t\x12\x37\n\x17trip_replacement_period\x18\x02 \x03(\x0b\x32\x16.TripReplacementPeriod\"\xa4\x01\n\x12NyctTripDescriptor\x12\x10\n\x08train_id\x18\x01 \x01(\t\x12\x13\n\x0bis_assigned\x18\x02 \x01(\x08\x12\x30\n\tdirection\x18\x03 \x01(\x0e\x32\x1d.NyctTripDescriptor.Direction\"5\n\tDirection\x12\t\n\x05NORTH\x10\x01\x12\x08\n\x04\x45\x41ST\x10\x02\x12\t\n\x05SOUTH\x10\x03\x12\x08\n\x04WEST\x10\x04\"C\n\x12NyctStopTimeUpdate\x12\x17\n\x0fscheduled_track\x18\x01 \x01(\t\x12\x14\n\x0c\x61\x63tual_track\x18\x02 \x01(\t:H\n\x10nyct_feed_header\x12\x1c.transit_realtime.FeedHeader\x18\xe9\x07 \x01(\x0b\x32\x0f.NyctFeedHeader:T\n\x14nyct_trip_descriptor\x12 .transit_realtime.TripDescriptor\x18\xe9\x07 \x01(\x0b\x32\x13.NyctTripDescriptor:`\n\x15nyct_stop_time_update\x12+.transit_realtime.TripUpdate.StopTimeUpdate\x18\xe9\x07 \x01(\x0b\x32\x13.NyctStopTimeUpdateB\x1d\n\x1b\x63om.google.transit.realtime') 24 | , 25 | dependencies=[gtfs__realtime__pb2.DESCRIPTOR,]) 26 | 27 | 28 | NYCT_FEED_HEADER_FIELD_NUMBER = 1001 29 | nyct_feed_header = _descriptor.FieldDescriptor( 30 | name='nyct_feed_header', full_name='nyct_feed_header', index=0, 31 | number=1001, type=11, cpp_type=10, label=1, 32 | has_default_value=False, default_value=None, 33 | message_type=None, enum_type=None, containing_type=None, 34 | is_extension=True, extension_scope=None, 35 | options=None, file=DESCRIPTOR) 36 | NYCT_TRIP_DESCRIPTOR_FIELD_NUMBER = 1001 37 | nyct_trip_descriptor = _descriptor.FieldDescriptor( 38 | name='nyct_trip_descriptor', full_name='nyct_trip_descriptor', index=1, 39 | number=1001, type=11, cpp_type=10, label=1, 40 | has_default_value=False, default_value=None, 41 | message_type=None, enum_type=None, containing_type=None, 42 | is_extension=True, extension_scope=None, 43 | options=None, file=DESCRIPTOR) 44 | NYCT_STOP_TIME_UPDATE_FIELD_NUMBER = 1001 45 | nyct_stop_time_update = _descriptor.FieldDescriptor( 46 | name='nyct_stop_time_update', full_name='nyct_stop_time_update', index=2, 47 | number=1001, type=11, cpp_type=10, label=1, 48 | has_default_value=False, default_value=None, 49 | message_type=None, enum_type=None, containing_type=None, 50 | is_extension=True, extension_scope=None, 51 | options=None, file=DESCRIPTOR) 52 | 53 | _NYCTTRIPDESCRIPTOR_DIRECTION = _descriptor.EnumDescriptor( 54 | name='Direction', 55 | full_name='NyctTripDescriptor.Direction', 56 | filename=None, 57 | file=DESCRIPTOR, 58 | values=[ 59 | _descriptor.EnumValueDescriptor( 60 | name='NORTH', index=0, number=1, 61 | options=None, 62 | type=None), 63 | _descriptor.EnumValueDescriptor( 64 | name='EAST', index=1, number=2, 65 | options=None, 66 | type=None), 67 | _descriptor.EnumValueDescriptor( 68 | name='SOUTH', index=2, number=3, 69 | options=None, 70 | type=None), 71 | _descriptor.EnumValueDescriptor( 72 | name='WEST', index=3, number=4, 73 | options=None, 74 | type=None), 75 | ], 76 | containing_type=None, 77 | options=None, 78 | serialized_start=358, 79 | serialized_end=411, 80 | ) 81 | _sym_db.RegisterEnumDescriptor(_NYCTTRIPDESCRIPTOR_DIRECTION) 82 | 83 | 84 | _TRIPREPLACEMENTPERIOD = _descriptor.Descriptor( 85 | name='TripReplacementPeriod', 86 | full_name='TripReplacementPeriod', 87 | filename=None, 88 | file=DESCRIPTOR, 89 | containing_type=None, 90 | fields=[ 91 | _descriptor.FieldDescriptor( 92 | name='route_id', full_name='TripReplacementPeriod.route_id', index=0, 93 | number=1, type=9, cpp_type=9, label=1, 94 | has_default_value=False, default_value=_b("").decode('utf-8'), 95 | message_type=None, enum_type=None, containing_type=None, 96 | is_extension=False, extension_scope=None, 97 | options=None, file=DESCRIPTOR), 98 | _descriptor.FieldDescriptor( 99 | name='replacement_period', full_name='TripReplacementPeriod.replacement_period', index=1, 100 | number=2, type=11, cpp_type=10, label=1, 101 | has_default_value=False, default_value=None, 102 | message_type=None, enum_type=None, containing_type=None, 103 | is_extension=False, extension_scope=None, 104 | options=None, file=DESCRIPTOR), 105 | ], 106 | extensions=[ 107 | ], 108 | nested_types=[], 109 | enum_types=[ 110 | ], 111 | options=None, 112 | is_extendable=False, 113 | syntax='proto2', 114 | extension_ranges=[], 115 | oneofs=[ 116 | ], 117 | serialized_start=42, 118 | serialized_end=140, 119 | ) 120 | 121 | 122 | _NYCTFEEDHEADER = _descriptor.Descriptor( 123 | name='NyctFeedHeader', 124 | full_name='NyctFeedHeader', 125 | filename=None, 126 | file=DESCRIPTOR, 127 | containing_type=None, 128 | fields=[ 129 | _descriptor.FieldDescriptor( 130 | name='nyct_subway_version', full_name='NyctFeedHeader.nyct_subway_version', index=0, 131 | number=1, type=9, cpp_type=9, label=2, 132 | has_default_value=False, default_value=_b("").decode('utf-8'), 133 | message_type=None, enum_type=None, containing_type=None, 134 | is_extension=False, extension_scope=None, 135 | options=None, file=DESCRIPTOR), 136 | _descriptor.FieldDescriptor( 137 | name='trip_replacement_period', full_name='NyctFeedHeader.trip_replacement_period', index=1, 138 | number=2, type=11, cpp_type=10, label=3, 139 | has_default_value=False, default_value=[], 140 | message_type=None, enum_type=None, containing_type=None, 141 | is_extension=False, extension_scope=None, 142 | options=None, file=DESCRIPTOR), 143 | ], 144 | extensions=[ 145 | ], 146 | nested_types=[], 147 | enum_types=[ 148 | ], 149 | options=None, 150 | is_extendable=False, 151 | syntax='proto2', 152 | extension_ranges=[], 153 | oneofs=[ 154 | ], 155 | serialized_start=142, 156 | serialized_end=244, 157 | ) 158 | 159 | 160 | _NYCTTRIPDESCRIPTOR = _descriptor.Descriptor( 161 | name='NyctTripDescriptor', 162 | full_name='NyctTripDescriptor', 163 | filename=None, 164 | file=DESCRIPTOR, 165 | containing_type=None, 166 | fields=[ 167 | _descriptor.FieldDescriptor( 168 | name='train_id', full_name='NyctTripDescriptor.train_id', index=0, 169 | number=1, type=9, cpp_type=9, label=1, 170 | has_default_value=False, default_value=_b("").decode('utf-8'), 171 | message_type=None, enum_type=None, containing_type=None, 172 | is_extension=False, extension_scope=None, 173 | options=None, file=DESCRIPTOR), 174 | _descriptor.FieldDescriptor( 175 | name='is_assigned', full_name='NyctTripDescriptor.is_assigned', index=1, 176 | number=2, type=8, cpp_type=7, label=1, 177 | has_default_value=False, default_value=False, 178 | message_type=None, enum_type=None, containing_type=None, 179 | is_extension=False, extension_scope=None, 180 | options=None, file=DESCRIPTOR), 181 | _descriptor.FieldDescriptor( 182 | name='direction', full_name='NyctTripDescriptor.direction', index=2, 183 | number=3, type=14, cpp_type=8, label=1, 184 | has_default_value=False, default_value=1, 185 | message_type=None, enum_type=None, containing_type=None, 186 | is_extension=False, extension_scope=None, 187 | options=None, file=DESCRIPTOR), 188 | ], 189 | extensions=[ 190 | ], 191 | nested_types=[], 192 | enum_types=[ 193 | _NYCTTRIPDESCRIPTOR_DIRECTION, 194 | ], 195 | options=None, 196 | is_extendable=False, 197 | syntax='proto2', 198 | extension_ranges=[], 199 | oneofs=[ 200 | ], 201 | serialized_start=247, 202 | serialized_end=411, 203 | ) 204 | 205 | 206 | _NYCTSTOPTIMEUPDATE = _descriptor.Descriptor( 207 | name='NyctStopTimeUpdate', 208 | full_name='NyctStopTimeUpdate', 209 | filename=None, 210 | file=DESCRIPTOR, 211 | containing_type=None, 212 | fields=[ 213 | _descriptor.FieldDescriptor( 214 | name='scheduled_track', full_name='NyctStopTimeUpdate.scheduled_track', index=0, 215 | number=1, type=9, cpp_type=9, label=1, 216 | has_default_value=False, default_value=_b("").decode('utf-8'), 217 | message_type=None, enum_type=None, containing_type=None, 218 | is_extension=False, extension_scope=None, 219 | options=None, file=DESCRIPTOR), 220 | _descriptor.FieldDescriptor( 221 | name='actual_track', full_name='NyctStopTimeUpdate.actual_track', index=1, 222 | number=2, type=9, cpp_type=9, label=1, 223 | has_default_value=False, default_value=_b("").decode('utf-8'), 224 | message_type=None, enum_type=None, containing_type=None, 225 | is_extension=False, extension_scope=None, 226 | options=None, file=DESCRIPTOR), 227 | ], 228 | extensions=[ 229 | ], 230 | nested_types=[], 231 | enum_types=[ 232 | ], 233 | options=None, 234 | is_extendable=False, 235 | syntax='proto2', 236 | extension_ranges=[], 237 | oneofs=[ 238 | ], 239 | serialized_start=413, 240 | serialized_end=480, 241 | ) 242 | 243 | _TRIPREPLACEMENTPERIOD.fields_by_name['replacement_period'].message_type = gtfs__realtime__pb2._TIMERANGE 244 | _NYCTFEEDHEADER.fields_by_name['trip_replacement_period'].message_type = _TRIPREPLACEMENTPERIOD 245 | _NYCTTRIPDESCRIPTOR.fields_by_name['direction'].enum_type = _NYCTTRIPDESCRIPTOR_DIRECTION 246 | _NYCTTRIPDESCRIPTOR_DIRECTION.containing_type = _NYCTTRIPDESCRIPTOR 247 | DESCRIPTOR.message_types_by_name['TripReplacementPeriod'] = _TRIPREPLACEMENTPERIOD 248 | DESCRIPTOR.message_types_by_name['NyctFeedHeader'] = _NYCTFEEDHEADER 249 | DESCRIPTOR.message_types_by_name['NyctTripDescriptor'] = _NYCTTRIPDESCRIPTOR 250 | DESCRIPTOR.message_types_by_name['NyctStopTimeUpdate'] = _NYCTSTOPTIMEUPDATE 251 | DESCRIPTOR.extensions_by_name['nyct_feed_header'] = nyct_feed_header 252 | DESCRIPTOR.extensions_by_name['nyct_trip_descriptor'] = nyct_trip_descriptor 253 | DESCRIPTOR.extensions_by_name['nyct_stop_time_update'] = nyct_stop_time_update 254 | _sym_db.RegisterFileDescriptor(DESCRIPTOR) 255 | 256 | TripReplacementPeriod = _reflection.GeneratedProtocolMessageType('TripReplacementPeriod', (_message.Message,), dict( 257 | DESCRIPTOR = _TRIPREPLACEMENTPERIOD, 258 | __module__ = 'nyct_subway_pb2' 259 | # @@protoc_insertion_point(class_scope:TripReplacementPeriod) 260 | )) 261 | _sym_db.RegisterMessage(TripReplacementPeriod) 262 | 263 | NyctFeedHeader = _reflection.GeneratedProtocolMessageType('NyctFeedHeader', (_message.Message,), dict( 264 | DESCRIPTOR = _NYCTFEEDHEADER, 265 | __module__ = 'nyct_subway_pb2' 266 | # @@protoc_insertion_point(class_scope:NyctFeedHeader) 267 | )) 268 | _sym_db.RegisterMessage(NyctFeedHeader) 269 | 270 | NyctTripDescriptor = _reflection.GeneratedProtocolMessageType('NyctTripDescriptor', (_message.Message,), dict( 271 | DESCRIPTOR = _NYCTTRIPDESCRIPTOR, 272 | __module__ = 'nyct_subway_pb2' 273 | # @@protoc_insertion_point(class_scope:NyctTripDescriptor) 274 | )) 275 | _sym_db.RegisterMessage(NyctTripDescriptor) 276 | 277 | NyctStopTimeUpdate = _reflection.GeneratedProtocolMessageType('NyctStopTimeUpdate', (_message.Message,), dict( 278 | DESCRIPTOR = _NYCTSTOPTIMEUPDATE, 279 | __module__ = 'nyct_subway_pb2' 280 | # @@protoc_insertion_point(class_scope:NyctStopTimeUpdate) 281 | )) 282 | _sym_db.RegisterMessage(NyctStopTimeUpdate) 283 | 284 | nyct_feed_header.message_type = _NYCTFEEDHEADER 285 | gtfs__realtime__pb2.FeedHeader.RegisterExtension(nyct_feed_header) 286 | nyct_trip_descriptor.message_type = _NYCTTRIPDESCRIPTOR 287 | gtfs__realtime__pb2.TripDescriptor.RegisterExtension(nyct_trip_descriptor) 288 | nyct_stop_time_update.message_type = _NYCTSTOPTIMEUPDATE 289 | gtfs__realtime__pb2.TripUpdate.StopTimeUpdate.RegisterExtension(nyct_stop_time_update) 290 | 291 | DESCRIPTOR.has_options = True 292 | DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\033com.google.transit.realtime')) 293 | # @@protoc_insertion_point(module_scope) 294 | -------------------------------------------------------------------------------- /citibikekafkastreams/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | 8 | io.confluent 9 | rest-utils-parent 10 | 5.0.0 11 | 12 | 13 | citibike-kafka-streams 14 | jar 15 | 16 | 17 | 18 | Apache License 2.0 19 | http://www.apache.org/licenses/LICENSE-2.0.html 20 | repo 21 | 22 | 23 | 24 | 25 | 26 | confluent 27 | https://packages.confluent.io/maven/ 28 | 29 | 30 | 31 | 32 | 33 | confluent 34 | https://packages.confluent.io/maven/ 35 | 36 | 37 | 38 | 41 | 42 | 0.13.4 43 | 1.8.2 44 | 0.9.2 45 | false 46 | false 47 | 1.8 48 | 5.0.0 49 | UTF-8 50 | ${kafka.scala.version}.8 51 | 2.2.6 52 | 53 | 54 | 55 | 56 | io.confluent 57 | kafka-streams-avro-serde 58 | ${confluent.version} 59 | 60 | 61 | io.confluent 62 | kafka-avro-serializer 63 | ${confluent.version} 64 | 65 | 66 | io.confluent 67 | kafka-schema-registry-client 68 | ${confluent.version} 69 | 70 | 71 | org.apache.kafka 72 | kafka-clients 73 | ${kafka.version} 74 | 75 | 76 | org.apache.kafka 77 | kafka-streams 78 | ${kafka.version} 79 | 80 | 81 | org.apache.kafka 82 | kafka-streams-scala_${kafka.scala.version} 83 | ${kafka.version} 84 | 85 | 86 | org.apache.avro 87 | avro 88 | ${avro.version} 89 | 90 | 91 | org.apache.avro 92 | avro-maven-plugin 93 | ${avro.version} 94 | 95 | 96 | org.scala-lang 97 | scala-library 98 | 99 | ${scala.version} 100 | 101 | 102 | 105 | com.101tec 106 | zkclient 107 | 0.9 108 | 109 | 110 | 111 | javax.ws.rs 112 | javax.ws.rs-api 113 | 2.1 114 | 115 | 116 | org.eclipse.jetty 117 | jetty-server 118 | ${jetty.version} 119 | 120 | 121 | org.eclipse.jetty 122 | jetty-servlet 123 | ${jetty.version} 124 | 125 | 126 | org.glassfish.jersey.containers 127 | jersey-container-servlet 128 | ${jersey.version} 129 | 130 | 131 | org.glassfish.jersey.inject 132 | jersey-hk2 133 | ${jersey.version} 134 | 135 | 136 | org.glassfish.jersey.media 137 | jersey-media-json-jackson 138 | ${jersey.version} 139 | 140 | 141 | com.fasterxml.jackson.core 142 | jackson-annotations 143 | ${jackson.version} 144 | 145 | 146 | com.fasterxml.jackson.core 147 | jackson-databind 148 | ${jackson.version} 149 | 150 | 151 | 152 | com.twitter 153 | algebird-core_${kafka.scala.version} 154 | ${algebird.version} 155 | 156 | 157 | com.twitter 158 | chill_${kafka.scala.version} 159 | ${chill.version} 160 | 161 | 162 | 163 | 164 | junit 165 | junit 166 | 4.12 167 | test 168 | 169 | 170 | org.assertj 171 | assertj-core 172 | 3.3.0 173 | test 174 | 175 | 176 | org.mockito 177 | mockito-core 178 | 2.7.14 179 | 180 | 181 | org.apache.kafka 182 | kafka_${kafka.scala.version} 183 | ${kafka.version} 184 | test 185 | 186 | 187 | org.apache.kafka 188 | kafka_${kafka.scala.version} 189 | ${kafka.version} 190 | test 191 | test 192 | 193 | 194 | org.apache.kafka 195 | kafka-clients 196 | ${kafka.version} 197 | test 198 | test 199 | 200 | 201 | org.apache.kafka 202 | kafka-streams 203 | ${kafka.version} 204 | test 205 | test 206 | 207 | 208 | org.apache.curator 209 | curator-test 210 | 2.9.0 211 | test 212 | 213 | 214 | io.confluent 215 | kafka-schema-registry 216 | ${confluent.version} 217 | test 218 | 219 | 220 | 221 | com.google.http-client 222 | google-http-client 223 | 1.23.0 224 | 225 | 226 | com.google.http-client 227 | google-http-client-jackson2 228 | 1.23.0 229 | 230 | 231 | org.apache.kafka 232 | connect-json 233 | 2.0.0 234 | 235 | 236 | io.confluent 237 | kafka-schema-registry 238 | ${confluent.version} 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | net.alchim31.maven 249 | scala-maven-plugin 250 | 3.2.1 251 | 252 | ${kafka.scala.version} 253 | ${scala.version} 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 265 | 266 | org.codehaus.mojo 267 | build-helper-maven-plugin 268 | 1.10 269 | 270 | 271 | add-source 272 | generate-sources 273 | 274 | add-source 275 | 276 | 277 | 278 | src/main/scala 279 | ${project.build.directory}/generated-sources 280 | 281 | 282 | 283 | 284 | add-test-source 285 | generate-test-sources 286 | 287 | add-test-source 288 | 289 | 290 | 291 | src/test/scala 292 | ${project.build.directory}/generated-sources 293 | 294 | 295 | 296 | 297 | 298 | 299 | 300 | 301 | net.alchim31.maven 302 | scala-maven-plugin 303 | 3.2.1 304 | 305 | 306 | 311 | -Xexperimental 312 | 313 | -feature 314 | 315 | -deprecation 316 | 317 | -unchecked 318 | 319 | -Xlint 320 | 321 | -Ywarn-adapted-args 322 | -Ywarn-dead-code 323 | 324 | 325 | 326 | 327 | 328 | compile 329 | testCompile 330 | 331 | 332 | 333 | 334 | 335 | 336 | org.apache.maven.plugins 337 | maven-compiler-plugin 338 | 3.3 339 | true 340 | 341 | ${java.version} 342 | ${java.version} 343 | 344 | 345 | 346 | 347 | org.apache.avro 348 | avro-maven-plugin 349 | ${avro.version} 350 | 351 | 352 | generate-sources 353 | 354 | schema 355 | 356 | 357 | src/main/resources/avro/com/cloudboxlabs/streams 358 | ${project.build.directory}/generated-sources 359 | String 360 | 361 | 362 | 363 | 364 | 365 | 366 | org.apache.maven.plugins 367 | maven-assembly-plugin 368 | 369 | 370 | src/assembly/development.xml 371 | src/assembly/package.xml 372 | src/assembly/standalone.xml 373 | 374 | 375 | 376 | io.confluent.kafka.schemaregistry.rest.SchemaRegistryMain 377 | 378 | 379 | false 380 | 381 | 382 | 383 | make-assembly 384 | package 385 | 386 | single 387 | 388 | 389 | 390 | 391 | 392 | 393 | org.apache.maven.plugins 394 | maven-surefire-plugin 395 | 2.19.1 396 | 397 | 1 398 | false 399 | 400 | 401 | 402 | 403 | 407 | org.jasig.maven 408 | maven-notice-plugin 409 | 1.0.6.1 410 | 411 | 412 | ../license-mappings.xml 413 | 414 | 415 | 416 | 417 | 418 | org.apache.maven.plugins 419 | maven-checkstyle-plugin 420 | 421 | true 422 | 423 | 424 | 425 | 426 | 427 | -------------------------------------------------------------------------------- /streaming/protos/gtfs-realtime.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The GTFS Specifications Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Protocol definition file for GTFS Realtime. 16 | // 17 | // GTFS Realtime lets transit agencies provide consumers with realtime 18 | // information about disruptions to their service (stations closed, lines not 19 | // operating, important delays etc), location of their vehicles and expected 20 | // arrival times. 21 | // 22 | // This protocol is published at: 23 | // https://github.com/google/transit/tree/master/gtfs-realtime 24 | 25 | syntax = "proto2"; 26 | option java_package = "com.google.transit.realtime"; 27 | package transit_realtime; 28 | 29 | // The contents of a feed message. 30 | // A feed is a continuous stream of feed messages. Each message in the stream is 31 | // obtained as a response to an appropriate HTTP GET request. 32 | // A realtime feed is always defined with relation to an existing GTFS feed. 33 | // All the entity ids are resolved with respect to the GTFS feed. 34 | // Note that "required" and "optional" as stated in this file refer to Protocol 35 | // Buffer cardinality, not semantic cardinality. See reference.md at 36 | // https://github.com/google/transit/tree/master/gtfs-realtime for field 37 | // semantic cardinality. 38 | message FeedMessage { 39 | // Metadata about this feed and feed message. 40 | required FeedHeader header = 1; 41 | 42 | // Contents of the feed. 43 | repeated FeedEntity entity = 2; 44 | 45 | // The extensions namespace allows 3rd-party developers to extend the 46 | // GTFS Realtime specification in order to add and evaluate new features and 47 | // modifications to the spec. 48 | extensions 1000 to 1999; 49 | } 50 | 51 | // Metadata about a feed, included in feed messages. 52 | message FeedHeader { 53 | // Version of the feed specification. 54 | // The current version is 2.0. 55 | required string gtfs_realtime_version = 1; 56 | 57 | // Determines whether the current fetch is incremental. Currently, 58 | // DIFFERENTIAL mode is unsupported and behavior is unspecified for feeds 59 | // that use this mode. There are discussions on the GTFS Realtime mailing 60 | // list around fully specifying the behavior of DIFFERENTIAL mode and the 61 | // documentation will be updated when those discussions are finalized. 62 | enum Incrementality { 63 | FULL_DATASET = 0; 64 | DIFFERENTIAL = 1; 65 | } 66 | optional Incrementality incrementality = 2 [default = FULL_DATASET]; 67 | 68 | // This timestamp identifies the moment when the content of this feed has been 69 | // created (in server time). In POSIX time (i.e., number of seconds since 70 | // January 1st 1970 00:00:00 UTC). 71 | optional uint64 timestamp = 3; 72 | 73 | // The extensions namespace allows 3rd-party developers to extend the 74 | // GTFS Realtime specification in order to add and evaluate new features and 75 | // modifications to the spec. 76 | extensions 1000 to 1999; 77 | } 78 | 79 | // A definition (or update) of an entity in the transit feed. 80 | message FeedEntity { 81 | // The ids are used only to provide incrementality support. The id should be 82 | // unique within a FeedMessage. Consequent FeedMessages may contain 83 | // FeedEntities with the same id. In case of a DIFFERENTIAL update the new 84 | // FeedEntity with some id will replace the old FeedEntity with the same id 85 | // (or delete it - see is_deleted below). 86 | // The actual GTFS entities (e.g. stations, routes, trips) referenced by the 87 | // feed must be specified by explicit selectors (see EntitySelector below for 88 | // more info). 89 | required string id = 1; 90 | 91 | // Whether this entity is to be deleted. Relevant only for incremental 92 | // fetches. 93 | optional bool is_deleted = 2 [default = false]; 94 | 95 | // Data about the entity itself. Exactly one of the following fields must be 96 | // present (unless the entity is being deleted). 97 | optional TripUpdate trip_update = 3; 98 | optional VehiclePosition vehicle = 4; 99 | optional Alert alert = 5; 100 | 101 | // The extensions namespace allows 3rd-party developers to extend the 102 | // GTFS Realtime Specification in order to add and evaluate new features and 103 | // modifications to the spec. 104 | extensions 1000 to 1999; 105 | } 106 | 107 | // 108 | // Entities used in the feed. 109 | // 110 | 111 | // Realtime update of the progress of a vehicle along a trip. 112 | // Depending on the value of ScheduleRelationship, a TripUpdate can specify: 113 | // - A trip that proceeds along the schedule. 114 | // - A trip that proceeds along a route but has no fixed schedule. 115 | // - A trip that have been added or removed with regard to schedule. 116 | // 117 | // The updates can be for future, predicted arrival/departure events, or for 118 | // past events that already occurred. 119 | // Normally, updates should get more precise and more certain (see 120 | // uncertainty below) as the events gets closer to current time. 121 | // Even if that is not possible, the information for past events should be 122 | // precise and certain. In particular, if an update points to time in the past 123 | // but its update's uncertainty is not 0, the client should conclude that the 124 | // update is a (wrong) prediction and that the trip has not completed yet. 125 | // 126 | // Note that the update can describe a trip that is already completed. 127 | // To this end, it is enough to provide an update for the last stop of the trip. 128 | // If the time of that is in the past, the client will conclude from that that 129 | // the whole trip is in the past (it is possible, although inconsequential, to 130 | // also provide updates for preceding stops). 131 | // This option is most relevant for a trip that has completed ahead of schedule, 132 | // but according to the schedule, the trip is still proceeding at the current 133 | // time. Removing the updates for this trip could make the client assume 134 | // that the trip is still proceeding. 135 | // Note that the feed provider is allowed, but not required, to purge past 136 | // updates - this is one case where this would be practically useful. 137 | message TripUpdate { 138 | // The Trip that this message applies to. There can be at most one 139 | // TripUpdate entity for each actual trip instance. 140 | // If there is none, that means there is no prediction information available. 141 | // It does *not* mean that the trip is progressing according to schedule. 142 | required TripDescriptor trip = 1; 143 | 144 | // Additional information on the vehicle that is serving this trip. 145 | optional VehicleDescriptor vehicle = 3; 146 | 147 | // Timing information for a single predicted event (either arrival or 148 | // departure). 149 | // Timing consists of delay and/or estimated time, and uncertainty. 150 | // - delay should be used when the prediction is given relative to some 151 | // existing schedule in GTFS. 152 | // - time should be given whether there is a predicted schedule or not. If 153 | // both time and delay are specified, time will take precedence 154 | // (although normally, time, if given for a scheduled trip, should be 155 | // equal to scheduled time in GTFS + delay). 156 | // 157 | // Uncertainty applies equally to both time and delay. 158 | // The uncertainty roughly specifies the expected error in true delay (but 159 | // note, we don't yet define its precise statistical meaning). It's possible 160 | // for the uncertainty to be 0, for example for trains that are driven under 161 | // computer timing control. 162 | message StopTimeEvent { 163 | // Delay (in seconds) can be positive (meaning that the vehicle is late) or 164 | // negative (meaning that the vehicle is ahead of schedule). Delay of 0 165 | // means that the vehicle is exactly on time. 166 | optional int32 delay = 1; 167 | 168 | // Event as absolute time. 169 | // In Unix time (i.e., number of seconds since January 1st 1970 00:00:00 170 | // UTC). 171 | optional int64 time = 2; 172 | 173 | // If uncertainty is omitted, it is interpreted as unknown. 174 | // If the prediction is unknown or too uncertain, the delay (or time) field 175 | // should be empty. In such case, the uncertainty field is ignored. 176 | // To specify a completely certain prediction, set its uncertainty to 0. 177 | optional int32 uncertainty = 3; 178 | 179 | // The extensions namespace allows 3rd-party developers to extend the 180 | // GTFS Realtime Specification in order to add and evaluate new features 181 | // and modifications to the spec. 182 | extensions 1000 to 1999; 183 | } 184 | 185 | // Realtime update for arrival and/or departure events for a given stop on a 186 | // trip. Updates can be supplied for both past and future events. 187 | // The producer is allowed, although not required, to drop past events. 188 | message StopTimeUpdate { 189 | // The update is linked to a specific stop either through stop_sequence or 190 | // stop_id, so one of the fields below must necessarily be set. 191 | // See the documentation in TripDescriptor for more information. 192 | 193 | // Must be the same as in stop_times.txt in the corresponding GTFS feed. 194 | optional uint32 stop_sequence = 1; 195 | // Must be the same as in stops.txt in the corresponding GTFS feed. 196 | optional string stop_id = 4; 197 | 198 | optional StopTimeEvent arrival = 2; 199 | optional StopTimeEvent departure = 3; 200 | 201 | // The relation between this StopTime and the static schedule. 202 | enum ScheduleRelationship { 203 | // The vehicle is proceeding in accordance with its static schedule of 204 | // stops, although not necessarily according to the times of the schedule. 205 | // At least one of arrival and departure must be provided. If the schedule 206 | // for this stop contains both arrival and departure times then so must 207 | // this update. 208 | SCHEDULED = 0; 209 | 210 | // The stop is skipped, i.e., the vehicle will not stop at this stop. 211 | // Arrival and departure are optional. 212 | SKIPPED = 1; 213 | 214 | // No data is given for this stop. The main intention for this value is to 215 | // give the predictions only for part of a trip, i.e., if the last update 216 | // for a trip has a NO_DATA specifier, then StopTimes for the rest of the 217 | // stops in the trip are considered to be unspecified as well. 218 | // Neither arrival nor departure should be supplied. 219 | NO_DATA = 2; 220 | } 221 | optional ScheduleRelationship schedule_relationship = 5 222 | [default = SCHEDULED]; 223 | 224 | // The extensions namespace allows 3rd-party developers to extend the 225 | // GTFS Realtime Specification in order to add and evaluate new features 226 | // and modifications to the spec. 227 | extensions 1000 to 1999; 228 | } 229 | 230 | // Updates to StopTimes for the trip (both future, i.e., predictions, and in 231 | // some cases, past ones, i.e., those that already happened). 232 | // The updates must be sorted by stop_sequence, and apply for all the 233 | // following stops of the trip up to the next specified one. 234 | // 235 | // Example 1: 236 | // For a trip with 20 stops, a StopTimeUpdate with arrival delay and departure 237 | // delay of 0 for stop_sequence of the current stop means that the trip is 238 | // exactly on time. 239 | // 240 | // Example 2: 241 | // For the same trip instance, 3 StopTimeUpdates are provided: 242 | // - delay of 5 min for stop_sequence 3 243 | // - delay of 1 min for stop_sequence 8 244 | // - delay of unspecified duration for stop_sequence 10 245 | // This will be interpreted as: 246 | // - stop_sequences 3,4,5,6,7 have delay of 5 min. 247 | // - stop_sequences 8,9 have delay of 1 min. 248 | // - stop_sequences 10,... have unknown delay. 249 | repeated StopTimeUpdate stop_time_update = 2; 250 | 251 | // Moment at which the vehicle's real-time progress was measured. In POSIX 252 | // time (i.e., the number of seconds since January 1st 1970 00:00:00 UTC). 253 | optional uint64 timestamp = 4; 254 | 255 | // The current schedule deviation for the trip. Delay should only be 256 | // specified when the prediction is given relative to some existing schedule 257 | // in GTFS. 258 | // 259 | // Delay (in seconds) can be positive (meaning that the vehicle is late) or 260 | // negative (meaning that the vehicle is ahead of schedule). Delay of 0 261 | // means that the vehicle is exactly on time. 262 | // 263 | // Delay information in StopTimeUpdates take precedent of trip-level delay 264 | // information, such that trip-level delay is only propagated until the next 265 | // stop along the trip with a StopTimeUpdate delay value specified. 266 | // 267 | // Feed providers are strongly encouraged to provide a TripUpdate.timestamp 268 | // value indicating when the delay value was last updated, in order to 269 | // evaluate the freshness of the data. 270 | // 271 | // NOTE: This field is still experimental, and subject to change. It may be 272 | // formally adopted in the future. 273 | optional int32 delay = 5; 274 | 275 | // The extensions namespace allows 3rd-party developers to extend the 276 | // GTFS Realtime Specification in order to add and evaluate new features and 277 | // modifications to the spec. 278 | extensions 1000 to 1999; 279 | } 280 | 281 | // Realtime positioning information for a given vehicle. 282 | message VehiclePosition { 283 | // The Trip that this vehicle is serving. 284 | // Can be empty or partial if the vehicle can not be identified with a given 285 | // trip instance. 286 | optional TripDescriptor trip = 1; 287 | 288 | // Additional information on the vehicle that is serving this trip. 289 | optional VehicleDescriptor vehicle = 8; 290 | 291 | // Current position of this vehicle. 292 | optional Position position = 2; 293 | 294 | // The stop sequence index of the current stop. The meaning of 295 | // current_stop_sequence (i.e., the stop that it refers to) is determined by 296 | // current_status. 297 | // If current_status is missing IN_TRANSIT_TO is assumed. 298 | optional uint32 current_stop_sequence = 3; 299 | // Identifies the current stop. The value must be the same as in stops.txt in 300 | // the corresponding GTFS feed. 301 | optional string stop_id = 7; 302 | 303 | enum VehicleStopStatus { 304 | // The vehicle is just about to arrive at the stop (on a stop 305 | // display, the vehicle symbol typically flashes). 306 | INCOMING_AT = 0; 307 | 308 | // The vehicle is standing at the stop. 309 | STOPPED_AT = 1; 310 | 311 | // The vehicle has departed and is in transit to the next stop. 312 | IN_TRANSIT_TO = 2; 313 | } 314 | // The exact status of the vehicle with respect to the current stop. 315 | // Ignored if current_stop_sequence is missing. 316 | optional VehicleStopStatus current_status = 4 [default = IN_TRANSIT_TO]; 317 | 318 | // Moment at which the vehicle's position was measured. In POSIX time 319 | // (i.e., number of seconds since January 1st 1970 00:00:00 UTC). 320 | optional uint64 timestamp = 5; 321 | 322 | // Congestion level that is affecting this vehicle. 323 | enum CongestionLevel { 324 | UNKNOWN_CONGESTION_LEVEL = 0; 325 | RUNNING_SMOOTHLY = 1; 326 | STOP_AND_GO = 2; 327 | CONGESTION = 3; 328 | SEVERE_CONGESTION = 4; // People leaving their cars. 329 | } 330 | optional CongestionLevel congestion_level = 6; 331 | 332 | // The degree of passenger occupancy of the vehicle. This field is still 333 | // experimental, and subject to change. It may be formally adopted in the 334 | // future. 335 | enum OccupancyStatus { 336 | // The vehicle is considered empty by most measures, and has few or no 337 | // passengers onboard, but is still accepting passengers. 338 | EMPTY = 0; 339 | 340 | // The vehicle has a relatively large percentage of seats available. 341 | // What percentage of free seats out of the total seats available is to be 342 | // considered large enough to fall into this category is determined at the 343 | // discretion of the producer. 344 | MANY_SEATS_AVAILABLE = 1; 345 | 346 | // The vehicle has a relatively small percentage of seats available. 347 | // What percentage of free seats out of the total seats available is to be 348 | // considered small enough to fall into this category is determined at the 349 | // discretion of the feed producer. 350 | FEW_SEATS_AVAILABLE = 2; 351 | 352 | // The vehicle can currently accommodate only standing passengers. 353 | STANDING_ROOM_ONLY = 3; 354 | 355 | // The vehicle can currently accommodate only standing passengers 356 | // and has limited space for them. 357 | CRUSHED_STANDING_ROOM_ONLY = 4; 358 | 359 | // The vehicle is considered full by most measures, but may still be 360 | // allowing passengers to board. 361 | FULL = 5; 362 | 363 | // The vehicle is not accepting additional passengers. 364 | NOT_ACCEPTING_PASSENGERS = 6; 365 | 366 | } 367 | optional OccupancyStatus occupancy_status = 9; 368 | 369 | // The extensions namespace allows 3rd-party developers to extend the 370 | // GTFS Realtime Specification in order to add and evaluate new features and 371 | // modifications to the spec. 372 | extensions 1000 to 1999; 373 | } 374 | 375 | // An alert, indicating some sort of incident in the public transit network. 376 | message Alert { 377 | // Time when the alert should be shown to the user. If missing, the 378 | // alert will be shown as long as it appears in the feed. 379 | // If multiple ranges are given, the alert will be shown during all of them. 380 | repeated TimeRange active_period = 1; 381 | 382 | // Entities whose users we should notify of this alert. 383 | repeated EntitySelector informed_entity = 5; 384 | 385 | // Cause of this alert. 386 | enum Cause { 387 | UNKNOWN_CAUSE = 1; 388 | OTHER_CAUSE = 2; // Not machine-representable. 389 | TECHNICAL_PROBLEM = 3; 390 | STRIKE = 4; // Public transit agency employees stopped working. 391 | DEMONSTRATION = 5; // People are blocking the streets. 392 | ACCIDENT = 6; 393 | HOLIDAY = 7; 394 | WEATHER = 8; 395 | MAINTENANCE = 9; 396 | CONSTRUCTION = 10; 397 | POLICE_ACTIVITY = 11; 398 | MEDICAL_EMERGENCY = 12; 399 | } 400 | optional Cause cause = 6 [default = UNKNOWN_CAUSE]; 401 | 402 | // What is the effect of this problem on the affected entity. 403 | enum Effect { 404 | NO_SERVICE = 1; 405 | REDUCED_SERVICE = 2; 406 | 407 | // We don't care about INsignificant delays: they are hard to detect, have 408 | // little impact on the user, and would clutter the results as they are too 409 | // frequent. 410 | SIGNIFICANT_DELAYS = 3; 411 | 412 | DETOUR = 4; 413 | ADDITIONAL_SERVICE = 5; 414 | MODIFIED_SERVICE = 6; 415 | OTHER_EFFECT = 7; 416 | UNKNOWN_EFFECT = 8; 417 | STOP_MOVED = 9; 418 | } 419 | optional Effect effect = 7 [default = UNKNOWN_EFFECT]; 420 | 421 | // The URL which provides additional information about the alert. 422 | optional TranslatedString url = 8; 423 | 424 | // Alert header. Contains a short summary of the alert text as plain-text. 425 | optional TranslatedString header_text = 10; 426 | 427 | // Full description for the alert as plain-text. The information in the 428 | // description should add to the information of the header. 429 | optional TranslatedString description_text = 11; 430 | 431 | // The extensions namespace allows 3rd-party developers to extend the 432 | // GTFS Realtime Specification in order to add and evaluate new features 433 | // and modifications to the spec. 434 | extensions 1000 to 1999; 435 | } 436 | 437 | // 438 | // Low level data structures used above. 439 | // 440 | 441 | // A time interval. The interval is considered active at time 't' if 't' is 442 | // greater than or equal to the start time and less than the end time. 443 | message TimeRange { 444 | // Start time, in POSIX time (i.e., number of seconds since January 1st 1970 445 | // 00:00:00 UTC). 446 | // If missing, the interval starts at minus infinity. 447 | optional uint64 start = 1; 448 | 449 | // End time, in POSIX time (i.e., number of seconds since January 1st 1970 450 | // 00:00:00 UTC). 451 | // If missing, the interval ends at plus infinity. 452 | optional uint64 end = 2; 453 | 454 | // The extensions namespace allows 3rd-party developers to extend the 455 | // GTFS Realtime Specification in order to add and evaluate new features and 456 | // modifications to the spec. 457 | extensions 1000 to 1999; 458 | } 459 | 460 | // A position. 461 | message Position { 462 | // Degrees North, in the WGS-84 coordinate system. 463 | required float latitude = 1; 464 | 465 | // Degrees East, in the WGS-84 coordinate system. 466 | required float longitude = 2; 467 | 468 | // Bearing, in degrees, clockwise from North, i.e., 0 is North and 90 is East. 469 | // This can be the compass bearing, or the direction towards the next stop 470 | // or intermediate location. 471 | // This should not be direction deduced from the sequence of previous 472 | // positions, which can be computed from previous data. 473 | optional float bearing = 3; 474 | 475 | // Odometer value, in meters. 476 | optional double odometer = 4; 477 | // Momentary speed measured by the vehicle, in meters per second. 478 | optional float speed = 5; 479 | 480 | // The extensions namespace allows 3rd-party developers to extend the 481 | // GTFS Realtime Specification in order to add and evaluate new features and 482 | // modifications to the spec. 483 | extensions 1000 to 1999; 484 | } 485 | 486 | // A descriptor that identifies an instance of a GTFS trip, or all instances of 487 | // a trip along a route. 488 | // - To specify a single trip instance, the trip_id (and if necessary, 489 | // start_time) is set. If route_id is also set, then it should be same as one 490 | // that the given trip corresponds to. 491 | // - To specify all the trips along a given route, only the route_id should be 492 | // set. Note that if the trip_id is not known, then stop sequence ids in 493 | // TripUpdate are not sufficient, and stop_ids must be provided as well. In 494 | // addition, absolute arrival/departure times must be provided. 495 | message TripDescriptor { 496 | // The trip_id from the GTFS feed that this selector refers to. 497 | // For non frequency-based trips, this field is enough to uniquely identify 498 | // the trip. For frequency-based trip, start_time and start_date might also be 499 | // necessary. 500 | optional string trip_id = 1; 501 | 502 | // The route_id from the GTFS that this selector refers to. 503 | optional string route_id = 5; 504 | 505 | // The direction_id from the GTFS feed trips.txt file, indicating the 506 | // direction of travel for trips this selector refers to. This field is 507 | // still experimental, and subject to change. It may be formally adopted in 508 | // the future. 509 | optional uint32 direction_id = 6; 510 | 511 | // The initially scheduled start time of this trip instance. 512 | // When the trip_id corresponds to a non-frequency-based trip, this field 513 | // should either be omitted or be equal to the value in the GTFS feed. When 514 | // the trip_id correponds to a frequency-based trip, the start_time must be 515 | // specified for trip updates and vehicle positions. If the trip corresponds 516 | // to exact_times=1 GTFS record, then start_time must be some multiple 517 | // (including zero) of headway_secs later than frequencies.txt start_time for 518 | // the corresponding time period. If the trip corresponds to exact_times=0, 519 | // then its start_time may be arbitrary, and is initially expected to be the 520 | // first departure of the trip. Once established, the start_time of this 521 | // frequency-based trip should be considered immutable, even if the first 522 | // departure time changes -- that time change may instead be reflected in a 523 | // StopTimeUpdate. 524 | // Format and semantics of the field is same as that of 525 | // GTFS/frequencies.txt/start_time, e.g., 11:15:35 or 25:15:35. 526 | optional string start_time = 2; 527 | // The scheduled start date of this trip instance. 528 | // Must be provided to disambiguate trips that are so late as to collide with 529 | // a scheduled trip on a next day. For example, for a train that departs 8:00 530 | // and 20:00 every day, and is 12 hours late, there would be two distinct 531 | // trips on the same time. 532 | // This field can be provided but is not mandatory for schedules in which such 533 | // collisions are impossible - for example, a service running on hourly 534 | // schedule where a vehicle that is one hour late is not considered to be 535 | // related to schedule anymore. 536 | // In YYYYMMDD format. 537 | optional string start_date = 3; 538 | 539 | // The relation between this trip and the static schedule. If a trip is done 540 | // in accordance with temporary schedule, not reflected in GTFS, then it 541 | // shouldn't be marked as SCHEDULED, but likely as ADDED. 542 | enum ScheduleRelationship { 543 | // Trip that is running in accordance with its GTFS schedule, or is close 544 | // enough to the scheduled trip to be associated with it. 545 | SCHEDULED = 0; 546 | 547 | // An extra trip that was added in addition to a running schedule, for 548 | // example, to replace a broken vehicle or to respond to sudden passenger 549 | // load. 550 | ADDED = 1; 551 | 552 | // A trip that is running with no schedule associated to it, for example, if 553 | // there is no schedule at all. 554 | UNSCHEDULED = 2; 555 | 556 | // A trip that existed in the schedule but was removed. 557 | CANCELED = 3; 558 | } 559 | optional ScheduleRelationship schedule_relationship = 4; 560 | 561 | // The extensions namespace allows 3rd-party developers to extend the 562 | // GTFS Realtime Specification in order to add and evaluate new features and 563 | // modifications to the spec. 564 | extensions 1000 to 1999; 565 | } 566 | 567 | // Identification information for the vehicle performing the trip. 568 | message VehicleDescriptor { 569 | // Internal system identification of the vehicle. Should be unique per 570 | // vehicle, and can be used for tracking the vehicle as it proceeds through 571 | // the system. 572 | optional string id = 1; 573 | 574 | // User visible label, i.e., something that must be shown to the passenger to 575 | // help identify the correct vehicle. 576 | optional string label = 2; 577 | 578 | // The license plate of the vehicle. 579 | optional string license_plate = 3; 580 | 581 | // The extensions namespace allows 3rd-party developers to extend the 582 | // GTFS Realtime Specification in order to add and evaluate new features and 583 | // modifications to the spec. 584 | extensions 1000 to 1999; 585 | } 586 | 587 | // A selector for an entity in a GTFS feed. 588 | message EntitySelector { 589 | // The values of the fields should correspond to the appropriate fields in the 590 | // GTFS feed. 591 | // At least one specifier must be given. If several are given, then the 592 | // matching has to apply to all the given specifiers. 593 | optional string agency_id = 1; 594 | optional string route_id = 2; 595 | // corresponds to route_type in GTFS. 596 | optional int32 route_type = 3; 597 | optional TripDescriptor trip = 4; 598 | optional string stop_id = 5; 599 | 600 | // The extensions namespace allows 3rd-party developers to extend the 601 | // GTFS Realtime Specification in order to add and evaluate new features and 602 | // modifications to the spec. 603 | extensions 1000 to 1999; 604 | } 605 | 606 | // An internationalized message containing per-language versions of a snippet of 607 | // text or a URL. 608 | // One of the strings from a message will be picked up. The resolution proceeds 609 | // as follows: 610 | // 1. If the UI language matches the language code of a translation, 611 | // the first matching translation is picked. 612 | // 2. If a default UI language (e.g., English) matches the language code of a 613 | // translation, the first matching translation is picked. 614 | // 3. If some translation has an unspecified language code, that translation is 615 | // picked. 616 | message TranslatedString { 617 | message Translation { 618 | // A UTF-8 string containing the message. 619 | required string text = 1; 620 | // BCP-47 language code. Can be omitted if the language is unknown or if 621 | // no i18n is done at all for the feed. At most one translation is 622 | // allowed to have an unspecified language tag. 623 | optional string language = 2; 624 | 625 | // The extensions namespace allows 3rd-party developers to extend the 626 | // GTFS Realtime Specification in order to add and evaluate new features and 627 | // modifications to the spec. 628 | extensions 1000 to 1999; 629 | } 630 | // At least one translation must be provided. 631 | repeated Translation translation = 1; 632 | 633 | // The extensions namespace allows 3rd-party developers to extend the 634 | // GTFS Realtime Specification in order to add and evaluate new features and 635 | // modifications to the spec. 636 | extensions 1000 to 1999; 637 | } -------------------------------------------------------------------------------- /streaming/static/mta_stations.csv: -------------------------------------------------------------------------------- 1 | Station ID,Complex ID,GTFS Stop ID,Division,Line,Stop Name,Borough,Daytime Routes,Structure,GTFS Latitude,GTFS Longitude 2 | 1,1,R01,BMT,Astoria,Astoria - Ditmars Blvd,Q,N W,Elevated,40.775036,-73.912034 3 | 2,2,R03,BMT,Astoria,Astoria Blvd,Q,N W,Elevated,40.770258,-73.917843 4 | 3,3,R04,BMT,Astoria,30 Av,Q,N W,Elevated,40.766779,-73.921479 5 | 4,4,R05,BMT,Astoria,Broadway,Q,N W,Elevated,40.76182,-73.925508 6 | 5,5,R06,BMT,Astoria,36 Av,Q,N W,Elevated,40.756804,-73.929575 7 | 6,6,R08,BMT,Astoria,39 Av,Q,N W,Elevated,40.752882,-73.932755 8 | 7,613,R11,BMT,Astoria,Lexington Av/59 St,M,N W R,Subway,40.76266,-73.967258 9 | 8,8,R13,BMT,Astoria,5 Av/59 St,M,N W R,Subway,40.764811,-73.973347 10 | 9,9,R14,BMT,Broadway - Brighton,57 St - 7 Av,M,N Q R W,Subway,40.764664,-73.980658 11 | 10,10,R15,BMT,Broadway - Brighton,49 St,M,N R W,Subway,40.759901,-73.984139 12 | 11,611,R16,BMT,Broadway - Brighton,Times Sq - 42 St,M,N Q R W,Subway,40.754672,-73.986754 13 | 12,607,R17,BMT,Broadway - Brighton,34 St - Herald Sq,M,N Q R W,Subway,40.749567,-73.98795 14 | 13,13,R18,BMT,Broadway - Brighton,28 St,M,R W,Subway,40.745494,-73.988691 15 | 14,14,R19,BMT,Broadway - Brighton,23 St,M,R W,Subway,40.741303,-73.989344 16 | 15,602,R20,BMT,Broadway - Brighton,14 St - Union Sq,M,N Q R W,Subway,40.735736,-73.990568 17 | 16,16,R21,BMT,Broadway - Brighton,8 St - NYU,M,R W,Subway,40.730328,-73.992629 18 | 17,17,R22,BMT,Broadway - Brighton,Prince St,M,R W,Subway,40.724329,-73.997702 19 | 18,623,R23,BMT,Broadway,Canal St,M,R W,Subway,40.719527,-74.001775 20 | 19,623,Q01,BMT,Manhattan Bridge,Canal St,M,N Q,Subway,40.718383,-74.00046 21 | 20,20,R24,BMT,Broadway,City Hall,M,R W,Subway,40.713282,-74.006978 22 | 21,21,R25,BMT,Broadway,Cortlandt St,M,R W,Subway,40.710668,-74.011029 23 | 22,22,R26,BMT,Broadway,Rector St,M,R W,Subway,40.70722,-74.013342 24 | 23,635,R27,BMT,Broadway,Whitehall St,M,R W,Subway,40.703087,-74.012994 25 | 24,620,R28,BMT,Broadway,Court St,Bk,R,Subway,40.6941,-73.991777 26 | 25,636,R29,BMT,Broadway,Jay St - MetroTech,Bk,R,Subway,40.69218,-73.985942 27 | 26,26,R30,BMT,Broadway - Brighton,DeKalb Av,Bk,B Q R,Subway,40.690635,-73.981824 28 | 27,617,R31,BMT,4th Av,Atlantic Av - Barclays Ctr,Bk,D N R,Subway,40.683666,-73.97881 29 | 28,28,R32,BMT,4th Av,Union St,Bk,R,Subway,40.677316,-73.98311 30 | 29,608,R33,BMT,4th Av,9 St,Bk,R,Subway,40.670847,-73.988302 31 | 30,30,R34,BMT,4th Av,Prospect Av,Bk,R,Subway,40.665414,-73.992872 32 | 31,31,R35,BMT,4th Av,25 St,Bk,R,Subway,40.660397,-73.998091 33 | 32,32,R36,BMT,4th Av,36 St,Bk,D N R,Subway,40.655144,-74.003549 34 | 33,33,R39,BMT,4th Av,45 St,Bk,R,Subway,40.648939,-74.010006 35 | 34,34,R40,BMT,4th Av,53 St,Bk,R,Subway,40.645069,-74.014034 36 | 35,35,R41,BMT,4th Av,59 St,Bk,N R,Subway,40.641362,-74.017881 37 | 36,36,R42,BMT,4th Av,Bay Ridge Av,Bk,R,Subway,40.634967,-74.023377 38 | 37,37,R43,BMT,4th Av,77 St,Bk,R,Subway,40.629742,-74.02551 39 | 38,38,R44,BMT,4th Av,86 St,Bk,R,Subway,40.622687,-74.028398 40 | 39,39,R45,BMT,4th Av,Bay Ridge - 95 St,Bk,R,Subway,40.616622,-74.030876 41 | 40,617,D24,BMT,Broadway - Brighton,Atlantic Av - Barclays Ctr,Bk,B Q,Subway,40.68446,-73.97689 42 | 41,41,D25,BMT,Broadway - Brighton,7 Av,Bk,B Q,Subway,40.67705,-73.972367 43 | 42,42,D26,BMT,Broadway - Brighton,Prospect Park,Bk,B Q,Open Cut,40.661614,-73.962246 44 | 43,43,D27,BMT,Broadway - Brighton,Parkside Av,Bk,B Q,Open Cut,40.655292,-73.961495 45 | 44,44,D28,BMT,Broadway - Brighton,Church Av,Bk,B Q,Open Cut,40.650527,-73.962982 46 | 45,45,D29,BMT,Broadway - Brighton,Beverley Rd,Bk,B Q,Open Cut,40.644031,-73.964492 47 | 46,46,D30,BMT,Broadway - Brighton,Cortelyou Rd,Bk,B Q,Open Cut,40.640927,-73.963891 48 | 47,47,D31,BMT,Broadway - Brighton,Newkirk Plaza,Bk,B Q,Open Cut,40.635082,-73.962793 49 | 48,48,D32,BMT,Broadway - Brighton,Avenue H,Bk,B Q,Open Cut,40.62927,-73.961639 50 | 49,49,D33,BMT,Broadway - Brighton,Avenue J,Bk,B Q,Open Cut,40.625039,-73.960803 51 | 50,50,D34,BMT,Broadway - Brighton,Avenue M,Bk,B Q,Open Cut,40.617618,-73.959399 52 | 51,51,D35,BMT,Broadway - Brighton,Kings Hwy,Bk,B Q,Open Cut,40.60867,-73.957734 53 | 52,52,D37,BMT,Broadway - Brighton,Avenue U,Bk,B Q,Open Cut,40.5993,-73.955929 54 | 53,53,D38,BMT,Broadway - Brighton,Neck Rd,Bk,B Q,Open Cut,40.595246,-73.955161 55 | 54,54,D39,BMT,Broadway - Brighton,Sheepshead Bay,Bk,B Q,Open Cut,40.586896,-73.954155 56 | 55,55,D40,BMT,Broadway - Brighton,Brighton Beach,Bk,B Q,Elevated,40.577621,-73.961376 57 | 56,56,D41,BMT,Broadway - Brighton,Ocean Pkwy,Bk,Q,Elevated,40.576312,-73.968501 58 | 57,57,D42,BMT,Broadway - Brighton,W 8 St - NY Aquarium,Bk,F Q,Elevated,40.576127,-73.975939 59 | 58,58,D43,BMT,Sea Beach / West End / Culver / Brighton,Coney Island - Stillwell Av,Bk,D F N Q,Viaduct,40.577422,-73.981233 60 | 59,59,B12,BMT,West End,9 Av,Bk,D,Open Cut,40.646292,-73.994324 61 | 60,60,B13,BMT,West End,Fort Hamilton Pkwy,Bk,D,Elevated,40.640914,-73.994304 62 | 61,61,B14,BMT,West End,50 St,Bk,D,Elevated,40.63626,-73.994791 63 | 62,62,B15,BMT,West End,55 St,Bk,D,Elevated,40.631435,-73.995476 64 | 63,615,B16,BMT,West End,62 St,Bk,D,Elevated,40.626472,-73.996895 65 | 64,64,B17,BMT,West End,71 St,Bk,D,Elevated,40.619589,-73.998864 66 | 65,65,B18,BMT,West End,79 St,Bk,D,Elevated,40.613501,-74.00061 67 | 66,66,B19,BMT,West End,18 Av,Bk,D,Elevated,40.607954,-74.001736 68 | 67,67,B20,BMT,West End,20 Av,Bk,D,Elevated,40.604556,-73.998168 69 | 68,68,B21,BMT,West End,Bay Pkwy,Bk,D,Elevated,40.601875,-73.993728 70 | 69,69,B22,BMT,West End,25 Av,Bk,D,Elevated,40.597704,-73.986829 71 | 70,70,B23,BMT,West End,Bay 50 St,Bk,D,Elevated,40.588841,-73.983765 72 | 71,71,N02,BMT,Sea Beach,8 Av,Bk,N,Open Cut,40.635064,-74.011719 73 | 72,72,N03,BMT,Sea Beach,Fort Hamilton Pkwy,Bk,N,Open Cut,40.631386,-74.005351 74 | 73,615,N04,BMT,Sea Beach,New Utrecht Av,Bk,N,Open Cut,40.624842,-73.996353 75 | 74,74,N05,BMT,Sea Beach,18 Av,Bk,N,Open Cut,40.620671,-73.990414 76 | 75,75,N06,BMT,Sea Beach,20 Av,Bk,N,Open Cut,40.61741,-73.985026 77 | 76,76,N07,BMT,Sea Beach,Bay Pkwy,Bk,N,Open Cut,40.611815,-73.981848 78 | 77,77,N08,BMT,Sea Beach,Kings Hwy,Bk,N,Open Cut,40.603923,-73.980353 79 | 78,78,N09,BMT,Sea Beach,Avenue U,Bk,N,Open Cut,40.597473,-73.979137 80 | 79,79,N10,BMT,Sea Beach,86 St,Bk,N,Open Cut,40.592721,-73.97823 81 | 80,80,J12,BMT,Jamaica,121 St,Q,J Z,Elevated,40.700492,-73.828294 82 | 81,81,J13,BMT,Jamaica,111 St,Q,J,Elevated,40.697418,-73.836345 83 | 82,82,J14,BMT,Jamaica,104 St,Q,J Z,Elevated,40.695178,-73.84433 84 | 83,83,J15,BMT,Jamaica,Woodhaven Blvd,Q,J Z,Elevated,40.693879,-73.851576 85 | 84,84,J16,BMT,Jamaica,85 St - Forest Pkwy,Q,J,Elevated,40.692435,-73.86001 86 | 85,85,J17,BMT,Jamaica,75 St,Q,J Z,Elevated,40.691324,-73.867139 87 | 86,86,J19,BMT,Jamaica,Cypress Hills,Bk,J,Elevated,40.689941,-73.87255 88 | 87,87,J20,BMT,Jamaica,Crescent St,Bk,J Z,Elevated,40.683194,-73.873785 89 | 88,88,J21,BMT,Jamaica,Norwood Av,Bk,J Z,Elevated,40.68141,-73.880039 90 | 89,89,J22,BMT,Jamaica,Cleveland St,Bk,J,Elevated,40.679947,-73.884639 91 | 90,90,J23,BMT,Jamaica,Van Siclen Av,Bk,J Z,Elevated,40.678024,-73.891688 92 | 91,91,J24,BMT,Jamaica,Alabama Av,Bk,J,Elevated,40.676992,-73.898654 93 | 92,621,J27,BMT,Jamaica,Broadway Jct,Bk,J Z,Elevated,40.679498,-73.904512 94 | 93,93,J28,BMT,Jamaica,Chauncey St,Bk,J Z,Elevated,40.682893,-73.910456 95 | 94,94,J29,BMT,Jamaica,Halsey St,Bk,J,Elevated,40.68637,-73.916559 96 | 95,95,J30,BMT,Jamaica,Gates Av,Bk,J Z,Elevated,40.68963,-73.92227 97 | 96,96,J31,BMT,Jamaica,Kosciuszko St,Bk,J,Elevated,40.693342,-73.928814 98 | 97,97,M11,BMT,Jamaica,Myrtle Av,Bk,J M Z,Elevated,40.697207,-73.935657 99 | 98,98,M12,BMT,Jamaica,Flushing Av,Bk,J M,Elevated,40.70026,-73.941126 100 | 99,99,M13,BMT,Jamaica,Lorimer St,Bk,J M,Elevated,40.703869,-73.947408 101 | 100,100,M14,BMT,Jamaica,Hewes St,Bk,J M,Elevated,40.70687,-73.953431 102 | 101,101,M16,BMT,Jamaica,Marcy Av,Bk,J M Z,Elevated,40.708359,-73.957757 103 | 102,625,M18,BMT,Jamaica,Essex St,M,J M Z,Subway,40.718315,-73.987437 104 | 103,103,M19,BMT,Jamaica,Bowery,M,J Z,Subway,40.72028,-73.993915 105 | 104,623,M20,BMT,Jamaica,Canal St,M,J Z,Subway,40.718092,-73.999892 106 | 105,622,M21,BMT,Jamaica,Chambers St,M,J Z,Subway,40.713243,-74.003401 107 | 106,628,M22,BMT,Jamaica,Fulton St,M,J Z,Subway,40.710374,-74.007582 108 | 107,107,M23,BMT,Jamaica,Broad St,M,J Z,Subway,40.706476,-74.011056 109 | 108,108,M01,BMT,Myrtle Av,Middle Village - Metropolitan Av,Q,M,Elevated,40.711396,-73.889601 110 | 109,109,M04,BMT,Myrtle Av,Fresh Pond Rd,Q,M,Elevated,40.706186,-73.895877 111 | 110,110,M05,BMT,Myrtle Av,Forest Av,Q,M,Elevated,40.704423,-73.903077 112 | 111,111,M06,BMT,Myrtle Av,Seneca Av,Q,M,Elevated,40.702762,-73.90774 113 | 112,630,M08,BMT,Myrtle Av,Myrtle - Wyckoff Avs,Bk,M,Elevated,40.69943,-73.912385 114 | 113,113,M09,BMT,Myrtle Av,Knickerbocker Av,Bk,M,Elevated,40.698664,-73.919711 115 | 114,114,M10,BMT,Myrtle Av,Central Av,Bk,M,Elevated,40.697857,-73.927397 116 | 115,618,L01,BMT,Canarsie,8 Av,M,L,Subway,40.739777,-74.002578 117 | 116,601,L02,BMT,Canarsie,6 Av,M,L,Subway,40.737335,-73.996786 118 | 117,602,L03,BMT,Canarsie,Union Sq - 14 St,M,L,Subway,40.734789,-73.99073 119 | 118,118,L05,BMT,Canarsie,3 Av,M,L,Subway,40.732849,-73.986122 120 | 119,119,L06,BMT,Canarsie,1 Av,M,L,Subway,40.730953,-73.981628 121 | 120,120,L08,BMT,Canarsie,Bedford Av,Bk,L,Subway,40.717304,-73.956872 122 | 121,629,L10,BMT,Canarsie,Lorimer St,Bk,L,Subway,40.714063,-73.950275 123 | 122,122,L11,BMT,Canarsie,Graham Av,Bk,L,Subway,40.714565,-73.944053 124 | 123,123,L12,BMT,Canarsie,Grand St,Bk,L,Subway,40.711926,-73.94067 125 | 124,124,L13,BMT,Canarsie,Montrose Av,Bk,L,Subway,40.707739,-73.93985 126 | 125,125,L14,BMT,Canarsie,Morgan Av,Bk,L,Subway,40.706152,-73.933147 127 | 126,126,L15,BMT,Canarsie,Jefferson St,Bk,L,Subway,40.706607,-73.922913 128 | 127,127,L16,BMT,Canarsie,DeKalb Av,Bk,L,Subway,40.703811,-73.918425 129 | 128,630,L17,BMT,Canarsie,Myrtle - Wyckoff Avs,Bk,L,Subway,40.699814,-73.911586 130 | 129,129,L19,BMT,Canarsie,Halsey St,Q,L,Subway,40.695602,-73.904084 131 | 130,130,L20,BMT,Canarsie,Wilson Av,Bk,L,Subway,40.688764,-73.904046 132 | 131,131,L21,BMT,Canarsie,Bushwick Av - Aberdeen St,Bk,L,Subway,40.682829,-73.905249 133 | 132,621,L22,BMT,Canarsie,Broadway Jct,Bk,L,Elevated,40.678856,-73.90324 134 | 133,133,L24,BMT,Canarsie,Atlantic Av,Bk,L,Elevated,40.675345,-73.903097 135 | 134,134,L25,BMT,Canarsie,Sutter Av,Bk,L,Elevated,40.669367,-73.901975 136 | 135,135,L26,BMT,Canarsie,Livonia Av,Bk,L,Elevated,40.664038,-73.900571 137 | 136,136,L27,BMT,Canarsie,New Lots Av,Bk,L,Elevated,40.658733,-73.899232 138 | 137,137,L28,BMT,Canarsie,E 105 St,Bk,L,At Grade,40.650573,-73.899485 139 | 138,138,L29,BMT,Canarsie,Canarsie - Rockaway Pkwy,Bk,L,At Grade,40.646654,-73.90185 140 | 139,627,S01,BMT,Franklin Shuttle,Franklin Av,Bk,S,Elevated,40.680596,-73.955827 141 | 141,141,S03,BMT,Franklin Shuttle,Park Pl,Bk,S,Open Cut,40.674772,-73.957624 142 | 142,626,S04,BMT,Franklin Shuttle,Botanic Garden,Bk,S,Open Cut,40.670343,-73.959245 143 | 143,143,A02,IND,8th Av - Fulton St,Inwood - 207 St,M,A,Subway,40.868072,-73.919899 144 | 144,144,A03,IND,8th Av - Fulton St,Dyckman St,M,A,Subway,40.865491,-73.927271 145 | 145,145,A05,IND,8th Av - Fulton St,190 St,M,A,Subway,40.859022,-73.93418 146 | 146,146,A06,IND,8th Av - Fulton St,181 St,M,A,Subway,40.851695,-73.937969 147 | 147,147,A07,IND,8th Av - Fulton St,175 St,M,A,Subway,40.847391,-73.939704 148 | 148,605,A09,IND,8th Av - Fulton St,168 St,M,A C,Subway,40.840719,-73.939561 149 | 149,149,A10,IND,8th Av - Fulton St,163 St - Amsterdam Av,M,C,Subway,40.836013,-73.939892 150 | 150,150,A11,IND,8th Av - Fulton St,155 St,M,C,Subway,40.830518,-73.941514 151 | 151,151,A12,IND,8th Av - Fulton St,145 St,M,A C,Subway,40.824783,-73.944216 152 | 151,151,D13,IND,Concourse,145 St,M,B D,Subway,40.824783,-73.944216 153 | 152,152,A14,IND,8th Av - Fulton St,135 St,M,B C,Subway,40.817894,-73.947649 154 | 153,153,A15,IND,8th Av - Fulton St,125 St,M,A B C D,Subway,40.811109,-73.952343 155 | 154,154,A16,IND,8th Av - Fulton St,116 St,M,B C,Subway,40.805085,-73.954882 156 | 155,155,A17,IND,8th Av - Fulton St,Cathedral Pkwy (110 St),M,B C,Subway,40.800603,-73.958161 157 | 156,156,A18,IND,8th Av - Fulton St,103 St,M,B C,Subway,40.796092,-73.961454 158 | 157,157,A19,IND,8th Av - Fulton St,96 St,M,B C,Subway,40.791642,-73.964696 159 | 158,158,A20,IND,8th Av - Fulton St,86 St,M,B C,Subway,40.785868,-73.968916 160 | 159,159,A21,IND,8th Av - Fulton St,81 St - Museum of Natural History,M,B C,Subway,40.781433,-73.972143 161 | 160,160,A22,IND,8th Av - Fulton St,72 St,M,B C,Subway,40.775594,-73.97641 162 | 161,614,A24,IND,8th Av - Fulton St,59 St - Columbus Circle,M,A B C D,Subway,40.768296,-73.981736 163 | 162,162,A25,IND,8th Av - Fulton St,50 St,M,C E,Subway,40.762456,-73.985984 164 | 163,611,A27,IND,8th Av - Fulton St,42 St - Port Authority Bus Terminal,M,A C E,Subway,40.757308,-73.989735 165 | 164,164,A28,IND,8th Av - Fulton St,34 St - Penn Station,M,A C E,Subway,40.752287,-73.993391 166 | 165,165,A30,IND,8th Av - Fulton St,23 St,M,C E,Subway,40.745906,-73.998041 167 | 166,618,A31,IND,8th Av - Fulton St,14 St,M,A C E,Subway,40.740893,-74.00169 168 | 167,167,A32,IND,8th Av - Fulton St,W 4 St,M,A C E,Subway,40.732338,-74.000495 169 | 167,167,D20,IND,6th Av - Culver,W 4 St,M,B D F M,Subway,40.732338,-74.000495 170 | 168,168,A33,IND,8th Av - Fulton St,Spring St,M,C E,Subway,40.726227,-74.003739 171 | 169,169,A34,IND,8th Av - Fulton St,Canal St,M,A C E,Subway,40.720824,-74.005229 172 | 170,624,A36,IND,8th Av - Fulton St,Chambers St,M,A C,Subway,40.714111,-74.008585 173 | 171,624,E01,IND,8th Av - Fulton St,World Trade Center,M,E,Subway,40.712582,-74.009781 174 | 172,628,A38,IND,8th Av - Fulton St,Fulton St,M,A C,Subway,40.710197,-74.007691 175 | 173,173,A40,IND,8th Av - Fulton St,High St,Bk,A C,Subway,40.699337,-73.990531 176 | 174,636,A41,IND,8th Av - Fulton St,Jay St - MetroTech,Bk,A C F,Subway,40.692338,-73.987342 177 | 175,175,A42,IND,8th Av - Fulton St,Hoyt - Schermerhorn Sts,Bk,A C G,Subway,40.688484,-73.985001 178 | 176,176,A43,IND,8th Av - Fulton St,Lafayette Av,Bk,C,Subway,40.686113,-73.973946 179 | 177,177,A44,IND,8th Av - Fulton St,Clinton - Washington Avs,Bk,C,Subway,40.683263,-73.965838 180 | 178,627,A45,IND,8th Av - Fulton St,Franklin Av,Bk,C,Subway,40.68138,-73.956848 181 | 179,179,A46,IND,8th Av - Fulton St,Nostrand Av,Bk,A C,Subway,40.680438,-73.950426 182 | 180,180,A47,IND,8th Av - Fulton St,Kingston - Throop Avs,Bk,C,Subway,40.679921,-73.940858 183 | 181,181,A48,IND,8th Av - Fulton St,Utica Av,Bk,A C,Subway,40.679364,-73.930729 184 | 182,182,A49,IND,8th Av - Fulton St,Ralph Av,Bk,C,Subway,40.678822,-73.920786 185 | 183,183,A50,IND,8th Av - Fulton St,Rockaway Av,Bk,C,Subway,40.67834,-73.911946 186 | 184,621,A51,IND,8th Av - Fulton St,Broadway Jct,Bk,A C,Subway,40.678334,-73.905316 187 | 185,185,A52,IND,8th Av - Fulton St,Liberty Av,Bk,C,Subway,40.674542,-73.896548 188 | 186,186,A53,IND,8th Av - Fulton St,Van Siclen Av,Bk,C,Subway,40.67271,-73.890358 189 | 187,187,A54,IND,8th Av - Fulton St,Shepherd Av,Bk,C,Subway,40.67413,-73.88075 190 | 188,188,A55,IND,8th Av - Fulton St,Euclid Av,Bk,A C,Subway,40.675377,-73.872106 191 | 189,189,A57,IND,Liberty Av,Grant Av,Bk,A,Subway,40.677044,-73.86505 192 | 190,190,A59,IND,Liberty Av,80 St,Q,A,Elevated,40.679371,-73.858992 193 | 191,191,A60,IND,Liberty Av,88 St,Q,A,Elevated,40.679843,-73.85147 194 | 192,192,A61,IND,Liberty Av,Rockaway Blvd,Q,A,Elevated,40.680429,-73.843853 195 | 193,193,A63,IND,Liberty Av,104 St,Q,A,Elevated,40.681711,-73.837683 196 | 194,194,A64,IND,Liberty Av,111 St,Q,A,Elevated,40.684331,-73.832163 197 | 195,195,A65,IND,Liberty Av,Ozone Park - Lefferts Blvd,Q,A,Elevated,40.685951,-73.825798 198 | 196,196,H01,IND,Rockaway,Aqueduct Racetrack,Q,A,At Grade,40.672097,-73.835919 199 | 197,197,H02,IND,Rockaway,Aqueduct - N Conduit Av,Q,A,At Grade,40.668234,-73.834058 200 | 198,198,H03,IND,Rockaway,Howard Beach - JFK Airport,Q,A,At Grade,40.660476,-73.830301 201 | 199,199,H04,IND,Rockaway,Broad Channel,Q,A S,At Grade,40.608382,-73.815925 202 | 199,199,H19,IND,Rockaway,Broad Channel,Q,A S,At Grade,40.608382,-73.815925 203 | 200,200,H12,IND,Rockaway,Beach 90 St,Q,A S,Viaduct,40.588034,-73.813641 204 | 201,201,H13,IND,Rockaway,Beach 98 St,Q,A S,Viaduct,40.585307,-73.820558 205 | 202,202,H14,IND,Rockaway,Beach 105 St,Q,A S,Viaduct,40.583209,-73.827559 206 | 203,203,H15,IND,Rockaway,Rockaway Park - Beach 116 St,Q,A S,At Grade,40.580903,-73.835592 207 | 204,204,H06,IND,Rockaway,Beach 67 St,Q,A,Viaduct,40.590927,-73.796924 208 | 205,205,H07,IND,Rockaway,Beach 60 St,Q,A,Viaduct,40.592374,-73.788522 209 | 206,206,H08,IND,Rockaway,Beach 44 St,Q,A,Viaduct,40.592943,-73.776013 210 | 207,207,H09,IND,Rockaway,Beach 36 St,Q,A,Viaduct,40.595398,-73.768175 211 | 208,208,H10,IND,Rockaway,Beach 25 St,Q,A,Viaduct,40.600066,-73.761353 212 | 209,209,H11,IND,Rockaway,Far Rockaway - Mott Av,Q,A,Viaduct,40.603995,-73.755405 213 | 210,210,D01,IND,Concourse,Norwood - 205 St,Bx,D,Subway,40.874811,-73.878855 214 | 211,211,D03,IND,Concourse,Bedford Park Blvd,Bx,B D,Subway,40.873244,-73.887138 215 | 212,212,D04,IND,Concourse,Kingsbridge Rd,Bx,B D,Subway,40.866978,-73.893509 216 | 213,213,D05,IND,Concourse,Fordham Rd,Bx,B D,Subway,40.861296,-73.897749 217 | 214,214,D06,IND,Concourse,182-183 Sts,Bx,B D,Subway,40.856093,-73.900741 218 | 215,215,D07,IND,Concourse,Tremont Av,Bx,B D,Subway,40.85041,-73.905227 219 | 216,216,D08,IND,Concourse,174-175 Sts,Bx,B D,Subway,40.8459,-73.910136 220 | 217,217,D09,IND,Concourse,170 St,Bx,B D,Subway,40.839306,-73.9134 221 | 218,218,D10,IND,Concourse,167 St,Bx,B D,Subway,40.833771,-73.91844 222 | 219,604,D11,IND,Concourse,161 St - Yankee Stadium,Bx,B D,Subway,40.827905,-73.925651 223 | 220,220,D12,IND,Concourse,155 St,M,B D,Subway,40.830135,-73.938209 224 | 221,221,B04,IND,63rd St,21 St - Queensbridge,Q,F,Subway,40.754203,-73.942836 225 | 222,222,B06,IND,63rd St,Roosevelt Island,M,F,Subway,40.759145,-73.95326 226 | 223,223,B08,IND,63rd St,Lexington Av/63 St,M,F Q,Subway,40.764629,-73.966113 227 | 224,224,B10,IND,6th Av - Culver,57 St,M,F,Subway,40.763972,-73.97745 228 | 225,225,D15,IND,6th Av - Culver,47-50 Sts - Rockefeller Ctr,M,B D F M,Subway,40.758663,-73.981329 229 | 226,609,D16,IND,6th Av - Culver,42 St - Bryant Pk,M,B D F M,Subway,40.754222,-73.984569 230 | 227,607,D17,IND,6th Av - Culver,34 St - Herald Sq,M,B D F M,Subway,40.749719,-73.987823 231 | 228,228,D18,IND,6th Av - Culver,23 St,M,F M,Subway,40.742878,-73.992821 232 | 229,601,D19,IND,6th Av - Culver,14 St,M,F M,Subway,40.738228,-73.996209 233 | 230,619,D21,IND,6th Av - Culver,Broadway-Lafayette St,M,B D F M,Subway,40.725297,-73.996204 234 | 231,231,D22,IND,6th Av - Culver,Grand St,M,B D,Subway,40.718267,-73.993753 235 | 232,232,F14,IND,6th Av - Culver,2 Av,M,F,Subway,40.723402,-73.989938 236 | 233,625,F15,IND,6th Av - Culver,Delancey St,M,F,Subway,40.718611,-73.988114 237 | 234,234,F16,IND,6th Av - Culver,East Broadway,M,F,Subway,40.713715,-73.990173 238 | 235,235,F18,IND,6th Av - Culver,York St,Bk,F,Subway,40.701397,-73.986751 239 | 236,236,F20,IND,6th Av - Culver,Bergen St,Bk,F G,Subway,40.686145,-73.990862 240 | 237,237,F21,IND,6th Av - Culver,Carroll St,Bk,F G,Subway,40.680303,-73.995048 241 | 238,238,F22,IND,6th Av - Culver,Smith - 9 Sts,Bk,F G,Viaduct,40.67358,-73.995959 242 | 239,608,F23,IND,6th Av - Culver,4 Av,Bk,F G,Viaduct,40.670272,-73.989779 243 | 240,240,F24,IND,6th Av - Culver,7 Av,Bk,F G,Subway,40.666271,-73.980305 244 | 241,241,F25,IND,6th Av - Culver,15 St - Prospect Park,Bk,F G,Subway,40.660365,-73.979493 245 | 242,242,F26,IND,6th Av - Culver,Fort Hamilton Pkwy,Bk,F G,Subway,40.650782,-73.975776 246 | 243,243,F27,IND,6th Av - Culver,Church Av,Bk,F,Subway,40.644041,-73.979678 247 | 244,244,F29,IND,6th Av - Culver,Ditmas Av,Bk,F,Elevated,40.636119,-73.978172 248 | 245,245,F30,IND,6th Av - Culver,18 Av,Bk,F,Elevated,40.629755,-73.976971 249 | 246,246,F31,IND,6th Av - Culver,Avenue I,Bk,F,Elevated,40.625322,-73.976127 250 | 247,247,F32,IND,6th Av - Culver,Bay Pkwy,Bk,F,Elevated,40.620769,-73.975264 251 | 248,248,F33,IND,6th Av - Culver,Avenue N,Bk,F,Elevated,40.61514,-73.974197 252 | 249,249,F34,IND,6th Av - Culver,Avenue P,Bk,F,Elevated,40.608944,-73.973022 253 | 250,250,F35,IND,6th Av - Culver,Kings Hwy,Bk,F,Elevated,40.603217,-73.972361 254 | 251,251,F36,IND,6th Av - Culver,Avenue U,Bk,F,Elevated,40.596063,-73.973357 255 | 252,252,F38,IND,6th Av - Culver,Avenue X,Bk,F,Elevated,40.58962,-73.97425 256 | 253,253,F39,IND,6th Av - Culver,Neptune Av,Bk,F,Elevated,40.581011,-73.974574 257 | 254,254,F01,IND,Queens Blvd,Jamaica - 179 St,Q,F,Subway,40.712646,-73.783817 258 | 255,255,F02,IND,Queens Blvd,169 St,Q,F,Subway,40.71047,-73.793604 259 | 256,256,F03,IND,Queens Blvd,Parsons Blvd,Q,F,Subway,40.707564,-73.803326 260 | 257,257,F04,IND,Queens Blvd,Sutphin Blvd,Q,F,Subway,40.70546,-73.810708 261 | 258,258,F05,IND,Queens Blvd,Briarwood - Van Wyck Blvd,Q,E F,Subway,40.709179,-73.820574 262 | 259,259,F06,IND,Queens Blvd,Kew Gardens - Union Tpke,Q,E F,Subway,40.714441,-73.831008 263 | 260,260,F07,IND,Queens Blvd,75 Av,Q,E F,Subway,40.718331,-73.837324 264 | 261,261,G08,IND,Queens Blvd,Forest Hills - 71 Av,Q,E F M R,Subway,40.721691,-73.844521 265 | 262,262,G09,IND,Queens Blvd,67 Av,Q,M R,Subway,40.726523,-73.852719 266 | 263,263,G10,IND,Queens Blvd,63 Dr - Rego Park,Q,M R,Subway,40.729846,-73.861604 267 | 264,264,G11,IND,Queens Blvd,Woodhaven Blvd,Q,M R,Subway,40.733106,-73.869229 268 | 265,265,G12,IND,Queens Blvd,Grand Av - Newtown,Q,M R,Subway,40.737015,-73.877223 269 | 266,266,G13,IND,Queens Blvd,Elmhurst Av,Q,M R,Subway,40.742454,-73.882017 270 | 267,616,G14,IND,Queens Blvd,Jackson Hts - Roosevelt Av,Q,E F M R,Subway,40.746644,-73.891338 271 | 268,268,G15,IND,Queens Blvd,65 St,Q,M R,Subway,40.749669,-73.898453 272 | 269,269,G16,IND,Queens Blvd,Northern Blvd,Q,M R,Subway,40.752885,-73.906006 273 | 270,270,G18,IND,Queens Blvd,46 St,Q,M R,Subway,40.756312,-73.913333 274 | 271,271,G19,IND,Queens Blvd,Steinway St,Q,M R,Subway,40.756879,-73.92074 275 | 272,272,G20,IND,Queens Blvd,36 St,Q,M R,Subway,40.752039,-73.928781 276 | 273,273,G21,IND,Queens Blvd,Queens Plaza,Q,E M R,Subway,40.748973,-73.937243 277 | 274,606,F09,IND,Queens Blvd,Court Sq,Q,E M,Subway,40.747846,-73.946 278 | 275,612,F11,IND,Queens Blvd,Lexington Av/53 St,M,E M,Subway,40.757552,-73.969055 279 | 276,276,F12,IND,Queens Blvd,5 Av/53 St,M,E M,Subway,40.760167,-73.975224 280 | 277,277,D14,IND,Queens Blvd,7 Av,M,B D E,Subway,40.762862,-73.981637 281 | 278,278,G05,IND,Queens - Archer,Jamaica Center - Parsons/Archer,Q,E J Z,Subway,40.702147,-73.801109 282 | 279,279,G06,IND,Queens - Archer,Sutphin Blvd - Archer Av - JFK Airport,Q,E J Z,Subway,40.700486,-73.807969 283 | 280,280,G07,IND,Queens - Archer,Jamaica - Van Wyck,Q,E,Subway,40.702566,-73.816859 284 | 281,606,G22,IND,Crosstown,Court Sq,Q,G,Subway,40.746554,-73.943832 285 | 282,282,G24,IND,Crosstown,21 St,Q,G,Subway,40.744065,-73.949724 286 | 283,283,G26,IND,Crosstown,Greenpoint Av,Bk,G,Subway,40.731352,-73.954449 287 | 284,284,G28,IND,Crosstown,Nassau Av,Bk,G,Subway,40.724635,-73.951277 288 | 285,629,G29,IND,Crosstown,Metropolitan Av,Bk,G,Subway,40.712792,-73.951418 289 | 286,286,G30,IND,Crosstown,Broadway,Bk,G,Subway,40.706092,-73.950308 290 | 287,287,G31,IND,Crosstown,Flushing Av,Bk,G,Subway,40.700377,-73.950234 291 | 288,288,G32,IND,Crosstown,Myrtle - Willoughby Avs,Bk,G,Subway,40.694568,-73.949046 292 | 289,289,G33,IND,Crosstown,Bedford - Nostrand Avs,Bk,G,Subway,40.689627,-73.953522 293 | 290,290,G34,IND,Crosstown,Classon Av,Bk,G,Subway,40.688873,-73.96007 294 | 291,291,G35,IND,Crosstown,Clinton - Washington Avs,Bk,G,Subway,40.688089,-73.966839 295 | 292,292,G36,IND,Crosstown,Fulton St,Bk,G,Subway,40.687119,-73.975375 296 | 293,293,101,IRT,Broadway - 7Av,Van Cortlandt Park - 242 St,Bx,1,Elevated,40.889248,-73.898583 297 | 294,294,103,IRT,Broadway - 7Av,238 St,Bx,1,Elevated,40.884667,-73.90087 298 | 295,295,104,IRT,Broadway - 7Av,231 St,Bx,1,Elevated,40.878856,-73.904834 299 | 296,296,106,IRT,Broadway - 7Av,Marble Hill - 225 St,M,1,Elevated,40.874561,-73.909831 300 | 297,297,107,IRT,Broadway - 7Av,215 St,M,1,Elevated,40.869444,-73.915279 301 | 298,298,108,IRT,Broadway - 7Av,207 St,M,1,Elevated,40.864621,-73.918822 302 | 299,299,109,IRT,Broadway - 7Av,Dyckman St,M,1,Elevated,40.860531,-73.925536 303 | 300,300,110,IRT,Broadway - 7Av,191 St,M,1,Subway,40.855225,-73.929412 304 | 301,301,111,IRT,Broadway - 7Av,181 St,M,1,Subway,40.849505,-73.933596 305 | 302,605,112,IRT,Broadway - 7Av,168 St - Washington Hts,M,1,Subway,40.840556,-73.940133 306 | 303,303,113,IRT,Broadway - 7Av,157 St,M,1,Subway,40.834041,-73.94489 307 | 304,304,114,IRT,Broadway - 7Av,145 St,M,1,Subway,40.826551,-73.95036 308 | 305,305,115,IRT,Broadway - 7Av,137 St - City College,M,1,Subway,40.822008,-73.953676 309 | 306,306,116,IRT,Broadway - 7Av,125 St,M,1,Elevated,40.815581,-73.958372 310 | 307,307,117,IRT,Broadway - 7Av,116 St - Columbia University,M,1,Subway,40.807722,-73.96411 311 | 308,308,118,IRT,Broadway - 7Av,Cathedral Pkwy,M,1,Subway,40.803967,-73.966847 312 | 309,309,119,IRT,Broadway - 7Av,103 St,M,1,Subway,40.799446,-73.968379 313 | 310,310,120,IRT,Broadway - 7Av,96 St,M,1 2 3,Subway,40.793919,-73.972323 314 | 311,311,121,IRT,Broadway - 7Av,86 St,M,1,Subway,40.788644,-73.976218 315 | 312,312,122,IRT,Broadway - 7Av,79 St,M,1,Subway,40.783934,-73.979917 316 | 313,313,123,IRT,Broadway - 7Av,72 St,M,1 2 3,Subway,40.778453,-73.98197 317 | 314,314,124,IRT,Broadway - 7Av,66 St - Lincoln Center,M,1,Subway,40.77344,-73.982209 318 | 315,614,125,IRT,Broadway - 7Av,59 St - Columbus Circle,M,1,Subway,40.768247,-73.981929 319 | 316,316,126,IRT,Broadway - 7Av,50 St,M,1,Subway,40.761728,-73.983849 320 | 317,611,127,IRT,Broadway - 7Av,Times Sq - 42 St,M,1 2 3,Subway,40.75529,-73.987495 321 | 318,318,128,IRT,Broadway - 7Av,34 St - Penn Station,M,1 2 3,Subway,40.750373,-73.991057 322 | 319,319,129,IRT,Broadway - 7Av,28 St,M,1,Subway,40.747215,-73.993365 323 | 320,320,130,IRT,Broadway - 7Av,23 St,M,1,Subway,40.744081,-73.995657 324 | 321,321,131,IRT,Broadway - 7Av,18 St,M,1,Subway,40.74104,-73.997871 325 | 322,601,132,IRT,Broadway - 7Av,14 St,M,1 2 3,Subway,40.737826,-74.000201 326 | 323,323,133,IRT,Broadway - 7Av,Christopher St - Sheridan Sq,M,1,Subway,40.733422,-74.002906 327 | 324,324,134,IRT,Broadway - 7Av,Houston St,M,1,Subway,40.728251,-74.005367 328 | 325,325,135,IRT,Broadway - 7Av,Canal St,M,1,Subway,40.722854,-74.006277 329 | 326,326,136,IRT,Broadway - 7Av,Franklin St,M,1,Subway,40.719318,-74.006886 330 | 327,327,137,IRT,Broadway - 7Av,Chambers St,M,1 2 3,Subway,40.715478,-74.009266 331 | 328,328,138,IRT,Broadway - 7Av,Cortlandt St,M,1,Subway,40.711835,-74.012188 332 | 329,329,139,IRT,Broadway - 7Av,Rector St,M,1,Subway,40.707513,-74.013783 333 | 330,635,142,IRT,Broadway - 7Av,South Ferry,M,1,Subway,40.702068,-74.013664 334 | 331,624,228,IRT,Clark St,Park Pl,M,2 3,Subway,40.713051,-74.008811 335 | 332,628,229,IRT,Clark St,Fulton St,M,2 3,Subway,40.709416,-74.006571 336 | 333,333,230,IRT,Clark St,Wall St,M,2 3,Subway,40.706821,-74.0091 337 | 334,334,231,IRT,Clark St,Clark St,Bk,2 3,Subway,40.697466,-73.993086 338 | 335,620,232,IRT,Clark St,Borough Hall,Bk,2 3,Subway,40.693219,-73.989998 339 | 336,336,233,IRT,Eastern Pky,Hoyt St,Bk,2 3,Subway,40.690545,-73.985065 340 | 337,337,234,IRT,Eastern Pky,Nevins St,Bk,2 3 4 5,Subway,40.688246,-73.980492 341 | 338,617,235,IRT,Eastern Pky,Atlantic Av - Barclays Ctr,Bk,2 3,Subway,40.684359,-73.977666 342 | 339,339,236,IRT,Eastern Pky,Bergen St,Bk,2 3,Subway,40.680829,-73.975098 343 | 340,340,237,IRT,Eastern Pky,Grand Army Plaza,Bk,2 3,Subway,40.675235,-73.971046 344 | 341,341,238,IRT,Eastern Pky,Eastern Pkwy - Brooklyn Museum,Bk,2 3,Subway,40.671987,-73.964375 345 | 342,626,239,IRT,Eastern Pky,Franklin Av,Bk,2 3 4 5,Subway,40.670682,-73.958131 346 | 343,343,248,IRT,Eastern Pky,Nostrand Av,Bk,3,Subway,40.669847,-73.950466 347 | 344,344,249,IRT,Eastern Pky,Kingston Av,Bk,3,Subway,40.669399,-73.942161 348 | 345,345,250,IRT,Eastern Pky,Crown Hts - Utica Av,Bk,3 4,Subway,40.668897,-73.932942 349 | 346,346,251,IRT,Eastern Pky,Sutter Av - Rutland Rd,Bk,3,Elevated,40.664717,-73.92261 350 | 347,347,252,IRT,Eastern Pky,Saratoga Av,Bk,3,Elevated,40.661453,-73.916327 351 | 348,348,253,IRT,Eastern Pky,Rockaway Av,Bk,3,Elevated,40.662549,-73.908946 352 | 349,349,254,IRT,Eastern Pky,Junius St,Bk,3,Elevated,40.663515,-73.902447 353 | 350,350,255,IRT,Eastern Pky,Pennsylvania Av,Bk,3,Elevated,40.664635,-73.894895 354 | 351,351,256,IRT,Eastern Pky,Van Siclen Av,Bk,3,Elevated,40.665449,-73.889395 355 | 352,352,257,IRT,Eastern Pky,New Lots Av,Bk,3,Elevated,40.666235,-73.884079 356 | 353,353,241,IRT,Nostrand,President St,Bk,2 5,Subway,40.667883,-73.950683 357 | 354,354,242,IRT,Nostrand,Sterling St,Bk,2 5,Subway,40.662742,-73.95085 358 | 355,355,243,IRT,Nostrand,Winthrop St,Bk,2 5,Subway,40.656652,-73.9502 359 | 356,356,244,IRT,Nostrand,Church Av,Bk,2 5,Subway,40.650843,-73.949575 360 | 357,357,245,IRT,Nostrand,Beverly Rd,Bk,2 5,Subway,40.645098,-73.948959 361 | 358,358,246,IRT,Nostrand,Newkirk Av,Bk,2 5,Subway,40.639967,-73.948411 362 | 359,359,247,IRT,Nostrand,Flatbush Av - Brooklyn College,Bk,2 5,Subway,40.632836,-73.947642 363 | 360,360,601,IRT,Pelham,Pelham Bay Park,Bx,6,Elevated,40.852462,-73.828121 364 | 361,361,602,IRT,Pelham,Buhre Av,Bx,6,Elevated,40.84681,-73.832569 365 | 362,362,603,IRT,Pelham,Middletown Rd,Bx,6,Elevated,40.843863,-73.836322 366 | 363,363,604,IRT,Pelham,Westchester Sq - E Tremont Av,Bx,6,Elevated,40.839892,-73.842952 367 | 364,364,606,IRT,Pelham,Zerega Av,Bx,6,Elevated,40.836488,-73.847036 368 | 365,365,607,IRT,Pelham,Castle Hill Av,Bx,6,Elevated,40.834255,-73.851222 369 | 366,366,608,IRT,Pelham,Parkchester,Bx,6,Elevated,40.833226,-73.860816 370 | 367,367,609,IRT,Pelham,St Lawrence Av,Bx,6,Elevated,40.831509,-73.867618 371 | 368,368,610,IRT,Pelham,Morrison Av- Sound View,Bx,6,Elevated,40.829521,-73.874516 372 | 369,369,611,IRT,Pelham,Elder Av,Bx,6,Elevated,40.828584,-73.879159 373 | 370,370,612,IRT,Pelham,Whitlock Av,Bx,6,Elevated,40.826525,-73.886283 374 | 371,371,613,IRT,Pelham,Hunts Point Av,Bx,6,Subway,40.820948,-73.890549 375 | 372,372,614,IRT,Pelham,Longwood Av,Bx,6,Subway,40.816104,-73.896435 376 | 373,373,615,IRT,Pelham,E 149 St,Bx,6,Subway,40.812118,-73.904098 377 | 374,374,616,IRT,Pelham,E 143 St - St Mary's St,Bx,6,Subway,40.808719,-73.907657 378 | 375,375,617,IRT,Pelham,Cypress Av,Bx,6,Subway,40.805368,-73.914042 379 | 376,376,618,IRT,Pelham,Brook Av,Bx,6,Subway,40.807566,-73.91924 380 | 377,377,619,IRT,Pelham,3 Av - 138 St,Bx,6,Subway,40.810476,-73.926138 381 | 378,378,401,IRT,Jerome Av,Woodlawn,Bx,4,Elevated,40.886037,-73.878751 382 | 379,379,402,IRT,Jerome Av,Mosholu Pkwy,Bx,4,Elevated,40.87975,-73.884655 383 | 380,380,405,IRT,Jerome Av,Bedford Park Blvd - Lehman College,Bx,4,Elevated,40.873412,-73.890064 384 | 381,381,406,IRT,Jerome Av,Kingsbridge Rd,Bx,4,Elevated,40.86776,-73.897174 385 | 382,382,407,IRT,Jerome Av,Fordham Rd,Bx,4,Elevated,40.862803,-73.901034 386 | 383,383,408,IRT,Jerome Av,183 St,Bx,4,Elevated,40.858407,-73.903879 387 | 384,384,409,IRT,Jerome Av,Burnside Av,Bx,4,Elevated,40.853453,-73.907684 388 | 385,385,410,IRT,Jerome Av,176 St,Bx,4,Elevated,40.84848,-73.911794 389 | 386,386,411,IRT,Jerome Av,Mt Eden Av,Bx,4,Elevated,40.844434,-73.914685 390 | 387,387,412,IRT,Jerome Av,170 St,Bx,4,Elevated,40.840075,-73.917791 391 | 388,388,413,IRT,Jerome Av,167 St,Bx,4,Elevated,40.835537,-73.9214 392 | 389,604,414,IRT,Jerome Av,161 St - Yankee Stadium,Bx,4,Elevated,40.827994,-73.925831 393 | 390,603,415,IRT,Jerome Av,149 St - Grand Concourse,Bx,4,Subway,40.818375,-73.927351 394 | 391,391,416,IRT,Jerome Av,138 St - Grand Concourse,Bx,4 5,Subway,40.813224,-73.929849 395 | 392,392,621,IRT,Lexington Av,125 St,M,4 5 6,Subway,40.804138,-73.937594 396 | 393,393,622,IRT,Lexington Av,116 St,M,6,Subway,40.798629,-73.941617 397 | 394,394,623,IRT,Lexington Av,110 St,M,6,Subway,40.79502,-73.94425 398 | 395,395,624,IRT,Lexington Av,103 St,M,6,Subway,40.7906,-73.947478 399 | 396,396,625,IRT,Lexington Av,96 St,M,6,Subway,40.785672,-73.95107 400 | 397,397,626,IRT,Lexington Av,86 St,M,4 5 6,Subway,40.779492,-73.955589 401 | 398,398,627,IRT,Lexington Av,77 St,M,6,Subway,40.77362,-73.959874 402 | 399,399,628,IRT,Lexington Av,68 St - Hunter College,M,6,Subway,40.768141,-73.96387 403 | 400,613,629,IRT,Lexington Av,59 St,M,4 5 6,Subway,40.762526,-73.967967 404 | 401,612,630,IRT,Lexington Av,51 St,M,6,Subway,40.757107,-73.97192 405 | 402,610,631,IRT,Lexington Av,Grand Central - 42 St,M,4 5 6,Subway,40.751776,-73.976848 406 | 403,403,632,IRT,Lexington Av,33 St,M,6,Subway,40.746081,-73.982076 407 | 404,404,633,IRT,Lexington Av,28 St,M,6,Subway,40.74307,-73.984264 408 | 405,405,634,IRT,Lexington Av,23 St,M,6,Subway,40.739864,-73.986599 409 | 406,602,635,IRT,Lexington Av,14 St - Union Sq,M,4 5 6,Subway,40.734673,-73.989951 410 | 407,407,636,IRT,Lexington Av,Astor Pl,M,6,Subway,40.730054,-73.99107 411 | 408,619,637,IRT,Lexington Av,Bleecker St,M,6,Subway,40.725915,-73.994659 412 | 409,409,638,IRT,Lexington Av,Spring St,M,6,Subway,40.722301,-73.997141 413 | 410,623,639,IRT,Lexington Av,Canal St,M,6,Subway,40.718803,-74.000193 414 | 411,622,640,IRT,Lexington Av,Brooklyn Bridge - City Hall,M,4 5 6,Subway,40.713065,-74.004131 415 | 412,628,418,IRT,Lexington Av,Fulton St,M,4 5,Subway,40.710368,-74.009509 416 | 413,413,419,IRT,Lexington Av,Wall St,M,4 5,Subway,40.707557,-74.011862 417 | 414,414,420,IRT,Lexington Av,Bowling Green,M,4 5,Subway,40.704817,-74.014065 418 | 415,620,423,IRT,Eastern Pky,Borough Hall,Bk,4 5,Subway,40.692404,-73.990151 419 | 416,416,201,IRT,Lenox - White Plains Rd,Wakefield - 241 St,Bx,2,Elevated,40.903125,-73.85062 420 | 417,417,204,IRT,Lenox - White Plains Rd,Nereid Av,Bx,2 5,Elevated,40.898379,-73.854376 421 | 418,418,205,IRT,Lenox - White Plains Rd,233 St,Bx,2 5,Elevated,40.893193,-73.857473 422 | 419,419,206,IRT,Lenox - White Plains Rd,225 St,Bx,2 5,Elevated,40.888022,-73.860341 423 | 420,420,207,IRT,Lenox - White Plains Rd,219 St,Bx,2 5,Elevated,40.883895,-73.862633 424 | 421,421,208,IRT,Lenox - White Plains Rd,Gun Hill Rd,Bx,2 5,Elevated,40.87785,-73.866256 425 | 422,422,209,IRT,Lenox - White Plains Rd,Burke Av,Bx,2 5,Elevated,40.871356,-73.867164 426 | 423,423,210,IRT,Lenox - White Plains Rd,Allerton Av,Bx,2 5,Elevated,40.865462,-73.867352 427 | 424,424,211,IRT,Lenox - White Plains Rd,Pelham Pkwy,Bx,2 5,Elevated,40.857192,-73.867615 428 | 425,425,212,IRT,Lenox - White Plains Rd,Bronx Park East,Bx,2 5,Elevated,40.848828,-73.868457 429 | 426,426,213,IRT,Lenox - White Plains Rd,E 180 St,Bx,2 5,Elevated,40.841894,-73.873488 430 | 427,427,214,IRT,Lenox - White Plains Rd,West Farms Sq - E Tremont Av,Bx,2 5,Elevated,40.840295,-73.880049 431 | 428,428,215,IRT,Lenox - White Plains Rd,174 St,Bx,2 5,Elevated,40.837288,-73.887734 432 | 429,429,216,IRT,Lenox - White Plains Rd,Freeman St,Bx,2 5,Elevated,40.829993,-73.891865 433 | 430,430,217,IRT,Lenox - White Plains Rd,Simpson St,Bx,2 5,Elevated,40.824073,-73.893064 434 | 431,431,218,IRT,Lenox - White Plains Rd,Intervale Av,Bx,2 5,Elevated,40.822181,-73.896736 435 | 432,432,219,IRT,Lenox - White Plains Rd,Prospect Av,Bx,2 5,Elevated,40.819585,-73.90177 436 | 433,433,220,IRT,Lenox - White Plains Rd,Jackson Av,Bx,2 5,Elevated,40.81649,-73.907807 437 | 434,434,221,IRT,Lenox - White Plains Rd,3 Av - 149 St,Bx,2 5,Subway,40.816109,-73.917757 438 | 435,603,222,IRT,Lenox - White Plains Rd,149 St - Grand Concourse,Bx,2 5,Subway,40.81841,-73.926718 439 | 436,436,301,IRT,Lenox - White Plains Rd,Harlem - 148 St,M,3,Subway,40.82388,-73.93647 440 | 437,437,302,IRT,Lenox - White Plains Rd,145 St,M,3,Subway,40.820421,-73.936245 441 | 438,438,224,IRT,Lenox - White Plains Rd,135 St,M,2 3,Subway,40.814229,-73.94077 442 | 439,439,225,IRT,Lenox - White Plains Rd,125 St,M,2 3,Subway,40.807754,-73.945495 443 | 440,440,226,IRT,Lenox - White Plains Rd,116 St,M,2 3,Subway,40.802098,-73.949625 444 | 441,441,227,IRT,Lenox - White Plains Rd,Central Park North (110 St),M,2 3,Subway,40.799075,-73.951822 445 | 442,442,501,IRT,Dyre Av,Eastchester - Dyre Av,Bx,5,At Grade,40.8883,-73.830834 446 | 443,443,502,IRT,Dyre Av,Baychester Av,Bx,5,Open Cut,40.878663,-73.838591 447 | 444,444,503,IRT,Dyre Av,Gun Hill Rd,Bx,5,Open Cut,40.869526,-73.846384 448 | 445,445,504,IRT,Dyre Av,Pelham Pkwy,Bx,5,Open Cut,40.858985,-73.855359 449 | 446,446,505,IRT,Dyre Av,Morris Park,Bx,5,Open Cut,40.854364,-73.860495 450 | 447,447,701,IRT,Flushing,Flushing - Main St,Q,7,Subway,40.7596,-73.83003 451 | 448,448,702,IRT,Flushing,Mets - Willets Point,Q,7,Elevated,40.754622,-73.845625 452 | 449,449,705,IRT,Flushing,111 St,Q,7,Elevated,40.75173,-73.855334 453 | 450,450,706,IRT,Flushing,103 St - Corona Plaza,Q,7,Elevated,40.749865,-73.8627 454 | 451,451,707,IRT,Flushing,Junction Blvd,Q,7,Elevated,40.749145,-73.869527 455 | 452,452,708,IRT,Flushing,90 St - Elmhurst Av,Q,7,Elevated,40.748408,-73.876613 456 | 453,453,709,IRT,Flushing,82 St - Jackson Hts,Q,7,Elevated,40.747659,-73.883697 457 | 454,616,710,IRT,Flushing,74 St - Broadway,Q,7,Elevated,40.746848,-73.891394 458 | 455,455,711,IRT,Flushing,69 St,Q,7,Elevated,40.746325,-73.896403 459 | 456,456,712,IRT,Flushing,Woodside - 61 St,Q,7,Elevated,40.74563,-73.902984 460 | 457,457,713,IRT,Flushing,52 St,Q,7,Elevated,40.744149,-73.912549 461 | 458,458,714,IRT,Flushing,46 St,Q,7,Elevated,40.743132,-73.918435 462 | 459,459,715,IRT,Flushing,40 St,Q,7,Elevated,40.743781,-73.924016 463 | 460,460,716,IRT,Flushing,33 St,Q,7,Elevated,40.744587,-73.930997 464 | 461,461,718,IRT,Flushing,Queensboro Plaza,Q,7,Elevated,40.750582,-73.940202 465 | 461,461,R09,BMT,Astoria,Queensboro Plaza,Q,N W,Elevated,40.750582,-73.940202 466 | 462,606,719,IRT,Flushing,Court Sq,Q,7,Elevated,40.747023,-73.945264 467 | 463,463,720,IRT,Flushing,Hunters Point Av,Q,7,Subway,40.742216,-73.948916 468 | 464,464,721,IRT,Flushing,Vernon Blvd - Jackson Av,Q,7,Subway,40.742626,-73.953581 469 | 465,610,723,IRT,Flushing,Grand Central - 42 St,M,7,Subway,40.751431,-73.976041 470 | 466,609,724,IRT,Flushing,5 Av,M,7,Subway,40.753821,-73.981963 471 | 467,611,725,IRT,Flushing,Times Sq - 42 St,M,7,Subway,40.755477,-73.987691 472 | 468,611,902,IRT,Lexington - Shuttle,Times Sq - 42 St,M,S,Subway,40.755983,-73.986229 473 | 469,610,901,IRT,Lexington - Shuttle,Grand Central - 42 St,M,S,Subway,40.752769,-73.979189 474 | 471,471,726,IRT,Flushing,34 St - 11 Av,M,7,Subway,40.755882,-74.00191 475 | 475,475,Q05,IND,Second Av,96 St,M,Q,Subway,40.784318,-73.947152 476 | 476,476,Q04,IND,Second Av,86 St,M,Q,Subway,40.777891,-73.951787 477 | 477,477,Q03,IND,Second Av,72 St,M,Q,Subway,40.768799,-73.958424 478 | 501,501,S31,SIR,Staten Island,St George,SI,SIR,Open Cut,40.643748,-74.073643 479 | 502,502,S30,SIR,Staten Island,Tompkinsville,SI,SIR,At Grade,40.636949,-74.074835 480 | 503,503,S29,SIR,Staten Island,Stapleton,SI,SIR,Elevated,40.627915,-74.075162 481 | 504,504,S28,SIR,Staten Island,Clifton,SI,SIR,Elevated,40.621319,-74.071402 482 | 505,505,S27,SIR,Staten Island,Grasmere,SI,SIR,Open Cut,40.603117,-74.084087 483 | 506,506,S26,SIR,Staten Island,Old Town,SI,SIR,Embankment,40.596612,-74.087368 484 | 507,507,S25,SIR,Staten Island,Dongan Hills,SI,SIR,Embankment,40.588849,-74.09609 485 | 508,508,S24,SIR,Staten Island,Jefferson Av,SI,SIR,Embankment,40.583591,-74.103338 486 | 509,509,S23,SIR,Staten Island,Grant City,SI,SIR,Open Cut,40.578965,-74.109704 487 | 510,510,S22,SIR,Staten Island,New Dorp,SI,SIR,Open Cut,40.57348,-74.11721 488 | 511,511,S21,SIR,Staten Island,Oakwood Heights,SI,SIR,Open Cut,40.56511,-74.12632 489 | 512,512,S20,SIR,Staten Island,Bay Terrace,SI,SIR,Embankment,40.5564,-74.136907 490 | 513,513,S19,SIR,Staten Island,Great Kills,SI,SIR,Open Cut,40.551231,-74.151399 491 | 514,514,S18,SIR,Staten Island,Eltingville,SI,SIR,Embankment,40.544601,-74.16457 492 | 515,515,S17,SIR,Staten Island,Annadale,SI,SIR,Open Cut,40.54046,-74.178217 493 | 516,516,S16,SIR,Staten Island,Huguenot,SI,SIR,Open Cut,40.533674,-74.191794 494 | 517,517,S15,SIR,Staten Island,Prince's Bay,SI,SIR,Open Cut,40.525507,-74.200064 495 | 518,518,S14,SIR,Staten Island,Pleasant Plains,SI,SIR,Embankment,40.52241,-74.217847 496 | 519,519,S13,SIR,Staten Island,Richmond Valley,SI,SIR,Open Cut,40.519631,-74.229141 497 | 522,522,S09,SIR,Staten Island,Tottenville,SI,SIR,At Grade,40.512764,-74.251961 498 | 523,523,S11,SIR,Staten Island,Arthur Kill,SI,SIR,At Grade,40.516578,-74.242096 --------------------------------------------------------------------------------