├── .gitattributes ├── .gitignore ├── LICENSE ├── README.md ├── dend ├── README.md ├── data_lake_with_spark │ ├── README.md │ ├── architecture.jpg │ ├── create_cluster.sh │ ├── default_emr_roles.png │ ├── etl.py │ ├── spartify.ipynb │ └── tests │ │ ├── README.md │ │ ├── app.py │ │ └── cities.csv ├── data_modeling_with_apache_cassandra │ ├── README.md │ ├── data_modeling_with_apache_cassandra.ipynb │ ├── event_data │ │ ├── 2018-11-01-events.csv │ │ ├── 2018-11-02-events.csv │ │ ├── 2018-11-03-events.csv │ │ ├── 2018-11-04-events.csv │ │ ├── 2018-11-05-events.csv │ │ ├── 2018-11-06-events.csv │ │ ├── 2018-11-07-events.csv │ │ ├── 2018-11-08-events.csv │ │ ├── 2018-11-09-events.csv │ │ ├── 2018-11-10-events.csv │ │ ├── 2018-11-11-events.csv │ │ ├── 2018-11-12-events.csv │ │ ├── 2018-11-13-events.csv │ │ ├── 2018-11-14-events.csv │ │ ├── 2018-11-15-events.csv │ │ ├── 2018-11-16-events.csv │ │ ├── 2018-11-17-events.csv │ │ ├── 2018-11-18-events.csv │ │ ├── 2018-11-19-events.csv │ │ ├── 2018-11-20-events.csv │ │ ├── 2018-11-21-events.csv │ │ ├── 2018-11-22-events.csv │ │ ├── 2018-11-23-events.csv │ │ ├── 2018-11-24-events.csv │ │ ├── 2018-11-25-events.csv │ │ ├── 2018-11-26-events.csv │ │ ├── 2018-11-27-events.csv │ │ ├── 2018-11-28-events.csv │ │ ├── 2018-11-29-events.csv │ │ └── 2018-11-30-events.csv │ └── event_datafile_new.csv ├── data_modeling_with_postgres │ ├── README.md │ ├── create_tables.py │ ├── data │ │ ├── log_data │ │ │ └── 2018 │ │ │ │ └── 11 │ │ │ │ ├── 2018-11-01-events.json │ │ │ │ ├── 2018-11-02-events.json │ │ │ │ ├── 2018-11-03-events.json │ │ │ │ ├── 2018-11-04-events.json │ │ │ │ ├── 2018-11-05-events.json │ │ │ │ ├── 2018-11-06-events.json │ │ │ │ ├── 2018-11-07-events.json │ │ │ │ ├── 2018-11-08-events.json │ │ │ │ ├── 2018-11-09-events.json │ │ │ │ ├── 2018-11-10-events.json │ │ │ │ ├── 2018-11-11-events.json │ │ │ │ ├── 2018-11-12-events.json │ │ │ │ ├── 2018-11-13-events.json │ │ │ │ ├── 2018-11-14-events.json │ │ │ │ ├── 2018-11-15-events.json │ │ │ │ ├── 2018-11-16-events.json │ │ │ │ ├── 2018-11-17-events.json │ │ │ │ ├── 2018-11-18-events.json │ │ │ │ ├── 2018-11-19-events.json │ │ │ │ ├── 2018-11-20-events.json │ │ │ │ ├── 2018-11-21-events.json │ │ │ │ ├── 2018-11-22-events.json │ │ │ │ ├── 2018-11-23-events.json │ │ │ │ ├── 2018-11-24-events.json │ │ │ │ ├── 2018-11-25-events.json │ │ │ │ ├── 2018-11-26-events.json │ │ │ │ ├── 2018-11-27-events.json │ │ │ │ ├── 2018-11-28-events.json │ │ │ │ ├── 2018-11-29-events.json │ │ │ │ └── 2018-11-30-events.json │ │ └── song_data │ │ │ └── A │ │ │ ├── A │ │ │ ├── A │ │ │ │ ├── TRAAAAW128F429D538.json │ │ │ │ ├── TRAAABD128F429CF47.json │ │ │ │ ├── TRAAADZ128F9348C2E.json │ │ │ │ ├── TRAAAEF128F4273421.json │ │ │ │ ├── TRAAAFD128F92F423A.json │ │ │ │ ├── TRAAAMO128F1481E7F.json │ │ │ │ ├── TRAAAMQ128F1460CD3.json │ │ │ │ ├── TRAAAPK128E0786D96.json │ │ │ │ ├── TRAAARJ128F9320760.json │ │ │ │ ├── TRAAAVG12903CFA543.json │ │ │ │ └── TRAAAVO128F93133D4.json │ │ │ ├── B │ │ │ │ ├── TRAABCL128F4286650.json │ │ │ │ ├── TRAABDL12903CAABBA.json │ │ │ │ ├── TRAABJL12903CDCF1A.json │ │ │ │ ├── TRAABJV128F1460C49.json │ │ │ │ ├── TRAABLR128F423B7E3.json │ │ │ │ ├── TRAABNV128F425CEE1.json │ │ │ │ ├── TRAABRB128F9306DD5.json │ │ │ │ ├── TRAABVM128F92CA9DC.json │ │ │ │ ├── TRAABXG128F9318EBD.json │ │ │ │ ├── TRAABYN12903CFD305.json │ │ │ │ └── TRAABYW128F4244559.json │ │ │ └── C │ │ │ │ ├── TRAACCG128F92E8A55.json │ │ │ │ ├── TRAACER128F4290F96.json │ │ │ │ ├── TRAACFV128F935E50B.json │ │ │ │ ├── TRAACHN128F1489601.json │ │ │ │ ├── TRAACIW12903CC0F6D.json │ │ │ │ ├── TRAACLV128F427E123.json │ │ │ │ ├── TRAACNS128F14A2DF5.json │ │ │ │ ├── TRAACOW128F933E35F.json │ │ │ │ ├── TRAACPE128F421C1B9.json │ │ │ │ ├── TRAACQT128F9331780.json │ │ │ │ ├── TRAACSL128F93462F4.json │ │ │ │ ├── TRAACTB12903CAAF15.json │ │ │ │ ├── TRAACVS128E078BE39.json │ │ │ │ └── TRAACZK128F4243829.json │ │ │ └── B │ │ │ ├── A │ │ │ ├── TRABACN128F425B784.json │ │ │ ├── TRABAFJ128F42AF24E.json │ │ │ ├── TRABAFP128F931E9A1.json │ │ │ ├── TRABAIO128F42938F9.json │ │ │ ├── TRABATO128F42627E9.json │ │ │ ├── TRABAVQ12903CBF7E0.json │ │ │ ├── TRABAWW128F4250A31.json │ │ │ ├── TRABAXL128F424FC50.json │ │ │ ├── TRABAXR128F426515F.json │ │ │ ├── TRABAXV128F92F6AE3.json │ │ │ └── TRABAZH128F930419A.json │ │ │ ├── B │ │ │ ├── TRABBAM128F429D223.json │ │ │ ├── TRABBBV128F42967D7.json │ │ │ ├── TRABBJE12903CDB442.json │ │ │ ├── TRABBKX128F4285205.json │ │ │ ├── TRABBLU128F93349CF.json │ │ │ ├── TRABBNP128F932546F.json │ │ │ ├── TRABBOP128F931B50D.json │ │ │ ├── TRABBOR128F4286200.json │ │ │ ├── TRABBTA128F933D304.json │ │ │ ├── TRABBVJ128F92F7EAA.json │ │ │ ├── TRABBXU128F92FEF48.json │ │ │ └── TRABBZN12903CD9297.json │ │ │ └── C │ │ │ ├── TRABCAJ12903CDFCC2.json │ │ │ ├── TRABCEC128F426456E.json │ │ │ ├── TRABCEI128F424C983.json │ │ │ ├── TRABCFL128F149BB0D.json │ │ │ ├── TRABCIX128F4265903.json │ │ │ ├── TRABCKL128F423A778.json │ │ │ ├── TRABCPZ128F4275C32.json │ │ │ ├── TRABCRU128F423F449.json │ │ │ ├── TRABCTK128F934B224.json │ │ │ ├── TRABCUQ128E0783E2B.json │ │ │ ├── TRABCXB128F4286BD3.json │ │ │ └── TRABCYE128F934CE1D.json │ ├── etl.ipynb │ ├── etl.py │ └── sql_queries.py ├── data_pipeline_with_airflow │ ├── .env │ ├── .gitignore │ ├── README.md │ ├── architecture.jpg │ ├── create_tables.sql │ ├── dags │ │ └── sparkify_pipeline.py │ ├── logs │ │ └── .gitkeep │ ├── pipeline.png │ └── plugins │ │ ├── data_quality.py │ │ ├── helpers │ │ └── sql_queries.py │ │ ├── load_dimension.py │ │ ├── load_fact.py │ │ └── stage_redshift.py ├── data_warehouse_with_redshift │ ├── README.md │ ├── architecture.jpg │ ├── config.ini │ ├── create_tables.py │ ├── etl.py │ ├── run_on_aws.py │ ├── sql_queries.py │ └── summarize_data.py └── dend_capstone_project │ ├── .gitignore │ ├── Dockerfile │ ├── README.md │ ├── airflow │ ├── .env │ ├── dags │ │ └── capstone_pipeline.py │ ├── logs │ │ └── .gitkeep │ ├── pipeline.jpg │ └── plugins │ │ ├── data_quality.py │ │ └── load_to_redshift.py │ ├── architecture.jpg │ ├── data_model.png │ ├── etl │ ├── etl.py │ └── start_emr_cluster.sh │ ├── redshift │ ├── config.ini │ ├── sql_queries.py │ └── start_db_on_redshift.py │ └── workspace │ ├── capital_bikeshare_system_map.png │ ├── data_exploration.ipynb │ └── datasets │ ├── capitalbikeshare_tripdata │ ├── new │ │ └── .gitkeep │ └── old │ │ └── .gitkeep │ ├── covid_data │ └── .gitkeep │ └── weather_data │ └── .gitkeep ├── dev_envs ├── airflow_docker │ ├── .env │ ├── .gitignore │ ├── README.md │ ├── airflow.sh │ ├── dags │ │ ├── aws_s3_to_redshift.py │ │ ├── bicycle_sharing_example.py │ │ ├── connections_and_hooks.py │ │ ├── context_and_templating.py │ │ ├── hello_world.py │ │ ├── sql_statements.py │ │ └── task_dependencies.py │ ├── docker-compose.yaml │ ├── logs │ │ └── .gitkeep │ └── plugins │ │ ├── facts_calculator.py │ │ ├── has_rows.py │ │ └── s3_to_redshift.py ├── kafka_docker │ ├── README.md │ ├── docker-compose.yml │ ├── java │ │ ├── .gitignore │ │ ├── README.md │ │ ├── config.properties │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ ├── ClientBase.java │ │ │ ├── ConsumerExample.java │ │ │ └── ProducerExample.java │ └── python │ │ ├── README.md │ │ ├── config.ini │ │ ├── example.py │ │ ├── example_with_avro.py │ │ └── requirements.txt └── spark_docker │ ├── .env │ ├── .gitignore │ ├── README.md │ ├── build.sh │ ├── cluster-base.Dockerfile │ ├── docker-compose.yml │ ├── java │ ├── Dockerfile │ ├── README.md │ └── java-app │ │ ├── pom.xml │ │ └── src │ │ └── main │ │ └── java │ │ └── LowerCase.java │ ├── jupyterlab.Dockerfile │ ├── maven.Dockerfile │ ├── pyspark.Dockerfile │ ├── python │ ├── Dockerfile │ ├── README.md │ └── python-app │ │ └── app.py │ ├── scala │ ├── Dockerfile │ ├── README.md │ └── scala-app │ │ ├── pom.xml │ │ └── src │ │ └── main │ │ └── scala │ │ └── LowerCase.scala │ ├── spark-base.Dockerfile │ ├── spark-master.Dockerfile │ ├── spark-worker.Dockerfile │ ├── spark.env │ └── workspace │ └── spark_test.ipynb └── dsnd ├── README.md ├── stedi_app_with_spark_streaming ├── README.md ├── architecture.jpg ├── docker-compose.yaml ├── event_kafka_stream.py ├── kafka_join.py ├── redis_kafka_stream.py ├── screenshots │ ├── risk_graph1.jpg │ └── risk_graph2.jpg ├── spark │ ├── .gitignore │ └── logs │ │ ├── event_stream.log │ │ ├── kafka_join.log │ │ ├── redis_stream.log │ │ ├── spark-master.log │ │ └── spark-worker.log ├── submit-event-kafka-stream.sh ├── submit-kafka-join.sh └── submit-redis-kafka-stream.sh └── transit_status_tracker_with_kafka ├── README.md ├── architecture.jpg ├── config.ini ├── docker-compose.yaml ├── load_stations.sql ├── logging.ini ├── misc └── screenshot.jpg ├── requirements.txt ├── simple_consumer.py ├── src ├── __init__.py ├── config.py ├── consumers │ ├── __init__.py │ ├── consumer.py │ ├── faust_stream.py │ ├── ksql.py │ ├── logger.py │ ├── server.py │ └── templates │ │ └── status.html ├── producers │ ├── __init__.py │ ├── connector.py │ ├── data │ │ ├── README.md │ │ ├── cta_stations.csv │ │ ├── ridership_curve.csv │ │ └── ridership_seed.csv │ ├── logger.py │ ├── models │ │ ├── __init__.py │ │ ├── line.py │ │ ├── producer.py │ │ ├── schemas │ │ │ ├── arrival_key.json │ │ │ ├── arrival_value.json │ │ │ ├── turnstile_key.json │ │ │ ├── turnstile_value.json │ │ │ ├── weather_key.json │ │ │ └── weather_value.json │ │ ├── station.py │ │ ├── timer.py │ │ ├── train.py │ │ ├── turnstile.py │ │ └── weather.py │ ├── simulation.py │ └── test │ │ ├── __init__.py │ │ └── test_simulator.py └── utils.py ├── start_faust_app.py ├── start_ksql.py ├── start_server.py └── start_simulation.py /.gitattributes: -------------------------------------------------------------------------------- 1 | *.ipynb linguist-vendored -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/README.md -------------------------------------------------------------------------------- /dend/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/README.md -------------------------------------------------------------------------------- /dend/data_lake_with_spark/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_lake_with_spark/README.md -------------------------------------------------------------------------------- /dend/data_lake_with_spark/architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_lake_with_spark/architecture.jpg -------------------------------------------------------------------------------- /dend/data_lake_with_spark/create_cluster.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_lake_with_spark/create_cluster.sh -------------------------------------------------------------------------------- /dend/data_lake_with_spark/default_emr_roles.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_lake_with_spark/default_emr_roles.png -------------------------------------------------------------------------------- /dend/data_lake_with_spark/etl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_lake_with_spark/etl.py -------------------------------------------------------------------------------- /dend/data_lake_with_spark/spartify.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_lake_with_spark/spartify.ipynb -------------------------------------------------------------------------------- /dend/data_lake_with_spark/tests/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_lake_with_spark/tests/README.md -------------------------------------------------------------------------------- /dend/data_lake_with_spark/tests/app.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_lake_with_spark/tests/app.py -------------------------------------------------------------------------------- /dend/data_lake_with_spark/tests/cities.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_lake_with_spark/tests/cities.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/README.md -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/data_modeling_with_apache_cassandra.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/data_modeling_with_apache_cassandra.ipynb -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-01-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-01-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-02-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-02-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-03-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-03-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-04-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-04-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-05-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-05-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-06-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-06-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-07-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-07-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-08-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-08-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-09-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-09-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-10-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-10-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-11-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-11-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-12-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-12-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-13-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-13-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-14-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-14-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-15-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-15-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-16-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-16-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-17-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-17-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-18-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-18-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-19-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-19-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-20-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-20-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-21-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-21-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-22-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-22-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-23-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-23-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-24-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-24-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-25-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-25-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-26-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-26-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-27-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-27-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-28-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-28-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-29-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-29-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_data/2018-11-30-events.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_data/2018-11-30-events.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_apache_cassandra/event_datafile_new.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_apache_cassandra/event_datafile_new.csv -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/README.md -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/create_tables.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/create_tables.py -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-01-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-01-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-02-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-02-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-03-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-03-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-04-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-04-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-05-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-05-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-06-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-06-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-07-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-07-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-08-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-08-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-09-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-09-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-10-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-10-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-11-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-11-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-12-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-12-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-13-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-13-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-14-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-14-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-15-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-15-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-16-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-16-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-17-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-17-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-18-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-18-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-19-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-19-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-20-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-20-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-21-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-21-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-22-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-22-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-23-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-23-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-24-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-24-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-25-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-25-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-26-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-26-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-27-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-27-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-28-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-28-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-29-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-29-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-30-events.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/log_data/2018/11/2018-11-30-events.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/A/TRAAAAW128F429D538.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/A/TRAAAAW128F429D538.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/A/TRAAABD128F429CF47.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/A/TRAAABD128F429CF47.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/A/TRAAADZ128F9348C2E.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/A/TRAAADZ128F9348C2E.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/A/TRAAAEF128F4273421.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/A/TRAAAEF128F4273421.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/A/TRAAAFD128F92F423A.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/A/TRAAAFD128F92F423A.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/A/TRAAAMO128F1481E7F.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/A/TRAAAMO128F1481E7F.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/A/TRAAAMQ128F1460CD3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/A/TRAAAMQ128F1460CD3.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/A/TRAAAPK128E0786D96.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/A/TRAAAPK128E0786D96.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/A/TRAAARJ128F9320760.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/A/TRAAARJ128F9320760.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/A/TRAAAVG12903CFA543.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/A/TRAAAVG12903CFA543.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/A/TRAAAVO128F93133D4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/A/TRAAAVO128F93133D4.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/B/TRAABCL128F4286650.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/B/TRAABCL128F4286650.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/B/TRAABDL12903CAABBA.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/B/TRAABDL12903CAABBA.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/B/TRAABJL12903CDCF1A.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/B/TRAABJL12903CDCF1A.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/B/TRAABJV128F1460C49.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/B/TRAABJV128F1460C49.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/B/TRAABLR128F423B7E3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/B/TRAABLR128F423B7E3.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/B/TRAABNV128F425CEE1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/B/TRAABNV128F425CEE1.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/B/TRAABRB128F9306DD5.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/B/TRAABRB128F9306DD5.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/B/TRAABVM128F92CA9DC.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/B/TRAABVM128F92CA9DC.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/B/TRAABXG128F9318EBD.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/B/TRAABXG128F9318EBD.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/B/TRAABYN12903CFD305.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/B/TRAABYN12903CFD305.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/B/TRAABYW128F4244559.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/B/TRAABYW128F4244559.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACCG128F92E8A55.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACCG128F92E8A55.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACER128F4290F96.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACER128F4290F96.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACFV128F935E50B.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACFV128F935E50B.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACHN128F1489601.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACHN128F1489601.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACIW12903CC0F6D.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACIW12903CC0F6D.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACLV128F427E123.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACLV128F427E123.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACNS128F14A2DF5.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACNS128F14A2DF5.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACOW128F933E35F.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACOW128F933E35F.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACPE128F421C1B9.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACPE128F421C1B9.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACQT128F9331780.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACQT128F9331780.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACSL128F93462F4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACSL128F93462F4.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACTB12903CAAF15.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACTB12903CAAF15.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACVS128E078BE39.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACVS128E078BE39.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACZK128F4243829.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/A/C/TRAACZK128F4243829.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/A/TRABACN128F425B784.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/A/TRABACN128F425B784.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/A/TRABAFJ128F42AF24E.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/A/TRABAFJ128F42AF24E.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/A/TRABAFP128F931E9A1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/A/TRABAFP128F931E9A1.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/A/TRABAIO128F42938F9.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/A/TRABAIO128F42938F9.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/A/TRABATO128F42627E9.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/A/TRABATO128F42627E9.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/A/TRABAVQ12903CBF7E0.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/A/TRABAVQ12903CBF7E0.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/A/TRABAWW128F4250A31.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/A/TRABAWW128F4250A31.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/A/TRABAXL128F424FC50.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/A/TRABAXL128F424FC50.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/A/TRABAXR128F426515F.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/A/TRABAXR128F426515F.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/A/TRABAXV128F92F6AE3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/A/TRABAXV128F92F6AE3.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/A/TRABAZH128F930419A.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/A/TRABAZH128F930419A.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/B/TRABBAM128F429D223.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/B/TRABBAM128F429D223.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/B/TRABBBV128F42967D7.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/B/TRABBBV128F42967D7.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/B/TRABBJE12903CDB442.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/B/TRABBJE12903CDB442.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/B/TRABBKX128F4285205.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/B/TRABBKX128F4285205.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/B/TRABBLU128F93349CF.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/B/TRABBLU128F93349CF.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/B/TRABBNP128F932546F.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/B/TRABBNP128F932546F.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/B/TRABBOP128F931B50D.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/B/TRABBOP128F931B50D.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/B/TRABBOR128F4286200.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/B/TRABBOR128F4286200.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/B/TRABBTA128F933D304.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/B/TRABBTA128F933D304.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/B/TRABBVJ128F92F7EAA.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/B/TRABBVJ128F92F7EAA.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/B/TRABBXU128F92FEF48.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/B/TRABBXU128F92FEF48.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/B/TRABBZN12903CD9297.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/B/TRABBZN12903CD9297.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/C/TRABCAJ12903CDFCC2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/C/TRABCAJ12903CDFCC2.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/C/TRABCEC128F426456E.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/C/TRABCEC128F426456E.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/C/TRABCEI128F424C983.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/C/TRABCEI128F424C983.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/C/TRABCFL128F149BB0D.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/C/TRABCFL128F149BB0D.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/C/TRABCIX128F4265903.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/C/TRABCIX128F4265903.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/C/TRABCKL128F423A778.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/C/TRABCKL128F423A778.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/C/TRABCPZ128F4275C32.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/C/TRABCPZ128F4275C32.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/C/TRABCRU128F423F449.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/C/TRABCRU128F423F449.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/C/TRABCTK128F934B224.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/C/TRABCTK128F934B224.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/C/TRABCUQ128E0783E2B.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/C/TRABCUQ128E0783E2B.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/C/TRABCXB128F4286BD3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/C/TRABCXB128F4286BD3.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/data/song_data/A/B/C/TRABCYE128F934CE1D.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/data/song_data/A/B/C/TRABCYE128F934CE1D.json -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/etl.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/etl.ipynb -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/etl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/etl.py -------------------------------------------------------------------------------- /dend/data_modeling_with_postgres/sql_queries.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_modeling_with_postgres/sql_queries.py -------------------------------------------------------------------------------- /dend/data_pipeline_with_airflow/.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_pipeline_with_airflow/.env -------------------------------------------------------------------------------- /dend/data_pipeline_with_airflow/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_pipeline_with_airflow/.gitignore -------------------------------------------------------------------------------- /dend/data_pipeline_with_airflow/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_pipeline_with_airflow/README.md -------------------------------------------------------------------------------- /dend/data_pipeline_with_airflow/architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_pipeline_with_airflow/architecture.jpg -------------------------------------------------------------------------------- /dend/data_pipeline_with_airflow/create_tables.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_pipeline_with_airflow/create_tables.sql -------------------------------------------------------------------------------- /dend/data_pipeline_with_airflow/dags/sparkify_pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_pipeline_with_airflow/dags/sparkify_pipeline.py -------------------------------------------------------------------------------- /dend/data_pipeline_with_airflow/logs/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dend/data_pipeline_with_airflow/pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_pipeline_with_airflow/pipeline.png -------------------------------------------------------------------------------- /dend/data_pipeline_with_airflow/plugins/data_quality.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_pipeline_with_airflow/plugins/data_quality.py -------------------------------------------------------------------------------- /dend/data_pipeline_with_airflow/plugins/helpers/sql_queries.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_pipeline_with_airflow/plugins/helpers/sql_queries.py -------------------------------------------------------------------------------- /dend/data_pipeline_with_airflow/plugins/load_dimension.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_pipeline_with_airflow/plugins/load_dimension.py -------------------------------------------------------------------------------- /dend/data_pipeline_with_airflow/plugins/load_fact.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_pipeline_with_airflow/plugins/load_fact.py -------------------------------------------------------------------------------- /dend/data_pipeline_with_airflow/plugins/stage_redshift.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_pipeline_with_airflow/plugins/stage_redshift.py -------------------------------------------------------------------------------- /dend/data_warehouse_with_redshift/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_warehouse_with_redshift/README.md -------------------------------------------------------------------------------- /dend/data_warehouse_with_redshift/architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_warehouse_with_redshift/architecture.jpg -------------------------------------------------------------------------------- /dend/data_warehouse_with_redshift/config.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_warehouse_with_redshift/config.ini -------------------------------------------------------------------------------- /dend/data_warehouse_with_redshift/create_tables.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_warehouse_with_redshift/create_tables.py -------------------------------------------------------------------------------- /dend/data_warehouse_with_redshift/etl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_warehouse_with_redshift/etl.py -------------------------------------------------------------------------------- /dend/data_warehouse_with_redshift/run_on_aws.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_warehouse_with_redshift/run_on_aws.py -------------------------------------------------------------------------------- /dend/data_warehouse_with_redshift/sql_queries.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_warehouse_with_redshift/sql_queries.py -------------------------------------------------------------------------------- /dend/data_warehouse_with_redshift/summarize_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/data_warehouse_with_redshift/summarize_data.py -------------------------------------------------------------------------------- /dend/dend_capstone_project/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/dend_capstone_project/.gitignore -------------------------------------------------------------------------------- /dend/dend_capstone_project/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/dend_capstone_project/Dockerfile -------------------------------------------------------------------------------- /dend/dend_capstone_project/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/dend_capstone_project/README.md -------------------------------------------------------------------------------- /dend/dend_capstone_project/airflow/.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/dend_capstone_project/airflow/.env -------------------------------------------------------------------------------- /dend/dend_capstone_project/airflow/dags/capstone_pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/dend_capstone_project/airflow/dags/capstone_pipeline.py -------------------------------------------------------------------------------- /dend/dend_capstone_project/airflow/logs/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dend/dend_capstone_project/airflow/pipeline.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/dend_capstone_project/airflow/pipeline.jpg -------------------------------------------------------------------------------- /dend/dend_capstone_project/airflow/plugins/data_quality.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/dend_capstone_project/airflow/plugins/data_quality.py -------------------------------------------------------------------------------- /dend/dend_capstone_project/airflow/plugins/load_to_redshift.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/dend_capstone_project/airflow/plugins/load_to_redshift.py -------------------------------------------------------------------------------- /dend/dend_capstone_project/architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/dend_capstone_project/architecture.jpg -------------------------------------------------------------------------------- /dend/dend_capstone_project/data_model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/dend_capstone_project/data_model.png -------------------------------------------------------------------------------- /dend/dend_capstone_project/etl/etl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/dend_capstone_project/etl/etl.py -------------------------------------------------------------------------------- /dend/dend_capstone_project/etl/start_emr_cluster.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/dend_capstone_project/etl/start_emr_cluster.sh -------------------------------------------------------------------------------- /dend/dend_capstone_project/redshift/config.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/dend_capstone_project/redshift/config.ini -------------------------------------------------------------------------------- /dend/dend_capstone_project/redshift/sql_queries.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/dend_capstone_project/redshift/sql_queries.py -------------------------------------------------------------------------------- /dend/dend_capstone_project/redshift/start_db_on_redshift.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/dend_capstone_project/redshift/start_db_on_redshift.py -------------------------------------------------------------------------------- /dend/dend_capstone_project/workspace/capital_bikeshare_system_map.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/dend_capstone_project/workspace/capital_bikeshare_system_map.png -------------------------------------------------------------------------------- /dend/dend_capstone_project/workspace/data_exploration.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dend/dend_capstone_project/workspace/data_exploration.ipynb -------------------------------------------------------------------------------- /dend/dend_capstone_project/workspace/datasets/capitalbikeshare_tripdata/new/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dend/dend_capstone_project/workspace/datasets/capitalbikeshare_tripdata/old/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dend/dend_capstone_project/workspace/datasets/covid_data/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dend/dend_capstone_project/workspace/datasets/weather_data/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dev_envs/airflow_docker/.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/airflow_docker/.env -------------------------------------------------------------------------------- /dev_envs/airflow_docker/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/airflow_docker/.gitignore -------------------------------------------------------------------------------- /dev_envs/airflow_docker/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/airflow_docker/README.md -------------------------------------------------------------------------------- /dev_envs/airflow_docker/airflow.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/airflow_docker/airflow.sh -------------------------------------------------------------------------------- /dev_envs/airflow_docker/dags/aws_s3_to_redshift.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/airflow_docker/dags/aws_s3_to_redshift.py -------------------------------------------------------------------------------- /dev_envs/airflow_docker/dags/bicycle_sharing_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/airflow_docker/dags/bicycle_sharing_example.py -------------------------------------------------------------------------------- /dev_envs/airflow_docker/dags/connections_and_hooks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/airflow_docker/dags/connections_and_hooks.py -------------------------------------------------------------------------------- /dev_envs/airflow_docker/dags/context_and_templating.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/airflow_docker/dags/context_and_templating.py -------------------------------------------------------------------------------- /dev_envs/airflow_docker/dags/hello_world.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/airflow_docker/dags/hello_world.py -------------------------------------------------------------------------------- /dev_envs/airflow_docker/dags/sql_statements.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/airflow_docker/dags/sql_statements.py -------------------------------------------------------------------------------- /dev_envs/airflow_docker/dags/task_dependencies.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/airflow_docker/dags/task_dependencies.py -------------------------------------------------------------------------------- /dev_envs/airflow_docker/docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/airflow_docker/docker-compose.yaml -------------------------------------------------------------------------------- /dev_envs/airflow_docker/logs/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dev_envs/airflow_docker/plugins/facts_calculator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/airflow_docker/plugins/facts_calculator.py -------------------------------------------------------------------------------- /dev_envs/airflow_docker/plugins/has_rows.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/airflow_docker/plugins/has_rows.py -------------------------------------------------------------------------------- /dev_envs/airflow_docker/plugins/s3_to_redshift.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/airflow_docker/plugins/s3_to_redshift.py -------------------------------------------------------------------------------- /dev_envs/kafka_docker/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/kafka_docker/README.md -------------------------------------------------------------------------------- /dev_envs/kafka_docker/docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/kafka_docker/docker-compose.yml -------------------------------------------------------------------------------- /dev_envs/kafka_docker/java/.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | -------------------------------------------------------------------------------- /dev_envs/kafka_docker/java/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/kafka_docker/java/README.md -------------------------------------------------------------------------------- /dev_envs/kafka_docker/java/config.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/kafka_docker/java/config.properties -------------------------------------------------------------------------------- /dev_envs/kafka_docker/java/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/kafka_docker/java/pom.xml -------------------------------------------------------------------------------- /dev_envs/kafka_docker/java/src/main/java/ClientBase.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/kafka_docker/java/src/main/java/ClientBase.java -------------------------------------------------------------------------------- /dev_envs/kafka_docker/java/src/main/java/ConsumerExample.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/kafka_docker/java/src/main/java/ConsumerExample.java -------------------------------------------------------------------------------- /dev_envs/kafka_docker/java/src/main/java/ProducerExample.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/kafka_docker/java/src/main/java/ProducerExample.java -------------------------------------------------------------------------------- /dev_envs/kafka_docker/python/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/kafka_docker/python/README.md -------------------------------------------------------------------------------- /dev_envs/kafka_docker/python/config.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/kafka_docker/python/config.ini -------------------------------------------------------------------------------- /dev_envs/kafka_docker/python/example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/kafka_docker/python/example.py -------------------------------------------------------------------------------- /dev_envs/kafka_docker/python/example_with_avro.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/kafka_docker/python/example_with_avro.py -------------------------------------------------------------------------------- /dev_envs/kafka_docker/python/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/kafka_docker/python/requirements.txt -------------------------------------------------------------------------------- /dev_envs/spark_docker/.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/spark_docker/.env -------------------------------------------------------------------------------- /dev_envs/spark_docker/.gitignore: -------------------------------------------------------------------------------- 1 | !.env -------------------------------------------------------------------------------- /dev_envs/spark_docker/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/spark_docker/README.md -------------------------------------------------------------------------------- /dev_envs/spark_docker/build.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/spark_docker/build.sh -------------------------------------------------------------------------------- /dev_envs/spark_docker/cluster-base.Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/spark_docker/cluster-base.Dockerfile -------------------------------------------------------------------------------- /dev_envs/spark_docker/docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/spark_docker/docker-compose.yml -------------------------------------------------------------------------------- /dev_envs/spark_docker/java/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/spark_docker/java/Dockerfile -------------------------------------------------------------------------------- /dev_envs/spark_docker/java/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/spark_docker/java/README.md -------------------------------------------------------------------------------- /dev_envs/spark_docker/java/java-app/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/spark_docker/java/java-app/pom.xml -------------------------------------------------------------------------------- /dev_envs/spark_docker/java/java-app/src/main/java/LowerCase.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/spark_docker/java/java-app/src/main/java/LowerCase.java -------------------------------------------------------------------------------- /dev_envs/spark_docker/jupyterlab.Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/spark_docker/jupyterlab.Dockerfile -------------------------------------------------------------------------------- /dev_envs/spark_docker/maven.Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/spark_docker/maven.Dockerfile -------------------------------------------------------------------------------- /dev_envs/spark_docker/pyspark.Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/spark_docker/pyspark.Dockerfile -------------------------------------------------------------------------------- /dev_envs/spark_docker/python/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/spark_docker/python/Dockerfile -------------------------------------------------------------------------------- /dev_envs/spark_docker/python/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/spark_docker/python/README.md -------------------------------------------------------------------------------- /dev_envs/spark_docker/python/python-app/app.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/spark_docker/python/python-app/app.py -------------------------------------------------------------------------------- /dev_envs/spark_docker/scala/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/spark_docker/scala/Dockerfile -------------------------------------------------------------------------------- /dev_envs/spark_docker/scala/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/spark_docker/scala/README.md -------------------------------------------------------------------------------- /dev_envs/spark_docker/scala/scala-app/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/spark_docker/scala/scala-app/pom.xml -------------------------------------------------------------------------------- /dev_envs/spark_docker/scala/scala-app/src/main/scala/LowerCase.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/spark_docker/scala/scala-app/src/main/scala/LowerCase.scala -------------------------------------------------------------------------------- /dev_envs/spark_docker/spark-base.Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/spark_docker/spark-base.Dockerfile -------------------------------------------------------------------------------- /dev_envs/spark_docker/spark-master.Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/spark_docker/spark-master.Dockerfile -------------------------------------------------------------------------------- /dev_envs/spark_docker/spark-worker.Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/spark_docker/spark-worker.Dockerfile -------------------------------------------------------------------------------- /dev_envs/spark_docker/spark.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/spark_docker/spark.env -------------------------------------------------------------------------------- /dev_envs/spark_docker/workspace/spark_test.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dev_envs/spark_docker/workspace/spark_test.ipynb -------------------------------------------------------------------------------- /dsnd/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/README.md -------------------------------------------------------------------------------- /dsnd/stedi_app_with_spark_streaming/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/stedi_app_with_spark_streaming/README.md -------------------------------------------------------------------------------- /dsnd/stedi_app_with_spark_streaming/architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/stedi_app_with_spark_streaming/architecture.jpg -------------------------------------------------------------------------------- /dsnd/stedi_app_with_spark_streaming/docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/stedi_app_with_spark_streaming/docker-compose.yaml -------------------------------------------------------------------------------- /dsnd/stedi_app_with_spark_streaming/event_kafka_stream.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/stedi_app_with_spark_streaming/event_kafka_stream.py -------------------------------------------------------------------------------- /dsnd/stedi_app_with_spark_streaming/kafka_join.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/stedi_app_with_spark_streaming/kafka_join.py -------------------------------------------------------------------------------- /dsnd/stedi_app_with_spark_streaming/redis_kafka_stream.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/stedi_app_with_spark_streaming/redis_kafka_stream.py -------------------------------------------------------------------------------- /dsnd/stedi_app_with_spark_streaming/screenshots/risk_graph1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/stedi_app_with_spark_streaming/screenshots/risk_graph1.jpg -------------------------------------------------------------------------------- /dsnd/stedi_app_with_spark_streaming/screenshots/risk_graph2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/stedi_app_with_spark_streaming/screenshots/risk_graph2.jpg -------------------------------------------------------------------------------- /dsnd/stedi_app_with_spark_streaming/spark/.gitignore: -------------------------------------------------------------------------------- 1 | jars/* -------------------------------------------------------------------------------- /dsnd/stedi_app_with_spark_streaming/spark/logs/event_stream.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/stedi_app_with_spark_streaming/spark/logs/event_stream.log -------------------------------------------------------------------------------- /dsnd/stedi_app_with_spark_streaming/spark/logs/kafka_join.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/stedi_app_with_spark_streaming/spark/logs/kafka_join.log -------------------------------------------------------------------------------- /dsnd/stedi_app_with_spark_streaming/spark/logs/redis_stream.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/stedi_app_with_spark_streaming/spark/logs/redis_stream.log -------------------------------------------------------------------------------- /dsnd/stedi_app_with_spark_streaming/spark/logs/spark-master.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/stedi_app_with_spark_streaming/spark/logs/spark-master.log -------------------------------------------------------------------------------- /dsnd/stedi_app_with_spark_streaming/spark/logs/spark-worker.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/stedi_app_with_spark_streaming/spark/logs/spark-worker.log -------------------------------------------------------------------------------- /dsnd/stedi_app_with_spark_streaming/submit-event-kafka-stream.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/stedi_app_with_spark_streaming/submit-event-kafka-stream.sh -------------------------------------------------------------------------------- /dsnd/stedi_app_with_spark_streaming/submit-kafka-join.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/stedi_app_with_spark_streaming/submit-kafka-join.sh -------------------------------------------------------------------------------- /dsnd/stedi_app_with_spark_streaming/submit-redis-kafka-stream.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/stedi_app_with_spark_streaming/submit-redis-kafka-stream.sh -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/README.md -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/architecture.jpg -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/config.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/config.ini -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/docker-compose.yaml -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/load_stations.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/load_stations.sql -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/logging.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/logging.ini -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/misc/screenshot.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/misc/screenshot.jpg -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/requirements.txt -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/simple_consumer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/simple_consumer.py -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/config.py -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/consumers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/consumers/consumer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/consumers/consumer.py -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/consumers/faust_stream.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/consumers/faust_stream.py -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/consumers/ksql.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/consumers/ksql.py -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/consumers/logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/consumers/logger.py -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/consumers/server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/consumers/server.py -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/consumers/templates/status.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/consumers/templates/status.html -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/producers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/producers/connector.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/producers/connector.py -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/producers/data/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/producers/data/README.md -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/producers/data/cta_stations.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/producers/data/cta_stations.csv -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/producers/data/ridership_curve.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/producers/data/ridership_curve.csv -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/producers/data/ridership_seed.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/producers/data/ridership_seed.csv -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/producers/logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/producers/logger.py -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/producers/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/producers/models/__init__.py -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/producers/models/line.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/producers/models/line.py -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/producers/models/producer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/producers/models/producer.py -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/producers/models/schemas/arrival_key.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/producers/models/schemas/arrival_key.json -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/producers/models/schemas/arrival_value.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/producers/models/schemas/arrival_value.json -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/producers/models/schemas/turnstile_key.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/producers/models/schemas/turnstile_key.json -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/producers/models/schemas/turnstile_value.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/producers/models/schemas/turnstile_value.json -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/producers/models/schemas/weather_key.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/producers/models/schemas/weather_key.json -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/producers/models/schemas/weather_value.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/producers/models/schemas/weather_value.json -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/producers/models/station.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/producers/models/station.py -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/producers/models/timer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/producers/models/timer.py -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/producers/models/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/producers/models/train.py -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/producers/models/turnstile.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/producers/models/turnstile.py -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/producers/models/weather.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/producers/models/weather.py -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/producers/simulation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/producers/simulation.py -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/producers/test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/producers/test/test_simulator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/producers/test/test_simulator.py -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/src/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/src/utils.py -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/start_faust_app.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/start_faust_app.py -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/start_ksql.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/start_ksql.py -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/start_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/start_server.py -------------------------------------------------------------------------------- /dsnd/transit_status_tracker_with_kafka/start_simulation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhujun98/data-engineering/HEAD/dsnd/transit_status_tracker_with_kafka/start_simulation.py --------------------------------------------------------------------------------