├── .gitignore ├── README.md ├── airflow ├── .env ├── .gitignore ├── Dockerfile ├── dags │ ├── dbt_test_dag.py │ ├── load_songs_dag.py │ ├── schema.py │ ├── sql │ │ ├── auth_events.sql │ │ ├── listen_events.sql │ │ └── page_view_events.sql │ ├── streamify_dag.py │ └── task_templates.py ├── docker-compose.yaml └── requirements.txt ├── dbt ├── .gitignore ├── README.md ├── analyses │ └── .gitkeep ├── dbt_project.yml ├── macros │ └── .gitkeep ├── models │ └── core │ │ ├── dim_artists.sql │ │ ├── dim_datetime.sql │ │ ├── dim_location.sql │ │ ├── dim_songs.sql │ │ ├── dim_users.sql │ │ ├── fact_streams.sql │ │ ├── schema.yml │ │ └── wide_streams.sql ├── packages.yml ├── profiles.yml ├── seeds │ ├── .gitkeep │ ├── songs.csv │ └── state_codes.csv ├── snapshots │ └── .gitkeep └── tests │ └── .gitkeep ├── eventsim ├── Dockerfile ├── README.md ├── data │ ├── .DS_Store │ ├── Gaz_zcta_national.txt │ ├── Top1000Surnames.csv │ ├── US.txt │ ├── listen_counts.txt.gz │ ├── songs_analysis.txt.gz │ ├── user agents.txt │ └── yob1990.txt ├── eventsim.sh ├── examples │ ├── alt-example-config.json │ └── example-config.json └── target │ └── eventsim-assembly-2.0.jar ├── images ├── Streamify-Architecture.jpg ├── airflow.jpg ├── dashboard.png ├── dbt.png ├── kafka.jpg ├── songs_dag.png ├── spark.jpg ├── streamify_dag.png └── topics.png ├── kafka ├── docker-compose.yml └── test_connection │ ├── data │ ├── rides.csv │ └── zones.csv │ └── produce_taxi_json.py ├── requirements.txt ├── scripts ├── airflow_startup.sh ├── eventsim_startup.sh ├── exec_commands.sh ├── spark_setup.sh └── vm_setup.sh ├── setup ├── airflow.md ├── debug.md ├── gcp.md ├── kafka.md ├── spark.md ├── ssh.md └── terraform.md ├── spark_streaming ├── schema.py ├── stream_all_events.py ├── streaming_functions.py └── test_connection │ └── stream_taxi_json.py └── terraform ├── .gitignore ├── main.tf └── variables.tf /.gitignore: -------------------------------------------------------------------------------- 1 | logs/ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/README.md -------------------------------------------------------------------------------- /airflow/.env: -------------------------------------------------------------------------------- 1 | AIRFLOW_UID=50000 2 | -------------------------------------------------------------------------------- /airflow/.gitignore: -------------------------------------------------------------------------------- 1 | logs/ 2 | plugins/ 3 | __pycache__/ -------------------------------------------------------------------------------- /airflow/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/airflow/Dockerfile -------------------------------------------------------------------------------- /airflow/dags/dbt_test_dag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/airflow/dags/dbt_test_dag.py -------------------------------------------------------------------------------- /airflow/dags/load_songs_dag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/airflow/dags/load_songs_dag.py -------------------------------------------------------------------------------- /airflow/dags/schema.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/airflow/dags/schema.py -------------------------------------------------------------------------------- /airflow/dags/sql/auth_events.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/airflow/dags/sql/auth_events.sql -------------------------------------------------------------------------------- /airflow/dags/sql/listen_events.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/airflow/dags/sql/listen_events.sql -------------------------------------------------------------------------------- /airflow/dags/sql/page_view_events.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/airflow/dags/sql/page_view_events.sql -------------------------------------------------------------------------------- /airflow/dags/streamify_dag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/airflow/dags/streamify_dag.py -------------------------------------------------------------------------------- /airflow/dags/task_templates.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/airflow/dags/task_templates.py -------------------------------------------------------------------------------- /airflow/docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/airflow/docker-compose.yaml -------------------------------------------------------------------------------- /airflow/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/airflow/requirements.txt -------------------------------------------------------------------------------- /dbt/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | .user.yml -------------------------------------------------------------------------------- /dbt/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/dbt/README.md -------------------------------------------------------------------------------- /dbt/analyses/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt/dbt_project.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/dbt/dbt_project.yml -------------------------------------------------------------------------------- /dbt/macros/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt/models/core/dim_artists.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/dbt/models/core/dim_artists.sql -------------------------------------------------------------------------------- /dbt/models/core/dim_datetime.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/dbt/models/core/dim_datetime.sql -------------------------------------------------------------------------------- /dbt/models/core/dim_location.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/dbt/models/core/dim_location.sql -------------------------------------------------------------------------------- /dbt/models/core/dim_songs.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/dbt/models/core/dim_songs.sql -------------------------------------------------------------------------------- /dbt/models/core/dim_users.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/dbt/models/core/dim_users.sql -------------------------------------------------------------------------------- /dbt/models/core/fact_streams.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/dbt/models/core/fact_streams.sql -------------------------------------------------------------------------------- /dbt/models/core/schema.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/dbt/models/core/schema.yml -------------------------------------------------------------------------------- /dbt/models/core/wide_streams.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/dbt/models/core/wide_streams.sql -------------------------------------------------------------------------------- /dbt/packages.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/dbt/packages.yml -------------------------------------------------------------------------------- /dbt/profiles.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/dbt/profiles.yml -------------------------------------------------------------------------------- /dbt/seeds/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt/seeds/songs.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/dbt/seeds/songs.csv -------------------------------------------------------------------------------- /dbt/seeds/state_codes.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/dbt/seeds/state_codes.csv -------------------------------------------------------------------------------- /dbt/snapshots/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt/tests/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eventsim/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/eventsim/Dockerfile -------------------------------------------------------------------------------- /eventsim/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/eventsim/README.md -------------------------------------------------------------------------------- /eventsim/data/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/eventsim/data/.DS_Store -------------------------------------------------------------------------------- /eventsim/data/Gaz_zcta_national.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/eventsim/data/Gaz_zcta_national.txt -------------------------------------------------------------------------------- /eventsim/data/Top1000Surnames.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/eventsim/data/Top1000Surnames.csv -------------------------------------------------------------------------------- /eventsim/data/US.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/eventsim/data/US.txt -------------------------------------------------------------------------------- /eventsim/data/listen_counts.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/eventsim/data/listen_counts.txt.gz -------------------------------------------------------------------------------- /eventsim/data/songs_analysis.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/eventsim/data/songs_analysis.txt.gz -------------------------------------------------------------------------------- /eventsim/data/user agents.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/eventsim/data/user agents.txt -------------------------------------------------------------------------------- /eventsim/data/yob1990.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/eventsim/data/yob1990.txt -------------------------------------------------------------------------------- /eventsim/eventsim.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/eventsim/eventsim.sh -------------------------------------------------------------------------------- /eventsim/examples/alt-example-config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/eventsim/examples/alt-example-config.json -------------------------------------------------------------------------------- /eventsim/examples/example-config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/eventsim/examples/example-config.json -------------------------------------------------------------------------------- /eventsim/target/eventsim-assembly-2.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/eventsim/target/eventsim-assembly-2.0.jar -------------------------------------------------------------------------------- /images/Streamify-Architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/images/Streamify-Architecture.jpg -------------------------------------------------------------------------------- /images/airflow.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/images/airflow.jpg -------------------------------------------------------------------------------- /images/dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/images/dashboard.png -------------------------------------------------------------------------------- /images/dbt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/images/dbt.png -------------------------------------------------------------------------------- /images/kafka.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/images/kafka.jpg -------------------------------------------------------------------------------- /images/songs_dag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/images/songs_dag.png -------------------------------------------------------------------------------- /images/spark.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/images/spark.jpg -------------------------------------------------------------------------------- /images/streamify_dag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/images/streamify_dag.png -------------------------------------------------------------------------------- /images/topics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/images/topics.png -------------------------------------------------------------------------------- /kafka/docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/kafka/docker-compose.yml -------------------------------------------------------------------------------- /kafka/test_connection/data/rides.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/kafka/test_connection/data/rides.csv -------------------------------------------------------------------------------- /kafka/test_connection/data/zones.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/kafka/test_connection/data/zones.csv -------------------------------------------------------------------------------- /kafka/test_connection/produce_taxi_json.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/kafka/test_connection/produce_taxi_json.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | kafka-python==1.4.6 2 | dbt-bigquery==1.0.0 -------------------------------------------------------------------------------- /scripts/airflow_startup.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/scripts/airflow_startup.sh -------------------------------------------------------------------------------- /scripts/eventsim_startup.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/scripts/eventsim_startup.sh -------------------------------------------------------------------------------- /scripts/exec_commands.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/scripts/exec_commands.sh -------------------------------------------------------------------------------- /scripts/spark_setup.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/scripts/spark_setup.sh -------------------------------------------------------------------------------- /scripts/vm_setup.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/scripts/vm_setup.sh -------------------------------------------------------------------------------- /setup/airflow.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/setup/airflow.md -------------------------------------------------------------------------------- /setup/debug.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/setup/debug.md -------------------------------------------------------------------------------- /setup/gcp.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/setup/gcp.md -------------------------------------------------------------------------------- /setup/kafka.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/setup/kafka.md -------------------------------------------------------------------------------- /setup/spark.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/setup/spark.md -------------------------------------------------------------------------------- /setup/ssh.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/setup/ssh.md -------------------------------------------------------------------------------- /setup/terraform.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/setup/terraform.md -------------------------------------------------------------------------------- /spark_streaming/schema.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/spark_streaming/schema.py -------------------------------------------------------------------------------- /spark_streaming/stream_all_events.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/spark_streaming/stream_all_events.py -------------------------------------------------------------------------------- /spark_streaming/streaming_functions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/spark_streaming/streaming_functions.py -------------------------------------------------------------------------------- /spark_streaming/test_connection/stream_taxi_json.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/spark_streaming/test_connection/stream_taxi_json.py -------------------------------------------------------------------------------- /terraform/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/terraform/.gitignore -------------------------------------------------------------------------------- /terraform/main.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/terraform/main.tf -------------------------------------------------------------------------------- /terraform/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurchavda/streamify/HEAD/terraform/variables.tf --------------------------------------------------------------------------------