├── .env ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── airflow-docker-compose.yaml ├── airflow ├── Dockerfile ├── README.md ├── dags │ ├── data2warehouse.py │ └── nyc_pipeline.py ├── include │ └── great_expectations │ │ ├── expectations │ │ ├── .ge_store_backend_id │ │ └── nyctaxi_suite.json │ │ ├── great_expectations.yml │ │ └── plugins │ │ └── custom_data_docs │ │ └── styles │ │ └── data_docs_custom_styles.css ├── postgresql-docker-compose.yaml └── requirements.txt ├── ansible ├── create_compute_instance.yaml ├── deploy_dataservice │ ├── deploy.yml │ ├── requirements.txt │ └── trino │ │ ├── catalog │ │ ├── datalake.properties │ │ └── lakehouse.properties │ │ └── etc │ │ ├── config.properties │ │ ├── jvm.config │ │ └── node.properties ├── deploy_monitoring │ └── deploy.yml └── inventory ├── datalake ├── README.md ├── minio_docker-compose.yml └── trino │ ├── catalog │ └── datalake.properties │ └── etc │ ├── config.properties │ ├── jvm.config │ └── node.properties ├── dbt_nyc ├── .gitignore ├── .user.yml ├── README.md ├── analyses │ └── .gitkeep ├── dbt_project.yml ├── macros │ ├── .gitkeep │ ├── decode_category.sql │ └── macr_docs.yml ├── models │ ├── dim_memory.sql │ ├── dim_payment.sql │ ├── dim_ratecode.sql │ ├── dim_vendor.sql │ ├── samll_passenger.sql │ ├── schema.yml │ ├── small_amount.sql │ ├── small_distance.sql │ └── yellow_taxi.sql ├── profiles.yml ├── seeds │ └── .gitkeep ├── snapshots │ └── .gitkeep └── tests │ └── .gitkeep ├── docker-compose.yml ├── imgs ├── airflow.png ├── airflow1.png ├── airflow2.png ├── airflow3.png ├── airflow4.png ├── airflow5.png ├── airflow6.png ├── airflow7.png ├── architecture.png ├── architecture1.png ├── create_compute_instance.gif ├── data.png ├── datadrawio.drawio.png ├── dbt.png ├── debezium-cdc.webp ├── debezium.png ├── deequ.png ├── final.png ├── gcp.png ├── gcp1.png ├── gcp2.png ├── gcp3.png ├── grafana.png ├── kafka-topic.png ├── kafka.png ├── kafka1.png ├── kafka2.1.png ├── kafka2.png ├── kafka_mess.png ├── kibana.png ├── minio.png ├── minio1.png ├── minio2.png ├── minio3.png ├── minio4.png ├── minio5.png ├── minio6.png ├── mle.drawio.png ├── mle2.drawio.png ├── ssh_key_out.gif ├── trino.png └── validation.png ├── jars ├── avro-1.11.1.jar ├── deequ-2.0.3-spark-3.3.jar ├── flink-avro-1.16.0.jar ├── flink-avro-1.17.1.jar ├── flink-avro-confluent-registry-1.16.0.jar ├── flink-avro-confluent-registry-1.17.1.jar ├── flink-connector-kafka-1.16.0.jar ├── flink-connector-kafka-1.17.1.jar ├── flink-table-api-java-1.16.0.jar ├── flink-table-api-java-1.17.1.jar ├── hadoop-aws-3.3.4.jar ├── jackson-annotations-2.14.2.jar ├── jackson-core-2.14.2.jar ├── jackson-databind-2.14.2.jar ├── kafka-clients-3.4.0.jar ├── kafka-connect-jdbc-10.6.4.jar ├── kafka-schema-registry-client-5.3.0.jar ├── postgresql-42.4.3.jar └── trino-jdbc-434.jar ├── monitoring ├── README.md ├── elk │ ├── .env │ ├── elasticsearch │ │ ├── .dockerignore │ │ ├── Dockerfile │ │ └── config │ │ │ └── elasticsearch.yml │ ├── elk-docker-compose.yml │ ├── extensions │ │ ├── README.md │ │ └── filebeat │ │ │ ├── .dockerignore │ │ │ ├── Dockerfile │ │ │ ├── README.md │ │ │ ├── config │ │ │ └── filebeat.yml │ │ │ └── filebeat-compose.yml │ ├── kibana │ │ ├── .dockerignore │ │ ├── Dockerfile │ │ └── config │ │ │ └── kibana.yml │ ├── run_env │ │ └── .gitkeep │ └── setup │ │ ├── .dockerignore │ │ ├── .gitignore │ │ ├── Dockerfile │ │ ├── entrypoint.sh │ │ ├── helpers.sh │ │ └── roles │ │ └── logstash_writer.json ├── grafana │ ├── config │ │ ├── dashboards.yaml │ │ └── datasources.yaml │ └── dashboards │ │ ├── 10585_rev1.json │ │ ├── 1860_rev31.json │ │ └── 893_rev5.json ├── prom-graf-docker-compose.yaml ├── prometheus │ └── config │ │ ├── alert-rules.yml │ │ └── prometheus.yml └── requirements.txt ├── notebooks ├── eda.ipynb ├── full_flow.ipynb └── reload_and_validate.ipynb ├── postgresql-docker-compose.yaml ├── pyspark ├── batch_processing.py ├── datalake2warehouse.py ├── parquet2warehouse.py └── validation.py ├── requirements.txt ├── src ├── db_scripts_generate.py ├── download_data.py ├── export_data_to_datalake.py ├── helpers.py ├── json_consume_message.py ├── load_s3.py ├── postgresql_client.py └── utils │ ├── create_table.py │ ├── database.py │ ├── drop_col.py │ ├── drop_table.py │ ├── insert_data.py │ └── merge_data.py ├── stream_processing ├── .gitignore ├── README.MD ├── flink │ ├── Dockerfile │ ├── avro_schemas │ │ └── schema_0.avsc │ ├── configs │ │ └── connect-timescaledb-sink.json │ ├── docker-compose_flink.yaml │ ├── generate_schemas.py │ ├── nyc_taxi.py │ └── produce.py ├── kafka │ ├── Dockerfile │ ├── README.md │ ├── configs │ │ └── postgresql-cdc.json │ ├── create_table.py │ ├── docker-compose.yml │ ├── insert_table.py │ ├── postgresql_client.py │ ├── producer.sh │ ├── readme.md │ ├── requirements.txt │ ├── run.sh │ └── stream.parquet ├── requirements.txt ├── run.sh └── scripts │ ├── datastream_api.py │ ├── json_consume_message.py │ ├── window_datastream_api.py │ └── window_datastream_api_example.py └── utils └── datalake.yaml /.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/.env -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/LICENSE -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/README.md -------------------------------------------------------------------------------- /airflow-docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/airflow-docker-compose.yaml -------------------------------------------------------------------------------- /airflow/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/airflow/Dockerfile -------------------------------------------------------------------------------- /airflow/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/airflow/README.md -------------------------------------------------------------------------------- /airflow/dags/data2warehouse.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/airflow/dags/data2warehouse.py -------------------------------------------------------------------------------- /airflow/dags/nyc_pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/airflow/dags/nyc_pipeline.py -------------------------------------------------------------------------------- /airflow/include/great_expectations/expectations/.ge_store_backend_id: -------------------------------------------------------------------------------- 1 | store_backend_id = 5e882009-0d2d-44af-9691-375879fb74dd 2 | -------------------------------------------------------------------------------- /airflow/include/great_expectations/expectations/nyctaxi_suite.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/airflow/include/great_expectations/expectations/nyctaxi_suite.json -------------------------------------------------------------------------------- /airflow/include/great_expectations/great_expectations.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/airflow/include/great_expectations/great_expectations.yml -------------------------------------------------------------------------------- /airflow/include/great_expectations/plugins/custom_data_docs/styles/data_docs_custom_styles.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/airflow/include/great_expectations/plugins/custom_data_docs/styles/data_docs_custom_styles.css -------------------------------------------------------------------------------- /airflow/postgresql-docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/airflow/postgresql-docker-compose.yaml -------------------------------------------------------------------------------- /airflow/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/airflow/requirements.txt -------------------------------------------------------------------------------- /ansible/create_compute_instance.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/ansible/create_compute_instance.yaml -------------------------------------------------------------------------------- /ansible/deploy_dataservice/deploy.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/ansible/deploy_dataservice/deploy.yml -------------------------------------------------------------------------------- /ansible/deploy_dataservice/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/ansible/deploy_dataservice/requirements.txt -------------------------------------------------------------------------------- /ansible/deploy_dataservice/trino/catalog/datalake.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/ansible/deploy_dataservice/trino/catalog/datalake.properties -------------------------------------------------------------------------------- /ansible/deploy_dataservice/trino/catalog/lakehouse.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/ansible/deploy_dataservice/trino/catalog/lakehouse.properties -------------------------------------------------------------------------------- /ansible/deploy_dataservice/trino/etc/config.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/ansible/deploy_dataservice/trino/etc/config.properties -------------------------------------------------------------------------------- /ansible/deploy_dataservice/trino/etc/jvm.config: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/ansible/deploy_dataservice/trino/etc/jvm.config -------------------------------------------------------------------------------- /ansible/deploy_dataservice/trino/etc/node.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/ansible/deploy_dataservice/trino/etc/node.properties -------------------------------------------------------------------------------- /ansible/deploy_monitoring/deploy.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/ansible/deploy_monitoring/deploy.yml -------------------------------------------------------------------------------- /ansible/inventory: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/ansible/inventory -------------------------------------------------------------------------------- /datalake/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/datalake/README.md -------------------------------------------------------------------------------- /datalake/minio_docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/datalake/minio_docker-compose.yml -------------------------------------------------------------------------------- /datalake/trino/catalog/datalake.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/datalake/trino/catalog/datalake.properties -------------------------------------------------------------------------------- /datalake/trino/etc/config.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/datalake/trino/etc/config.properties -------------------------------------------------------------------------------- /datalake/trino/etc/jvm.config: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/datalake/trino/etc/jvm.config -------------------------------------------------------------------------------- /datalake/trino/etc/node.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/datalake/trino/etc/node.properties -------------------------------------------------------------------------------- /dbt_nyc/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | -------------------------------------------------------------------------------- /dbt_nyc/.user.yml: -------------------------------------------------------------------------------- 1 | id: 0a906bbd-50d9-43e9-9976-6966d6ac69b4 2 | -------------------------------------------------------------------------------- /dbt_nyc/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/dbt_nyc/README.md -------------------------------------------------------------------------------- /dbt_nyc/analyses/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt_nyc/dbt_project.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/dbt_nyc/dbt_project.yml -------------------------------------------------------------------------------- /dbt_nyc/macros/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt_nyc/macros/decode_category.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/dbt_nyc/macros/decode_category.sql -------------------------------------------------------------------------------- /dbt_nyc/macros/macr_docs.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/dbt_nyc/macros/macr_docs.yml -------------------------------------------------------------------------------- /dbt_nyc/models/dim_memory.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/dbt_nyc/models/dim_memory.sql -------------------------------------------------------------------------------- /dbt_nyc/models/dim_payment.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/dbt_nyc/models/dim_payment.sql -------------------------------------------------------------------------------- /dbt_nyc/models/dim_ratecode.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/dbt_nyc/models/dim_ratecode.sql -------------------------------------------------------------------------------- /dbt_nyc/models/dim_vendor.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/dbt_nyc/models/dim_vendor.sql -------------------------------------------------------------------------------- /dbt_nyc/models/samll_passenger.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/dbt_nyc/models/samll_passenger.sql -------------------------------------------------------------------------------- /dbt_nyc/models/schema.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/dbt_nyc/models/schema.yml -------------------------------------------------------------------------------- /dbt_nyc/models/small_amount.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/dbt_nyc/models/small_amount.sql -------------------------------------------------------------------------------- /dbt_nyc/models/small_distance.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/dbt_nyc/models/small_distance.sql -------------------------------------------------------------------------------- /dbt_nyc/models/yellow_taxi.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/dbt_nyc/models/yellow_taxi.sql -------------------------------------------------------------------------------- /dbt_nyc/profiles.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/dbt_nyc/profiles.yml -------------------------------------------------------------------------------- /dbt_nyc/seeds/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt_nyc/snapshots/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt_nyc/tests/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/docker-compose.yml -------------------------------------------------------------------------------- /imgs/airflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/airflow.png -------------------------------------------------------------------------------- /imgs/airflow1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/airflow1.png -------------------------------------------------------------------------------- /imgs/airflow2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/airflow2.png -------------------------------------------------------------------------------- /imgs/airflow3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/airflow3.png -------------------------------------------------------------------------------- /imgs/airflow4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/airflow4.png -------------------------------------------------------------------------------- /imgs/airflow5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/airflow5.png -------------------------------------------------------------------------------- /imgs/airflow6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/airflow6.png -------------------------------------------------------------------------------- /imgs/airflow7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/airflow7.png -------------------------------------------------------------------------------- /imgs/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/architecture.png -------------------------------------------------------------------------------- /imgs/architecture1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/architecture1.png -------------------------------------------------------------------------------- /imgs/create_compute_instance.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/create_compute_instance.gif -------------------------------------------------------------------------------- /imgs/data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/data.png -------------------------------------------------------------------------------- /imgs/datadrawio.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/datadrawio.drawio.png -------------------------------------------------------------------------------- /imgs/dbt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/dbt.png -------------------------------------------------------------------------------- /imgs/debezium-cdc.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/debezium-cdc.webp -------------------------------------------------------------------------------- /imgs/debezium.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/debezium.png -------------------------------------------------------------------------------- /imgs/deequ.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/deequ.png -------------------------------------------------------------------------------- /imgs/final.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/final.png -------------------------------------------------------------------------------- /imgs/gcp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/gcp.png -------------------------------------------------------------------------------- /imgs/gcp1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/gcp1.png -------------------------------------------------------------------------------- /imgs/gcp2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/gcp2.png -------------------------------------------------------------------------------- /imgs/gcp3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/gcp3.png -------------------------------------------------------------------------------- /imgs/grafana.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/grafana.png -------------------------------------------------------------------------------- /imgs/kafka-topic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/kafka-topic.png -------------------------------------------------------------------------------- /imgs/kafka.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/kafka.png -------------------------------------------------------------------------------- /imgs/kafka1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/kafka1.png -------------------------------------------------------------------------------- /imgs/kafka2.1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/kafka2.1.png -------------------------------------------------------------------------------- /imgs/kafka2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/kafka2.png -------------------------------------------------------------------------------- /imgs/kafka_mess.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/kafka_mess.png -------------------------------------------------------------------------------- /imgs/kibana.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/kibana.png -------------------------------------------------------------------------------- /imgs/minio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/minio.png -------------------------------------------------------------------------------- /imgs/minio1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/minio1.png -------------------------------------------------------------------------------- /imgs/minio2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/minio2.png -------------------------------------------------------------------------------- /imgs/minio3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/minio3.png -------------------------------------------------------------------------------- /imgs/minio4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/minio4.png -------------------------------------------------------------------------------- /imgs/minio5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/minio5.png -------------------------------------------------------------------------------- /imgs/minio6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/minio6.png -------------------------------------------------------------------------------- /imgs/mle.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/mle.drawio.png -------------------------------------------------------------------------------- /imgs/mle2.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/mle2.drawio.png -------------------------------------------------------------------------------- /imgs/ssh_key_out.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/ssh_key_out.gif -------------------------------------------------------------------------------- /imgs/trino.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/trino.png -------------------------------------------------------------------------------- /imgs/validation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/imgs/validation.png -------------------------------------------------------------------------------- /jars/avro-1.11.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/jars/avro-1.11.1.jar -------------------------------------------------------------------------------- /jars/deequ-2.0.3-spark-3.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/jars/deequ-2.0.3-spark-3.3.jar -------------------------------------------------------------------------------- /jars/flink-avro-1.16.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/jars/flink-avro-1.16.0.jar -------------------------------------------------------------------------------- /jars/flink-avro-1.17.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/jars/flink-avro-1.17.1.jar -------------------------------------------------------------------------------- /jars/flink-avro-confluent-registry-1.16.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/jars/flink-avro-confluent-registry-1.16.0.jar -------------------------------------------------------------------------------- /jars/flink-avro-confluent-registry-1.17.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/jars/flink-avro-confluent-registry-1.17.1.jar -------------------------------------------------------------------------------- /jars/flink-connector-kafka-1.16.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/jars/flink-connector-kafka-1.16.0.jar -------------------------------------------------------------------------------- /jars/flink-connector-kafka-1.17.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/jars/flink-connector-kafka-1.17.1.jar -------------------------------------------------------------------------------- /jars/flink-table-api-java-1.16.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/jars/flink-table-api-java-1.16.0.jar -------------------------------------------------------------------------------- /jars/flink-table-api-java-1.17.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/jars/flink-table-api-java-1.17.1.jar -------------------------------------------------------------------------------- /jars/hadoop-aws-3.3.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/jars/hadoop-aws-3.3.4.jar -------------------------------------------------------------------------------- /jars/jackson-annotations-2.14.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/jars/jackson-annotations-2.14.2.jar -------------------------------------------------------------------------------- /jars/jackson-core-2.14.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/jars/jackson-core-2.14.2.jar -------------------------------------------------------------------------------- /jars/jackson-databind-2.14.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/jars/jackson-databind-2.14.2.jar -------------------------------------------------------------------------------- /jars/kafka-clients-3.4.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/jars/kafka-clients-3.4.0.jar -------------------------------------------------------------------------------- /jars/kafka-connect-jdbc-10.6.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/jars/kafka-connect-jdbc-10.6.4.jar -------------------------------------------------------------------------------- /jars/kafka-schema-registry-client-5.3.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/jars/kafka-schema-registry-client-5.3.0.jar -------------------------------------------------------------------------------- /jars/postgresql-42.4.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/jars/postgresql-42.4.3.jar -------------------------------------------------------------------------------- /jars/trino-jdbc-434.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/jars/trino-jdbc-434.jar -------------------------------------------------------------------------------- /monitoring/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/README.md -------------------------------------------------------------------------------- /monitoring/elk/.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/elk/.env -------------------------------------------------------------------------------- /monitoring/elk/elasticsearch/.dockerignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/elk/elasticsearch/.dockerignore -------------------------------------------------------------------------------- /monitoring/elk/elasticsearch/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/elk/elasticsearch/Dockerfile -------------------------------------------------------------------------------- /monitoring/elk/elasticsearch/config/elasticsearch.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/elk/elasticsearch/config/elasticsearch.yml -------------------------------------------------------------------------------- /monitoring/elk/elk-docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/elk/elk-docker-compose.yml -------------------------------------------------------------------------------- /monitoring/elk/extensions/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/elk/extensions/README.md -------------------------------------------------------------------------------- /monitoring/elk/extensions/filebeat/.dockerignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/elk/extensions/filebeat/.dockerignore -------------------------------------------------------------------------------- /monitoring/elk/extensions/filebeat/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/elk/extensions/filebeat/Dockerfile -------------------------------------------------------------------------------- /monitoring/elk/extensions/filebeat/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/elk/extensions/filebeat/README.md -------------------------------------------------------------------------------- /monitoring/elk/extensions/filebeat/config/filebeat.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/elk/extensions/filebeat/config/filebeat.yml -------------------------------------------------------------------------------- /monitoring/elk/extensions/filebeat/filebeat-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/elk/extensions/filebeat/filebeat-compose.yml -------------------------------------------------------------------------------- /monitoring/elk/kibana/.dockerignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/elk/kibana/.dockerignore -------------------------------------------------------------------------------- /monitoring/elk/kibana/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/elk/kibana/Dockerfile -------------------------------------------------------------------------------- /monitoring/elk/kibana/config/kibana.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/elk/kibana/config/kibana.yml -------------------------------------------------------------------------------- /monitoring/elk/run_env/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /monitoring/elk/setup/.dockerignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/elk/setup/.dockerignore -------------------------------------------------------------------------------- /monitoring/elk/setup/.gitignore: -------------------------------------------------------------------------------- 1 | /state/ 2 | -------------------------------------------------------------------------------- /monitoring/elk/setup/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/elk/setup/Dockerfile -------------------------------------------------------------------------------- /monitoring/elk/setup/entrypoint.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/elk/setup/entrypoint.sh -------------------------------------------------------------------------------- /monitoring/elk/setup/helpers.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/elk/setup/helpers.sh -------------------------------------------------------------------------------- /monitoring/elk/setup/roles/logstash_writer.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/elk/setup/roles/logstash_writer.json -------------------------------------------------------------------------------- /monitoring/grafana/config/dashboards.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/grafana/config/dashboards.yaml -------------------------------------------------------------------------------- /monitoring/grafana/config/datasources.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/grafana/config/datasources.yaml -------------------------------------------------------------------------------- /monitoring/grafana/dashboards/10585_rev1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/grafana/dashboards/10585_rev1.json -------------------------------------------------------------------------------- /monitoring/grafana/dashboards/1860_rev31.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/grafana/dashboards/1860_rev31.json -------------------------------------------------------------------------------- /monitoring/grafana/dashboards/893_rev5.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/grafana/dashboards/893_rev5.json -------------------------------------------------------------------------------- /monitoring/prom-graf-docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/prom-graf-docker-compose.yaml -------------------------------------------------------------------------------- /monitoring/prometheus/config/alert-rules.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/prometheus/config/alert-rules.yml -------------------------------------------------------------------------------- /monitoring/prometheus/config/prometheus.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/prometheus/config/prometheus.yml -------------------------------------------------------------------------------- /monitoring/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/monitoring/requirements.txt -------------------------------------------------------------------------------- /notebooks/eda.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/notebooks/eda.ipynb -------------------------------------------------------------------------------- /notebooks/full_flow.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/notebooks/full_flow.ipynb -------------------------------------------------------------------------------- /notebooks/reload_and_validate.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/notebooks/reload_and_validate.ipynb -------------------------------------------------------------------------------- /postgresql-docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/postgresql-docker-compose.yaml -------------------------------------------------------------------------------- /pyspark/batch_processing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/pyspark/batch_processing.py -------------------------------------------------------------------------------- /pyspark/datalake2warehouse.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/pyspark/datalake2warehouse.py -------------------------------------------------------------------------------- /pyspark/parquet2warehouse.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/pyspark/parquet2warehouse.py -------------------------------------------------------------------------------- /pyspark/validation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/pyspark/validation.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/requirements.txt -------------------------------------------------------------------------------- /src/db_scripts_generate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/src/db_scripts_generate.py -------------------------------------------------------------------------------- /src/download_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/src/download_data.py -------------------------------------------------------------------------------- /src/export_data_to_datalake.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/src/export_data_to_datalake.py -------------------------------------------------------------------------------- /src/helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/src/helpers.py -------------------------------------------------------------------------------- /src/json_consume_message.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/src/json_consume_message.py -------------------------------------------------------------------------------- /src/load_s3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/src/load_s3.py -------------------------------------------------------------------------------- /src/postgresql_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/src/postgresql_client.py -------------------------------------------------------------------------------- /src/utils/create_table.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/src/utils/create_table.py -------------------------------------------------------------------------------- /src/utils/database.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/src/utils/database.py -------------------------------------------------------------------------------- /src/utils/drop_col.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/src/utils/drop_col.py -------------------------------------------------------------------------------- /src/utils/drop_table.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/src/utils/drop_table.py -------------------------------------------------------------------------------- /src/utils/insert_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/src/utils/insert_data.py -------------------------------------------------------------------------------- /src/utils/merge_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/src/utils/merge_data.py -------------------------------------------------------------------------------- /stream_processing/.gitignore: -------------------------------------------------------------------------------- 1 | old-dc.yml 2 | .DS_Store -------------------------------------------------------------------------------- /stream_processing/README.MD: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/README.MD -------------------------------------------------------------------------------- /stream_processing/flink/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/flink/Dockerfile -------------------------------------------------------------------------------- /stream_processing/flink/avro_schemas/schema_0.avsc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/flink/avro_schemas/schema_0.avsc -------------------------------------------------------------------------------- /stream_processing/flink/configs/connect-timescaledb-sink.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/flink/configs/connect-timescaledb-sink.json -------------------------------------------------------------------------------- /stream_processing/flink/docker-compose_flink.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/flink/docker-compose_flink.yaml -------------------------------------------------------------------------------- /stream_processing/flink/generate_schemas.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/flink/generate_schemas.py -------------------------------------------------------------------------------- /stream_processing/flink/nyc_taxi.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/flink/nyc_taxi.py -------------------------------------------------------------------------------- /stream_processing/flink/produce.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/flink/produce.py -------------------------------------------------------------------------------- /stream_processing/kafka/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/kafka/Dockerfile -------------------------------------------------------------------------------- /stream_processing/kafka/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/kafka/README.md -------------------------------------------------------------------------------- /stream_processing/kafka/configs/postgresql-cdc.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/kafka/configs/postgresql-cdc.json -------------------------------------------------------------------------------- /stream_processing/kafka/create_table.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/kafka/create_table.py -------------------------------------------------------------------------------- /stream_processing/kafka/docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/kafka/docker-compose.yml -------------------------------------------------------------------------------- /stream_processing/kafka/insert_table.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/kafka/insert_table.py -------------------------------------------------------------------------------- /stream_processing/kafka/postgresql_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/kafka/postgresql_client.py -------------------------------------------------------------------------------- /stream_processing/kafka/producer.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/kafka/producer.sh -------------------------------------------------------------------------------- /stream_processing/kafka/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/kafka/readme.md -------------------------------------------------------------------------------- /stream_processing/kafka/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/kafka/requirements.txt -------------------------------------------------------------------------------- /stream_processing/kafka/run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/kafka/run.sh -------------------------------------------------------------------------------- /stream_processing/kafka/stream.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/kafka/stream.parquet -------------------------------------------------------------------------------- /stream_processing/requirements.txt: -------------------------------------------------------------------------------- 1 | apache-flink==1.17.1 -------------------------------------------------------------------------------- /stream_processing/run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/run.sh -------------------------------------------------------------------------------- /stream_processing/scripts/datastream_api.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/scripts/datastream_api.py -------------------------------------------------------------------------------- /stream_processing/scripts/json_consume_message.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/scripts/json_consume_message.py -------------------------------------------------------------------------------- /stream_processing/scripts/window_datastream_api.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/scripts/window_datastream_api.py -------------------------------------------------------------------------------- /stream_processing/scripts/window_datastream_api_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/stream_processing/scripts/window_datastream_api_example.py -------------------------------------------------------------------------------- /utils/datalake.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luongphambao/nyc-taxi-feature-store/HEAD/utils/datalake.yaml --------------------------------------------------------------------------------