├── .env ├── .gitattributes ├── .github └── workflows │ └── main.yml ├── .gitignore ├── .idea ├── big-data-engineering-project1.iml └── modules.xml ├── LICENSE ├── README.md ├── docker-compose-bigdata.yml ├── docs ├── doc │ ├── business_data_research.md │ ├── data-warehouse-development-specification │ │ ├── data-table-lifecycle-management-specification.md │ │ ├── data-warehouse-layering-specification.md │ │ ├── development-specification.md │ │ ├── partitioning-column-naming-conventions.md │ │ └── table-naming-convertions.md │ ├── development │ │ ├── SparkConnectorWriteToCK.md │ │ ├── data_pipeline │ │ │ ├── Config.md │ │ │ └── Utils.md │ │ ├── infra │ │ │ └── zookeeper-ha-cluster-format.md │ │ └── warehouse_modeling │ │ │ └── dwh-dev-ods.md │ ├── dwh-modelling-arch-diagram.md │ ├── dwh-modelling-sop.md │ ├── error-handling │ │ ├── 04_MAR_2025.md │ │ ├── 05_MAR_2025.md │ │ ├── 06_MAR_2025.md │ │ ├── 06_MAR_2025_02.md │ │ ├── 07_MAR_2025.md │ │ ├── 08-MAR-2025.md │ │ ├── 10-MAR-2025.md │ │ ├── 10-MAR-2025_02.md │ │ ├── 13-MAR-2025.md │ │ ├── 17-MAR-2025.md │ │ ├── 21-MAR-2025.md │ │ ├── 22-MAR-2025-2.md │ │ ├── 22-MAR-2025.md │ │ ├── future-fix.md │ │ └── image │ │ │ ├── 07_MAR_2025 │ │ │ ├── 1741396230363.png │ │ │ ├── 1741396348637.png │ │ │ ├── 1741396357527.png │ │ │ ├── 1741396403474.png │ │ │ ├── 1741396564052.png │ │ │ ├── 1741396658624.png │ │ │ ├── 1741396708285.png │ │ │ ├── 1741396794600.png │ │ │ ├── 1741397074160.png │ │ │ └── img_1.png │ │ │ ├── 08-MAR-2025 │ │ │ └── 1741467916690.png │ │ │ └── 10-MAR-2025 │ │ │ ├── 1741631354779.png │ │ │ └── 1741631491093.png │ ├── optimization │ │ └── reducing-spark-console-log-levels.md │ ├── tech-architecture.md │ └── testing │ │ ├── setup_custom_logger.md │ │ ├── spark_connect_oracle.md │ │ └── test_oracle_connect.md └── image │ ├── business_data_research │ └── ER.png │ ├── dwh-modelling-sop │ ├── DWH SOP.jpg │ └── DWH-modelling.jpg │ ├── error-handling │ └── future-fix │ │ └── no-logs.png │ └── tech-architecture │ └── data-pipeline-architecture.jpg ├── mysql-metadata-restore.sh ├── mysql-metastore-dump.sh ├── push-to-ghcr.sh ├── src ├── README.md ├── README │ ├── README │ │ ├── Test1-launch container.png │ │ ├── Test10-run spark script.png │ │ ├── Test10-run spark script2.png │ │ ├── Test2-launch hadoop cluster through script.png │ │ ├── Test3-hadoopmaster.png │ │ ├── Test3-hadoopworker1.png │ │ ├── Test3-hadoopworker2.png │ │ ├── Test4-check zookeeper status.png │ │ ├── Test5-cluster nodes.png │ │ ├── Test5-data nodes.png │ │ ├── Test5-overview.png │ │ ├── Test6-restore metadata in mysql.png │ │ ├── Test7-hive status.png │ │ ├── Test8-start spark service.png │ │ ├── Test9-connect to spark.png │ │ ├── airflow_web_ui.png │ │ ├── airflow_web_ui_login.png │ │ ├── grafana_web_ui.png │ │ ├── img.png │ │ ├── img_1.png │ │ ├── img_10.png │ │ ├── img_11.png │ │ ├── img_12.png │ │ ├── img_13.png │ │ ├── img_14.png │ │ ├── img_16.png │ │ ├── img_2.png │ │ ├── img_3.png │ │ ├── img_4.png │ │ ├── img_5.png │ │ └── img_6.png │ ├── batch_processing.md │ ├── quick-start.md │ ├── scheduler.md │ ├── scripts.md │ ├── snippets.md │ ├── spark-downstream.md │ ├── spark-upstream.md │ └── warehouse_modelling.md ├── batch_processing │ ├── __init__.py │ ├── configs │ │ ├── __init__.py │ │ ├── get_env.py │ │ ├── job_configs.py │ │ ├── logging_config.py │ │ └── sql_file.py │ ├── jobs │ │ ├── __init__.py │ │ ├── dwd_to_dim.py │ │ ├── dwd_to_dwm.py │ │ ├── dwm_to_dws.py │ │ └── ods_to_dwd.py │ ├── sql │ │ ├── all_partition_table_refresh.sql │ │ ├── dim │ │ │ ├── dml_dim_campaign_discount_fpd.sql │ │ │ ├── dml_dim_category_fpd.sql │ │ │ ├── dml_dim_customer_fpd.sql │ │ │ ├── dml_dim_payment_method_fpd.sql │ │ │ ├── dml_dim_product_fpd.sql │ │ │ └── dml_dim_supplier_fpd.sql │ │ ├── dwd │ │ │ ├── dml_dwd_campaign_product_subcategory_fpd.sql │ │ │ ├── dml_dwd_category_fpd.sql │ │ │ ├── dml_dwd_customer_fpd.sql │ │ │ ├── dml_dwd_customer_product_ratings_ipd.sql │ │ │ ├── dml_dwd_marketing_campaigns_fpd.sql │ │ │ ├── dml_dwd_orderitem_ipd.sql │ │ │ ├── dml_dwd_orders_ipd.sql │ │ │ ├── dml_dwd_payment_method_fpd.sql │ │ │ ├── dml_dwd_product_fpd.sql │ │ │ ├── dml_dwd_returns_ipd.sql │ │ │ ├── dml_dwd_subcategory_fpd.sql │ │ │ └── dml_dwd_supplier_fpd.sql │ │ ├── dwm │ │ │ └── dml_dwm_orders_with_items_ipd.sql │ │ └── dws │ │ │ └── dml_dws_orders_detailed_info_wide_ipd.sql │ └── utils │ │ ├── __init__.py │ │ ├── logging_utils.py │ │ ├── read_sql.py │ │ └── sql_render_partition_date.py ├── bi │ └── Dashboard-V2.pbix ├── data_pipeline │ ├── __init__.py │ ├── configs │ │ ├── .env │ │ ├── .env.secret │ │ ├── __init__.py │ │ ├── database_connection_config.py │ │ ├── get_env.py │ │ ├── hadoop_env_config.py │ │ ├── hive2ck_tables_config.py │ │ ├── job_configs.py │ │ ├── logging_config.py │ │ ├── oracle2hive_tables_config.py │ │ └── spark_env_config.py │ ├── core │ │ ├── __init__.py │ │ ├── extract_oracle.py │ │ ├── load_ck.py │ │ ├── load_hdfs.py │ │ ├── parse_field_type.py │ │ ├── spark_downstream.py │ │ └── spark_upstream.py │ └── utils │ │ ├── __init__.py │ │ ├── hdfs_utils.py │ │ ├── logging_utils.py │ │ └── oracle_database_utils.py ├── infra │ ├── airflow-config │ │ ├── airflow.cfg │ │ └── webserver_config.py │ ├── backup │ │ └── 2025-03-03 │ │ │ ├── core-site.xml │ │ │ ├── hdfs-site.xml │ │ │ ├── hive-site.xml │ │ │ ├── yarn-env.sh │ │ │ └── yarn-site.xml │ ├── hadoop-config │ │ ├── capacity-scheduler.xml │ │ ├── configuration.xsl │ │ ├── container-executor.cfg │ │ ├── core-site.xml │ │ ├── hadoop-env.cmd │ │ ├── hadoop-env.sh │ │ ├── hadoop-metrics2.properties │ │ ├── hadoop-policy.xml │ │ ├── hadoop-user-functions.sh.example │ │ ├── hdfs-site.xml │ │ ├── hosts │ │ │ └── hosts │ │ ├── httpfs-env.sh │ │ ├── httpfs-log4j.properties │ │ ├── httpfs-signature.secret │ │ ├── httpfs-site.xml │ │ ├── kms-acls.xml │ │ ├── kms-env.sh │ │ ├── kms-log4j.properties │ │ ├── kms-site.xml │ │ ├── log4j.properties │ │ ├── mapred-env.cmd │ │ ├── mapred-env.sh │ │ ├── mapred-queues.xml.template │ │ ├── mapred-site.xml │ │ ├── shellprofile.d │ │ │ └── example.sh │ │ ├── ssl-client.xml.example │ │ ├── ssl-server.xml.example │ │ ├── user_ec_policies.xml.template │ │ ├── workers │ │ ├── yarn-env.cmd │ │ ├── yarn-env.sh │ │ ├── yarn-site.xml │ │ └── yarnservice-log4j.properties │ ├── hive-config │ │ ├── Dockerfile │ │ ├── core-site.xml │ │ ├── hadoop-env.sh │ │ ├── hdfs-site.xml │ │ ├── hive-site.xml │ │ ├── hiveservices.sh │ │ └── mysql-connector-java-8.0.30.jar │ ├── monitoring-config │ │ ├── alertmanager.yml │ │ ├── grafana-dashboards │ │ │ ├── hadoop-cluster-node-exporter-grafana-dashboard.json │ │ │ └── mysqld-exporter-grafana-dashboard.json │ │ ├── grafana.ini │ │ ├── prometheus.yml │ │ └── prometheus_datasource.yml │ ├── mysql-config │ │ └── mysqld-exporter │ │ │ ├── my.cnf │ │ │ └── mysql_exporter_user_auth.sql │ ├── mysql-metastore-dump │ │ └── all_databases_backup.sql │ ├── spark-config │ │ ├── log4j2.properties │ │ ├── spark-defaults.conf │ │ └── spark-env.sh │ ├── ssh-config │ │ └── config │ └── zookeeper-config │ │ ├── dataDir.1 │ │ └── myid │ │ ├── dataDir.2 │ │ └── myid │ │ ├── dataDir.3 │ │ └── myid │ │ └── zoo.cfg ├── main_batch_processing.py ├── main_data_pipeline.py ├── scheduler │ └── dags │ │ ├── configs │ │ ├── __init__.py │ │ ├── dags_config.py │ │ ├── hql_files_path_config.py │ │ └── spark_config.py │ │ ├── dag_backfill_refresh.py │ │ ├── dag_batch_process.py │ │ ├── dag_etl_oltp2dwh.py │ │ ├── hql │ │ ├── dwd_partition_tbls_refresh.hql │ │ ├── dwm_partition_tbls_refresh.hql │ │ ├── dws_partition_tbls_refresh.hql │ │ └── ods_partition_tbls_refresh.hql │ │ └── utils │ │ ├── __init__.py │ │ ├── build_beeline_command.py │ │ ├── build_spark_submit_command.py │ │ └── read_sql.py ├── scripts │ ├── airflow │ │ ├── start-airflow.sh │ │ └── stop-airflow.sh │ ├── hadoop-master │ │ ├── my-start-NM-DN.sh │ │ ├── my-start-NN-ZKFC.sh │ │ ├── my-start-RM.sh │ │ ├── my-start-historyserver.sh │ │ ├── my-start-node-exporter.sh │ │ ├── my-start-zk.sh │ │ ├── my-stop-hadoop-cluster.sh │ │ └── my-stop-node-exporter.sh │ ├── hadoop-worker1 │ │ ├── my-start-NM-DN.sh │ │ ├── my-start-RM.sh │ │ ├── my-start-node-exporter.sh │ │ ├── my-start-zk.sh │ │ ├── my-stop-hadoop-cluster.sh │ │ └── my-stop-node-exporter.sh │ ├── hadoop-worker2 │ │ ├── my-start-NM-DN.sh │ │ ├── my-start-NN-ZKFC.sh │ │ ├── my-start-node-exporter.sh │ │ ├── my-start-zk.sh │ │ ├── my-stop-hadoop-cluster.sh │ │ └── my-stop-node-exporter.sh │ ├── hive │ │ ├── start-hive-services.sh │ │ └── stop-hive-services.sh │ ├── monitoring │ │ ├── start-monitoring-services.sh │ │ └── stop-monitoring-services.sh │ ├── mysql-hive-metastore │ │ ├── dump-mysql.sh │ │ ├── restore-mysql.sh │ │ ├── start-mysqld-exporter.sh │ │ └── stop-mysqld-exporter.sh │ └── spark │ │ ├── start-spark-services.sh │ │ └── stop-spark-services.sh ├── snippets │ ├── README.md │ ├── airflow │ │ └── airflow-commands.sh │ ├── docker │ │ ├── docker-images.md │ │ └── linux-container-init.md │ ├── hadoop_cluster_launching │ │ └── hadoop-cluster-zk-HA.md │ ├── hive │ │ ├── hive-launch.sh │ │ └── hiveuser_root_auth_grant.sql │ ├── mysql │ │ ├── hive-metadata-query-refresh.md │ │ └── mysql-commadns.md │ └── pyspark │ │ ├── python-package.sh │ │ ├── spark-submit-data-processing.sh │ │ ├── spark-submit.sh │ │ └── start_thriftserver.sh └── warehouse_modeling │ ├── ads_clickhouse │ ├── ck_ddl_ads_orders_detailed_info_wide_ipd.sql │ └── ck_ddl_ads_schema.sql │ ├── dim │ ├── ddl_dim_campaign_discount_fpd.hql │ ├── ddl_dim_category_fpd.hql │ ├── ddl_dim_customer_fpd.hql │ ├── ddl_dim_payment_method_fpd.hql │ ├── ddl_dim_product_fpd.hql │ ├── ddl_dim_schema.hql │ ├── ddl_dim_supplier_fpd.hql │ └── hdfs │ │ ├── hdfs_dim_env_initial.sh │ │ └── upload_to_hadoop_cluster.sh │ ├── dwd │ ├── ddl_dwd_campaign_product_subcategory_fpd.hql │ ├── ddl_dwd_category_fpd.hql │ ├── ddl_dwd_customer_fpd.hql │ ├── ddl_dwd_customer_product_ratings_ipd.hql │ ├── ddl_dwd_marketing_campaigns_fpd.hql │ ├── ddl_dwd_orderitem_ipd.hql │ ├── ddl_dwd_orders_ipd.hql │ ├── ddl_dwd_payment_method_fpd.hql │ ├── ddl_dwd_product_fpd.hql │ ├── ddl_dwd_returns_ipd.hql │ ├── ddl_dwd_schema.hql │ ├── ddl_dwd_subcategory_fpd.hql │ ├── ddl_dwd_supplier_fpd.hql │ ├── hdfs │ │ ├── hdfs_dwd_env_initial.sh │ │ └── upload_to_hadoop_cluster.sh │ └── hive_config.hql │ ├── dwm │ ├── ddl_dwm_orders_with_items_ipd.hql │ ├── ddl_dwm_schema.hql │ └── hdfs │ │ ├── hdfs_dwm_env_initial.sh │ │ └── upload_to_hadoop_cluster.sh │ ├── dws │ ├── ddl_dws_orders_detailed_info_wide_ipd.hql │ ├── ddl_dws_schema.hql │ └── hdfs │ │ ├── hdfs_dws_env_initial.sh │ │ └── upload_to_hadoop_cluster.sh │ └── ods │ ├── avro_schema │ ├── ods_campaign_product_subcategory_fpd.avsc │ ├── ods_category_fpd.avsc │ ├── ods_customer_fpd.avsc │ ├── ods_customer_product_ratings_ipd.avsc │ ├── ods_marketing_campaigns_fpd.avsc │ ├── ods_orderitem_ipd.avsc │ ├── ods_orders_ipd.avsc │ ├── ods_payment_method_fpd.avsc │ ├── ods_product_fpd.avsc │ ├── ods_returns_ipd.avsc │ ├── ods_subcategory_fpd.avsc │ └── ods_supplier_fpd.avsc │ ├── ddl_ods_campaign_product_subcategory_fpd.hql │ ├── ddl_ods_category_fpd.hql │ ├── ddl_ods_customer_fpd.hql │ ├── ddl_ods_customer_product_ratings_ipd.hql │ ├── ddl_ods_marketing_campaigns_fpd.hql │ ├── ddl_ods_orderitem_ipd.hql │ ├── ddl_ods_orders_ipd.hql │ ├── ddl_ods_payment_method_fpd.hql │ ├── ddl_ods_product_fpd.hql │ ├── ddl_ods_returns_ipd.hql │ ├── ddl_ods_schema.hql │ ├── ddl_ods_subcategory_fpd.hql │ ├── ddl_ods_supplier_fpd.hql │ └── hdfs │ ├── hdfs_ods_env_initial.sh │ └── upload_to_hadoop_cluster.sh ├── start-data-clients.sh ├── start-hadoop-cluster.sh ├── start-other-services.sh ├── stop-data-clients.sh ├── stop-hadoop-cluster.sh ├── stop-other-services.sh └── test ├── batch_processing ├── dml_sql │ └── dml_dwd_customer_product_ratings_ipd.sql ├── exec_dml_sql.py ├── generate_hive_compitable_parquet.py ├── read_parquet.py ├── read_sql.py └── transform_ods2dwd.py └── data_pipeline ├── setup_custom_logger.py ├── spark_connect_oracle.py ├── spark_extract_from_oracle.py ├── spark_load_to_hive.py ├── spark_read_avro.py ├── spark_write_parquet_data_to_clickhouse.py └── test_oracle_connect.py /.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/.env -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/.gitattributes -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/.github/workflows/main.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/.gitignore -------------------------------------------------------------------------------- /.idea/big-data-engineering-project1.iml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/.idea/big-data-engineering-project1.iml -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/.idea/modules.xml -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/README.md -------------------------------------------------------------------------------- /docker-compose-bigdata.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docker-compose-bigdata.yml -------------------------------------------------------------------------------- /docs/doc/business_data_research.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/business_data_research.md -------------------------------------------------------------------------------- /docs/doc/data-warehouse-development-specification/data-table-lifecycle-management-specification.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/data-warehouse-development-specification/data-table-lifecycle-management-specification.md -------------------------------------------------------------------------------- /docs/doc/data-warehouse-development-specification/data-warehouse-layering-specification.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/data-warehouse-development-specification/data-warehouse-layering-specification.md -------------------------------------------------------------------------------- /docs/doc/data-warehouse-development-specification/development-specification.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/data-warehouse-development-specification/development-specification.md -------------------------------------------------------------------------------- /docs/doc/data-warehouse-development-specification/partitioning-column-naming-conventions.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/data-warehouse-development-specification/partitioning-column-naming-conventions.md -------------------------------------------------------------------------------- /docs/doc/data-warehouse-development-specification/table-naming-convertions.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/data-warehouse-development-specification/table-naming-convertions.md -------------------------------------------------------------------------------- /docs/doc/development/SparkConnectorWriteToCK.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/development/SparkConnectorWriteToCK.md -------------------------------------------------------------------------------- /docs/doc/development/data_pipeline/Config.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/development/data_pipeline/Config.md -------------------------------------------------------------------------------- /docs/doc/development/data_pipeline/Utils.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/development/data_pipeline/Utils.md -------------------------------------------------------------------------------- /docs/doc/development/infra/zookeeper-ha-cluster-format.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/development/infra/zookeeper-ha-cluster-format.md -------------------------------------------------------------------------------- /docs/doc/development/warehouse_modeling/dwh-dev-ods.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/development/warehouse_modeling/dwh-dev-ods.md -------------------------------------------------------------------------------- /docs/doc/dwh-modelling-arch-diagram.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/dwh-modelling-arch-diagram.md -------------------------------------------------------------------------------- /docs/doc/dwh-modelling-sop.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/dwh-modelling-sop.md -------------------------------------------------------------------------------- /docs/doc/error-handling/04_MAR_2025.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/04_MAR_2025.md -------------------------------------------------------------------------------- /docs/doc/error-handling/05_MAR_2025.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/05_MAR_2025.md -------------------------------------------------------------------------------- /docs/doc/error-handling/06_MAR_2025.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/06_MAR_2025.md -------------------------------------------------------------------------------- /docs/doc/error-handling/06_MAR_2025_02.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/06_MAR_2025_02.md -------------------------------------------------------------------------------- /docs/doc/error-handling/07_MAR_2025.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/07_MAR_2025.md -------------------------------------------------------------------------------- /docs/doc/error-handling/08-MAR-2025.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/08-MAR-2025.md -------------------------------------------------------------------------------- /docs/doc/error-handling/10-MAR-2025.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/10-MAR-2025.md -------------------------------------------------------------------------------- /docs/doc/error-handling/10-MAR-2025_02.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/10-MAR-2025_02.md -------------------------------------------------------------------------------- /docs/doc/error-handling/13-MAR-2025.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/13-MAR-2025.md -------------------------------------------------------------------------------- /docs/doc/error-handling/17-MAR-2025.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/17-MAR-2025.md -------------------------------------------------------------------------------- /docs/doc/error-handling/21-MAR-2025.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/21-MAR-2025.md -------------------------------------------------------------------------------- /docs/doc/error-handling/22-MAR-2025-2.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/22-MAR-2025-2.md -------------------------------------------------------------------------------- /docs/doc/error-handling/22-MAR-2025.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/22-MAR-2025.md -------------------------------------------------------------------------------- /docs/doc/error-handling/future-fix.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/future-fix.md -------------------------------------------------------------------------------- /docs/doc/error-handling/image/07_MAR_2025/1741396230363.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/image/07_MAR_2025/1741396230363.png -------------------------------------------------------------------------------- /docs/doc/error-handling/image/07_MAR_2025/1741396348637.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/image/07_MAR_2025/1741396348637.png -------------------------------------------------------------------------------- /docs/doc/error-handling/image/07_MAR_2025/1741396357527.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/image/07_MAR_2025/1741396357527.png -------------------------------------------------------------------------------- /docs/doc/error-handling/image/07_MAR_2025/1741396403474.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/image/07_MAR_2025/1741396403474.png -------------------------------------------------------------------------------- /docs/doc/error-handling/image/07_MAR_2025/1741396564052.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/image/07_MAR_2025/1741396564052.png -------------------------------------------------------------------------------- /docs/doc/error-handling/image/07_MAR_2025/1741396658624.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/image/07_MAR_2025/1741396658624.png -------------------------------------------------------------------------------- /docs/doc/error-handling/image/07_MAR_2025/1741396708285.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/image/07_MAR_2025/1741396708285.png -------------------------------------------------------------------------------- /docs/doc/error-handling/image/07_MAR_2025/1741396794600.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/image/07_MAR_2025/1741396794600.png -------------------------------------------------------------------------------- /docs/doc/error-handling/image/07_MAR_2025/1741397074160.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/image/07_MAR_2025/1741397074160.png -------------------------------------------------------------------------------- /docs/doc/error-handling/image/07_MAR_2025/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/image/07_MAR_2025/img_1.png -------------------------------------------------------------------------------- /docs/doc/error-handling/image/08-MAR-2025/1741467916690.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/image/08-MAR-2025/1741467916690.png -------------------------------------------------------------------------------- /docs/doc/error-handling/image/10-MAR-2025/1741631354779.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/image/10-MAR-2025/1741631354779.png -------------------------------------------------------------------------------- /docs/doc/error-handling/image/10-MAR-2025/1741631491093.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/error-handling/image/10-MAR-2025/1741631491093.png -------------------------------------------------------------------------------- /docs/doc/optimization/reducing-spark-console-log-levels.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/optimization/reducing-spark-console-log-levels.md -------------------------------------------------------------------------------- /docs/doc/tech-architecture.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/tech-architecture.md -------------------------------------------------------------------------------- /docs/doc/testing/setup_custom_logger.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/testing/setup_custom_logger.md -------------------------------------------------------------------------------- /docs/doc/testing/spark_connect_oracle.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/testing/spark_connect_oracle.md -------------------------------------------------------------------------------- /docs/doc/testing/test_oracle_connect.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/doc/testing/test_oracle_connect.md -------------------------------------------------------------------------------- /docs/image/business_data_research/ER.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/image/business_data_research/ER.png -------------------------------------------------------------------------------- /docs/image/dwh-modelling-sop/DWH SOP.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/image/dwh-modelling-sop/DWH SOP.jpg -------------------------------------------------------------------------------- /docs/image/dwh-modelling-sop/DWH-modelling.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/image/dwh-modelling-sop/DWH-modelling.jpg -------------------------------------------------------------------------------- /docs/image/error-handling/future-fix/no-logs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/image/error-handling/future-fix/no-logs.png -------------------------------------------------------------------------------- /docs/image/tech-architecture/data-pipeline-architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/docs/image/tech-architecture/data-pipeline-architecture.jpg -------------------------------------------------------------------------------- /mysql-metadata-restore.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/mysql-metadata-restore.sh -------------------------------------------------------------------------------- /mysql-metastore-dump.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/mysql-metastore-dump.sh -------------------------------------------------------------------------------- /push-to-ghcr.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/push-to-ghcr.sh -------------------------------------------------------------------------------- /src/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README.md -------------------------------------------------------------------------------- /src/README/README/Test1-launch container.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/Test1-launch container.png -------------------------------------------------------------------------------- /src/README/README/Test10-run spark script.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/Test10-run spark script.png -------------------------------------------------------------------------------- /src/README/README/Test10-run spark script2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/Test10-run spark script2.png -------------------------------------------------------------------------------- /src/README/README/Test2-launch hadoop cluster through script.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/Test2-launch hadoop cluster through script.png -------------------------------------------------------------------------------- /src/README/README/Test3-hadoopmaster.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/Test3-hadoopmaster.png -------------------------------------------------------------------------------- /src/README/README/Test3-hadoopworker1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/Test3-hadoopworker1.png -------------------------------------------------------------------------------- /src/README/README/Test3-hadoopworker2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/Test3-hadoopworker2.png -------------------------------------------------------------------------------- /src/README/README/Test4-check zookeeper status.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/Test4-check zookeeper status.png -------------------------------------------------------------------------------- /src/README/README/Test5-cluster nodes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/Test5-cluster nodes.png -------------------------------------------------------------------------------- /src/README/README/Test5-data nodes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/Test5-data nodes.png -------------------------------------------------------------------------------- /src/README/README/Test5-overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/Test5-overview.png -------------------------------------------------------------------------------- /src/README/README/Test6-restore metadata in mysql.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/Test6-restore metadata in mysql.png -------------------------------------------------------------------------------- /src/README/README/Test7-hive status.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/Test7-hive status.png -------------------------------------------------------------------------------- /src/README/README/Test8-start spark service.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/Test8-start spark service.png -------------------------------------------------------------------------------- /src/README/README/Test9-connect to spark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/Test9-connect to spark.png -------------------------------------------------------------------------------- /src/README/README/airflow_web_ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/airflow_web_ui.png -------------------------------------------------------------------------------- /src/README/README/airflow_web_ui_login.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/airflow_web_ui_login.png -------------------------------------------------------------------------------- /src/README/README/grafana_web_ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/grafana_web_ui.png -------------------------------------------------------------------------------- /src/README/README/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/img.png -------------------------------------------------------------------------------- /src/README/README/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/img_1.png -------------------------------------------------------------------------------- /src/README/README/img_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/img_10.png -------------------------------------------------------------------------------- /src/README/README/img_11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/img_11.png -------------------------------------------------------------------------------- /src/README/README/img_12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/img_12.png -------------------------------------------------------------------------------- /src/README/README/img_13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/img_13.png -------------------------------------------------------------------------------- /src/README/README/img_14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/img_14.png -------------------------------------------------------------------------------- /src/README/README/img_16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/img_16.png -------------------------------------------------------------------------------- /src/README/README/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/img_2.png -------------------------------------------------------------------------------- /src/README/README/img_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/img_3.png -------------------------------------------------------------------------------- /src/README/README/img_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/img_4.png -------------------------------------------------------------------------------- /src/README/README/img_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/img_5.png -------------------------------------------------------------------------------- /src/README/README/img_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/README/img_6.png -------------------------------------------------------------------------------- /src/README/batch_processing.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/batch_processing.md -------------------------------------------------------------------------------- /src/README/quick-start.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/quick-start.md -------------------------------------------------------------------------------- /src/README/scheduler.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/scheduler.md -------------------------------------------------------------------------------- /src/README/scripts.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/scripts.md -------------------------------------------------------------------------------- /src/README/snippets.md: -------------------------------------------------------------------------------- 1 | # snippets -------------------------------------------------------------------------------- /src/README/spark-downstream.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/spark-downstream.md -------------------------------------------------------------------------------- /src/README/spark-upstream.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/spark-upstream.md -------------------------------------------------------------------------------- /src/README/warehouse_modelling.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/README/warehouse_modelling.md -------------------------------------------------------------------------------- /src/batch_processing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/__init__.py -------------------------------------------------------------------------------- /src/batch_processing/configs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/configs/__init__.py -------------------------------------------------------------------------------- /src/batch_processing/configs/get_env.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/configs/get_env.py -------------------------------------------------------------------------------- /src/batch_processing/configs/job_configs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/configs/job_configs.py -------------------------------------------------------------------------------- /src/batch_processing/configs/logging_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/configs/logging_config.py -------------------------------------------------------------------------------- /src/batch_processing/configs/sql_file.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/configs/sql_file.py -------------------------------------------------------------------------------- /src/batch_processing/jobs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/jobs/__init__.py -------------------------------------------------------------------------------- /src/batch_processing/jobs/dwd_to_dim.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/jobs/dwd_to_dim.py -------------------------------------------------------------------------------- /src/batch_processing/jobs/dwd_to_dwm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/jobs/dwd_to_dwm.py -------------------------------------------------------------------------------- /src/batch_processing/jobs/dwm_to_dws.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/jobs/dwm_to_dws.py -------------------------------------------------------------------------------- /src/batch_processing/jobs/ods_to_dwd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/jobs/ods_to_dwd.py -------------------------------------------------------------------------------- /src/batch_processing/sql/all_partition_table_refresh.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/sql/all_partition_table_refresh.sql -------------------------------------------------------------------------------- /src/batch_processing/sql/dim/dml_dim_campaign_discount_fpd.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/sql/dim/dml_dim_campaign_discount_fpd.sql -------------------------------------------------------------------------------- /src/batch_processing/sql/dim/dml_dim_category_fpd.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/sql/dim/dml_dim_category_fpd.sql -------------------------------------------------------------------------------- /src/batch_processing/sql/dim/dml_dim_customer_fpd.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/sql/dim/dml_dim_customer_fpd.sql -------------------------------------------------------------------------------- /src/batch_processing/sql/dim/dml_dim_payment_method_fpd.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/sql/dim/dml_dim_payment_method_fpd.sql -------------------------------------------------------------------------------- /src/batch_processing/sql/dim/dml_dim_product_fpd.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/sql/dim/dml_dim_product_fpd.sql -------------------------------------------------------------------------------- /src/batch_processing/sql/dim/dml_dim_supplier_fpd.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/sql/dim/dml_dim_supplier_fpd.sql -------------------------------------------------------------------------------- /src/batch_processing/sql/dwd/dml_dwd_campaign_product_subcategory_fpd.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/sql/dwd/dml_dwd_campaign_product_subcategory_fpd.sql -------------------------------------------------------------------------------- /src/batch_processing/sql/dwd/dml_dwd_category_fpd.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/sql/dwd/dml_dwd_category_fpd.sql -------------------------------------------------------------------------------- /src/batch_processing/sql/dwd/dml_dwd_customer_fpd.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/sql/dwd/dml_dwd_customer_fpd.sql -------------------------------------------------------------------------------- /src/batch_processing/sql/dwd/dml_dwd_customer_product_ratings_ipd.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/sql/dwd/dml_dwd_customer_product_ratings_ipd.sql -------------------------------------------------------------------------------- /src/batch_processing/sql/dwd/dml_dwd_marketing_campaigns_fpd.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/sql/dwd/dml_dwd_marketing_campaigns_fpd.sql -------------------------------------------------------------------------------- /src/batch_processing/sql/dwd/dml_dwd_orderitem_ipd.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/sql/dwd/dml_dwd_orderitem_ipd.sql -------------------------------------------------------------------------------- /src/batch_processing/sql/dwd/dml_dwd_orders_ipd.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/sql/dwd/dml_dwd_orders_ipd.sql -------------------------------------------------------------------------------- /src/batch_processing/sql/dwd/dml_dwd_payment_method_fpd.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/sql/dwd/dml_dwd_payment_method_fpd.sql -------------------------------------------------------------------------------- /src/batch_processing/sql/dwd/dml_dwd_product_fpd.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/sql/dwd/dml_dwd_product_fpd.sql -------------------------------------------------------------------------------- /src/batch_processing/sql/dwd/dml_dwd_returns_ipd.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/sql/dwd/dml_dwd_returns_ipd.sql -------------------------------------------------------------------------------- /src/batch_processing/sql/dwd/dml_dwd_subcategory_fpd.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/sql/dwd/dml_dwd_subcategory_fpd.sql -------------------------------------------------------------------------------- /src/batch_processing/sql/dwd/dml_dwd_supplier_fpd.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/sql/dwd/dml_dwd_supplier_fpd.sql -------------------------------------------------------------------------------- /src/batch_processing/sql/dwm/dml_dwm_orders_with_items_ipd.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/sql/dwm/dml_dwm_orders_with_items_ipd.sql -------------------------------------------------------------------------------- /src/batch_processing/sql/dws/dml_dws_orders_detailed_info_wide_ipd.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/sql/dws/dml_dws_orders_detailed_info_wide_ipd.sql -------------------------------------------------------------------------------- /src/batch_processing/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/utils/__init__.py -------------------------------------------------------------------------------- /src/batch_processing/utils/logging_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/utils/logging_utils.py -------------------------------------------------------------------------------- /src/batch_processing/utils/read_sql.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/utils/read_sql.py -------------------------------------------------------------------------------- /src/batch_processing/utils/sql_render_partition_date.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/batch_processing/utils/sql_render_partition_date.py -------------------------------------------------------------------------------- /src/bi/Dashboard-V2.pbix: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/bi/Dashboard-V2.pbix -------------------------------------------------------------------------------- /src/data_pipeline/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/data_pipeline/__init__.py -------------------------------------------------------------------------------- /src/data_pipeline/configs/.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/data_pipeline/configs/.env -------------------------------------------------------------------------------- /src/data_pipeline/configs/.env.secret: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/data_pipeline/configs/.env.secret -------------------------------------------------------------------------------- /src/data_pipeline/configs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/data_pipeline/configs/__init__.py -------------------------------------------------------------------------------- /src/data_pipeline/configs/database_connection_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/data_pipeline/configs/database_connection_config.py -------------------------------------------------------------------------------- /src/data_pipeline/configs/get_env.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/data_pipeline/configs/get_env.py -------------------------------------------------------------------------------- /src/data_pipeline/configs/hadoop_env_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/data_pipeline/configs/hadoop_env_config.py -------------------------------------------------------------------------------- /src/data_pipeline/configs/hive2ck_tables_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/data_pipeline/configs/hive2ck_tables_config.py -------------------------------------------------------------------------------- /src/data_pipeline/configs/job_configs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/data_pipeline/configs/job_configs.py -------------------------------------------------------------------------------- /src/data_pipeline/configs/logging_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/data_pipeline/configs/logging_config.py -------------------------------------------------------------------------------- /src/data_pipeline/configs/oracle2hive_tables_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/data_pipeline/configs/oracle2hive_tables_config.py -------------------------------------------------------------------------------- /src/data_pipeline/configs/spark_env_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/data_pipeline/configs/spark_env_config.py -------------------------------------------------------------------------------- /src/data_pipeline/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/data_pipeline/core/__init__.py -------------------------------------------------------------------------------- /src/data_pipeline/core/extract_oracle.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/data_pipeline/core/extract_oracle.py -------------------------------------------------------------------------------- /src/data_pipeline/core/load_ck.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/data_pipeline/core/load_ck.py -------------------------------------------------------------------------------- /src/data_pipeline/core/load_hdfs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/data_pipeline/core/load_hdfs.py -------------------------------------------------------------------------------- /src/data_pipeline/core/parse_field_type.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/data_pipeline/core/parse_field_type.py -------------------------------------------------------------------------------- /src/data_pipeline/core/spark_downstream.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/data_pipeline/core/spark_downstream.py -------------------------------------------------------------------------------- /src/data_pipeline/core/spark_upstream.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/data_pipeline/core/spark_upstream.py -------------------------------------------------------------------------------- /src/data_pipeline/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/data_pipeline/utils/__init__.py -------------------------------------------------------------------------------- /src/data_pipeline/utils/hdfs_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/data_pipeline/utils/hdfs_utils.py -------------------------------------------------------------------------------- /src/data_pipeline/utils/logging_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/data_pipeline/utils/logging_utils.py -------------------------------------------------------------------------------- /src/data_pipeline/utils/oracle_database_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/data_pipeline/utils/oracle_database_utils.py -------------------------------------------------------------------------------- /src/infra/airflow-config/airflow.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/airflow-config/airflow.cfg -------------------------------------------------------------------------------- /src/infra/airflow-config/webserver_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/airflow-config/webserver_config.py -------------------------------------------------------------------------------- /src/infra/backup/2025-03-03/core-site.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/backup/2025-03-03/core-site.xml -------------------------------------------------------------------------------- /src/infra/backup/2025-03-03/hdfs-site.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/backup/2025-03-03/hdfs-site.xml -------------------------------------------------------------------------------- /src/infra/backup/2025-03-03/hive-site.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/backup/2025-03-03/hive-site.xml -------------------------------------------------------------------------------- /src/infra/backup/2025-03-03/yarn-env.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/backup/2025-03-03/yarn-env.sh -------------------------------------------------------------------------------- /src/infra/backup/2025-03-03/yarn-site.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/backup/2025-03-03/yarn-site.xml -------------------------------------------------------------------------------- /src/infra/hadoop-config/capacity-scheduler.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/capacity-scheduler.xml -------------------------------------------------------------------------------- /src/infra/hadoop-config/configuration.xsl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/configuration.xsl -------------------------------------------------------------------------------- /src/infra/hadoop-config/container-executor.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/container-executor.cfg -------------------------------------------------------------------------------- /src/infra/hadoop-config/core-site.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/core-site.xml -------------------------------------------------------------------------------- /src/infra/hadoop-config/hadoop-env.cmd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/hadoop-env.cmd -------------------------------------------------------------------------------- /src/infra/hadoop-config/hadoop-env.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/hadoop-env.sh -------------------------------------------------------------------------------- /src/infra/hadoop-config/hadoop-metrics2.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/hadoop-metrics2.properties -------------------------------------------------------------------------------- /src/infra/hadoop-config/hadoop-policy.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/hadoop-policy.xml -------------------------------------------------------------------------------- /src/infra/hadoop-config/hadoop-user-functions.sh.example: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/hadoop-user-functions.sh.example -------------------------------------------------------------------------------- /src/infra/hadoop-config/hdfs-site.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/hdfs-site.xml -------------------------------------------------------------------------------- /src/infra/hadoop-config/hosts/hosts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/hosts/hosts -------------------------------------------------------------------------------- /src/infra/hadoop-config/httpfs-env.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/httpfs-env.sh -------------------------------------------------------------------------------- /src/infra/hadoop-config/httpfs-log4j.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/httpfs-log4j.properties -------------------------------------------------------------------------------- /src/infra/hadoop-config/httpfs-signature.secret: -------------------------------------------------------------------------------- 1 | hadoop httpfs secret 2 | -------------------------------------------------------------------------------- /src/infra/hadoop-config/httpfs-site.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/httpfs-site.xml -------------------------------------------------------------------------------- /src/infra/hadoop-config/kms-acls.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/kms-acls.xml -------------------------------------------------------------------------------- /src/infra/hadoop-config/kms-env.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/kms-env.sh -------------------------------------------------------------------------------- /src/infra/hadoop-config/kms-log4j.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/kms-log4j.properties -------------------------------------------------------------------------------- /src/infra/hadoop-config/kms-site.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/kms-site.xml -------------------------------------------------------------------------------- /src/infra/hadoop-config/log4j.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/log4j.properties -------------------------------------------------------------------------------- /src/infra/hadoop-config/mapred-env.cmd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/mapred-env.cmd -------------------------------------------------------------------------------- /src/infra/hadoop-config/mapred-env.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/mapred-env.sh -------------------------------------------------------------------------------- /src/infra/hadoop-config/mapred-queues.xml.template: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/mapred-queues.xml.template -------------------------------------------------------------------------------- /src/infra/hadoop-config/mapred-site.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/mapred-site.xml -------------------------------------------------------------------------------- /src/infra/hadoop-config/shellprofile.d/example.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/shellprofile.d/example.sh -------------------------------------------------------------------------------- /src/infra/hadoop-config/ssl-client.xml.example: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/ssl-client.xml.example -------------------------------------------------------------------------------- /src/infra/hadoop-config/ssl-server.xml.example: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/ssl-server.xml.example -------------------------------------------------------------------------------- /src/infra/hadoop-config/user_ec_policies.xml.template: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/user_ec_policies.xml.template -------------------------------------------------------------------------------- /src/infra/hadoop-config/workers: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/workers -------------------------------------------------------------------------------- /src/infra/hadoop-config/yarn-env.cmd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/yarn-env.cmd -------------------------------------------------------------------------------- /src/infra/hadoop-config/yarn-env.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/yarn-env.sh -------------------------------------------------------------------------------- /src/infra/hadoop-config/yarn-site.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/yarn-site.xml -------------------------------------------------------------------------------- /src/infra/hadoop-config/yarnservice-log4j.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hadoop-config/yarnservice-log4j.properties -------------------------------------------------------------------------------- /src/infra/hive-config/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hive-config/Dockerfile -------------------------------------------------------------------------------- /src/infra/hive-config/core-site.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hive-config/core-site.xml -------------------------------------------------------------------------------- /src/infra/hive-config/hadoop-env.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hive-config/hadoop-env.sh -------------------------------------------------------------------------------- /src/infra/hive-config/hdfs-site.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hive-config/hdfs-site.xml -------------------------------------------------------------------------------- /src/infra/hive-config/hive-site.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hive-config/hive-site.xml -------------------------------------------------------------------------------- /src/infra/hive-config/hiveservices.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hive-config/hiveservices.sh -------------------------------------------------------------------------------- /src/infra/hive-config/mysql-connector-java-8.0.30.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/hive-config/mysql-connector-java-8.0.30.jar -------------------------------------------------------------------------------- /src/infra/monitoring-config/alertmanager.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/monitoring-config/alertmanager.yml -------------------------------------------------------------------------------- /src/infra/monitoring-config/grafana-dashboards/hadoop-cluster-node-exporter-grafana-dashboard.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/monitoring-config/grafana-dashboards/hadoop-cluster-node-exporter-grafana-dashboard.json -------------------------------------------------------------------------------- /src/infra/monitoring-config/grafana-dashboards/mysqld-exporter-grafana-dashboard.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/monitoring-config/grafana-dashboards/mysqld-exporter-grafana-dashboard.json -------------------------------------------------------------------------------- /src/infra/monitoring-config/grafana.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/monitoring-config/grafana.ini -------------------------------------------------------------------------------- /src/infra/monitoring-config/prometheus.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/monitoring-config/prometheus.yml -------------------------------------------------------------------------------- /src/infra/monitoring-config/prometheus_datasource.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/monitoring-config/prometheus_datasource.yml -------------------------------------------------------------------------------- /src/infra/mysql-config/mysqld-exporter/my.cnf: -------------------------------------------------------------------------------- 1 | [client] 2 | user=mysql_exporter 3 | password=123456 4 | -------------------------------------------------------------------------------- /src/infra/mysql-config/mysqld-exporter/mysql_exporter_user_auth.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/mysql-config/mysqld-exporter/mysql_exporter_user_auth.sql -------------------------------------------------------------------------------- /src/infra/mysql-metastore-dump/all_databases_backup.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/mysql-metastore-dump/all_databases_backup.sql -------------------------------------------------------------------------------- /src/infra/spark-config/log4j2.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/spark-config/log4j2.properties -------------------------------------------------------------------------------- /src/infra/spark-config/spark-defaults.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/spark-config/spark-defaults.conf -------------------------------------------------------------------------------- /src/infra/spark-config/spark-env.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/spark-config/spark-env.sh -------------------------------------------------------------------------------- /src/infra/ssh-config/config: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/ssh-config/config -------------------------------------------------------------------------------- /src/infra/zookeeper-config/dataDir.1/myid: -------------------------------------------------------------------------------- 1 | 1 -------------------------------------------------------------------------------- /src/infra/zookeeper-config/dataDir.2/myid: -------------------------------------------------------------------------------- 1 | 2 -------------------------------------------------------------------------------- /src/infra/zookeeper-config/dataDir.3/myid: -------------------------------------------------------------------------------- 1 | 3 -------------------------------------------------------------------------------- /src/infra/zookeeper-config/zoo.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/infra/zookeeper-config/zoo.cfg -------------------------------------------------------------------------------- /src/main_batch_processing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/main_batch_processing.py -------------------------------------------------------------------------------- /src/main_data_pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/main_data_pipeline.py -------------------------------------------------------------------------------- /src/scheduler/dags/configs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scheduler/dags/configs/__init__.py -------------------------------------------------------------------------------- /src/scheduler/dags/configs/dags_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scheduler/dags/configs/dags_config.py -------------------------------------------------------------------------------- /src/scheduler/dags/configs/hql_files_path_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scheduler/dags/configs/hql_files_path_config.py -------------------------------------------------------------------------------- /src/scheduler/dags/configs/spark_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scheduler/dags/configs/spark_config.py -------------------------------------------------------------------------------- /src/scheduler/dags/dag_backfill_refresh.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scheduler/dags/dag_backfill_refresh.py -------------------------------------------------------------------------------- /src/scheduler/dags/dag_batch_process.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scheduler/dags/dag_batch_process.py -------------------------------------------------------------------------------- /src/scheduler/dags/dag_etl_oltp2dwh.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scheduler/dags/dag_etl_oltp2dwh.py -------------------------------------------------------------------------------- /src/scheduler/dags/hql/dwd_partition_tbls_refresh.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scheduler/dags/hql/dwd_partition_tbls_refresh.hql -------------------------------------------------------------------------------- /src/scheduler/dags/hql/dwm_partition_tbls_refresh.hql: -------------------------------------------------------------------------------- 1 | MSCK REPAIR TABLE dwm.dwm_orders_with_items_ipd; -------------------------------------------------------------------------------- /src/scheduler/dags/hql/dws_partition_tbls_refresh.hql: -------------------------------------------------------------------------------- 1 | MSCK REPAIR TABLE dws.dws_orders_detailed_info_wide_ipd; -------------------------------------------------------------------------------- /src/scheduler/dags/hql/ods_partition_tbls_refresh.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scheduler/dags/hql/ods_partition_tbls_refresh.hql -------------------------------------------------------------------------------- /src/scheduler/dags/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scheduler/dags/utils/__init__.py -------------------------------------------------------------------------------- /src/scheduler/dags/utils/build_beeline_command.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scheduler/dags/utils/build_beeline_command.py -------------------------------------------------------------------------------- /src/scheduler/dags/utils/build_spark_submit_command.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scheduler/dags/utils/build_spark_submit_command.py -------------------------------------------------------------------------------- /src/scheduler/dags/utils/read_sql.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scheduler/dags/utils/read_sql.py -------------------------------------------------------------------------------- /src/scripts/airflow/start-airflow.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/airflow/start-airflow.sh -------------------------------------------------------------------------------- /src/scripts/airflow/stop-airflow.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/airflow/stop-airflow.sh -------------------------------------------------------------------------------- /src/scripts/hadoop-master/my-start-NM-DN.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/hadoop-master/my-start-NM-DN.sh -------------------------------------------------------------------------------- /src/scripts/hadoop-master/my-start-NN-ZKFC.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/hadoop-master/my-start-NN-ZKFC.sh -------------------------------------------------------------------------------- /src/scripts/hadoop-master/my-start-RM.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/hadoop-master/my-start-RM.sh -------------------------------------------------------------------------------- /src/scripts/hadoop-master/my-start-historyserver.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/hadoop-master/my-start-historyserver.sh -------------------------------------------------------------------------------- /src/scripts/hadoop-master/my-start-node-exporter.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/hadoop-master/my-start-node-exporter.sh -------------------------------------------------------------------------------- /src/scripts/hadoop-master/my-start-zk.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/hadoop-master/my-start-zk.sh -------------------------------------------------------------------------------- /src/scripts/hadoop-master/my-stop-hadoop-cluster.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/hadoop-master/my-stop-hadoop-cluster.sh -------------------------------------------------------------------------------- /src/scripts/hadoop-master/my-stop-node-exporter.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/hadoop-master/my-stop-node-exporter.sh -------------------------------------------------------------------------------- /src/scripts/hadoop-worker1/my-start-NM-DN.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/hadoop-worker1/my-start-NM-DN.sh -------------------------------------------------------------------------------- /src/scripts/hadoop-worker1/my-start-RM.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/hadoop-worker1/my-start-RM.sh -------------------------------------------------------------------------------- /src/scripts/hadoop-worker1/my-start-node-exporter.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/hadoop-worker1/my-start-node-exporter.sh -------------------------------------------------------------------------------- /src/scripts/hadoop-worker1/my-start-zk.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/hadoop-worker1/my-start-zk.sh -------------------------------------------------------------------------------- /src/scripts/hadoop-worker1/my-stop-hadoop-cluster.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/hadoop-worker1/my-stop-hadoop-cluster.sh -------------------------------------------------------------------------------- /src/scripts/hadoop-worker1/my-stop-node-exporter.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/hadoop-worker1/my-stop-node-exporter.sh -------------------------------------------------------------------------------- /src/scripts/hadoop-worker2/my-start-NM-DN.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/hadoop-worker2/my-start-NM-DN.sh -------------------------------------------------------------------------------- /src/scripts/hadoop-worker2/my-start-NN-ZKFC.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/hadoop-worker2/my-start-NN-ZKFC.sh -------------------------------------------------------------------------------- /src/scripts/hadoop-worker2/my-start-node-exporter.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/hadoop-worker2/my-start-node-exporter.sh -------------------------------------------------------------------------------- /src/scripts/hadoop-worker2/my-start-zk.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/hadoop-worker2/my-start-zk.sh -------------------------------------------------------------------------------- /src/scripts/hadoop-worker2/my-stop-hadoop-cluster.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/hadoop-worker2/my-stop-hadoop-cluster.sh -------------------------------------------------------------------------------- /src/scripts/hadoop-worker2/my-stop-node-exporter.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/hadoop-worker2/my-stop-node-exporter.sh -------------------------------------------------------------------------------- /src/scripts/hive/start-hive-services.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/hive/start-hive-services.sh -------------------------------------------------------------------------------- /src/scripts/hive/stop-hive-services.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/hive/stop-hive-services.sh -------------------------------------------------------------------------------- /src/scripts/monitoring/start-monitoring-services.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/monitoring/start-monitoring-services.sh -------------------------------------------------------------------------------- /src/scripts/monitoring/stop-monitoring-services.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/monitoring/stop-monitoring-services.sh -------------------------------------------------------------------------------- /src/scripts/mysql-hive-metastore/dump-mysql.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/mysql-hive-metastore/dump-mysql.sh -------------------------------------------------------------------------------- /src/scripts/mysql-hive-metastore/restore-mysql.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/mysql-hive-metastore/restore-mysql.sh -------------------------------------------------------------------------------- /src/scripts/mysql-hive-metastore/start-mysqld-exporter.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/mysql-hive-metastore/start-mysqld-exporter.sh -------------------------------------------------------------------------------- /src/scripts/mysql-hive-metastore/stop-mysqld-exporter.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/mysql-hive-metastore/stop-mysqld-exporter.sh -------------------------------------------------------------------------------- /src/scripts/spark/start-spark-services.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/spark/start-spark-services.sh -------------------------------------------------------------------------------- /src/scripts/spark/stop-spark-services.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/scripts/spark/stop-spark-services.sh -------------------------------------------------------------------------------- /src/snippets/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/snippets/README.md -------------------------------------------------------------------------------- /src/snippets/airflow/airflow-commands.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/snippets/airflow/airflow-commands.sh -------------------------------------------------------------------------------- /src/snippets/docker/docker-images.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/snippets/docker/docker-images.md -------------------------------------------------------------------------------- /src/snippets/docker/linux-container-init.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/snippets/docker/linux-container-init.md -------------------------------------------------------------------------------- /src/snippets/hadoop_cluster_launching/hadoop-cluster-zk-HA.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/snippets/hadoop_cluster_launching/hadoop-cluster-zk-HA.md -------------------------------------------------------------------------------- /src/snippets/hive/hive-launch.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/snippets/hive/hive-launch.sh -------------------------------------------------------------------------------- /src/snippets/hive/hiveuser_root_auth_grant.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/snippets/hive/hiveuser_root_auth_grant.sql -------------------------------------------------------------------------------- /src/snippets/mysql/hive-metadata-query-refresh.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/snippets/mysql/hive-metadata-query-refresh.md -------------------------------------------------------------------------------- /src/snippets/mysql/mysql-commadns.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/snippets/mysql/mysql-commadns.md -------------------------------------------------------------------------------- /src/snippets/pyspark/python-package.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/snippets/pyspark/python-package.sh -------------------------------------------------------------------------------- /src/snippets/pyspark/spark-submit-data-processing.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/snippets/pyspark/spark-submit-data-processing.sh -------------------------------------------------------------------------------- /src/snippets/pyspark/spark-submit.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/snippets/pyspark/spark-submit.sh -------------------------------------------------------------------------------- /src/snippets/pyspark/start_thriftserver.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/snippets/pyspark/start_thriftserver.sh -------------------------------------------------------------------------------- /src/warehouse_modeling/ads_clickhouse/ck_ddl_ads_orders_detailed_info_wide_ipd.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ads_clickhouse/ck_ddl_ads_orders_detailed_info_wide_ipd.sql -------------------------------------------------------------------------------- /src/warehouse_modeling/ads_clickhouse/ck_ddl_ads_schema.sql: -------------------------------------------------------------------------------- 1 | CREATE DATABASE IF NOT EXISTS ads_data_mart_ecomerce; -------------------------------------------------------------------------------- /src/warehouse_modeling/dim/ddl_dim_campaign_discount_fpd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dim/ddl_dim_campaign_discount_fpd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dim/ddl_dim_category_fpd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dim/ddl_dim_category_fpd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dim/ddl_dim_customer_fpd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dim/ddl_dim_customer_fpd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dim/ddl_dim_payment_method_fpd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dim/ddl_dim_payment_method_fpd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dim/ddl_dim_product_fpd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dim/ddl_dim_product_fpd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dim/ddl_dim_schema.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dim/ddl_dim_schema.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dim/ddl_dim_supplier_fpd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dim/ddl_dim_supplier_fpd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dim/hdfs/hdfs_dim_env_initial.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dim/hdfs/hdfs_dim_env_initial.sh -------------------------------------------------------------------------------- /src/warehouse_modeling/dim/hdfs/upload_to_hadoop_cluster.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dim/hdfs/upload_to_hadoop_cluster.sh -------------------------------------------------------------------------------- /src/warehouse_modeling/dwd/ddl_dwd_campaign_product_subcategory_fpd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dwd/ddl_dwd_campaign_product_subcategory_fpd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dwd/ddl_dwd_category_fpd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dwd/ddl_dwd_category_fpd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dwd/ddl_dwd_customer_fpd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dwd/ddl_dwd_customer_fpd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dwd/ddl_dwd_customer_product_ratings_ipd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dwd/ddl_dwd_customer_product_ratings_ipd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dwd/ddl_dwd_marketing_campaigns_fpd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dwd/ddl_dwd_marketing_campaigns_fpd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dwd/ddl_dwd_orderitem_ipd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dwd/ddl_dwd_orderitem_ipd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dwd/ddl_dwd_orders_ipd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dwd/ddl_dwd_orders_ipd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dwd/ddl_dwd_payment_method_fpd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dwd/ddl_dwd_payment_method_fpd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dwd/ddl_dwd_product_fpd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dwd/ddl_dwd_product_fpd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dwd/ddl_dwd_returns_ipd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dwd/ddl_dwd_returns_ipd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dwd/ddl_dwd_schema.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dwd/ddl_dwd_schema.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dwd/ddl_dwd_subcategory_fpd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dwd/ddl_dwd_subcategory_fpd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dwd/ddl_dwd_supplier_fpd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dwd/ddl_dwd_supplier_fpd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dwd/hdfs/hdfs_dwd_env_initial.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dwd/hdfs/hdfs_dwd_env_initial.sh -------------------------------------------------------------------------------- /src/warehouse_modeling/dwd/hdfs/upload_to_hadoop_cluster.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dwd/hdfs/upload_to_hadoop_cluster.sh -------------------------------------------------------------------------------- /src/warehouse_modeling/dwd/hive_config.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dwd/hive_config.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dwm/ddl_dwm_orders_with_items_ipd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dwm/ddl_dwm_orders_with_items_ipd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dwm/ddl_dwm_schema.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dwm/ddl_dwm_schema.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dwm/hdfs/hdfs_dwm_env_initial.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dwm/hdfs/hdfs_dwm_env_initial.sh -------------------------------------------------------------------------------- /src/warehouse_modeling/dwm/hdfs/upload_to_hadoop_cluster.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dwm/hdfs/upload_to_hadoop_cluster.sh -------------------------------------------------------------------------------- /src/warehouse_modeling/dws/ddl_dws_orders_detailed_info_wide_ipd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dws/ddl_dws_orders_detailed_info_wide_ipd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dws/ddl_dws_schema.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dws/ddl_dws_schema.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/dws/hdfs/hdfs_dws_env_initial.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dws/hdfs/hdfs_dws_env_initial.sh -------------------------------------------------------------------------------- /src/warehouse_modeling/dws/hdfs/upload_to_hadoop_cluster.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/dws/hdfs/upload_to_hadoop_cluster.sh -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/avro_schema/ods_campaign_product_subcategory_fpd.avsc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/avro_schema/ods_campaign_product_subcategory_fpd.avsc -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/avro_schema/ods_category_fpd.avsc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/avro_schema/ods_category_fpd.avsc -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/avro_schema/ods_customer_fpd.avsc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/avro_schema/ods_customer_fpd.avsc -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/avro_schema/ods_customer_product_ratings_ipd.avsc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/avro_schema/ods_customer_product_ratings_ipd.avsc -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/avro_schema/ods_marketing_campaigns_fpd.avsc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/avro_schema/ods_marketing_campaigns_fpd.avsc -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/avro_schema/ods_orderitem_ipd.avsc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/avro_schema/ods_orderitem_ipd.avsc -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/avro_schema/ods_orders_ipd.avsc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/avro_schema/ods_orders_ipd.avsc -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/avro_schema/ods_payment_method_fpd.avsc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/avro_schema/ods_payment_method_fpd.avsc -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/avro_schema/ods_product_fpd.avsc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/avro_schema/ods_product_fpd.avsc -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/avro_schema/ods_returns_ipd.avsc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/avro_schema/ods_returns_ipd.avsc -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/avro_schema/ods_subcategory_fpd.avsc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/avro_schema/ods_subcategory_fpd.avsc -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/avro_schema/ods_supplier_fpd.avsc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/avro_schema/ods_supplier_fpd.avsc -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/ddl_ods_campaign_product_subcategory_fpd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/ddl_ods_campaign_product_subcategory_fpd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/ddl_ods_category_fpd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/ddl_ods_category_fpd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/ddl_ods_customer_fpd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/ddl_ods_customer_fpd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/ddl_ods_customer_product_ratings_ipd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/ddl_ods_customer_product_ratings_ipd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/ddl_ods_marketing_campaigns_fpd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/ddl_ods_marketing_campaigns_fpd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/ddl_ods_orderitem_ipd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/ddl_ods_orderitem_ipd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/ddl_ods_orders_ipd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/ddl_ods_orders_ipd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/ddl_ods_payment_method_fpd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/ddl_ods_payment_method_fpd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/ddl_ods_product_fpd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/ddl_ods_product_fpd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/ddl_ods_returns_ipd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/ddl_ods_returns_ipd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/ddl_ods_schema.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/ddl_ods_schema.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/ddl_ods_subcategory_fpd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/ddl_ods_subcategory_fpd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/ddl_ods_supplier_fpd.hql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/ddl_ods_supplier_fpd.hql -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/hdfs/hdfs_ods_env_initial.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/hdfs/hdfs_ods_env_initial.sh -------------------------------------------------------------------------------- /src/warehouse_modeling/ods/hdfs/upload_to_hadoop_cluster.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/src/warehouse_modeling/ods/hdfs/upload_to_hadoop_cluster.sh -------------------------------------------------------------------------------- /start-data-clients.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/start-data-clients.sh -------------------------------------------------------------------------------- /start-hadoop-cluster.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/start-hadoop-cluster.sh -------------------------------------------------------------------------------- /start-other-services.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/start-other-services.sh -------------------------------------------------------------------------------- /stop-data-clients.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/stop-data-clients.sh -------------------------------------------------------------------------------- /stop-hadoop-cluster.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/stop-hadoop-cluster.sh -------------------------------------------------------------------------------- /stop-other-services.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/stop-other-services.sh -------------------------------------------------------------------------------- /test/batch_processing/dml_sql/dml_dwd_customer_product_ratings_ipd.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/test/batch_processing/dml_sql/dml_dwd_customer_product_ratings_ipd.sql -------------------------------------------------------------------------------- /test/batch_processing/exec_dml_sql.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/test/batch_processing/exec_dml_sql.py -------------------------------------------------------------------------------- /test/batch_processing/generate_hive_compitable_parquet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/test/batch_processing/generate_hive_compitable_parquet.py -------------------------------------------------------------------------------- /test/batch_processing/read_parquet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/test/batch_processing/read_parquet.py -------------------------------------------------------------------------------- /test/batch_processing/read_sql.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/test/batch_processing/read_sql.py -------------------------------------------------------------------------------- /test/batch_processing/transform_ods2dwd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/test/batch_processing/transform_ods2dwd.py -------------------------------------------------------------------------------- /test/data_pipeline/setup_custom_logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/test/data_pipeline/setup_custom_logger.py -------------------------------------------------------------------------------- /test/data_pipeline/spark_connect_oracle.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/test/data_pipeline/spark_connect_oracle.py -------------------------------------------------------------------------------- /test/data_pipeline/spark_extract_from_oracle.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/test/data_pipeline/spark_extract_from_oracle.py -------------------------------------------------------------------------------- /test/data_pipeline/spark_load_to_hive.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/test/data_pipeline/spark_load_to_hive.py -------------------------------------------------------------------------------- /test/data_pipeline/spark_read_avro.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/test/data_pipeline/spark_read_avro.py -------------------------------------------------------------------------------- /test/data_pipeline/spark_write_parquet_data_to_clickhouse.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/test/data_pipeline/spark_write_parquet_data_to_clickhouse.py -------------------------------------------------------------------------------- /test/data_pipeline/test_oracle_connect.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Smars-Bin-Hu/EComDWH-BatchDataProcessingPlatform/HEAD/test/data_pipeline/test_oracle_connect.py --------------------------------------------------------------------------------