├── .gitignore ├── .idea ├── .gitignore ├── aws.xml ├── inspectionProfiles │ ├── Project_Default.xml │ └── profiles_settings.xml ├── misc.xml ├── modules.xml ├── streaming-sales-generator.iml └── vcs.xml ├── .vscode └── settings.json ├── LICENSE.md ├── README.md ├── apache_pinot_examples ├── README.md ├── apache_superset │ ├── Dockerfile │ └── config.py ├── configs_schemas │ ├── _unused │ │ ├── purchases-aggregated-config.json │ │ └── purchases-aggregated-schema.json │ ├── products-config.json │ ├── products-schema.json │ ├── purchases-config.json │ ├── purchases-enriched-config.json │ ├── purchases-enriched-schema.json │ └── purchases-schema.json ├── pinot_sample_notebook.ipynb ├── pinot_sample_script.py └── screengrabs │ ├── pinot_ui.png │ └── superset.png ├── apache_spark_examples ├── README.md ├── requirements.txt ├── spark_batch_kafka.py ├── spark_batch_nb.ipynb ├── spark_streaming_kafka.py ├── spark_streaming_kafka_running_totals.py └── spark_streaming_nb.ipynb ├── diagram ├── streaming_workflow.drawio └── streaming_workflow_intro.png ├── docker ├── flink-pinot-superset-stack.yml └── spark-kstreams-stack.yml ├── sales_generator ├── __init__.py ├── config │ ├── __init__.py │ └── kafka.py ├── configuration │ └── configuration.ini ├── consumer.py ├── data │ └── products.csv ├── models │ ├── __init__.py │ ├── inventory.py │ ├── product.py │ └── purchase.py └── producer.py ├── sample_batch_data ├── sample_data_large.json ├── sample_data_small.json └── sample_date_new.json └── screengrabs └── dashboard.png /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/.gitignore -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/.idea/.gitignore -------------------------------------------------------------------------------- /.idea/aws.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/.idea/aws.xml -------------------------------------------------------------------------------- /.idea/inspectionProfiles/Project_Default.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/.idea/inspectionProfiles/Project_Default.xml -------------------------------------------------------------------------------- /.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/.idea/inspectionProfiles/profiles_settings.xml -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/.idea/misc.xml -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/.idea/modules.xml -------------------------------------------------------------------------------- /.idea/streaming-sales-generator.iml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/.idea/streaming-sales-generator.iml -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/.idea/vcs.xml -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/.vscode/settings.json -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/LICENSE.md -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/README.md -------------------------------------------------------------------------------- /apache_pinot_examples/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/apache_pinot_examples/README.md -------------------------------------------------------------------------------- /apache_pinot_examples/apache_superset/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/apache_pinot_examples/apache_superset/Dockerfile -------------------------------------------------------------------------------- /apache_pinot_examples/apache_superset/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/apache_pinot_examples/apache_superset/config.py -------------------------------------------------------------------------------- /apache_pinot_examples/configs_schemas/_unused/purchases-aggregated-config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/apache_pinot_examples/configs_schemas/_unused/purchases-aggregated-config.json -------------------------------------------------------------------------------- /apache_pinot_examples/configs_schemas/_unused/purchases-aggregated-schema.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/apache_pinot_examples/configs_schemas/_unused/purchases-aggregated-schema.json -------------------------------------------------------------------------------- /apache_pinot_examples/configs_schemas/products-config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/apache_pinot_examples/configs_schemas/products-config.json -------------------------------------------------------------------------------- /apache_pinot_examples/configs_schemas/products-schema.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/apache_pinot_examples/configs_schemas/products-schema.json -------------------------------------------------------------------------------- /apache_pinot_examples/configs_schemas/purchases-config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/apache_pinot_examples/configs_schemas/purchases-config.json -------------------------------------------------------------------------------- /apache_pinot_examples/configs_schemas/purchases-enriched-config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/apache_pinot_examples/configs_schemas/purchases-enriched-config.json -------------------------------------------------------------------------------- /apache_pinot_examples/configs_schemas/purchases-enriched-schema.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/apache_pinot_examples/configs_schemas/purchases-enriched-schema.json -------------------------------------------------------------------------------- /apache_pinot_examples/configs_schemas/purchases-schema.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/apache_pinot_examples/configs_schemas/purchases-schema.json -------------------------------------------------------------------------------- /apache_pinot_examples/pinot_sample_notebook.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/apache_pinot_examples/pinot_sample_notebook.ipynb -------------------------------------------------------------------------------- /apache_pinot_examples/pinot_sample_script.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/apache_pinot_examples/pinot_sample_script.py -------------------------------------------------------------------------------- /apache_pinot_examples/screengrabs/pinot_ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/apache_pinot_examples/screengrabs/pinot_ui.png -------------------------------------------------------------------------------- /apache_pinot_examples/screengrabs/superset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/apache_pinot_examples/screengrabs/superset.png -------------------------------------------------------------------------------- /apache_spark_examples/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/apache_spark_examples/README.md -------------------------------------------------------------------------------- /apache_spark_examples/requirements.txt: -------------------------------------------------------------------------------- 1 | kafka-python -------------------------------------------------------------------------------- /apache_spark_examples/spark_batch_kafka.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/apache_spark_examples/spark_batch_kafka.py -------------------------------------------------------------------------------- /apache_spark_examples/spark_batch_nb.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/apache_spark_examples/spark_batch_nb.ipynb -------------------------------------------------------------------------------- /apache_spark_examples/spark_streaming_kafka.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/apache_spark_examples/spark_streaming_kafka.py -------------------------------------------------------------------------------- /apache_spark_examples/spark_streaming_kafka_running_totals.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/apache_spark_examples/spark_streaming_kafka_running_totals.py -------------------------------------------------------------------------------- /apache_spark_examples/spark_streaming_nb.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/apache_spark_examples/spark_streaming_nb.ipynb -------------------------------------------------------------------------------- /diagram/streaming_workflow.drawio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/diagram/streaming_workflow.drawio -------------------------------------------------------------------------------- /diagram/streaming_workflow_intro.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/diagram/streaming_workflow_intro.png -------------------------------------------------------------------------------- /docker/flink-pinot-superset-stack.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/docker/flink-pinot-superset-stack.yml -------------------------------------------------------------------------------- /docker/spark-kstreams-stack.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/docker/spark-kstreams-stack.yml -------------------------------------------------------------------------------- /sales_generator/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sales_generator/config/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sales_generator/config/kafka.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/sales_generator/config/kafka.py -------------------------------------------------------------------------------- /sales_generator/configuration/configuration.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/sales_generator/configuration/configuration.ini -------------------------------------------------------------------------------- /sales_generator/consumer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/sales_generator/consumer.py -------------------------------------------------------------------------------- /sales_generator/data/products.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/sales_generator/data/products.csv -------------------------------------------------------------------------------- /sales_generator/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sales_generator/models/inventory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/sales_generator/models/inventory.py -------------------------------------------------------------------------------- /sales_generator/models/product.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/sales_generator/models/product.py -------------------------------------------------------------------------------- /sales_generator/models/purchase.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/sales_generator/models/purchase.py -------------------------------------------------------------------------------- /sales_generator/producer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/sales_generator/producer.py -------------------------------------------------------------------------------- /sample_batch_data/sample_data_large.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/sample_batch_data/sample_data_large.json -------------------------------------------------------------------------------- /sample_batch_data/sample_data_small.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/sample_batch_data/sample_data_small.json -------------------------------------------------------------------------------- /sample_batch_data/sample_date_new.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/sample_batch_data/sample_date_new.json -------------------------------------------------------------------------------- /screengrabs/dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/streaming-sales-generator/HEAD/screengrabs/dashboard.png --------------------------------------------------------------------------------