├── .dockerignore ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── azure-pipelines.yml ├── deploy ├── Dockerfile ├── azuredeploy.json ├── azuredeploy.parameters.json ├── databricks │ ├── config │ │ ├── cluster.config.json │ │ ├── job.batchscoring.config.json │ │ ├── job.streamdatagen.config.json │ │ ├── job.streamscoring.config.json │ │ ├── run.downloaddata.config.json │ │ ├── run.etl.config.json │ │ └── run.trainmodelall.config.json │ ├── configure_databricks.sh │ └── create_secrets.sh └── deploy.sh ├── images ├── FileStore │ ├── GBTModel.PNG │ ├── LogRegCVPipeline.PNG │ ├── MLPipeline.PNG │ ├── PCAAnomalyPipeline.PNG │ ├── RandomForestPipeline.PNG │ ├── TransformPipeline.PNG │ └── transformation_and_actions.PNG └── archi.PNG ├── notebooks ├── .gitkeep └── databricks_notebooks │ ├── 00_demo_hello_spark.py │ ├── 01_download_data.py │ ├── 02_ETL.py │ ├── 03_explore_data.py │ ├── 04_trainmodel_multiple.scala │ ├── 04_trainmodel_pca_w_custom.scala │ ├── 04_trainmodel_pca_wo_custom.scala │ ├── 05_batch_scoring.scala │ ├── 06a_streaming_datagen.scala │ └── 06b_streaming_scoring.scala ├── references └── Lace Lofranco - Building Advanced Analytics Pipelines with Azure Databricks.pdf ├── requirements.txt ├── setup.py └── test_environment.py /.dockerignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/.dockerignore -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/LICENSE -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/README.md -------------------------------------------------------------------------------- /azure-pipelines.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/azure-pipelines.yml -------------------------------------------------------------------------------- /deploy/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/deploy/Dockerfile -------------------------------------------------------------------------------- /deploy/azuredeploy.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/deploy/azuredeploy.json -------------------------------------------------------------------------------- /deploy/azuredeploy.parameters.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/deploy/azuredeploy.parameters.json -------------------------------------------------------------------------------- /deploy/databricks/config/cluster.config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/deploy/databricks/config/cluster.config.json -------------------------------------------------------------------------------- /deploy/databricks/config/job.batchscoring.config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/deploy/databricks/config/job.batchscoring.config.json -------------------------------------------------------------------------------- /deploy/databricks/config/job.streamdatagen.config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/deploy/databricks/config/job.streamdatagen.config.json -------------------------------------------------------------------------------- /deploy/databricks/config/job.streamscoring.config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/deploy/databricks/config/job.streamscoring.config.json -------------------------------------------------------------------------------- /deploy/databricks/config/run.downloaddata.config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/deploy/databricks/config/run.downloaddata.config.json -------------------------------------------------------------------------------- /deploy/databricks/config/run.etl.config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/deploy/databricks/config/run.etl.config.json -------------------------------------------------------------------------------- /deploy/databricks/config/run.trainmodelall.config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/deploy/databricks/config/run.trainmodelall.config.json -------------------------------------------------------------------------------- /deploy/databricks/configure_databricks.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/deploy/databricks/configure_databricks.sh -------------------------------------------------------------------------------- /deploy/databricks/create_secrets.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/deploy/databricks/create_secrets.sh -------------------------------------------------------------------------------- /deploy/deploy.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/deploy/deploy.sh -------------------------------------------------------------------------------- /images/FileStore/GBTModel.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/images/FileStore/GBTModel.PNG -------------------------------------------------------------------------------- /images/FileStore/LogRegCVPipeline.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/images/FileStore/LogRegCVPipeline.PNG -------------------------------------------------------------------------------- /images/FileStore/MLPipeline.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/images/FileStore/MLPipeline.PNG -------------------------------------------------------------------------------- /images/FileStore/PCAAnomalyPipeline.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/images/FileStore/PCAAnomalyPipeline.PNG -------------------------------------------------------------------------------- /images/FileStore/RandomForestPipeline.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/images/FileStore/RandomForestPipeline.PNG -------------------------------------------------------------------------------- /images/FileStore/TransformPipeline.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/images/FileStore/TransformPipeline.PNG -------------------------------------------------------------------------------- /images/FileStore/transformation_and_actions.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/images/FileStore/transformation_and_actions.PNG -------------------------------------------------------------------------------- /images/archi.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/images/archi.PNG -------------------------------------------------------------------------------- /notebooks/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /notebooks/databricks_notebooks/00_demo_hello_spark.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/notebooks/databricks_notebooks/00_demo_hello_spark.py -------------------------------------------------------------------------------- /notebooks/databricks_notebooks/01_download_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/notebooks/databricks_notebooks/01_download_data.py -------------------------------------------------------------------------------- /notebooks/databricks_notebooks/02_ETL.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/notebooks/databricks_notebooks/02_ETL.py -------------------------------------------------------------------------------- /notebooks/databricks_notebooks/03_explore_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/notebooks/databricks_notebooks/03_explore_data.py -------------------------------------------------------------------------------- /notebooks/databricks_notebooks/04_trainmodel_multiple.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/notebooks/databricks_notebooks/04_trainmodel_multiple.scala -------------------------------------------------------------------------------- /notebooks/databricks_notebooks/04_trainmodel_pca_w_custom.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/notebooks/databricks_notebooks/04_trainmodel_pca_w_custom.scala -------------------------------------------------------------------------------- /notebooks/databricks_notebooks/04_trainmodel_pca_wo_custom.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/notebooks/databricks_notebooks/04_trainmodel_pca_wo_custom.scala -------------------------------------------------------------------------------- /notebooks/databricks_notebooks/05_batch_scoring.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/notebooks/databricks_notebooks/05_batch_scoring.scala -------------------------------------------------------------------------------- /notebooks/databricks_notebooks/06a_streaming_datagen.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/notebooks/databricks_notebooks/06a_streaming_datagen.scala -------------------------------------------------------------------------------- /notebooks/databricks_notebooks/06b_streaming_scoring.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/notebooks/databricks_notebooks/06b_streaming_scoring.scala -------------------------------------------------------------------------------- /references/Lace Lofranco - Building Advanced Analytics Pipelines with Azure Databricks.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/references/Lace Lofranco - Building Advanced Analytics Pipelines with Azure Databricks.pdf -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/requirements.txt -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/setup.py -------------------------------------------------------------------------------- /test_environment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devlace/azure-databricks-anomaly/HEAD/test_environment.py --------------------------------------------------------------------------------