├── .gitignore ├── README.md ├── aft ├── models │ ├── aft │ │ ├── data │ │ │ ├── _SUCCESS │ │ │ ├── __SUCCESS.crc │ │ │ ├── _part-00000-d0fcb17b-f728-41cf-bb14-3469c26578b5-c000.snappy.parquet.crc │ │ │ └── part-00000-d0fcb17b-f728-41cf-bb14-3469c26578b5-c000.snappy.parquet │ │ └── metadata │ │ │ ├── _SUCCESS │ │ │ ├── __SUCCESS.crc │ │ │ ├── _part-00000.crc │ │ │ └── part-00000 │ ├── formula │ │ ├── data │ │ │ ├── _SUCCESS │ │ │ ├── __SUCCESS.crc │ │ │ ├── _part-00000-e541380b-1452-4493-812d-c1a2c0666f06-c000.snappy.parquet.crc │ │ │ └── part-00000-e541380b-1452-4493-812d-c1a2c0666f06-c000.snappy.parquet │ │ ├── metadata │ │ │ ├── _SUCCESS │ │ │ ├── __SUCCESS.crc │ │ │ ├── _part-00000.crc │ │ │ └── part-00000 │ │ └── pipelineModel │ │ │ ├── metadata │ │ │ ├── _SUCCESS │ │ │ ├── __SUCCESS.crc │ │ │ ├── _part-00000.crc │ │ │ └── part-00000 │ │ │ └── stages │ │ │ ├── 0_RFormula_c37eef0b2ea9 │ │ │ └── metadata │ │ │ │ ├── _SUCCESS │ │ │ │ ├── __SUCCESS.crc │ │ │ │ ├── _part-00000.crc │ │ │ │ └── part-00000 │ │ │ ├── 1_vectorAttrRewriter_03dd9672fa79 │ │ │ ├── data │ │ │ │ ├── _SUCCESS │ │ │ │ ├── __SUCCESS.crc │ │ │ │ ├── _part-00000-fa95e1a4-c68d-4494-a5b8-305862b9a807-c000.snappy.parquet.crc │ │ │ │ └── part-00000-fa95e1a4-c68d-4494-a5b8-305862b9a807-c000.snappy.parquet │ │ │ └── metadata │ │ │ │ ├── _SUCCESS │ │ │ │ ├── __SUCCESS.crc │ │ │ │ ├── _part-00000.crc │ │ │ │ └── part-00000 │ │ │ └── 2_columnPruner_16c8a03db342 │ │ │ ├── data │ │ │ ├── _SUCCESS │ │ │ ├── __SUCCESS.crc │ │ │ ├── _part-00000-54605c1f-0685-46e8-b36e-cfdf72550943-c000.snappy.parquet.crc │ │ │ └── part-00000-54605c1f-0685-46e8-b36e-cfdf72550943-c000.snappy.parquet │ │ │ └── metadata │ │ │ ├── _SUCCESS │ │ │ ├── __SUCCESS.crc │ │ │ ├── _part-00000.crc │ │ │ └── part-00000 │ └── scaler │ │ ├── data │ │ ├── _SUCCESS │ │ ├── __SUCCESS.crc │ │ ├── _part-00000-3e561200-d04b-4b1e-bc3d-a21838a1c66e-c000.snappy.parquet.crc │ │ └── part-00000-3e561200-d04b-4b1e-bc3d-a21838a1c66e-c000.snappy.parquet │ │ └── metadata │ │ ├── _SUCCESS │ │ ├── __SUCCESS.crc │ │ ├── _part-00000.crc │ │ └── part-00000 ├── project_aft_inference.ipynb ├── project_aft_prep_train.ipynb ├── project_data_analysis.ipynb ├── test_x.csv ├── test_y.csv └── train.csv ├── bin ├── local_kafka.sh ├── start-kafka-shell.sh ├── start_engine_alert_monitor.sh ├── start_engine_monitor.sh ├── start_engine_sim.sh └── start_services.sh ├── data ├── test_x.csv ├── test_y.csv └── train.csv ├── doc └── images │ ├── data_fragment.png │ ├── overview.png │ ├── sample_results.png │ └── spark_ml.png ├── docker-compose.yml ├── engine_util.py ├── engine_viz.py ├── environment.yml ├── kafka ├── engine_cycle_consumer_struct.py ├── engine_cycle_producer.py └── generic_consumer.py └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/README.md -------------------------------------------------------------------------------- /aft/models/aft/data/_SUCCESS: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /aft/models/aft/data/__SUCCESS.crc: -------------------------------------------------------------------------------- 1 | crc -------------------------------------------------------------------------------- /aft/models/aft/data/_part-00000-d0fcb17b-f728-41cf-bb14-3469c26578b5-c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/models/aft/data/_part-00000-d0fcb17b-f728-41cf-bb14-3469c26578b5-c000.snappy.parquet.crc -------------------------------------------------------------------------------- /aft/models/aft/data/part-00000-d0fcb17b-f728-41cf-bb14-3469c26578b5-c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/models/aft/data/part-00000-d0fcb17b-f728-41cf-bb14-3469c26578b5-c000.snappy.parquet -------------------------------------------------------------------------------- /aft/models/aft/metadata/_SUCCESS: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /aft/models/aft/metadata/__SUCCESS.crc: -------------------------------------------------------------------------------- 1 | crc -------------------------------------------------------------------------------- /aft/models/aft/metadata/_part-00000.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/models/aft/metadata/_part-00000.crc -------------------------------------------------------------------------------- /aft/models/aft/metadata/part-00000: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/models/aft/metadata/part-00000 -------------------------------------------------------------------------------- /aft/models/formula/data/_SUCCESS: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /aft/models/formula/data/__SUCCESS.crc: -------------------------------------------------------------------------------- 1 | crc -------------------------------------------------------------------------------- /aft/models/formula/data/_part-00000-e541380b-1452-4493-812d-c1a2c0666f06-c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/models/formula/data/_part-00000-e541380b-1452-4493-812d-c1a2c0666f06-c000.snappy.parquet.crc -------------------------------------------------------------------------------- /aft/models/formula/data/part-00000-e541380b-1452-4493-812d-c1a2c0666f06-c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/models/formula/data/part-00000-e541380b-1452-4493-812d-c1a2c0666f06-c000.snappy.parquet -------------------------------------------------------------------------------- /aft/models/formula/metadata/_SUCCESS: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /aft/models/formula/metadata/__SUCCESS.crc: -------------------------------------------------------------------------------- 1 | crc -------------------------------------------------------------------------------- /aft/models/formula/metadata/_part-00000.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/models/formula/metadata/_part-00000.crc -------------------------------------------------------------------------------- /aft/models/formula/metadata/part-00000: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/models/formula/metadata/part-00000 -------------------------------------------------------------------------------- /aft/models/formula/pipelineModel/metadata/_SUCCESS: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /aft/models/formula/pipelineModel/metadata/__SUCCESS.crc: -------------------------------------------------------------------------------- 1 | crc -------------------------------------------------------------------------------- /aft/models/formula/pipelineModel/metadata/_part-00000.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/models/formula/pipelineModel/metadata/_part-00000.crc -------------------------------------------------------------------------------- /aft/models/formula/pipelineModel/metadata/part-00000: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/models/formula/pipelineModel/metadata/part-00000 -------------------------------------------------------------------------------- /aft/models/formula/pipelineModel/stages/0_RFormula_c37eef0b2ea9/metadata/_SUCCESS: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /aft/models/formula/pipelineModel/stages/0_RFormula_c37eef0b2ea9/metadata/__SUCCESS.crc: -------------------------------------------------------------------------------- 1 | crc -------------------------------------------------------------------------------- /aft/models/formula/pipelineModel/stages/0_RFormula_c37eef0b2ea9/metadata/_part-00000.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/models/formula/pipelineModel/stages/0_RFormula_c37eef0b2ea9/metadata/_part-00000.crc -------------------------------------------------------------------------------- /aft/models/formula/pipelineModel/stages/0_RFormula_c37eef0b2ea9/metadata/part-00000: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/models/formula/pipelineModel/stages/0_RFormula_c37eef0b2ea9/metadata/part-00000 -------------------------------------------------------------------------------- /aft/models/formula/pipelineModel/stages/1_vectorAttrRewriter_03dd9672fa79/data/_SUCCESS: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /aft/models/formula/pipelineModel/stages/1_vectorAttrRewriter_03dd9672fa79/data/__SUCCESS.crc: -------------------------------------------------------------------------------- 1 | crc -------------------------------------------------------------------------------- /aft/models/formula/pipelineModel/stages/1_vectorAttrRewriter_03dd9672fa79/data/_part-00000-fa95e1a4-c68d-4494-a5b8-305862b9a807-c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/models/formula/pipelineModel/stages/1_vectorAttrRewriter_03dd9672fa79/data/_part-00000-fa95e1a4-c68d-4494-a5b8-305862b9a807-c000.snappy.parquet.crc -------------------------------------------------------------------------------- /aft/models/formula/pipelineModel/stages/1_vectorAttrRewriter_03dd9672fa79/data/part-00000-fa95e1a4-c68d-4494-a5b8-305862b9a807-c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/models/formula/pipelineModel/stages/1_vectorAttrRewriter_03dd9672fa79/data/part-00000-fa95e1a4-c68d-4494-a5b8-305862b9a807-c000.snappy.parquet -------------------------------------------------------------------------------- /aft/models/formula/pipelineModel/stages/1_vectorAttrRewriter_03dd9672fa79/metadata/_SUCCESS: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /aft/models/formula/pipelineModel/stages/1_vectorAttrRewriter_03dd9672fa79/metadata/__SUCCESS.crc: -------------------------------------------------------------------------------- 1 | crc -------------------------------------------------------------------------------- /aft/models/formula/pipelineModel/stages/1_vectorAttrRewriter_03dd9672fa79/metadata/_part-00000.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/models/formula/pipelineModel/stages/1_vectorAttrRewriter_03dd9672fa79/metadata/_part-00000.crc -------------------------------------------------------------------------------- /aft/models/formula/pipelineModel/stages/1_vectorAttrRewriter_03dd9672fa79/metadata/part-00000: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/models/formula/pipelineModel/stages/1_vectorAttrRewriter_03dd9672fa79/metadata/part-00000 -------------------------------------------------------------------------------- /aft/models/formula/pipelineModel/stages/2_columnPruner_16c8a03db342/data/_SUCCESS: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /aft/models/formula/pipelineModel/stages/2_columnPruner_16c8a03db342/data/__SUCCESS.crc: -------------------------------------------------------------------------------- 1 | crc -------------------------------------------------------------------------------- /aft/models/formula/pipelineModel/stages/2_columnPruner_16c8a03db342/data/_part-00000-54605c1f-0685-46e8-b36e-cfdf72550943-c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/models/formula/pipelineModel/stages/2_columnPruner_16c8a03db342/data/_part-00000-54605c1f-0685-46e8-b36e-cfdf72550943-c000.snappy.parquet.crc -------------------------------------------------------------------------------- /aft/models/formula/pipelineModel/stages/2_columnPruner_16c8a03db342/data/part-00000-54605c1f-0685-46e8-b36e-cfdf72550943-c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/models/formula/pipelineModel/stages/2_columnPruner_16c8a03db342/data/part-00000-54605c1f-0685-46e8-b36e-cfdf72550943-c000.snappy.parquet -------------------------------------------------------------------------------- /aft/models/formula/pipelineModel/stages/2_columnPruner_16c8a03db342/metadata/_SUCCESS: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /aft/models/formula/pipelineModel/stages/2_columnPruner_16c8a03db342/metadata/__SUCCESS.crc: -------------------------------------------------------------------------------- 1 | crc -------------------------------------------------------------------------------- /aft/models/formula/pipelineModel/stages/2_columnPruner_16c8a03db342/metadata/_part-00000.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/models/formula/pipelineModel/stages/2_columnPruner_16c8a03db342/metadata/_part-00000.crc -------------------------------------------------------------------------------- /aft/models/formula/pipelineModel/stages/2_columnPruner_16c8a03db342/metadata/part-00000: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/models/formula/pipelineModel/stages/2_columnPruner_16c8a03db342/metadata/part-00000 -------------------------------------------------------------------------------- /aft/models/scaler/data/_SUCCESS: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /aft/models/scaler/data/__SUCCESS.crc: -------------------------------------------------------------------------------- 1 | crc -------------------------------------------------------------------------------- /aft/models/scaler/data/_part-00000-3e561200-d04b-4b1e-bc3d-a21838a1c66e-c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/models/scaler/data/_part-00000-3e561200-d04b-4b1e-bc3d-a21838a1c66e-c000.snappy.parquet.crc -------------------------------------------------------------------------------- /aft/models/scaler/data/part-00000-3e561200-d04b-4b1e-bc3d-a21838a1c66e-c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/models/scaler/data/part-00000-3e561200-d04b-4b1e-bc3d-a21838a1c66e-c000.snappy.parquet -------------------------------------------------------------------------------- /aft/models/scaler/metadata/_SUCCESS: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /aft/models/scaler/metadata/__SUCCESS.crc: -------------------------------------------------------------------------------- 1 | crc -------------------------------------------------------------------------------- /aft/models/scaler/metadata/_part-00000.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/models/scaler/metadata/_part-00000.crc -------------------------------------------------------------------------------- /aft/models/scaler/metadata/part-00000: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/models/scaler/metadata/part-00000 -------------------------------------------------------------------------------- /aft/project_aft_inference.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/project_aft_inference.ipynb -------------------------------------------------------------------------------- /aft/project_aft_prep_train.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/project_aft_prep_train.ipynb -------------------------------------------------------------------------------- /aft/project_data_analysis.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/project_data_analysis.ipynb -------------------------------------------------------------------------------- /aft/test_x.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/test_x.csv -------------------------------------------------------------------------------- /aft/test_y.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/test_y.csv -------------------------------------------------------------------------------- /aft/train.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/aft/train.csv -------------------------------------------------------------------------------- /bin/local_kafka.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/bin/local_kafka.sh -------------------------------------------------------------------------------- /bin/start-kafka-shell.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/bin/start-kafka-shell.sh -------------------------------------------------------------------------------- /bin/start_engine_alert_monitor.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/bin/start_engine_alert_monitor.sh -------------------------------------------------------------------------------- /bin/start_engine_monitor.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/bin/start_engine_monitor.sh -------------------------------------------------------------------------------- /bin/start_engine_sim.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/bin/start_engine_sim.sh -------------------------------------------------------------------------------- /bin/start_services.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/bin/start_services.sh -------------------------------------------------------------------------------- /data/test_x.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/data/test_x.csv -------------------------------------------------------------------------------- /data/test_y.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/data/test_y.csv -------------------------------------------------------------------------------- /data/train.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/data/train.csv -------------------------------------------------------------------------------- /doc/images/data_fragment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/doc/images/data_fragment.png -------------------------------------------------------------------------------- /doc/images/overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/doc/images/overview.png -------------------------------------------------------------------------------- /doc/images/sample_results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/doc/images/sample_results.png -------------------------------------------------------------------------------- /doc/images/spark_ml.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/doc/images/spark_ml.png -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/docker-compose.yml -------------------------------------------------------------------------------- /engine_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/engine_util.py -------------------------------------------------------------------------------- /engine_viz.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/engine_viz.py -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/environment.yml -------------------------------------------------------------------------------- /kafka/engine_cycle_consumer_struct.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/kafka/engine_cycle_consumer_struct.py -------------------------------------------------------------------------------- /kafka/engine_cycle_producer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/kafka/engine_cycle_producer.py -------------------------------------------------------------------------------- /kafka/generic_consumer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/kafka/generic_consumer.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sabderra/predictive-maintenance-spark/HEAD/requirements.txt --------------------------------------------------------------------------------