├── .circleci └── config.yml ├── .gitignore ├── .idea ├── dbnavigator.xml ├── inspectionProfiles │ ├── Project_Default.xml │ └── profiles_settings.xml ├── modules.xml ├── wafer_circleci.iml └── workspace.xml ├── DataTransform_Training ├── DataTransformation.py ├── __init__.py └── __pycache__ │ ├── DataTransformation.cpython-36.pyc │ └── DataTransformation.cpython-37.pyc ├── DataTransformation_Prediction ├── DataTransformationPrediction.py ├── __init__.py └── __pycache__ │ ├── DataTransformationPrediction.cpython-36.pyc │ └── DataTransformationPrediction.cpython-37.pyc ├── DataTypeValidation_Insertion_Prediction ├── DataTypeValidationPrediction.py ├── __init__.py └── __pycache__ │ ├── DataTypeValidationPrediction.cpython-36.pyc │ └── DataTypeValidationPrediction.cpython-37.pyc ├── DataTypeValidation_Insertion_Training ├── DataTypeValidation.py ├── __init__.py └── __pycache__ │ ├── DataTypeValidation.cpython-36.pyc │ └── DataTypeValidation.cpython-37.pyc ├── Dockerfile ├── LLD ├── Problem Statement.docx ├── Screenshots │ ├── Screenshot at 2021-02-25 11-48-32.png │ ├── Screenshot at 2021-02-25 11-52-17.png │ └── Screenshot at 2021-02-25 11-52-44.png ├── wafer_fault_detection.docx ├── ~$fer_fault_detection.docx └── ~$oblem Statement.docx ├── Prediction_Batch_files ├── Wafer_13012020_141000.csv ├── Wafer_14012020_113045.csv ├── Wafer_15010_130532.csv ├── wafer_07012020_041011.csv ├── wafer_13012020_090817.csv ├── wafer_16012020_051629.csv ├── wafer_20012020_090819.csv ├── wafer_20022020_090716.csv ├── wafer_21012020_080913.csv ├── wafer_22022020_041119.csv ├── wafer_23012020_011008.csv ├── wafer_23012020_041211.csv ├── wafer_27012020_080911.csv ├── wafer_28012020_051011.csv ├── wafer_28012020_090817.csv ├── wafer_28042020_031911.csv ├── wafer_29012020_050617.csv ├── wafer_29012020_060756.csv └── wafer_31012020_090811.csv ├── Prediction_Database └── Prediction.db ├── Prediction_FileFromDB └── InputFile.csv ├── Prediction_Logs ├── DataBaseConnectionLog.txt ├── DbInsertLog.txt ├── DbTableCreateLog.txt ├── ExportToCsv.txt ├── GeneralLog.txt ├── Prediction_Log.txt ├── columnValidationLog.txt ├── dataTransformLog.txt ├── missingValuesInColumn.txt └── nameValidationLog.txt ├── Prediction_Output_File └── Predictions.csv ├── Prediction_Raw_Data_Validation ├── __init__.py ├── __pycache__ │ ├── predictionDataValidation.cpython-36.pyc │ └── predictionDataValidation.cpython-37.pyc └── predictionDataValidation.py ├── Procfile ├── README.md ├── Training_Batch_Files ├── Wafer12_20012.csv ├── Wafer_07012020_000000.csv ├── Wafer_07012020_223345.csv ├── Wafer_08012020_120000.csv ├── Wafer_10012020_131534.csv ├── Wafer_11012020_151432.csv ├── Wafer_12012020_111.csv ├── Wafer_12012020_111213.csv ├── Wafer_13012020_141000.csv ├── Wafer_14012020_113045.csv ├── Wafer_15010_130532.csv ├── Wafer_15012020_130532.csv ├── Wafer_18012020_121532.csv ├── Wafer_19012020_141432.csv ├── Wafer_20012020_135132.csv ├── Wafer_21012020_143634.csv ├── Wafer_22012020_173245.csv ├── Wafer_23012020_163456.csv ├── Wafer_24012020_150121.csv ├── Wafer_25012020_142112.csv ├── Wafer_fault_detection.csv ├── Wafer_text_130532.csv ├── Waferdefault_15010_130532.csv ├── wafer_07012020_041011.csv ├── wafer_13012020_090817.csv ├── wafer_16012020_051629.csv ├── wafer_16012020_134553.csv ├── wafer_17012020_125434.csv ├── wafer_20012020_090819.csv ├── wafer_20022020_090716.csv ├── wafer_21012020_080913.csv ├── wafer_22022020_041119.csv ├── wafer_23012020_011008.csv ├── wafer_23012020_041211.csv ├── wafer_27012020_080911.csv ├── wafer_28012020_051011.csv ├── wafer_28012020_090817.csv ├── wafer_28042020_031911.csv ├── wafer_29012020_050617.csv ├── wafer_29012020_060756.csv └── wafer_31012020_090811.csv ├── Training_Database └── Training.db ├── Training_FileFromDB └── InputFile.csv ├── Training_Logs ├── DataBaseConnectionLog.txt ├── DbInsertLog.txt ├── DbTableCreateLog.txt ├── ExportToCsv.txt ├── GeneralLog.txt ├── ModelTrainingLog.txt ├── Training_Main_Log.txt ├── columnValidationLog.txt ├── dataTransformLog.txt ├── missingValuesInColumn.txt ├── nameValidationLog.txt └── valuesfromSchemaValidationLog.txt ├── Training_Raw_data_validation ├── __init__.py ├── __pycache__ │ ├── rawValidation.cpython-36.pyc │ └── rawValidation.cpython-37.pyc └── rawValidation.py ├── __pycache__ ├── predictFromModel.cpython-37.pyc ├── prediction_Validation_Insertion.cpython-37.pyc ├── trainingModel.cpython-37.pyc └── training_Validation_Insertion.cpython-37.pyc ├── application_logging ├── __init__.py ├── __pycache__ │ ├── logger.cpython-36.pyc │ └── logger.cpython-37.pyc └── logger.py ├── best_model_finder ├── __init__.py ├── __pycache__ │ ├── tuner.cpython-36.pyc │ └── tuner.cpython-37.pyc └── tuner.py ├── data_ingestion ├── __init__.py ├── __pycache__ │ ├── data_loader.cpython-36.pyc │ ├── data_loader.cpython-37.pyc │ ├── data_loader_prediction.cpython-36.pyc │ └── data_loader_prediction.cpython-37.pyc ├── data_loader.py └── data_loader_prediction.py ├── data_preprocessing ├── __init__.py ├── __pycache__ │ ├── clustering.cpython-36.pyc │ ├── clustering.cpython-37.pyc │ ├── preprocessing.cpython-36.pyc │ ├── preprocessing.cpython-37.pyc │ └── preprocessing_pred.cpython-36.pyc ├── clustering.py └── preprocessing.py ├── file_operations ├── __init__.py ├── __pycache__ │ ├── file_methods.cpython-36.pyc │ └── file_methods.cpython-37.pyc └── file_methods.py ├── flask_monitoringdashboard.db ├── main.py ├── models ├── KMeans │ └── KMeans.sav ├── RandomForest2 │ └── RandomForest2.sav ├── XGBoost0 │ └── XGBoost0.sav └── XGBoost1 │ └── XGBoost1.sav ├── pattern.py ├── predictFromModel.py ├── prediction_Validation_Insertion.py ├── preprocessing_data ├── K-Means_Elbow.PNG └── null_values.csv ├── requirements.txt ├── runtime.txt ├── schema_prediction.json ├── schema_training.json ├── setup.py ├── templates └── index.html ├── testing.py ├── tests ├── __init__.py ├── __pycache__ │ └── test_script.cpython-37-pytest-6.2.5.pyc └── test_script.py ├── trainingModel.py └── training_Validation_Insertion.py /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | orbs: 3 | heroku: circleci/heroku@1.0.1 4 | jobs: 5 | build-and-test: 6 | executor: heroku/default 7 | docker: 8 | - image: circleci/python:3.6.2-stretch-browsers 9 | auth: 10 | username: mydockerhub-user 11 | password: $DOCKERHUB_PASSWORD # context / project UI env-var reference 12 | steps: 13 | - checkout 14 | - restore_cache: 15 | key: deps1-{{ .Branch }}-{{ checksum "requirements.txt" }} 16 | - run: 17 | name: Install Python deps in a venv 18 | command: | 19 | echo 'export TAG=0.1.${CIRCLE_BUILD_NUM}' >> $BASH_ENV 20 | echo 'export IMAGE_NAME=${DOCKER_IMAGE_NAME}' >> $BASH_ENV 21 | python3 -m venv venv 22 | . venv/bin/activate 23 | pip install --upgrade pip 24 | pip install -r requirements.txt 25 | - save_cache: 26 | key: deps1-{{ .Branch }}-{{ checksum "requirements.txt" }} 27 | paths: 28 | - "venv" 29 | - run: 30 | command: | 31 | . venv/bin/activate 32 | python -m pytest -v tests/test_script.py 33 | - store_artifacts: 34 | path: test-reports/ 35 | destination: tr1 36 | - store_test_results: 37 | path: test-reports/ 38 | - setup_remote_docker: 39 | version: 19.03.13 40 | - run: 41 | name: Build and push Docker image 42 | command: | 43 | docker build -t $DOCKERHUB_USER/$IMAGE_NAME:$TAG . 44 | echo $DOCKER_HUB_PASSWORD_USER | docker login -u $DOCKERHUB_USER --password-stdin 45 | docker push $DOCKERHUB_USER/$IMAGE_NAME:$TAG 46 | deploy: 47 | executor: heroku/default 48 | steps: 49 | - checkout 50 | - run: 51 | name: Storing previous commit 52 | command: | 53 | git rev-parse HEAD > ./commit.txt 54 | - heroku/install 55 | - setup_remote_docker: 56 | version: 18.06.0-ce 57 | - run: 58 | name: Pushing to heroku registry 59 | command: | 60 | heroku container:login 61 | #heroku ps:scale web=1 -a $HEROKU_APP_NAME 62 | heroku container:push web -a $HEROKU_APP_NAME 63 | heroku container:release web -a $HEROKU_APP_NAME 64 | 65 | workflows: 66 | build-test-deploy: 67 | jobs: 68 | - build-and-test 69 | - deploy: 70 | requires: 71 | - build-and-test 72 | filters: 73 | branches: 74 | only: 75 | - main -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | wafer -------------------------------------------------------------------------------- /.idea/inspectionProfiles/Project_Default.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 13 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/wafer_circleci.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 12 | -------------------------------------------------------------------------------- /.idea/workspace.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 10 | 11 | 13 | 14 | 15 | 16 | 19 | 26 | 27 | 28 | 29 | 30 | 1652450431169 31 | 35 | 36 | 37 | 38 | 47 | 48 | -------------------------------------------------------------------------------- /DataTransform_Training/DataTransformation.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from os import listdir 3 | import pandas 4 | from application_logging.logger import App_Logger 5 | 6 | 7 | class dataTransform: 8 | 9 | """ 10 | This class shall be used for transforming the Good Raw Training Data before loading it in Database!!. 11 | 12 | Written By: iNeuron Intelligence 13 | Version: 1.0 14 | Revisions: None 15 | 16 | """ 17 | 18 | def __init__(self): 19 | self.goodDataPath = "Training_Raw_files_validated/Good_Raw" 20 | self.logger = App_Logger() 21 | 22 | 23 | def replaceMissingWithNull(self): 24 | """ 25 | Method Name: replaceMissingWithNull 26 | Description: This method replaces the missing values in columns with "NULL" to 27 | store in the table. We are using substring in the first column to 28 | keep only "Integer" data for ease up the loading. 29 | This column is anyways going to be removed during training. 30 | 31 | Written By: iNeuron Intelligence 32 | Version: 1.0 33 | Revisions: None 34 | 35 | """ 36 | 37 | log_file = open("Training_Logs/dataTransformLog.txt", 'a+') 38 | try: 39 | onlyfiles = [f for f in listdir(self.goodDataPath)] 40 | for file in onlyfiles: 41 | csv = pandas.read_csv(self.goodDataPath+"/" + file) 42 | csv.fillna('NULL',inplace=True) 43 | # #csv.update("'"+ csv['Wafer'] +"'") 44 | # csv.update(csv['Wafer'].astype(str)) 45 | csv['Wafer'] = csv['Wafer'].str[6:] 46 | csv.to_csv(self.goodDataPath+ "/" + file, index=None, header=True) 47 | self.logger.log(log_file," %s: File Transformed successfully!!" % file) 48 | #log_file.write("Current Date :: %s" %date +"\t" + "Current time:: %s" % current_time + "\t \t" + + "\n") 49 | except Exception as e: 50 | self.logger.log(log_file, "Data Transformation failed because:: %s" % e) 51 | #log_file.write("Current Date :: %s" %date +"\t" +"Current time:: %s" % current_time + "\t \t" + "Data Transformation failed because:: %s" % e + "\n") 52 | log_file.close() 53 | log_file.close() 54 | -------------------------------------------------------------------------------- /DataTransform_Training/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/DataTransform_Training/__init__.py -------------------------------------------------------------------------------- /DataTransform_Training/__pycache__/DataTransformation.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/DataTransform_Training/__pycache__/DataTransformation.cpython-36.pyc -------------------------------------------------------------------------------- /DataTransform_Training/__pycache__/DataTransformation.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/DataTransform_Training/__pycache__/DataTransformation.cpython-37.pyc -------------------------------------------------------------------------------- /DataTransformation_Prediction/DataTransformationPrediction.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from os import listdir 3 | import pandas 4 | from application_logging.logger import App_Logger 5 | 6 | 7 | class dataTransformPredict: 8 | 9 | """ 10 | This class shall be used for transforming the Good Raw Training Data before loading it in Database!!. 11 | 12 | Written By: iNeuron Intelligence 13 | Version: 1.0 14 | Revisions: None 15 | 16 | """ 17 | 18 | def __init__(self): 19 | self.goodDataPath = "Prediction_Raw_Files_Validated/Good_Raw" 20 | self.logger = App_Logger() 21 | 22 | 23 | def replaceMissingWithNull(self): 24 | 25 | """ 26 | Method Name: replaceMissingWithNull 27 | Description: This method replaces the missing values in columns with "NULL" to 28 | store in the table. We are using substring in the first column to 29 | keep only "Integer" data for ease up the loading. 30 | This column is anyways going to be removed during prediction. 31 | 32 | Written By: iNeuron Intelligence 33 | Version: 1.0 34 | Revisions: None 35 | 36 | """ 37 | 38 | try: 39 | log_file = open("Prediction_Logs/dataTransformLog.txt", 'a+') 40 | onlyfiles = [f for f in listdir(self.goodDataPath)] 41 | for file in onlyfiles: 42 | csv = pandas.read_csv(self.goodDataPath+"/" + file) 43 | csv.fillna('NULL',inplace=True) 44 | # #csv.update("'"+ csv['Wafer'] +"'") 45 | # csv.update(csv['Wafer'].astype(str)) 46 | csv['Wafer'] = csv['Wafer'].str[6:] 47 | csv.to_csv(self.goodDataPath+ "/" + file, index=None, header=True) 48 | self.logger.log(log_file," %s: File Transformed successfully!!" % file) 49 | #log_file.write("Current Date :: %s" %date +"\t" + "Current time:: %s" % current_time + "\t \t" + + "\n") 50 | 51 | except Exception as e: 52 | self.logger.log(log_file, "Data Transformation failed because:: %s" % e) 53 | #log_file.write("Current Date :: %s" %date +"\t" +"Current time:: %s" % current_time + "\t \t" + "Data Transformation failed because:: %s" % e + "\n") 54 | log_file.close() 55 | raise e 56 | log_file.close() 57 | -------------------------------------------------------------------------------- /DataTransformation_Prediction/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/DataTransformation_Prediction/__init__.py -------------------------------------------------------------------------------- /DataTransformation_Prediction/__pycache__/DataTransformationPrediction.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/DataTransformation_Prediction/__pycache__/DataTransformationPrediction.cpython-36.pyc -------------------------------------------------------------------------------- /DataTransformation_Prediction/__pycache__/DataTransformationPrediction.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/DataTransformation_Prediction/__pycache__/DataTransformationPrediction.cpython-37.pyc -------------------------------------------------------------------------------- /DataTypeValidation_Insertion_Prediction/DataTypeValidationPrediction.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import sqlite3 3 | from datetime import datetime 4 | from os import listdir 5 | import os 6 | import csv 7 | from application_logging.logger import App_Logger 8 | 9 | 10 | class dBOperation: 11 | """ 12 | This class shall be used for handling all the SQL operations. 13 | 14 | Written By: iNeuron Intelligence 15 | Version: 1.0 16 | Revisions: None 17 | 18 | """ 19 | 20 | def __init__(self): 21 | self.path = 'Prediction_Database/' 22 | self.badFilePath = "Prediction_Raw_Files_Validated/Bad_Raw" 23 | self.goodFilePath = "Prediction_Raw_Files_Validated/Good_Raw" 24 | self.logger = App_Logger() 25 | 26 | 27 | def dataBaseConnection(self,DatabaseName): 28 | 29 | """ 30 | Method Name: dataBaseConnection 31 | Description: This method creates the database with the given name and if Database already exists then opens the connection to the DB. 32 | Output: Connection to the DB 33 | On Failure: Raise ConnectionError 34 | 35 | Written By: iNeuron Intelligence 36 | Version: 1.0 37 | Revisions: None 38 | 39 | """ 40 | try: 41 | conn = sqlite3.connect(self.path+DatabaseName+'.db') 42 | 43 | file = open("Prediction_Logs/DataBaseConnectionLog.txt", 'a+') 44 | self.logger.log(file, "Opened %s database successfully" % DatabaseName) 45 | file.close() 46 | except ConnectionError: 47 | file = open("Prediction_Logs/DataBaseConnectionLog.txt", 'a+') 48 | self.logger.log(file, "Error while connecting to database: %s" %ConnectionError) 49 | file.close() 50 | raise ConnectionError 51 | return conn 52 | 53 | def createTableDb(self,DatabaseName,column_names): 54 | 55 | """ 56 | Method Name: createTableDb 57 | Description: This method creates a table in the given database which will be used to insert the Good data after raw data validation. 58 | Output: None 59 | On Failure: Raise Exception 60 | 61 | Written By: iNeuron Intelligence 62 | Version: 1.0 63 | Revisions: None 64 | 65 | """ 66 | try: 67 | conn = self.dataBaseConnection(DatabaseName) 68 | conn.execute('DROP TABLE IF EXISTS Good_Raw_Data;') 69 | 70 | for key in column_names.keys(): 71 | type = column_names[key] 72 | 73 | # we will remove the column of string datatype before loading as it is not needed for training 74 | #in try block we check if the table exists, if yes then add columns to the table 75 | # else in catch block we create the table 76 | try: 77 | #cur = cur.execute("SELECT name FROM {dbName} WHERE type='table' AND name='Good_Raw_Data'".format(dbName=DatabaseName)) 78 | conn.execute('ALTER TABLE Good_Raw_Data ADD COLUMN "{column_name}" {dataType}'.format(column_name=key,dataType=type)) 79 | except: 80 | conn.execute('CREATE TABLE Good_Raw_Data ({column_name} {dataType})'.format(column_name=key, dataType=type)) 81 | 82 | conn.close() 83 | 84 | file = open("Prediction_Logs/DbTableCreateLog.txt", 'a+') 85 | self.logger.log(file, "Tables created successfully!!") 86 | file.close() 87 | 88 | file = open("Prediction_Logs/DataBaseConnectionLog.txt", 'a+') 89 | self.logger.log(file, "Closed %s database successfully" % DatabaseName) 90 | file.close() 91 | 92 | except Exception as e: 93 | file = open("Prediction_Logs/DbTableCreateLog.txt", 'a+') 94 | self.logger.log(file, "Error while creating table: %s " % e) 95 | file.close() 96 | conn.close() 97 | file = open("Prediction_Logs/DataBaseConnectionLog.txt", 'a+') 98 | self.logger.log(file, "Closed %s database successfully" % DatabaseName) 99 | file.close() 100 | raise e 101 | 102 | 103 | def insertIntoTableGoodData(self,Database): 104 | 105 | """ 106 | Method Name: insertIntoTableGoodData 107 | Description: This method inserts the Good data files from the Good_Raw folder into the 108 | above created table. 109 | Output: None 110 | On Failure: Raise Exception 111 | 112 | Written By: iNeuron Intelligence 113 | Version: 1.0 114 | Revisions: None 115 | 116 | """ 117 | 118 | conn = self.dataBaseConnection(Database) 119 | goodFilePath= self.goodFilePath 120 | badFilePath = self.badFilePath 121 | onlyfiles = [f for f in listdir(goodFilePath)] 122 | log_file = open("Prediction_Logs/DbInsertLog.txt", 'a+') 123 | 124 | for file in onlyfiles: 125 | try: 126 | 127 | with open(goodFilePath+'/'+file, "r") as f: 128 | next(f) 129 | reader = csv.reader(f, delimiter="\n") 130 | for line in enumerate(reader): 131 | for list_ in (line[1]): 132 | try: 133 | conn.execute('INSERT INTO Good_Raw_Data values ({values})'.format(values=(list_))) 134 | self.logger.log(log_file," %s: File loaded successfully!!" % file) 135 | conn.commit() 136 | except Exception as e: 137 | raise e 138 | 139 | except Exception as e: 140 | 141 | conn.rollback() 142 | self.logger.log(log_file,"Error while creating table: %s " % e) 143 | shutil.move(goodFilePath+'/' + file, badFilePath) 144 | self.logger.log(log_file, "File Moved Successfully %s" % file) 145 | log_file.close() 146 | conn.close() 147 | raise e 148 | 149 | conn.close() 150 | log_file.close() 151 | 152 | 153 | def selectingDatafromtableintocsv(self,Database): 154 | 155 | """ 156 | Method Name: selectingDatafromtableintocsv 157 | Description: This method exports the data in GoodData table as a CSV file. in a given location. 158 | above created . 159 | Output: None 160 | On Failure: Raise Exception 161 | 162 | Written By: iNeuron Intelligence 163 | Version: 1.0 164 | Revisions: None 165 | 166 | """ 167 | 168 | self.fileFromDb = 'Prediction_FileFromDB/' 169 | self.fileName = 'InputFile.csv' 170 | log_file = open("Prediction_Logs/ExportToCsv.txt", 'a+') 171 | try: 172 | conn = self.dataBaseConnection(Database) 173 | sqlSelect = "SELECT * FROM Good_Raw_Data" 174 | cursor = conn.cursor() 175 | 176 | cursor.execute(sqlSelect) 177 | 178 | results = cursor.fetchall() 179 | 180 | #Get the headers of the csv file 181 | headers = [i[0] for i in cursor.description] 182 | 183 | #Make the CSV ouput directory 184 | if not os.path.isdir(self.fileFromDb): 185 | os.makedirs(self.fileFromDb) 186 | 187 | # Open CSV file for writing. 188 | csvFile = csv.writer(open(self.fileFromDb + self.fileName, 'w', newline=''),delimiter=',', lineterminator='\r\n',quoting=csv.QUOTE_ALL, escapechar='\\') 189 | 190 | # Add the headers and data to the CSV file. 191 | csvFile.writerow(headers) 192 | csvFile.writerows(results) 193 | 194 | self.logger.log(log_file, "File exported successfully!!!") 195 | 196 | except Exception as e: 197 | self.logger.log(log_file, "File exporting failed. Error : %s" %e) 198 | raise e 199 | 200 | 201 | 202 | 203 | 204 | -------------------------------------------------------------------------------- /DataTypeValidation_Insertion_Prediction/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/DataTypeValidation_Insertion_Prediction/__init__.py -------------------------------------------------------------------------------- /DataTypeValidation_Insertion_Prediction/__pycache__/DataTypeValidationPrediction.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/DataTypeValidation_Insertion_Prediction/__pycache__/DataTypeValidationPrediction.cpython-36.pyc -------------------------------------------------------------------------------- /DataTypeValidation_Insertion_Prediction/__pycache__/DataTypeValidationPrediction.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/DataTypeValidation_Insertion_Prediction/__pycache__/DataTypeValidationPrediction.cpython-37.pyc -------------------------------------------------------------------------------- /DataTypeValidation_Insertion_Training/DataTypeValidation.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import sqlite3 3 | from datetime import datetime 4 | from os import listdir 5 | import os 6 | import csv 7 | from application_logging.logger import App_Logger 8 | 9 | 10 | class dBOperation: 11 | """ 12 | This class shall be used for handling all the SQL operations. 13 | 14 | Written By: iNeuron Intelligence 15 | Version: 1.0 16 | Revisions: None 17 | 18 | """ 19 | def __init__(self): 20 | self.path = 'Training_Database/' 21 | self.badFilePath = "Training_Raw_files_validated/Bad_Raw" 22 | self.goodFilePath = "Training_Raw_files_validated/Good_Raw" 23 | self.logger = App_Logger() 24 | 25 | 26 | def dataBaseConnection(self,DatabaseName): 27 | 28 | """ 29 | Method Name: dataBaseConnection 30 | Description: This method creates the database with the given name and if Database already exists then opens the connection to the DB. 31 | Output: Connection to the DB 32 | On Failure: Raise ConnectionError 33 | 34 | Written By: iNeuron Intelligence 35 | Version: 1.0 36 | Revisions: None 37 | 38 | """ 39 | try: 40 | conn = sqlite3.connect(self.path+DatabaseName+'.db') 41 | 42 | file = open("Training_Logs/DataBaseConnectionLog.txt", 'a+') 43 | self.logger.log(file, "Opened %s database successfully" % DatabaseName) 44 | file.close() 45 | except ConnectionError: 46 | file = open("Training_Logs/DataBaseConnectionLog.txt", 'a+') 47 | self.logger.log(file, "Error while connecting to database: %s" %ConnectionError) 48 | file.close() 49 | raise ConnectionError 50 | return conn 51 | 52 | def createTableDb(self,DatabaseName,column_names): 53 | """ 54 | Method Name: createTableDb 55 | Description: This method creates a table in the given database which will be used to insert the Good data after raw data validation. 56 | Output: None 57 | On Failure: Raise Exception 58 | 59 | Written By: iNeuron Intelligence 60 | Version: 1.0 61 | Revisions: None 62 | 63 | """ 64 | try: 65 | conn = self.dataBaseConnection(DatabaseName) 66 | c=conn.cursor() 67 | c.execute("SELECT count(name) FROM sqlite_master WHERE type = 'table'AND name = 'Good_Raw_Data'") 68 | if c.fetchone()[0] ==1: 69 | conn.close() 70 | file = open("Training_Logs/DbTableCreateLog.txt", 'a+') 71 | self.logger.log(file, "Tables created successfully!!") 72 | file.close() 73 | 74 | file = open("Training_Logs/DataBaseConnectionLog.txt", 'a+') 75 | self.logger.log(file, "Closed %s database successfully" % DatabaseName) 76 | file.close() 77 | 78 | else: 79 | 80 | for key in column_names.keys(): 81 | type = column_names[key] 82 | 83 | #in try block we check if the table exists, if yes then add columns to the table 84 | # else in catch block we will create the table 85 | try: 86 | #cur = cur.execute("SELECT name FROM {dbName} WHERE type='table' AND name='Good_Raw_Data'".format(dbName=DatabaseName)) 87 | conn.execute('ALTER TABLE Good_Raw_Data ADD COLUMN "{column_name}" {dataType}'.format(column_name=key,dataType=type)) 88 | except: 89 | conn.execute('CREATE TABLE Good_Raw_Data ({column_name} {dataType})'.format(column_name=key, dataType=type)) 90 | 91 | 92 | conn.close() 93 | 94 | file = open("Training_Logs/DbTableCreateLog.txt", 'a+') 95 | self.logger.log(file, "Tables created successfully!!") 96 | file.close() 97 | 98 | file = open("Training_Logs/DataBaseConnectionLog.txt", 'a+') 99 | self.logger.log(file, "Closed %s database successfully" % DatabaseName) 100 | file.close() 101 | 102 | except Exception as e: 103 | file = open("Training_Logs/DbTableCreateLog.txt", 'a+') 104 | self.logger.log(file, "Error while creating table: %s " % e) 105 | file.close() 106 | conn.close() 107 | file = open("Training_Logs/DataBaseConnectionLog.txt", 'a+') 108 | self.logger.log(file, "Closed %s database successfully" % DatabaseName) 109 | file.close() 110 | raise e 111 | 112 | 113 | def insertIntoTableGoodData(self,Database): 114 | 115 | """ 116 | Method Name: insertIntoTableGoodData 117 | Description: This method inserts the Good data files from the Good_Raw folder into the 118 | above created table. 119 | Output: None 120 | On Failure: Raise Exception 121 | 122 | Written By: iNeuron Intelligence 123 | Version: 1.0 124 | Revisions: None 125 | 126 | """ 127 | 128 | conn = self.dataBaseConnection(Database) 129 | goodFilePath= self.goodFilePath 130 | badFilePath = self.badFilePath 131 | onlyfiles = [f for f in listdir(goodFilePath)] 132 | log_file = open("Training_Logs/DbInsertLog.txt", 'a+') 133 | 134 | for file in onlyfiles: 135 | try: 136 | with open(goodFilePath+'/'+file, "r") as f: 137 | next(f) 138 | reader = csv.reader(f, delimiter="\n") 139 | for line in enumerate(reader): 140 | for list_ in (line[1]): 141 | try: 142 | conn.execute('INSERT INTO Good_Raw_Data values ({values})'.format(values=(list_))) 143 | self.logger.log(log_file," %s: File loaded successfully!!" % file) 144 | conn.commit() 145 | except Exception as e: 146 | raise e 147 | 148 | except Exception as e: 149 | 150 | conn.rollback() 151 | self.logger.log(log_file,"Error while creating table: %s " % e) 152 | shutil.move(goodFilePath+'/' + file, badFilePath) 153 | self.logger.log(log_file, "File Moved Successfully %s" % file) 154 | log_file.close() 155 | conn.close() 156 | 157 | conn.close() 158 | log_file.close() 159 | 160 | 161 | def selectingDatafromtableintocsv(self,Database): 162 | 163 | """ 164 | Method Name: selectingDatafromtableintocsv 165 | Description: This method exports the data in GoodData table as a CSV file. in a given location. 166 | above created . 167 | Output: None 168 | On Failure: Raise Exception 169 | 170 | Written By: iNeuron Intelligence 171 | Version: 1.0 172 | Revisions: None 173 | 174 | """ 175 | 176 | self.fileFromDb = 'Training_FileFromDB/' 177 | self.fileName = 'InputFile.csv' 178 | log_file = open("Training_Logs/ExportToCsv.txt", 'a+') 179 | try: 180 | conn = self.dataBaseConnection(Database) 181 | sqlSelect = "SELECT * FROM Good_Raw_Data" 182 | cursor = conn.cursor() 183 | 184 | cursor.execute(sqlSelect) 185 | 186 | results = cursor.fetchall() 187 | # Get the headers of the csv file 188 | headers = [i[0] for i in cursor.description] 189 | 190 | #Make the CSV ouput directory 191 | if not os.path.isdir(self.fileFromDb): 192 | os.makedirs(self.fileFromDb) 193 | 194 | # Open CSV file for writing. 195 | csvFile = csv.writer(open(self.fileFromDb + self.fileName, 'w', newline=''),delimiter=',', lineterminator='\r\n',quoting=csv.QUOTE_ALL, escapechar='\\') 196 | 197 | # Add the headers and data to the CSV file. 198 | csvFile.writerow(headers) 199 | csvFile.writerows(results) 200 | 201 | self.logger.log(log_file, "File exported successfully!!!") 202 | log_file.close() 203 | 204 | except Exception as e: 205 | self.logger.log(log_file, "File exporting failed. Error : %s" %e) 206 | log_file.close() 207 | 208 | 209 | 210 | 211 | 212 | -------------------------------------------------------------------------------- /DataTypeValidation_Insertion_Training/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/DataTypeValidation_Insertion_Training/__init__.py -------------------------------------------------------------------------------- /DataTypeValidation_Insertion_Training/__pycache__/DataTypeValidation.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/DataTypeValidation_Insertion_Training/__pycache__/DataTypeValidation.cpython-36.pyc -------------------------------------------------------------------------------- /DataTypeValidation_Insertion_Training/__pycache__/DataTypeValidation.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/DataTypeValidation_Insertion_Training/__pycache__/DataTypeValidation.cpython-37.pyc -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7 2 | COPY . /app 3 | WORKDIR /app 4 | RUN pip install -r requirements.txt 5 | ENTRYPOINT [ "python" ] 6 | CMD [ "main.py" ] -------------------------------------------------------------------------------- /LLD/Problem Statement.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/LLD/Problem Statement.docx -------------------------------------------------------------------------------- /LLD/Screenshots/Screenshot at 2021-02-25 11-48-32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/LLD/Screenshots/Screenshot at 2021-02-25 11-48-32.png -------------------------------------------------------------------------------- /LLD/Screenshots/Screenshot at 2021-02-25 11-52-17.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/LLD/Screenshots/Screenshot at 2021-02-25 11-52-17.png -------------------------------------------------------------------------------- /LLD/Screenshots/Screenshot at 2021-02-25 11-52-44.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/LLD/Screenshots/Screenshot at 2021-02-25 11-52-44.png -------------------------------------------------------------------------------- /LLD/wafer_fault_detection.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/LLD/wafer_fault_detection.docx -------------------------------------------------------------------------------- /LLD/~$fer_fault_detection.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/LLD/~$fer_fault_detection.docx -------------------------------------------------------------------------------- /LLD/~$oblem Statement.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/LLD/~$oblem Statement.docx -------------------------------------------------------------------------------- /Prediction_Database/Prediction.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/Prediction_Database/Prediction.db -------------------------------------------------------------------------------- /Prediction_Logs/DataBaseConnectionLog.txt: -------------------------------------------------------------------------------- 1 | 2020-10-09/16:14:39 Opened Prediction database successfully 2 | 2020-10-09/16:16:20 Closed Prediction database successfully 3 | 2020-10-09/16:16:20 Opened Prediction database successfully 4 | 2020-10-09/16:18:24 Opened Prediction database successfully 5 | 2020-10-09/18:19:41 Opened Prediction database successfully 6 | 2020-10-09/18:21:48 Closed Prediction database successfully 7 | 2020-10-09/18:21:48 Opened Prediction database successfully 8 | 2020-10-09/18:24:03 Opened Prediction database successfully 9 | 2020-10-15/10:29:34 Opened Prediction database successfully 10 | 2020-10-15/10:30:40 Closed Prediction database successfully 11 | 2020-10-15/10:30:40 Opened Prediction database successfully 12 | 2020-10-15/10:32:02 Opened Prediction database successfully 13 | 2021-02-05/21:26:23 Opened Prediction database successfully 14 | 2021-02-05/21:27:26 Closed Prediction database successfully 15 | 2021-02-05/21:27:26 Opened Prediction database successfully 16 | 2021-02-05/21:28:57 Opened Prediction database successfully 17 | 2021-02-07/22:34:55 Opened Prediction database successfully 18 | 2021-02-07/22:36:11 Closed Prediction database successfully 19 | 2021-02-07/22:36:11 Opened Prediction database successfully 20 | 2021-02-07/22:37:29 Opened Prediction database successfully 21 | 2021-02-07/22:40:47 Opened Prediction database successfully 22 | 2021-02-07/22:41:42 Closed Prediction database successfully 23 | 2021-02-07/22:41:42 Opened Prediction database successfully 24 | 2021-02-07/22:42:55 Opened Prediction database successfully 25 | 2021-02-25/11:49:38 Opened Prediction database successfully 26 | 2021-02-25/11:50:26 Closed Prediction database successfully 27 | 2021-02-25/11:50:26 Opened Prediction database successfully 28 | 2021-02-25/11:51:33 Opened Prediction database successfully 29 | 2021-07-12/10:12:24 Opened Prediction database successfully 30 | 2021-07-12/10:13:17 Closed Prediction database successfully 31 | 2021-07-12/10:13:17 Opened Prediction database successfully 32 | 2021-07-12/10:14:54 Opened Prediction database successfully 33 | 2021-07-12/10:15:52 Opened Prediction database successfully 34 | 2021-07-12/10:16:48 Closed Prediction database successfully 35 | 2021-07-12/10:16:48 Opened Prediction database successfully 36 | 2021-07-12/10:18:06 Opened Prediction database successfully 37 | 2021-07-20/10:20:28 Opened Prediction database successfully 38 | 2021-07-20/10:21:22 Closed Prediction database successfully 39 | 2021-07-20/10:21:22 Opened Prediction database successfully 40 | 2021-07-20/10:22:45 Opened Prediction database successfully 41 | 2021-11-20/00:07:27 Opened Prediction database successfully 42 | 2021-11-20/00:08:20 Closed Prediction database successfully 43 | 2021-11-20/00:08:20 Opened Prediction database successfully 44 | 2021-11-20/00:09:35 Opened Prediction database successfully 45 | 2021-11-20/00:10:37 Opened Prediction database successfully 46 | 2021-11-20/00:11:33 Closed Prediction database successfully 47 | 2021-11-20/00:11:33 Opened Prediction database successfully 48 | 2021-11-20/00:12:55 Opened Prediction database successfully 49 | 2021-11-20/00:13:37 Opened Prediction database successfully 50 | 2021-11-20/00:14:37 Closed Prediction database successfully 51 | 2021-11-20/00:14:37 Opened Prediction database successfully 52 | 2021-11-20/00:16:00 Opened Prediction database successfully 53 | -------------------------------------------------------------------------------- /Prediction_Logs/DbTableCreateLog.txt: -------------------------------------------------------------------------------- 1 | 2020-10-09/16:16:20 Tables created successfully!! 2 | 2020-10-09/18:21:48 Tables created successfully!! 3 | 2020-10-15/10:30:40 Tables created successfully!! 4 | 2021-02-05/21:27:26 Tables created successfully!! 5 | 2021-02-07/22:36:11 Tables created successfully!! 6 | 2021-02-07/22:41:42 Tables created successfully!! 7 | 2021-02-25/11:50:26 Tables created successfully!! 8 | 2021-07-12/10:13:17 Tables created successfully!! 9 | 2021-07-12/10:16:48 Tables created successfully!! 10 | 2021-07-20/10:21:22 Tables created successfully!! 11 | 2021-11-20/00:08:20 Tables created successfully!! 12 | 2021-11-20/00:11:33 Tables created successfully!! 13 | 2021-11-20/00:14:37 Tables created successfully!! 14 | -------------------------------------------------------------------------------- /Prediction_Logs/ExportToCsv.txt: -------------------------------------------------------------------------------- 1 | 2020-10-09/16:18:27 File exported successfully!!! 2 | 2020-10-09/18:24:05 File exported successfully!!! 3 | 2020-10-15/10:32:02 File exported successfully!!! 4 | 2021-02-05/21:28:58 File exported successfully!!! 5 | 2021-02-07/22:37:29 File exported successfully!!! 6 | 2021-02-07/22:42:55 File exported successfully!!! 7 | 2021-02-25/11:51:34 File exported successfully!!! 8 | 2021-07-12/10:14:55 File exported successfully!!! 9 | 2021-07-12/10:18:07 File exported successfully!!! 10 | 2021-07-20/10:22:46 File exported successfully!!! 11 | 2021-11-20/00:09:36 File exported successfully!!! 12 | 2021-11-20/00:12:56 File exported successfully!!! 13 | 2021-11-20/00:16:00 File exported successfully!!! 14 | -------------------------------------------------------------------------------- /Prediction_Logs/GeneralLog.txt: -------------------------------------------------------------------------------- 1 | 2020-10-09/16:18:24 GoodRaw directory deleted successfully!!! 2 | 2020-10-09/16:18:24 Bad files moved to archive 3 | 2020-10-09/16:18:24 Bad Raw Data Folder Deleted successfully!! 4 | 2020-10-09/18:24:02 GoodRaw directory deleted successfully!!! 5 | 2020-10-09/18:24:03 Bad files moved to archive 6 | 2020-10-09/18:24:03 Bad Raw Data Folder Deleted successfully!! 7 | 2020-10-15/10:32:02 GoodRaw directory deleted successfully!!! 8 | 2020-10-15/10:32:02 Bad files moved to archive 9 | 2020-10-15/10:32:02 Bad Raw Data Folder Deleted successfully!! 10 | 2021-02-05/21:28:57 GoodRaw directory deleted successfully!!! 11 | 2021-02-05/21:28:57 Bad files moved to archive 12 | 2021-02-05/21:28:57 Bad Raw Data Folder Deleted successfully!! 13 | 2021-02-07/22:37:29 GoodRaw directory deleted successfully!!! 14 | 2021-02-07/22:37:29 Bad files moved to archive 15 | 2021-02-07/22:37:29 Bad Raw Data Folder Deleted successfully!! 16 | 2021-02-07/22:40:35 BadRaw directory deleted before starting validation!!! 17 | 2021-02-07/22:40:35 GoodRaw directory deleted successfully!!! 18 | 2021-02-07/22:42:55 GoodRaw directory deleted successfully!!! 19 | 2021-02-07/22:42:55 Bad files moved to archive 20 | 2021-02-07/22:42:55 Bad Raw Data Folder Deleted successfully!! 21 | 2021-02-25/11:51:33 GoodRaw directory deleted successfully!!! 22 | 2021-02-25/11:51:33 Bad files moved to archive 23 | 2021-02-25/11:51:33 Bad Raw Data Folder Deleted successfully!! 24 | 2021-07-12/10:14:54 GoodRaw directory deleted successfully!!! 25 | 2021-07-12/10:14:54 Bad files moved to archive 26 | 2021-07-12/10:14:54 Bad Raw Data Folder Deleted successfully!! 27 | 2021-07-12/10:18:06 GoodRaw directory deleted successfully!!! 28 | 2021-07-12/10:18:06 Bad files moved to archive 29 | 2021-07-12/10:18:06 Bad Raw Data Folder Deleted successfully!! 30 | 2021-07-20/10:22:45 GoodRaw directory deleted successfully!!! 31 | 2021-07-20/10:22:45 Bad files moved to archive 32 | 2021-07-20/10:22:45 Bad Raw Data Folder Deleted successfully!! 33 | 2021-11-20/00:09:35 GoodRaw directory deleted successfully!!! 34 | 2021-11-20/00:09:35 Bad files moved to archive 35 | 2021-11-20/00:09:35 Bad Raw Data Folder Deleted successfully!! 36 | 2021-11-20/00:10:10 BadRaw directory deleted before starting validation!!! 37 | 2021-11-20/00:10:10 GoodRaw directory deleted successfully!!! 38 | 2021-11-20/00:10:11 BadRaw directory deleted before starting validation!!! 39 | 2021-11-20/00:10:11 GoodRaw directory deleted successfully!!! 40 | 2021-11-20/00:10:27 BadRaw directory deleted before starting validation!!! 41 | 2021-11-20/00:10:27 GoodRaw directory deleted successfully!!! 42 | 2021-11-20/00:10:28 BadRaw directory deleted before starting validation!!! 43 | 2021-11-20/00:10:28 GoodRaw directory deleted successfully!!! 44 | 2021-11-20/00:10:28 BadRaw directory deleted before starting validation!!! 45 | 2021-11-20/00:10:28 GoodRaw directory deleted successfully!!! 46 | 2021-11-20/00:10:29 BadRaw directory deleted before starting validation!!! 47 | 2021-11-20/00:10:29 GoodRaw directory deleted successfully!!! 48 | 2021-11-20/00:12:55 GoodRaw directory deleted successfully!!! 49 | 2021-11-20/00:12:55 Bad files moved to archive 50 | 2021-11-20/00:12:55 Bad Raw Data Folder Deleted successfully!! 51 | 2021-11-20/00:16:00 GoodRaw directory deleted successfully!!! 52 | 2021-11-20/00:16:00 Bad files moved to archive 53 | 2021-11-20/00:16:00 Bad Raw Data Folder Deleted successfully!! 54 | -------------------------------------------------------------------------------- /Prediction_Logs/columnValidationLog.txt: -------------------------------------------------------------------------------- 1 | 2020-10-09/16:14:12 Column Length Validation Started!! 2 | 2020-10-09/16:14:19 Column Length Validation Completed!! 3 | 2020-10-09/18:19:14 Column Length Validation Started!! 4 | 2020-10-09/18:19:21 Column Length Validation Completed!! 5 | 2020-10-15/10:29:24 Column Length Validation Started!! 6 | 2020-10-15/10:29:26 Column Length Validation Completed!! 7 | 2021-02-05/21:26:14 Column Length Validation Started!! 8 | 2021-02-05/21:26:16 Column Length Validation Completed!! 9 | 2021-02-07/22:34:42 Column Length Validation Started!! 10 | 2021-02-07/22:34:45 Column Length Validation Completed!! 11 | 2021-02-07/22:40:35 Column Length Validation Started!! 12 | 2021-02-07/22:40:37 Column Length Validation Completed!! 13 | 2021-02-25/11:49:31 Column Length Validation Started!! 14 | 2021-02-25/11:49:32 Column Length Validation Completed!! 15 | 2021-07-12/10:12:16 Column Length Validation Started!! 16 | 2021-07-12/10:12:18 Column Length Validation Completed!! 17 | 2021-07-12/10:15:43 Column Length Validation Started!! 18 | 2021-07-12/10:15:45 Column Length Validation Completed!! 19 | 2021-07-20/10:20:19 Column Length Validation Started!! 20 | 2021-07-20/10:20:21 Column Length Validation Completed!! 21 | 2021-11-20/00:07:19 Column Length Validation Started!! 22 | 2021-11-20/00:07:21 Column Length Validation Completed!! 23 | 2021-11-20/00:10:29 Column Length Validation Started!! 24 | 2021-11-20/00:10:31 Column Length Validation Completed!! 25 | 2021-11-20/00:13:30 Column Length Validation Started!! 26 | 2021-11-20/00:13:31 Column Length Validation Completed!! 27 | -------------------------------------------------------------------------------- /Prediction_Logs/dataTransformLog.txt: -------------------------------------------------------------------------------- 1 | 2020-10-09/16:14:27 wafer_13012020_090817.csv: File Transformed successfully!! 2 | 2020-10-09/16:14:29 wafer_16012020_051629.csv: File Transformed successfully!! 3 | 2020-10-09/16:14:30 wafer_20022020_090716.csv: File Transformed successfully!! 4 | 2020-10-09/16:14:31 wafer_21012020_080913.csv: File Transformed successfully!! 5 | 2020-10-09/16:14:33 wafer_23012020_041211.csv: File Transformed successfully!! 6 | 2020-10-09/16:14:34 wafer_27012020_080911.csv: File Transformed successfully!! 7 | 2020-10-09/16:14:36 wafer_28012020_090817.csv: File Transformed successfully!! 8 | 2020-10-09/16:14:38 wafer_28042020_031911.csv: File Transformed successfully!! 9 | 2020-10-09/16:14:39 wafer_29012020_050617.csv: File Transformed successfully!! 10 | 2020-10-09/18:19:29 wafer_13012020_090817.csv: File Transformed successfully!! 11 | 2020-10-09/18:19:31 wafer_16012020_051629.csv: File Transformed successfully!! 12 | 2020-10-09/18:19:32 wafer_20022020_090716.csv: File Transformed successfully!! 13 | 2020-10-09/18:19:33 wafer_21012020_080913.csv: File Transformed successfully!! 14 | 2020-10-09/18:19:35 wafer_23012020_041211.csv: File Transformed successfully!! 15 | 2020-10-09/18:19:36 wafer_27012020_080911.csv: File Transformed successfully!! 16 | 2020-10-09/18:19:38 wafer_28012020_090817.csv: File Transformed successfully!! 17 | 2020-10-09/18:19:40 wafer_28042020_031911.csv: File Transformed successfully!! 18 | 2020-10-09/18:19:41 wafer_29012020_050617.csv: File Transformed successfully!! 19 | 2020-10-15/10:29:29 wafer_13012020_090817.csv: File Transformed successfully!! 20 | 2020-10-15/10:29:30 wafer_20022020_090716.csv: File Transformed successfully!! 21 | 2020-10-15/10:29:30 wafer_23012020_041211.csv: File Transformed successfully!! 22 | 2020-10-15/10:29:31 wafer_28042020_031911.csv: File Transformed successfully!! 23 | 2020-10-15/10:29:32 wafer_16012020_051629.csv: File Transformed successfully!! 24 | 2020-10-15/10:29:32 wafer_21012020_080913.csv: File Transformed successfully!! 25 | 2020-10-15/10:29:33 wafer_27012020_080911.csv: File Transformed successfully!! 26 | 2020-10-15/10:29:34 wafer_28012020_090817.csv: File Transformed successfully!! 27 | 2020-10-15/10:29:34 wafer_29012020_050617.csv: File Transformed successfully!! 28 | 2021-02-05/21:26:18 wafer_13012020_090817.csv: File Transformed successfully!! 29 | 2021-02-05/21:26:19 wafer_20022020_090716.csv: File Transformed successfully!! 30 | 2021-02-05/21:26:20 wafer_23012020_041211.csv: File Transformed successfully!! 31 | 2021-02-05/21:26:21 wafer_28042020_031911.csv: File Transformed successfully!! 32 | 2021-02-05/21:26:22 wafer_16012020_051629.csv: File Transformed successfully!! 33 | 2021-02-05/21:26:22 wafer_21012020_080913.csv: File Transformed successfully!! 34 | 2021-02-05/21:26:22 wafer_27012020_080911.csv: File Transformed successfully!! 35 | 2021-02-05/21:26:23 wafer_28012020_090817.csv: File Transformed successfully!! 36 | 2021-02-05/21:26:23 wafer_29012020_050617.csv: File Transformed successfully!! 37 | 2021-02-07/22:34:48 wafer_13012020_090817.csv: File Transformed successfully!! 38 | 2021-02-07/22:34:49 wafer_20022020_090716.csv: File Transformed successfully!! 39 | 2021-02-07/22:34:50 wafer_23012020_041211.csv: File Transformed successfully!! 40 | 2021-02-07/22:34:51 wafer_28042020_031911.csv: File Transformed successfully!! 41 | 2021-02-07/22:34:51 wafer_16012020_051629.csv: File Transformed successfully!! 42 | 2021-02-07/22:34:52 wafer_21012020_080913.csv: File Transformed successfully!! 43 | 2021-02-07/22:34:53 wafer_27012020_080911.csv: File Transformed successfully!! 44 | 2021-02-07/22:34:54 wafer_28012020_090817.csv: File Transformed successfully!! 45 | 2021-02-07/22:34:55 wafer_29012020_050617.csv: File Transformed successfully!! 46 | 2021-02-07/22:40:40 wafer_13012020_090817.csv: File Transformed successfully!! 47 | 2021-02-07/22:40:41 wafer_20022020_090716.csv: File Transformed successfully!! 48 | 2021-02-07/22:40:41 wafer_23012020_041211.csv: File Transformed successfully!! 49 | 2021-02-07/22:40:43 wafer_28042020_031911.csv: File Transformed successfully!! 50 | 2021-02-07/22:40:44 wafer_16012020_051629.csv: File Transformed successfully!! 51 | 2021-02-07/22:40:45 wafer_21012020_080913.csv: File Transformed successfully!! 52 | 2021-02-07/22:40:45 wafer_27012020_080911.csv: File Transformed successfully!! 53 | 2021-02-07/22:40:47 wafer_28012020_090817.csv: File Transformed successfully!! 54 | 2021-02-07/22:40:47 wafer_29012020_050617.csv: File Transformed successfully!! 55 | 2021-02-25/11:49:35 wafer_13012020_090817.csv: File Transformed successfully!! 56 | 2021-02-25/11:49:35 wafer_20022020_090716.csv: File Transformed successfully!! 57 | 2021-02-25/11:49:35 wafer_23012020_041211.csv: File Transformed successfully!! 58 | 2021-02-25/11:49:36 wafer_28042020_031911.csv: File Transformed successfully!! 59 | 2021-02-25/11:49:36 wafer_16012020_051629.csv: File Transformed successfully!! 60 | 2021-02-25/11:49:37 wafer_21012020_080913.csv: File Transformed successfully!! 61 | 2021-02-25/11:49:37 wafer_27012020_080911.csv: File Transformed successfully!! 62 | 2021-02-25/11:49:38 wafer_28012020_090817.csv: File Transformed successfully!! 63 | 2021-02-25/11:49:38 wafer_29012020_050617.csv: File Transformed successfully!! 64 | 2021-07-12/10:12:20 wafer_13012020_090817.csv: File Transformed successfully!! 65 | 2021-07-12/10:12:21 wafer_20022020_090716.csv: File Transformed successfully!! 66 | 2021-07-12/10:12:21 wafer_23012020_041211.csv: File Transformed successfully!! 67 | 2021-07-12/10:12:22 wafer_28042020_031911.csv: File Transformed successfully!! 68 | 2021-07-12/10:12:23 wafer_16012020_051629.csv: File Transformed successfully!! 69 | 2021-07-12/10:12:23 wafer_21012020_080913.csv: File Transformed successfully!! 70 | 2021-07-12/10:12:23 wafer_27012020_080911.csv: File Transformed successfully!! 71 | 2021-07-12/10:12:24 wafer_28012020_090817.csv: File Transformed successfully!! 72 | 2021-07-12/10:12:24 wafer_29012020_050617.csv: File Transformed successfully!! 73 | 2021-07-12/10:15:48 wafer_13012020_090817.csv: File Transformed successfully!! 74 | 2021-07-12/10:15:48 wafer_20022020_090716.csv: File Transformed successfully!! 75 | 2021-07-12/10:15:49 wafer_23012020_041211.csv: File Transformed successfully!! 76 | 2021-07-12/10:15:50 wafer_28042020_031911.csv: File Transformed successfully!! 77 | 2021-07-12/10:15:50 wafer_16012020_051629.csv: File Transformed successfully!! 78 | 2021-07-12/10:15:51 wafer_21012020_080913.csv: File Transformed successfully!! 79 | 2021-07-12/10:15:51 wafer_27012020_080911.csv: File Transformed successfully!! 80 | 2021-07-12/10:15:52 wafer_28012020_090817.csv: File Transformed successfully!! 81 | 2021-07-12/10:15:52 wafer_29012020_050617.csv: File Transformed successfully!! 82 | 2021-07-20/10:20:24 wafer_13012020_090817.csv: File Transformed successfully!! 83 | 2021-07-20/10:20:24 wafer_20022020_090716.csv: File Transformed successfully!! 84 | 2021-07-20/10:20:25 wafer_23012020_041211.csv: File Transformed successfully!! 85 | 2021-07-20/10:20:26 wafer_28042020_031911.csv: File Transformed successfully!! 86 | 2021-07-20/10:20:26 wafer_16012020_051629.csv: File Transformed successfully!! 87 | 2021-07-20/10:20:27 wafer_21012020_080913.csv: File Transformed successfully!! 88 | 2021-07-20/10:20:27 wafer_27012020_080911.csv: File Transformed successfully!! 89 | 2021-07-20/10:20:28 wafer_28012020_090817.csv: File Transformed successfully!! 90 | 2021-07-20/10:20:28 wafer_29012020_050617.csv: File Transformed successfully!! 91 | 2021-11-20/00:07:23 wafer_13012020_090817.csv: File Transformed successfully!! 92 | 2021-11-20/00:07:23 wafer_20022020_090716.csv: File Transformed successfully!! 93 | 2021-11-20/00:07:24 wafer_23012020_041211.csv: File Transformed successfully!! 94 | 2021-11-20/00:07:25 wafer_28042020_031911.csv: File Transformed successfully!! 95 | 2021-11-20/00:07:25 wafer_16012020_051629.csv: File Transformed successfully!! 96 | 2021-11-20/00:07:26 wafer_21012020_080913.csv: File Transformed successfully!! 97 | 2021-11-20/00:07:26 wafer_27012020_080911.csv: File Transformed successfully!! 98 | 2021-11-20/00:07:27 wafer_28012020_090817.csv: File Transformed successfully!! 99 | 2021-11-20/00:07:27 wafer_29012020_050617.csv: File Transformed successfully!! 100 | 2021-11-20/00:10:33 wafer_13012020_090817.csv: File Transformed successfully!! 101 | 2021-11-20/00:10:33 wafer_20022020_090716.csv: File Transformed successfully!! 102 | 2021-11-20/00:10:34 wafer_23012020_041211.csv: File Transformed successfully!! 103 | 2021-11-20/00:10:35 wafer_28042020_031911.csv: File Transformed successfully!! 104 | 2021-11-20/00:10:35 wafer_16012020_051629.csv: File Transformed successfully!! 105 | 2021-11-20/00:10:35 wafer_21012020_080913.csv: File Transformed successfully!! 106 | 2021-11-20/00:10:36 wafer_27012020_080911.csv: File Transformed successfully!! 107 | 2021-11-20/00:10:36 wafer_28012020_090817.csv: File Transformed successfully!! 108 | 2021-11-20/00:10:37 wafer_29012020_050617.csv: File Transformed successfully!! 109 | 2021-11-20/00:13:34 wafer_13012020_090817.csv: File Transformed successfully!! 110 | 2021-11-20/00:13:34 wafer_20022020_090716.csv: File Transformed successfully!! 111 | 2021-11-20/00:13:34 wafer_23012020_041211.csv: File Transformed successfully!! 112 | 2021-11-20/00:13:35 wafer_28042020_031911.csv: File Transformed successfully!! 113 | 2021-11-20/00:13:35 wafer_16012020_051629.csv: File Transformed successfully!! 114 | 2021-11-20/00:13:36 wafer_21012020_080913.csv: File Transformed successfully!! 115 | 2021-11-20/00:13:36 wafer_27012020_080911.csv: File Transformed successfully!! 116 | 2021-11-20/00:13:37 wafer_28012020_090817.csv: File Transformed successfully!! 117 | 2021-11-20/00:13:37 wafer_29012020_050617.csv: File Transformed successfully!! 118 | -------------------------------------------------------------------------------- /Prediction_Logs/missingValuesInColumn.txt: -------------------------------------------------------------------------------- 1 | 2020-10-09/16:14:19 Missing Values Validation Started!! 2 | 2020-10-09/16:14:19 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_07012020_041011.csv 3 | 2020-10-09/16:14:21 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_20012020_090819.csv 4 | 2020-10-09/16:14:22 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_22022020_041119.csv 5 | 2020-10-09/16:14:22 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_23012020_011008.csv 6 | 2020-10-09/16:14:24 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_28012020_051011.csv 7 | 2020-10-09/16:14:26 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_29012020_060756.csv 8 | 2020-10-09/16:14:26 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_31012020_090811.csv 9 | 2020-10-09/18:19:21 Missing Values Validation Started!! 10 | 2020-10-09/18:19:22 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_07012020_041011.csv 11 | 2020-10-09/18:19:23 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_20012020_090819.csv 12 | 2020-10-09/18:19:24 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_22022020_041119.csv 13 | 2020-10-09/18:19:24 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_23012020_011008.csv 14 | 2020-10-09/18:19:26 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_28012020_051011.csv 15 | 2020-10-09/18:19:27 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_29012020_060756.csv 16 | 2020-10-09/18:19:27 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_31012020_090811.csv 17 | 2020-10-15/10:29:26 Missing Values Validation Started!! 18 | 2020-10-15/10:29:26 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_23012020_011008.csv 19 | 2020-10-15/10:29:27 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_22022020_041119.csv 20 | 2020-10-15/10:29:27 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_29012020_060756.csv 21 | 2020-10-15/10:29:27 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_28012020_051011.csv 22 | 2020-10-15/10:29:27 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_07012020_041011.csv 23 | 2020-10-15/10:29:27 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_20012020_090819.csv 24 | 2020-10-15/10:29:27 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_31012020_090811.csv 25 | 2021-02-05/21:26:16 Missing Values Validation Started!! 26 | 2021-02-05/21:26:16 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_23012020_011008.csv 27 | 2021-02-05/21:26:16 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_22022020_041119.csv 28 | 2021-02-05/21:26:16 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_29012020_060756.csv 29 | 2021-02-05/21:26:16 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_28012020_051011.csv 30 | 2021-02-05/21:26:17 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_07012020_041011.csv 31 | 2021-02-05/21:26:17 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_20012020_090819.csv 32 | 2021-02-05/21:26:17 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_31012020_090811.csv 33 | 2021-02-07/22:34:45 Missing Values Validation Started!! 34 | 2021-02-07/22:34:45 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_23012020_011008.csv 35 | 2021-02-07/22:34:45 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_22022020_041119.csv 36 | 2021-02-07/22:34:46 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_29012020_060756.csv 37 | 2021-02-07/22:34:46 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_28012020_051011.csv 38 | 2021-02-07/22:34:46 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_07012020_041011.csv 39 | 2021-02-07/22:34:46 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_20012020_090819.csv 40 | 2021-02-07/22:34:46 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_31012020_090811.csv 41 | 2021-02-07/22:40:37 Missing Values Validation Started!! 42 | 2021-02-07/22:40:37 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_23012020_011008.csv 43 | 2021-02-07/22:40:37 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_22022020_041119.csv 44 | 2021-02-07/22:40:38 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_29012020_060756.csv 45 | 2021-02-07/22:40:38 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_28012020_051011.csv 46 | 2021-02-07/22:40:38 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_07012020_041011.csv 47 | 2021-02-07/22:40:38 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_20012020_090819.csv 48 | 2021-02-07/22:40:38 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_31012020_090811.csv 49 | 2021-02-25/11:49:32 Missing Values Validation Started!! 50 | 2021-02-25/11:49:33 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_23012020_011008.csv 51 | 2021-02-25/11:49:33 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_22022020_041119.csv 52 | 2021-02-25/11:49:33 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_29012020_060756.csv 53 | 2021-02-25/11:49:33 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_28012020_051011.csv 54 | 2021-02-25/11:49:33 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_07012020_041011.csv 55 | 2021-02-25/11:49:33 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_20012020_090819.csv 56 | 2021-02-25/11:49:33 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_31012020_090811.csv 57 | 2021-07-12/10:12:18 Missing Values Validation Started!! 58 | 2021-07-12/10:12:18 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_23012020_011008.csv 59 | 2021-07-12/10:12:18 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_22022020_041119.csv 60 | 2021-07-12/10:12:18 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_29012020_060756.csv 61 | 2021-07-12/10:12:18 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_28012020_051011.csv 62 | 2021-07-12/10:12:19 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_07012020_041011.csv 63 | 2021-07-12/10:12:19 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_20012020_090819.csv 64 | 2021-07-12/10:12:19 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_31012020_090811.csv 65 | 2021-07-12/10:15:45 Missing Values Validation Started!! 66 | 2021-07-12/10:15:45 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_23012020_011008.csv 67 | 2021-07-12/10:15:45 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_22022020_041119.csv 68 | 2021-07-12/10:15:45 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_29012020_060756.csv 69 | 2021-07-12/10:15:46 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_28012020_051011.csv 70 | 2021-07-12/10:15:46 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_07012020_041011.csv 71 | 2021-07-12/10:15:46 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_20012020_090819.csv 72 | 2021-07-12/10:15:46 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_31012020_090811.csv 73 | 2021-07-20/10:20:21 Missing Values Validation Started!! 74 | 2021-07-20/10:20:22 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_23012020_011008.csv 75 | 2021-07-20/10:20:22 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_22022020_041119.csv 76 | 2021-07-20/10:20:22 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_29012020_060756.csv 77 | 2021-07-20/10:20:22 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_28012020_051011.csv 78 | 2021-07-20/10:20:22 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_07012020_041011.csv 79 | 2021-07-20/10:20:23 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_20012020_090819.csv 80 | 2021-07-20/10:20:23 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_31012020_090811.csv 81 | 2021-11-20/00:07:21 Missing Values Validation Started!! 82 | 2021-11-20/00:07:21 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_23012020_011008.csv 83 | 2021-11-20/00:07:21 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_22022020_041119.csv 84 | 2021-11-20/00:07:21 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_29012020_060756.csv 85 | 2021-11-20/00:07:21 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_28012020_051011.csv 86 | 2021-11-20/00:07:21 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_07012020_041011.csv 87 | 2021-11-20/00:07:22 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_20012020_090819.csv 88 | 2021-11-20/00:07:22 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_31012020_090811.csv 89 | 2021-11-20/00:10:31 Missing Values Validation Started!! 90 | 2021-11-20/00:10:31 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_23012020_011008.csv 91 | 2021-11-20/00:10:31 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_22022020_041119.csv 92 | 2021-11-20/00:10:31 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_29012020_060756.csv 93 | 2021-11-20/00:10:31 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_28012020_051011.csv 94 | 2021-11-20/00:10:32 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_07012020_041011.csv 95 | 2021-11-20/00:10:32 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_20012020_090819.csv 96 | 2021-11-20/00:10:32 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_31012020_090811.csv 97 | 2021-11-20/00:13:31 Missing Values Validation Started!! 98 | 2021-11-20/00:13:32 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_23012020_011008.csv 99 | 2021-11-20/00:13:32 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_22022020_041119.csv 100 | 2021-11-20/00:13:32 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_29012020_060756.csv 101 | 2021-11-20/00:13:32 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_28012020_051011.csv 102 | 2021-11-20/00:13:32 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_07012020_041011.csv 103 | 2021-11-20/00:13:32 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_20012020_090819.csv 104 | 2021-11-20/00:13:32 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_31012020_090811.csv 105 | -------------------------------------------------------------------------------- /Prediction_Output_File/Predictions.csv: -------------------------------------------------------------------------------- 1 | ,Wafer,Prediction 2 | 0,1201.0,-1 3 | 1,1203.0,-1 4 | 2,1204.0,-1 5 | 3,1205.0,-1 6 | 4,1206.0,-1 7 | 5,1207.0,-1 8 | 6,1210.0,-1 9 | 7,1211.0,-1 10 | 8,1212.0,-1 11 | 9,1215.0,-1 12 | 10,1216.0,-1 13 | 11,1217.0,-1 14 | 12,1218.0,-1 15 | 13,1220.0,-1 16 | 14,1221.0,-1 17 | 15,1222.0,-1 18 | 16,1223.0,-1 19 | 17,1224.0,-1 20 | 18,1226.0,-1 21 | 19,1227.0,-1 22 | 20,1228.0,-1 23 | 21,1230.0,-1 24 | 22,1231.0,-1 25 | 23,1234.0,-1 26 | 24,1236.0,-1 27 | 25,1237.0,-1 28 | 26,1238.0,-1 29 | 27,1240.0,-1 30 | 28,1243.0,-1 31 | 29,1244.0,-1 32 | 30,1245.0,-1 33 | 31,1246.0,-1 34 | 32,1247.0,-1 35 | 33,1248.0,-1 36 | 34,1254.0,-1 37 | 35,1255.0,-1 38 | 36,1256.0,-1 39 | 37,1257.0,-1 40 | 38,1258.0,-1 41 | 39,1259.0,-1 42 | 40,1260.0,-1 43 | 41,1261.0,-1 44 | 42,1264.0,-1 45 | 43,1265.0,-1 46 | 44,1268.0,-1 47 | 45,1269.0,-1 48 | 46,1270.0,-1 49 | 47,1271.0,-1 50 | 48,1272.0,-1 51 | 49,1274.0,-1 52 | 50,1275.0,-1 53 | 51,1277.0,-1 54 | 52,1278.0,-1 55 | 53,1279.0,-1 56 | 54,1280.0,-1 57 | 55,1282.0,-1 58 | 56,1283.0,-1 59 | 57,1284.0,-1 60 | 58,1286.0,-1 61 | 59,1288.0,-1 62 | 60,1291.0,-1 63 | 61,1292.0,-1 64 | 62,1296.0,-1 65 | 63,1297.0,-1 66 | 64,1298.0,-1 67 | 65,1001.0,-1 68 | 66,1002.0,-1 69 | 67,1003.0,-1 70 | 68,1004.0,-1 71 | 69,1006.0,-1 72 | 70,1007.0,-1 73 | 71,1010.0,-1 74 | 72,1013.0,-1 75 | 73,1014.0,1 76 | 74,1015.0,-1 77 | 75,1016.0,-1 78 | 76,1017.0,-1 79 | 77,1019.0,-1 80 | 78,1020.0,-1 81 | 79,1021.0,-1 82 | 80,1024.0,-1 83 | 81,1025.0,-1 84 | 82,1026.0,-1 85 | 83,1029.0,-1 86 | 84,1031.0,-1 87 | 85,1032.0,-1 88 | 86,1034.0,-1 89 | 87,1036.0,-1 90 | 88,1037.0,-1 91 | 89,1038.0,-1 92 | 90,1039.0,-1 93 | 91,1041.0,-1 94 | 92,1043.0,-1 95 | 93,1045.0,-1 96 | 94,1046.0,-1 97 | 95,1047.0,1 98 | 96,1048.0,-1 99 | 97,1050.0,1 100 | 98,1054.0,-1 101 | 99,1055.0,-1 102 | 100,1058.0,-1 103 | 101,1059.0,-1 104 | 102,1060.0,-1 105 | 103,1061.0,-1 106 | 104,1062.0,-1 107 | 105,1064.0,-1 108 | 106,1065.0,-1 109 | 107,1066.0,-1 110 | 108,1067.0,-1 111 | 109,1068.0,-1 112 | 110,1069.0,-1 113 | 111,1070.0,-1 114 | 112,1071.0,-1 115 | 113,1072.0,-1 116 | 114,1074.0,-1 117 | 115,1075.0,-1 118 | 116,1077.0,-1 119 | 117,1078.0,-1 120 | 118,1079.0,-1 121 | 119,1080.0,-1 122 | 120,1082.0,-1 123 | 121,1084.0,-1 124 | 122,1085.0,-1 125 | 123,1087.0,-1 126 | 124,1088.0,-1 127 | 125,1089.0,-1 128 | 126,1090.0,-1 129 | 127,1091.0,-1 130 | 128,1093.0,-1 131 | 129,1096.0,-1 132 | 130,1097.0,-1 133 | 131,1098.0,-1 134 | 132,1099.0,-1 135 | 133,1100.0,-1 136 | 134,1501.0,-1 137 | 135,1502.0,-1 138 | 136,1503.0,-1 139 | 137,1504.0,-1 140 | 138,1505.0,-1 141 | 139,1506.0,-1 142 | 140,1507.0,-1 143 | 141,1508.0,-1 144 | 142,1512.0,-1 145 | 143,1513.0,-1 146 | 144,1514.0,-1 147 | 145,1517.0,-1 148 | 146,1518.0,-1 149 | 147,1519.0,-1 150 | 148,1522.0,-1 151 | 149,1523.0,-1 152 | 150,1524.0,-1 153 | 151,1525.0,-1 154 | 152,1526.0,-1 155 | 153,1527.0,-1 156 | 154,1528.0,-1 157 | 155,1529.0,-1 158 | 156,1530.0,-1 159 | 157,1531.0,-1 160 | 158,1532.0,-1 161 | 159,1533.0,-1 162 | 160,1534.0,-1 163 | 161,1535.0,-1 164 | 162,1536.0,-1 165 | 163,1537.0,-1 166 | 164,902.0,-1 167 | 165,904.0,-1 168 | 166,907.0,-1 169 | 167,908.0,-1 170 | 168,915.0,-1 171 | 169,916.0,-1 172 | 170,919.0,-1 173 | 171,920.0,-1 174 | 172,922.0,-1 175 | 173,923.0,-1 176 | 174,924.0,-1 177 | 175,925.0,-1 178 | 176,927.0,-1 179 | 177,929.0,-1 180 | 178,930.0,-1 181 | 179,931.0,-1 182 | 180,932.0,-1 183 | 181,936.0,-1 184 | 182,937.0,-1 185 | 183,938.0,-1 186 | 184,939.0,-1 187 | 185,940.0,-1 188 | 186,941.0,-1 189 | 187,942.0,-1 190 | 188,944.0,-1 191 | 189,946.0,-1 192 | 190,947.0,-1 193 | 191,948.0,-1 194 | 192,949.0,-1 195 | 193,950.0,-1 196 | 194,951.0,-1 197 | 195,952.0,-1 198 | 196,953.0,-1 199 | 197,954.0,-1 200 | 198,957.0,-1 201 | 199,960.0,-1 202 | 200,961.0,-1 203 | 201,962.0,-1 204 | 202,963.0,-1 205 | 203,964.0,-1 206 | 204,965.0,-1 207 | 205,968.0,-1 208 | 206,969.0,-1 209 | 207,970.0,-1 210 | 208,973.0,-1 211 | 209,974.0,-1 212 | 210,975.0,-1 213 | 211,976.0,-1 214 | 212,979.0,-1 215 | 213,980.0,-1 216 | 214,981.0,-1 217 | 215,982.0,-1 218 | 216,983.0,-1 219 | 217,984.0,-1 220 | 218,985.0,-1 221 | 219,986.0,-1 222 | 220,988.0,-1 223 | 221,989.0,-1 224 | 222,991.0,-1 225 | 223,992.0,-1 226 | 224,993.0,-1 227 | 225,994.0,-1 228 | 226,996.0,-1 229 | 227,997.0,-1 230 | 228,998.0,-1 231 | 229,999.0,-1 232 | 230,801.0,-1 233 | 231,802.0,-1 234 | 232,803.0,-1 235 | 233,804.0,-1 236 | 234,805.0,-1 237 | 235,810.0,-1 238 | 236,811.0,-1 239 | 237,812.0,-1 240 | 238,814.0,-1 241 | 239,815.0,-1 242 | 240,822.0,-1 243 | 241,823.0,-1 244 | 242,825.0,-1 245 | 243,826.0,-1 246 | 244,827.0,-1 247 | 245,828.0,-1 248 | 246,829.0,-1 249 | 247,830.0,-1 250 | 248,831.0,-1 251 | 249,833.0,-1 252 | 250,834.0,-1 253 | 251,837.0,-1 254 | 252,840.0,-1 255 | 253,841.0,-1 256 | 254,842.0,-1 257 | 255,843.0,-1 258 | 256,844.0,-1 259 | 257,845.0,-1 260 | 258,847.0,-1 261 | 259,849.0,-1 262 | 260,852.0,-1 263 | 261,853.0,-1 264 | 262,854.0,-1 265 | 263,855.0,-1 266 | 264,856.0,-1 267 | 265,857.0,-1 268 | 266,858.0,-1 269 | 267,860.0,-1 270 | 268,863.0,-1 271 | 269,864.0,-1 272 | 270,865.0,-1 273 | 271,866.0,-1 274 | 272,867.0,-1 275 | 273,869.0,-1 276 | 274,870.0,-1 277 | 275,871.0,-1 278 | 276,872.0,-1 279 | 277,875.0,-1 280 | 278,877.0,-1 281 | 279,881.0,-1 282 | 280,882.0,-1 283 | 281,883.0,-1 284 | 282,885.0,-1 285 | 283,886.0,-1 286 | 284,887.0,-1 287 | 285,888.0,-1 288 | 286,889.0,-1 289 | 287,890.0,-1 290 | 288,895.0,-1 291 | 289,896.0,-1 292 | 290,897.0,-1 293 | 291,899.0,-1 294 | 292,1401.0,-1 295 | 293,1403.0,-1 296 | 294,1405.0,-1 297 | 295,1407.0,-1 298 | 296,1408.0,-1 299 | 297,1409.0,-1 300 | 298,1411.0,-1 301 | 299,1412.0,-1 302 | 300,1413.0,-1 303 | 301,1414.0,-1 304 | 302,1415.0,-1 305 | 303,1416.0,-1 306 | 304,1417.0,-1 307 | 305,1418.0,-1 308 | 306,1419.0,-1 309 | 307,1421.0,-1 310 | 308,1422.0,-1 311 | 309,1424.0,-1 312 | 310,1425.0,-1 313 | 311,1426.0,-1 314 | 312,1427.0,-1 315 | 313,1428.0,-1 316 | 314,1429.0,-1 317 | 315,1430.0,-1 318 | 316,1431.0,-1 319 | 317,1432.0,-1 320 | 318,1433.0,-1 321 | 319,1434.0,-1 322 | 320,1435.0,-1 323 | 321,1436.0,-1 324 | 322,1437.0,-1 325 | 323,1438.0,-1 326 | 324,1439.0,-1 327 | 325,1440.0,-1 328 | 326,1442.0,-1 329 | 327,1443.0,-1 330 | 328,1447.0,-1 331 | 329,1448.0,-1 332 | 330,1450.0,-1 333 | 331,1452.0,-1 334 | 332,1453.0,-1 335 | 333,1458.0,-1 336 | 334,1459.0,-1 337 | 335,1460.0,-1 338 | 336,1461.0,-1 339 | 337,1464.0,-1 340 | 338,1466.0,1 341 | 339,1467.0,-1 342 | 340,1468.0,-1 343 | 341,1470.0,-1 344 | 342,1473.0,-1 345 | 343,1474.0,-1 346 | 344,1475.0,-1 347 | 345,1476.0,-1 348 | 346,1477.0,-1 349 | 347,1478.0,-1 350 | 348,1480.0,-1 351 | 349,1481.0,-1 352 | 350,1483.0,-1 353 | 351,1484.0,-1 354 | 352,1485.0,-1 355 | 353,1486.0,-1 356 | 354,1489.0,-1 357 | 355,1490.0,-1 358 | 356,1491.0,-1 359 | 357,1492.0,-1 360 | 358,1493.0,-1 361 | 359,1494.0,-1 362 | 360,1496.0,-1 363 | 361,1498.0,-1 364 | 362,1500.0,-1 365 | 363,1102.0,-1 366 | 364,1104.0,-1 367 | 365,1106.0,-1 368 | 366,1109.0,-1 369 | 367,1110.0,-1 370 | 368,1111.0,-1 371 | 369,1113.0,-1 372 | 370,1116.0,-1 373 | 371,1119.0,-1 374 | 372,1121.0,-1 375 | 373,1124.0,-1 376 | 374,1125.0,-1 377 | 375,1127.0,-1 378 | 376,1131.0,-1 379 | 377,1132.0,-1 380 | 378,1133.0,1 381 | 379,1134.0,-1 382 | 380,1135.0,-1 383 | 381,1137.0,-1 384 | 382,1138.0,-1 385 | 383,1139.0,-1 386 | 384,1142.0,-1 387 | 385,1143.0,-1 388 | 386,1144.0,-1 389 | 387,1145.0,-1 390 | 388,1146.0,-1 391 | 389,1148.0,-1 392 | 390,1151.0,-1 393 | 391,1152.0,-1 394 | 392,1155.0,-1 395 | 393,1156.0,-1 396 | 394,1158.0,-1 397 | 395,1159.0,-1 398 | 396,1160.0,-1 399 | 397,1162.0,-1 400 | 398,1163.0,-1 401 | 399,1164.0,-1 402 | 400,1165.0,-1 403 | 401,1169.0,-1 404 | 402,1171.0,-1 405 | 403,1172.0,-1 406 | 404,1175.0,-1 407 | 405,1178.0,-1 408 | 406,1181.0,-1 409 | 407,1182.0,-1 410 | 408,1185.0,-1 411 | 409,1187.0,-1 412 | 410,1188.0,-1 413 | 411,1189.0,-1 414 | 412,1190.0,-1 415 | 413,1191.0,-1 416 | 414,1193.0,-1 417 | 415,1194.0,-1 418 | 416,1196.0,-1 419 | 417,1198.0,-1 420 | 418,1199.0,-1 421 | 419,1200.0,-1 422 | 420,701.0,-1 423 | 421,702.0,1 424 | 422,704.0,-1 425 | 423,705.0,-1 426 | 424,706.0,-1 427 | 425,707.0,-1 428 | 426,712.0,-1 429 | 427,713.0,-1 430 | 428,715.0,-1 431 | 429,718.0,-1 432 | 430,719.0,-1 433 | 431,720.0,1 434 | 432,721.0,-1 435 | 433,723.0,1 436 | 434,724.0,-1 437 | 435,726.0,-1 438 | 436,729.0,-1 439 | 437,730.0,-1 440 | 438,731.0,-1 441 | 439,732.0,-1 442 | 440,734.0,-1 443 | 441,735.0,-1 444 | 442,736.0,-1 445 | 443,739.0,-1 446 | 444,742.0,-1 447 | 445,743.0,1 448 | 446,744.0,-1 449 | 447,745.0,-1 450 | 448,747.0,-1 451 | 449,753.0,-1 452 | 450,754.0,1 453 | 451,757.0,-1 454 | 452,758.0,-1 455 | 453,759.0,-1 456 | 454,760.0,-1 457 | 455,764.0,-1 458 | 456,765.0,-1 459 | 457,766.0,-1 460 | 458,767.0,-1 461 | 459,768.0,-1 462 | 460,770.0,-1 463 | 461,772.0,-1 464 | 462,773.0,-1 465 | 463,774.0,-1 466 | 464,775.0,-1 467 | 465,776.0,-1 468 | 466,781.0,-1 469 | 467,782.0,-1 470 | 468,783.0,-1 471 | 469,787.0,-1 472 | 470,788.0,-1 473 | 471,789.0,-1 474 | 472,790.0,-1 475 | 473,791.0,-1 476 | 474,793.0,-1 477 | 475,794.0,-1 478 | 476,796.0,-1 479 | 477,797.0,-1 480 | 478,799.0,-1 481 | 479,800.0,-1 482 | 480,1301.0,-1 483 | 481,1302.0,-1 484 | 482,1303.0,-1 485 | 483,1306.0,-1 486 | 484,1307.0,-1 487 | 485,1308.0,-1 488 | 486,1309.0,-1 489 | 487,1310.0,-1 490 | 488,1311.0,-1 491 | 489,1314.0,-1 492 | 490,1315.0,-1 493 | 491,1316.0,-1 494 | 492,1317.0,-1 495 | 493,1320.0,-1 496 | 494,1324.0,-1 497 | 495,1325.0,-1 498 | 496,1326.0,-1 499 | 497,1328.0,-1 500 | 498,1329.0,-1 501 | 499,1333.0,-1 502 | 500,1336.0,-1 503 | 501,1337.0,-1 504 | 502,1339.0,-1 505 | 503,1341.0,-1 506 | 504,1345.0,-1 507 | 505,1346.0,-1 508 | 506,1347.0,-1 509 | 507,1348.0,-1 510 | 508,1349.0,-1 511 | 509,1352.0,-1 512 | 510,1353.0,-1 513 | 511,1355.0,-1 514 | 512,1358.0,-1 515 | 513,1360.0,-1 516 | 514,1361.0,-1 517 | 515,1362.0,-1 518 | 516,1363.0,-1 519 | 517,1365.0,-1 520 | 518,1366.0,-1 521 | 519,1367.0,-1 522 | 520,1368.0,-1 523 | 521,1370.0,-1 524 | 522,1372.0,-1 525 | 523,1375.0,-1 526 | 524,1376.0,-1 527 | 525,1379.0,-1 528 | 526,1380.0,1 529 | 527,1381.0,-1 530 | 528,1383.0,-1 531 | 529,1384.0,-1 532 | 530,1385.0,-1 533 | 531,1387.0,-1 534 | 532,1388.0,-1 535 | 533,1389.0,-1 536 | 534,1391.0,-1 537 | 535,1392.0,-1 538 | 536,1393.0,-1 539 | 537,1394.0,-1 540 | 538,1395.0,-1 541 | 539,1397.0,-1 542 | 540,1399.0,-1 543 | ,Wafer,Prediction 544 | 0,1202.0,-1 545 | 1,1287.0,-1 546 | 2,1290.0,-1 547 | 3,1294.0,-1 548 | 4,1299.0,-1 549 | 5,1300.0,-1 550 | 6,1005.0,-1 551 | 7,1012.0,-1 552 | 8,1022.0,-1 553 | 9,1027.0,-1 554 | 10,1033.0,-1 555 | 11,1044.0,-1 556 | 12,1052.0,-1 557 | 13,1063.0,-1 558 | 14,1073.0,-1 559 | 15,1076.0,-1 560 | 16,1092.0,-1 561 | 17,1511.0,-1 562 | 18,1516.0,-1 563 | 19,1521.0,-1 564 | 20,905.0,-1 565 | 21,914.0,-1 566 | 22,926.0,-1 567 | 23,934.0,-1 568 | 24,935.0,-1 569 | 25,945.0,-1 570 | 26,955.0,-1 571 | 27,956.0,-1 572 | 28,958.0,-1 573 | 29,967.0,-1 574 | 30,971.0,-1 575 | 31,972.0,-1 576 | 32,978.0,-1 577 | 33,987.0,-1 578 | 34,1000.0,-1 579 | 35,816.0,-1 580 | 36,818.0,-1 581 | 37,835.0,-1 582 | 38,846.0,-1 583 | 39,850.0,-1 584 | 40,862.0,-1 585 | 41,868.0,-1 586 | 42,892.0,-1 587 | 43,900.0,-1 588 | 44,1449.0,-1 589 | 45,1465.0,-1 590 | 46,1487.0,-1 591 | 47,1488.0,-1 592 | 48,1497.0,-1 593 | 49,1499.0,-1 594 | 50,1101.0,-1 595 | 51,1105.0,-1 596 | 52,1114.0,-1 597 | 53,1122.0,-1 598 | 54,1123.0,-1 599 | 55,1126.0,-1 600 | 56,1140.0,-1 601 | 57,1150.0,-1 602 | 58,1168.0,-1 603 | 59,1170.0,-1 604 | 60,1176.0,-1 605 | 61,1179.0,-1 606 | 62,1180.0,-1 607 | 63,1184.0,-1 608 | 64,1186.0,-1 609 | 65,708.0,-1 610 | 66,710.0,-1 611 | 67,716.0,-1 612 | 68,725.0,-1 613 | 69,727.0,-1 614 | 70,737.0,-1 615 | 71,741.0,-1 616 | 72,746.0,-1 617 | 73,748.0,-1 618 | 74,751.0,-1 619 | 75,752.0,-1 620 | 76,756.0,-1 621 | 77,771.0,-1 622 | 78,778.0,-1 623 | 79,785.0,-1 624 | 80,795.0,-1 625 | 81,1304.0,-1 626 | 82,1305.0,-1 627 | 83,1319.0,-1 628 | 84,1321.0,-1 629 | 85,1332.0,-1 630 | 86,1350.0,-1 631 | 87,1354.0,-1 632 | 88,1356.0,-1 633 | 89,1369.0,-1 634 | 90,1390.0,-1 635 | 91,1398.0,-1 636 | ,Wafer,Prediction 637 | 0,1208.0,-1 638 | 1,1209.0,-1 639 | 2,1213.0,-1 640 | 3,1214.0,1 641 | 4,1219.0,-1 642 | 5,1225.0,-1 643 | 6,1229.0,-1 644 | 7,1232.0,-1 645 | 8,1233.0,1 646 | 9,1235.0,-1 647 | 10,1239.0,-1 648 | 11,1241.0,-1 649 | 12,1242.0,1 650 | 13,1249.0,-1 651 | 14,1250.0,-1 652 | 15,1251.0,-1 653 | 16,1252.0,-1 654 | 17,1253.0,-1 655 | 18,1262.0,-1 656 | 19,1263.0,-1 657 | 20,1266.0,-1 658 | 21,1267.0,-1 659 | 22,1273.0,1 660 | 23,1276.0,-1 661 | 24,1281.0,1 662 | 25,1285.0,-1 663 | 26,1289.0,1 664 | 27,1293.0,-1 665 | 28,1295.0,-1 666 | 29,1008.0,-1 667 | 30,1009.0,-1 668 | 31,1011.0,-1 669 | 32,1018.0,-1 670 | 33,1023.0,-1 671 | 34,1028.0,-1 672 | 35,1030.0,-1 673 | 36,1035.0,-1 674 | 37,1040.0,1 675 | 38,1042.0,-1 676 | 39,1049.0,-1 677 | 40,1051.0,-1 678 | 41,1053.0,-1 679 | 42,1056.0,-1 680 | 43,1057.0,-1 681 | 44,1081.0,-1 682 | 45,1083.0,-1 683 | 46,1086.0,-1 684 | 47,1094.0,-1 685 | 48,1095.0,-1 686 | 49,1509.0,-1 687 | 50,1510.0,-1 688 | 51,1515.0,-1 689 | 52,1520.0,-1 690 | 53,901.0,-1 691 | 54,903.0,-1 692 | 55,906.0,-1 693 | 56,909.0,-1 694 | 57,910.0,-1 695 | 58,911.0,-1 696 | 59,912.0,-1 697 | 60,913.0,-1 698 | 61,917.0,-1 699 | 62,918.0,-1 700 | 63,921.0,-1 701 | 64,928.0,-1 702 | 65,933.0,-1 703 | 66,943.0,-1 704 | 67,959.0,-1 705 | 68,966.0,-1 706 | 69,977.0,-1 707 | 70,990.0,-1 708 | 71,995.0,-1 709 | 72,806.0,1 710 | 73,807.0,-1 711 | 74,808.0,1 712 | 75,809.0,-1 713 | 76,813.0,-1 714 | 77,817.0,-1 715 | 78,819.0,-1 716 | 79,820.0,-1 717 | 80,821.0,-1 718 | 81,824.0,-1 719 | 82,832.0,-1 720 | 83,836.0,-1 721 | 84,838.0,-1 722 | 85,839.0,-1 723 | 86,848.0,-1 724 | 87,851.0,-1 725 | 88,859.0,-1 726 | 89,861.0,-1 727 | 90,873.0,-1 728 | 91,874.0,-1 729 | 92,876.0,-1 730 | 93,878.0,-1 731 | 94,879.0,1 732 | 95,880.0,-1 733 | 96,884.0,-1 734 | 97,891.0,-1 735 | 98,893.0,-1 736 | 99,894.0,-1 737 | 100,898.0,-1 738 | 101,1402.0,-1 739 | 102,1404.0,-1 740 | 103,1406.0,-1 741 | 104,1410.0,-1 742 | 105,1420.0,-1 743 | 106,1423.0,-1 744 | 107,1441.0,-1 745 | 108,1444.0,-1 746 | 109,1445.0,-1 747 | 110,1446.0,-1 748 | 111,1451.0,-1 749 | 112,1454.0,-1 750 | 113,1455.0,-1 751 | 114,1456.0,-1 752 | 115,1457.0,-1 753 | 116,1462.0,-1 754 | 117,1463.0,-1 755 | 118,1469.0,-1 756 | 119,1471.0,-1 757 | 120,1472.0,-1 758 | 121,1479.0,-1 759 | 122,1482.0,-1 760 | 123,1495.0,-1 761 | 124,1103.0,-1 762 | 125,1107.0,-1 763 | 126,1108.0,-1 764 | 127,1112.0,-1 765 | 128,1115.0,-1 766 | 129,1117.0,-1 767 | 130,1118.0,-1 768 | 131,1120.0,-1 769 | 132,1128.0,-1 770 | 133,1129.0,-1 771 | 134,1130.0,-1 772 | 135,1136.0,-1 773 | 136,1141.0,-1 774 | 137,1147.0,-1 775 | 138,1149.0,-1 776 | 139,1153.0,-1 777 | 140,1154.0,-1 778 | 141,1157.0,-1 779 | 142,1161.0,-1 780 | 143,1166.0,-1 781 | 144,1167.0,-1 782 | 145,1173.0,-1 783 | 146,1174.0,-1 784 | 147,1177.0,-1 785 | 148,1183.0,-1 786 | 149,1192.0,-1 787 | 150,1195.0,-1 788 | 151,1197.0,-1 789 | 152,703.0,-1 790 | 153,709.0,-1 791 | 154,711.0,-1 792 | 155,714.0,-1 793 | 156,717.0,-1 794 | 157,722.0,-1 795 | 158,728.0,-1 796 | 159,733.0,-1 797 | 160,738.0,-1 798 | 161,740.0,-1 799 | 162,749.0,-1 800 | 163,750.0,-1 801 | 164,755.0,-1 802 | 165,761.0,-1 803 | 166,762.0,-1 804 | 167,763.0,-1 805 | 168,769.0,-1 806 | 169,777.0,-1 807 | 170,779.0,-1 808 | 171,780.0,-1 809 | 172,784.0,-1 810 | 173,786.0,-1 811 | 174,792.0,-1 812 | 175,798.0,-1 813 | 176,1312.0,-1 814 | 177,1313.0,-1 815 | 178,1318.0,-1 816 | 179,1322.0,-1 817 | 180,1323.0,-1 818 | 181,1327.0,-1 819 | 182,1330.0,-1 820 | 183,1331.0,-1 821 | 184,1334.0,-1 822 | 185,1335.0,-1 823 | 186,1338.0,-1 824 | 187,1340.0,-1 825 | 188,1342.0,-1 826 | 189,1343.0,-1 827 | 190,1344.0,-1 828 | 191,1351.0,-1 829 | 192,1357.0,-1 830 | 193,1359.0,-1 831 | 194,1364.0,-1 832 | 195,1371.0,-1 833 | 196,1373.0,-1 834 | 197,1374.0,-1 835 | 198,1377.0,-1 836 | 199,1378.0,-1 837 | 200,1382.0,-1 838 | 201,1386.0,-1 839 | 202,1396.0,-1 840 | -------------------------------------------------------------------------------- /Prediction_Raw_Data_Validation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/Prediction_Raw_Data_Validation/__init__.py -------------------------------------------------------------------------------- /Prediction_Raw_Data_Validation/__pycache__/predictionDataValidation.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/Prediction_Raw_Data_Validation/__pycache__/predictionDataValidation.cpython-36.pyc -------------------------------------------------------------------------------- /Prediction_Raw_Data_Validation/__pycache__/predictionDataValidation.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/Prediction_Raw_Data_Validation/__pycache__/predictionDataValidation.cpython-37.pyc -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: gunicorn main:app -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Wafer Fault Detection 2 | 3 | #### Problem Statement: 4 | 5 | Wafer (In electronics), also called a slice or substrate, is a thin slice of semiconductor, 6 | such as a crystalline silicon (c-Si), used for fabricationof integrated circuits and in photovoltaics, 7 | to manufacture solar cells. 8 | 9 | The inputs of various sensors for different wafers have been provided. 10 | The goal is to build a machine learning model which predicts whether a wafer needs to be replaced or not 11 | (i.e whether it is working or not) nased on the inputs from various sensors. 12 | There are two classes: +1 and -1. 13 | +1: Means that the wafer is in a working condition and it doesn't need to be replaced. 14 | -1: Means that the wafer is faulty and it needa to be replaced. 15 | 16 | #### Data Description 17 | 18 | The client will send data in multiple sets of files in batches at a given location. 19 | Data will contain Wafer names and 590 columns of different sensor values for each wafer. 20 | The last column will have the "Good/Bad" value for each wafer. 21 | 22 | Apart from training files, we laso require a "schema" file from the client, which contain all the 23 | relevant information about the training files such as: 24 | 25 | Name of the files, Length of Date value in FileName, Length of Time value in FileName, NUmber of Columnns, 26 | Name of Columns, and their dataype. 27 | 28 | #### Data Validation 29 | 30 | In This step, we perform different sets of validation on the given set of training files. 31 | 32 | Name Validation: We validate the name of the files based on the given name in the schema file. We have 33 | created a regex patterg as per the name given in the schema fileto use for validation. After validating 34 | the pattern in the name, we check for the length of the date in the file name as well as the length of time 35 | in the file name. If all the values are as per requirements, we move such files to "Good_Data_Folder" else 36 | we move such files to "Bad_Data_Folder." 37 | 38 | Number of Columns: We validate the number of columns present in the files, and if it doesn't match with the 39 | value given in the schema file, then the file id moves to "Bad_Data_Folder." 40 | 41 | Name of Columns: The name of the columns is validated and should be the same as given in the schema file. 42 | If not, then the file is moved to "Bad_Data_Folder". 43 | 44 | The datatype of columns: The datatype of columns is given in the schema file. This is validated when we insert 45 | the files into Database. If the datatype is wrong, then the file is moved to "Bad_Data_Folder." 46 | 47 | Null values in columns: If any of the columns in a file have all the values as NULL or missing, we discard such 48 | a file and move it to "Bad_Data_Folder". 49 | 50 | #### Data Insertion in Database 51 | 52 | Database Creation and Connection: Create a database with the given name passed. If the database is already created, 53 | open the connection to the database. 54 | 55 | Table creation in the database: Table with name - "Good_Data", is created in the database for inserting the files 56 | in the "Good_Data_Folder" based on given column names and datatype in the schema file. If the table is already 57 | present, then the new table is not created and new files are inserted in the already present table as we want 58 | training to be done on new as well as old training files. 59 | 60 | Insertion of file in the table: All the files in the "Good_Data_Folder" are inserted in the above-created table. If 61 | any file has invalid data type in any of the columns, the file is not loaded in the table and is moved to 62 | "Bad_Data_Folder". 63 | 64 | #### Model Training 65 | 66 | Data Export from Db: The data in a stored database is exported as a CSV file to be used for model training. 67 | 68 | Data Preprocessing: 69 | Check for null values in the columns. If present, impute the null values using the KNN imputer. 70 | 71 | Check if any column has zero standard deviation, remove such columns as they don't give any information during 72 | model training. 73 | 74 | Clustering: KMeans algorithm is used to create clusters in the preprocessed data. The optimum number of clusters 75 | is selected 76 | 77 | 78 | ## Create a file "Dockerfile" with below content 79 | 80 | ``` 81 | FROM python:3.7 82 | COPY . /app 83 | WORKDIR /app 84 | RUN pip install -r requirements.txt 85 | ENTRYPOINT [ "python" ] 86 | CMD [ "main.py" ] 87 | ``` 88 | 89 | ## Create a "Procfile" with following content 90 | ``` 91 | web: gunicorn main:app 92 | ``` 93 | 94 | ## create a file ".circleci\config.yml" with following content 95 | ``` 96 | version: 2.1 97 | orbs: 98 | heroku: circleci/heroku@1.0.1 99 | jobs: 100 | build-and-test: 101 | executor: heroku/default 102 | docker: 103 | - image: circleci/python:3.6.2-stretch-browsers 104 | auth: 105 | username: mydockerhub-user 106 | password: $DOCKERHUB_PASSWORD # context / project UI env-var reference 107 | steps: 108 | - checkout 109 | - restore_cache: 110 | key: deps1-{{ .Branch }}-{{ checksum "requirements.txt" }} 111 | - run: 112 | name: Install Python deps in a venv 113 | command: | 114 | echo 'export TAG=0.1.${CIRCLE_BUILD_NUM}' >> $BASH_ENV 115 | echo 'export IMAGE_NAME=python-circleci-docker' >> $BASH_ENV 116 | python3 -m venv venv 117 | . venv/bin/activate 118 | pip install --upgrade pip 119 | pip install -r requirements.txt 120 | - save_cache: 121 | key: deps1-{{ .Branch }}-{{ checksum "requirements.txt" }} 122 | paths: 123 | - "venv" 124 | - run: 125 | command: | 126 | . venv/bin/activate 127 | python -m pytest -v tests/test_script.py 128 | - store_artifacts: 129 | path: test-reports/ 130 | destination: tr1 131 | - store_test_results: 132 | path: test-reports/ 133 | - setup_remote_docker: 134 | version: 19.03.13 135 | - run: 136 | name: Build and push Docker image 137 | command: | 138 | docker build -t $DOCKERHUB_USER/$IMAGE_NAME:$TAG . 139 | docker login -u $DOCKERHUB_USER -p $DOCKER_HUB_PASSWORD_USER docker.io 140 | docker push $DOCKERHUB_USER/$IMAGE_NAME:$TAG 141 | deploy: 142 | executor: heroku/default 143 | steps: 144 | - checkout 145 | - run: 146 | name: Storing previous commit 147 | command: | 148 | git rev-parse HEAD > ./commit.txt 149 | - heroku/install 150 | - setup_remote_docker: 151 | version: 18.06.0-ce 152 | - run: 153 | name: Pushing to heroku registry 154 | command: | 155 | heroku container:login 156 | #heroku ps:scale web=1 -a $HEROKU_APP_NAME 157 | heroku container:push web -a $HEROKU_APP_NAME 158 | heroku container:release web -a $HEROKU_APP_NAME 159 | 160 | workflows: 161 | build-test-deploy: 162 | jobs: 163 | - build-and-test 164 | - deploy: 165 | requires: 166 | - build-and-test 167 | filters: 168 | branches: 169 | only: 170 | - main 171 | ``` 172 | ## to create requirements.txt 173 | 174 | ```buildoutcfg 175 | pip freeze>requirements.txt 176 | ``` 177 | 178 | ## initialize git repo 179 | 180 | ``` 181 | git init 182 | git add . 183 | git commit -m "first commit" 184 | git branch -M main 185 | git remote add origin 186 | git push -u origin main 187 | ``` 188 | 189 | ## create a account at circle ci 190 | 191 | Circle CI 192 | 193 | ## setup your project 194 | 195 | Setup project 196 | 197 | ## Select project setting in CircleCI and below environment variable 198 | 199 | ``` 200 | DOCKERHUB_USER 201 | DOCKER_HUB_PASSWORD_USER 202 | HEROKU_API_KEY 203 | HEROKU_APP_NAME 204 | HEROKU_EMAIL_ADDRESS 205 | DOCKER_IMAGE_NAME=wafercircle3270303 206 | ``` 207 | 208 | 209 | ## to update the modification 210 | 211 | ``` 212 | git add . 213 | git commit -m "proper message" 214 | git push 215 | ``` 216 | 217 | 218 | ## #docker login -u $DOCKERHUB_USER -p $DOCKER_HUB_PASSWORD_USER docker.io 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | -------------------------------------------------------------------------------- /Training_Database/Training.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/Training_Database/Training.db -------------------------------------------------------------------------------- /Training_Logs/DataBaseConnectionLog.txt: -------------------------------------------------------------------------------- 1 | 2020-10-09/16:23:38 Opened Training database successfully 2 | 2020-10-09/16:25:20 Closed Training database successfully 3 | 2020-10-09/16:25:20 Opened Training database successfully 4 | 2020-10-09/16:27:36 Opened Training database successfully 5 | 2020-10-09/16:50:46 Opened Training database successfully 6 | 2020-10-09/16:50:47 Closed Training database successfully 7 | 2020-10-09/16:50:47 Opened Training database successfully 8 | 2020-10-09/16:53:26 Opened Training database successfully 9 | 2020-10-15/11:11:44 Opened Training database successfully 10 | 2020-10-15/11:11:44 Closed Training database successfully 11 | 2020-10-15/11:11:44 Opened Training database successfully 12 | 2020-10-15/11:13:48 Opened Training database successfully 13 | -------------------------------------------------------------------------------- /Training_Logs/DbTableCreateLog.txt: -------------------------------------------------------------------------------- 1 | 2020-10-09/16:25:20 Tables created successfully!! 2 | 2020-10-09/16:50:47 Tables created successfully!! 3 | 2020-10-15/11:11:44 Tables created successfully!! 4 | -------------------------------------------------------------------------------- /Training_Logs/ExportToCsv.txt: -------------------------------------------------------------------------------- 1 | 2020-10-09/16:27:39 File exported successfully!!! 2 | 2020-10-09/16:53:32 File exported successfully!!! 3 | 2020-10-15/11:13:49 File exported successfully!!! 4 | -------------------------------------------------------------------------------- /Training_Logs/GeneralLog.txt: -------------------------------------------------------------------------------- 1 | 2020-10-09/16:27:36 GoodRaw directory deleted successfully!!! 2 | 2020-10-09/16:27:36 Bad files moved to archive 3 | 2020-10-09/16:27:36 Bad Raw Data Folder Deleted successfully!! 4 | 2020-10-09/16:53:26 GoodRaw directory deleted successfully!!! 5 | 2020-10-09/16:53:26 Bad files moved to archive 6 | 2020-10-09/16:53:26 Bad Raw Data Folder Deleted successfully!! 7 | 2020-10-15/11:13:48 GoodRaw directory deleted successfully!!! 8 | 2020-10-15/11:13:48 Bad files moved to archive 9 | 2020-10-15/11:13:48 Bad Raw Data Folder Deleted successfully!! 10 | -------------------------------------------------------------------------------- /Training_Logs/ModelTrainingLog.txt: -------------------------------------------------------------------------------- 1 | 2020-10-09/16:27:39 Start of Training 2 | 2020-10-09/16:27:39 Entered the get_data method of the Data_Getter class 3 | 2020-10-09/16:27:40 Data Load Successful.Exited the get_data method of the Data_Getter class 4 | 2020-10-09/16:27:40 Entered the remove_columns method of the Preprocessor class 5 | 2020-10-09/16:27:40 Column removal Successful.Exited the remove_columns method of the Preprocessor class 6 | 2020-10-09/16:27:40 Entered the separate_label_feature method of the Preprocessor class 7 | 2020-10-09/16:27:40 Label Separation Successful. Exited the separate_label_feature method of the Preprocessor class 8 | 2020-10-09/16:27:40 Entered the is_null_present method of the Preprocessor class 9 | 2020-10-09/16:27:40 Finding missing values is a success.Data written to the null values file. Exited the is_null_present method of the Preprocessor class 10 | 2020-10-09/16:27:40 Entered the impute_missing_values method of the Preprocessor class 11 | 2020-10-09/16:27:44 Imputing missing values Successful. Exited the impute_missing_values method of the Preprocessor class 12 | 2020-10-09/16:27:44 Entered the get_columns_with_zero_std_deviation method of the Preprocessor class 13 | 2020-10-09/16:27:49 Column search for Standard Deviation of Zero Successful. Exited the get_columns_with_zero_std_deviation method of the Preprocessor class 14 | 2020-10-09/16:27:49 Entered the remove_columns method of the Preprocessor class 15 | 2020-10-09/16:27:49 Column removal Successful.Exited the remove_columns method of the Preprocessor class 16 | 2020-10-09/16:27:49 Entered the elbow_plot method of the KMeansClustering class 17 | 2020-10-09/16:28:02 The optimum number of clusters is: 3 . Exited the elbow_plot method of the KMeansClustering class 18 | 2020-10-09/16:28:02 Entered the create_clusters method of the KMeansClustering class 19 | 2020-10-09/16:28:03 Entered the save_model method of the File_Operation class 20 | 2020-10-09/16:28:03 Model File KMeans saved. Exited the save_model method of the Model_Finder class 21 | 2020-10-09/16:28:03 succesfully created 3clusters. Exited the create_clusters method of the KMeansClustering class 22 | 2020-10-09/16:28:03 Entered the get_best_model method of the Model_Finder class 23 | 2020-10-09/16:28:03 Entered the get_best_params_for_xgboost method of the Model_Finder class 24 | 2020-10-09/16:41:13 XGBoost best params: {'learning_rate': 0.5, 'max_depth': 3, 'n_estimators': 10}. Exited the get_best_params_for_xgboost method of the Model_Finder class 25 | 2020-10-09/16:41:13 AUC for XGBoost:0.5 26 | 2020-10-09/16:41:13 Entered the get_best_params_for_random_forest method of the Model_Finder class 27 | 2020-10-09/16:42:13 Random Forest best params: {'criterion': 'gini', 'max_depth': 2, 'max_features': 'auto', 'n_estimators': 10}. Exited the get_best_params_for_random_forest method of the Model_Finder class 28 | 2020-10-09/16:42:13 AUC for RF:0.5 29 | 2020-10-09/16:42:13 Entered the save_model method of the File_Operation class 30 | 2020-10-09/16:42:13 Model File RandomForest0 saved. Exited the save_model method of the Model_Finder class 31 | 2020-10-09/16:42:13 Entered the get_best_model method of the Model_Finder class 32 | 2020-10-09/16:42:13 Entered the get_best_params_for_xgboost method of the Model_Finder class 33 | 2020-10-09/16:44:46 XGBoost best params: {'learning_rate': 0.5, 'max_depth': 3, 'n_estimators': 10}. Exited the get_best_params_for_xgboost method of the Model_Finder class 34 | 2020-10-09/16:44:46 Accuracy for XGBoost:1.0 35 | 2020-10-09/16:44:46 Entered the get_best_params_for_random_forest method of the Model_Finder class 36 | 2020-10-09/16:45:30 Random Forest best params: {'criterion': 'gini', 'max_depth': 2, 'max_features': 'auto', 'n_estimators': 10}. Exited the get_best_params_for_random_forest method of the Model_Finder class 37 | 2020-10-09/16:45:30 Accuracy for RF:1.0 38 | 2020-10-09/16:45:30 Entered the save_model method of the File_Operation class 39 | 2020-10-09/16:45:30 Model File RandomForest1 saved. Exited the save_model method of the Model_Finder class 40 | 2020-10-09/16:45:30 Entered the get_best_model method of the Model_Finder class 41 | 2020-10-09/16:45:30 Entered the get_best_params_for_xgboost method of the Model_Finder class 42 | 2020-10-09/16:49:27 XGBoost best params: {'learning_rate': 0.5, 'max_depth': 3, 'n_estimators': 50}. Exited the get_best_params_for_xgboost method of the Model_Finder class 43 | 2020-10-09/16:49:27 AUC for XGBoost:0.5 44 | 2020-10-09/16:49:27 Entered the get_best_params_for_random_forest method of the Model_Finder class 45 | 2020-10-09/16:50:23 Random Forest best params: {'criterion': 'gini', 'max_depth': 2, 'max_features': 'auto', 'n_estimators': 50}. Exited the get_best_params_for_random_forest method of the Model_Finder class 46 | 2020-10-09/16:50:23 AUC for RF:0.5 47 | 2020-10-09/16:50:23 Entered the save_model method of the File_Operation class 48 | 2020-10-09/16:50:23 Model File RandomForest2 saved. Exited the save_model method of the Model_Finder class 49 | 2020-10-09/16:50:23 Successful End of Training 50 | 2020-10-09/16:53:32 Start of Training 51 | 2020-10-09/16:53:32 Entered the get_data method of the Data_Getter class 52 | 2020-10-09/16:53:33 Data Load Successful.Exited the get_data method of the Data_Getter class 53 | 2020-10-09/16:53:33 Entered the remove_columns method of the Preprocessor class 54 | 2020-10-09/16:53:33 Column removal Successful.Exited the remove_columns method of the Preprocessor class 55 | 2020-10-09/16:53:33 Entered the separate_label_feature method of the Preprocessor class 56 | 2020-10-09/16:53:33 Label Separation Successful. Exited the separate_label_feature method of the Preprocessor class 57 | 2020-10-09/16:53:33 Entered the is_null_present method of the Preprocessor class 58 | 2020-10-09/16:53:33 Finding missing values is a success.Data written to the null values file. Exited the is_null_present method of the Preprocessor class 59 | 2020-10-09/16:53:33 Entered the impute_missing_values method of the Preprocessor class 60 | 2020-10-09/16:53:43 Imputing missing values Successful. Exited the impute_missing_values method of the Preprocessor class 61 | 2020-10-09/16:53:43 Entered the get_columns_with_zero_std_deviation method of the Preprocessor class 62 | 2020-10-09/16:53:46 Column search for Standard Deviation of Zero Successful. Exited the get_columns_with_zero_std_deviation method of the Preprocessor class 63 | 2020-10-09/16:53:46 Entered the remove_columns method of the Preprocessor class 64 | 2020-10-09/16:53:46 Column removal Successful.Exited the remove_columns method of the Preprocessor class 65 | 2020-10-09/16:53:46 Entered the elbow_plot method of the KMeansClustering class 66 | 2020-10-09/16:54:00 The optimum number of clusters is: 3 . Exited the elbow_plot method of the KMeansClustering class 67 | 2020-10-09/16:54:00 Entered the create_clusters method of the KMeansClustering class 68 | 2020-10-09/16:54:01 Entered the save_model method of the File_Operation class 69 | 2020-10-09/16:54:01 Model File KMeans saved. Exited the save_model method of the Model_Finder class 70 | 2020-10-09/16:54:01 succesfully created 3clusters. Exited the create_clusters method of the KMeansClustering class 71 | 2020-10-09/16:54:01 Entered the get_best_model method of the Model_Finder class 72 | 2020-10-09/16:54:01 Entered the get_best_params_for_xgboost method of the Model_Finder class 73 | 2020-10-09/17:24:38 XGBoost best params: {'learning_rate': 0.5, 'max_depth': 3, 'n_estimators': 50}. Exited the get_best_params_for_xgboost method of the Model_Finder class 74 | 2020-10-09/17:24:38 AUC for XGBoost:0.8461538461538461 75 | 2020-10-09/17:24:38 Entered the get_best_params_for_random_forest method of the Model_Finder class 76 | 2020-10-09/17:26:02 Random Forest best params: {'criterion': 'gini', 'max_depth': 2, 'max_features': 'auto', 'n_estimators': 10}. Exited the get_best_params_for_random_forest method of the Model_Finder class 77 | 2020-10-09/17:26:02 AUC for RF:0.5 78 | 2020-10-09/17:26:02 Entered the save_model method of the File_Operation class 79 | 2020-10-09/17:26:02 Model File XGBoost2 saved. Exited the save_model method of the Model_Finder class 80 | 2020-10-09/17:26:02 Entered the get_best_model method of the Model_Finder class 81 | 2020-10-09/17:26:02 Entered the get_best_params_for_xgboost method of the Model_Finder class 82 | 2020-10-09/17:30:02 XGBoost best params: {'learning_rate': 0.5, 'max_depth': 3, 'n_estimators': 10}. Exited the get_best_params_for_xgboost method of the Model_Finder class 83 | 2020-10-09/17:30:02 AUC for XGBoost:1.0 84 | 2020-10-09/17:30:02 Entered the get_best_params_for_random_forest method of the Model_Finder class 85 | 2020-10-09/17:31:09 Random Forest best params: {'criterion': 'gini', 'max_depth': 3, 'max_features': 'auto', 'n_estimators': 100}. Exited the get_best_params_for_random_forest method of the Model_Finder class 86 | 2020-10-09/17:31:10 AUC for RF:0.875 87 | 2020-10-09/17:31:10 Entered the save_model method of the File_Operation class 88 | 2020-10-09/17:31:10 Model File XGBoost0 saved. Exited the save_model method of the Model_Finder class 89 | 2020-10-09/17:31:10 Entered the get_best_model method of the Model_Finder class 90 | 2020-10-09/17:31:10 Entered the get_best_params_for_xgboost method of the Model_Finder class 91 | 2020-10-09/17:39:48 XGBoost best params: {'learning_rate': 0.5, 'max_depth': 3, 'n_estimators': 50}. Exited the get_best_params_for_xgboost method of the Model_Finder class 92 | 2020-10-09/17:39:48 AUC for XGBoost:0.8493909191583611 93 | 2020-10-09/17:39:48 Entered the get_best_params_for_random_forest method of the Model_Finder class 94 | 2020-10-09/17:40:43 Random Forest best params: {'criterion': 'entropy', 'max_depth': 3, 'max_features': 'auto', 'n_estimators': 50}. Exited the get_best_params_for_random_forest method of the Model_Finder class 95 | 2020-10-09/17:40:43 AUC for RF:0.5714285714285714 96 | 2020-10-09/17:40:43 Entered the save_model method of the File_Operation class 97 | 2020-10-09/17:40:43 Model File XGBoost1 saved. Exited the save_model method of the Model_Finder class 98 | 2020-10-09/17:40:43 Successful End of Training 99 | 2020-10-15/11:13:50 Start of Training 100 | 2020-10-15/11:13:50 Entered the get_data method of the Data_Getter class 101 | 2020-10-15/11:13:50 Data Load Successful.Exited the get_data method of the Data_Getter class 102 | 2020-10-15/11:13:50 Entered the remove_columns method of the Preprocessor class 103 | 2020-10-15/11:13:50 Column removal Successful.Exited the remove_columns method of the Preprocessor class 104 | 2020-10-15/11:13:50 Entered the separate_label_feature method of the Preprocessor class 105 | 2020-10-15/11:13:50 Label Separation Successful. Exited the separate_label_feature method of the Preprocessor class 106 | 2020-10-15/11:13:50 Entered the is_null_present method of the Preprocessor class 107 | 2020-10-15/11:13:50 Finding missing values is a success.Data written to the null values file. Exited the is_null_present method of the Preprocessor class 108 | 2020-10-15/11:13:50 Entered the impute_missing_values method of the Preprocessor class 109 | 2020-10-15/11:14:06 Imputing missing values Successful. Exited the impute_missing_values method of the Preprocessor class 110 | 2020-10-15/11:14:06 Entered the get_columns_with_zero_std_deviation method of the Preprocessor class 111 | 2020-10-15/11:14:08 Column search for Standard Deviation of Zero Successful. Exited the get_columns_with_zero_std_deviation method of the Preprocessor class 112 | 2020-10-15/11:14:08 Entered the remove_columns method of the Preprocessor class 113 | 2020-10-15/11:14:08 Column removal Successful.Exited the remove_columns method of the Preprocessor class 114 | 2020-10-15/11:14:08 Entered the elbow_plot method of the KMeansClustering class 115 | 2020-10-15/11:14:21 The optimum number of clusters is: 3 . Exited the elbow_plot method of the KMeansClustering class 116 | 2020-10-15/11:14:21 Entered the create_clusters method of the KMeansClustering class 117 | 2020-10-15/11:14:22 Entered the save_model method of the File_Operation class 118 | 2020-10-15/11:14:22 Model File KMeans saved. Exited the save_model method of the Model_Finder class 119 | 2020-10-15/11:14:22 succesfully created 3clusters. Exited the create_clusters method of the KMeansClustering class 120 | 2020-10-15/11:14:22 Entered the get_best_model method of the Model_Finder class 121 | 2020-10-15/11:14:22 Entered the get_best_params_for_xgboost method of the Model_Finder class 122 | 2020-10-15/11:29:14 XGBoost best params: {'learning_rate': 0.5, 'max_depth': 5, 'n_estimators': 10}. Exited the get_best_params_for_xgboost method of the Model_Finder class 123 | 2020-10-15/11:29:14 AUC for XGBoost:1.0 124 | 2020-10-15/11:29:14 Entered the get_best_params_for_random_forest method of the Model_Finder class 125 | 2020-10-15/11:29:54 Random Forest best params: {'criterion': 'gini', 'max_depth': 3, 'max_features': 'log2', 'n_estimators': 10}. Exited the get_best_params_for_random_forest method of the Model_Finder class 126 | 2020-10-15/11:29:54 AUC for RF:0.5 127 | 2020-10-15/11:29:54 Entered the save_model method of the File_Operation class 128 | 2020-10-15/11:29:54 Model File XGBoost2 saved. Exited the save_model method of the Model_Finder class 129 | 2020-10-15/11:29:54 Entered the get_best_model method of the Model_Finder class 130 | 2020-10-15/11:29:54 Entered the get_best_params_for_xgboost method of the Model_Finder class 131 | 2020-10-15/11:31:33 XGBoost best params: {'learning_rate': 0.5, 'max_depth': 3, 'n_estimators': 10}. Exited the get_best_params_for_xgboost method of the Model_Finder class 132 | 2020-10-15/11:31:33 AUC for XGBoost:1.0 133 | 2020-10-15/11:31:33 Entered the get_best_params_for_random_forest method of the Model_Finder class 134 | 2020-10-15/11:32:00 Random Forest best params: {'criterion': 'entropy', 'max_depth': 3, 'max_features': 'auto', 'n_estimators': 100}. Exited the get_best_params_for_random_forest method of the Model_Finder class 135 | 2020-10-15/11:32:00 AUC for RF:0.8333333333333333 136 | 2020-10-15/11:32:00 Entered the save_model method of the File_Operation class 137 | 2020-10-15/11:32:00 Model File XGBoost0 saved. Exited the save_model method of the Model_Finder class 138 | 2020-10-15/11:32:00 Entered the get_best_model method of the Model_Finder class 139 | 2020-10-15/11:32:00 Entered the get_best_params_for_xgboost method of the Model_Finder class 140 | 2020-10-15/11:35:34 XGBoost best params: {'learning_rate': 0.5, 'max_depth': 3, 'n_estimators': 10}. Exited the get_best_params_for_xgboost method of the Model_Finder class 141 | 2020-10-15/11:35:34 AUC for XGBoost:1.0 142 | 2020-10-15/11:35:34 Entered the get_best_params_for_random_forest method of the Model_Finder class 143 | 2020-10-15/11:35:59 Random Forest best params: {'criterion': 'entropy', 'max_depth': 3, 'max_features': 'auto', 'n_estimators': 10}. Exited the get_best_params_for_random_forest method of the Model_Finder class 144 | 2020-10-15/11:35:59 AUC for RF:0.625 145 | 2020-10-15/11:35:59 Entered the save_model method of the File_Operation class 146 | 2020-10-15/11:35:59 Model File XGBoost1 saved. Exited the save_model method of the Model_Finder class 147 | 2020-10-15/11:35:59 Successful End of Training 148 | -------------------------------------------------------------------------------- /Training_Logs/Training_Main_Log.txt: -------------------------------------------------------------------------------- 1 | 2020-10-09/16:23:16 Start of Validation on files!! 2 | 2020-10-09/16:23:29 Raw Data Validation Complete!! 3 | 2020-10-09/16:23:29 Starting Data Transforamtion!! 4 | 2020-10-09/16:23:38 DataTransformation Completed!!! 5 | 2020-10-09/16:23:38 Creating Training_Database and tables on the basis of given schema!!! 6 | 2020-10-09/16:25:20 Table creation Completed!! 7 | 2020-10-09/16:25:20 Insertion of Data into Table started!!!! 8 | 2020-10-09/16:27:36 Insertion in Table completed!!! 9 | 2020-10-09/16:27:36 Deleting Good Data Folder!!! 10 | 2020-10-09/16:27:36 Good_Data folder deleted!!! 11 | 2020-10-09/16:27:36 Moving bad files to Archive and deleting Bad_Data folder!!! 12 | 2020-10-09/16:27:36 Bad files moved to archive!! Bad folder Deleted!! 13 | 2020-10-09/16:27:36 Validation Operation completed!! 14 | 2020-10-09/16:27:36 Extracting csv file from table 15 | 2020-10-09/16:50:23 Start of Validation on files!! 16 | 2020-10-09/16:50:33 Raw Data Validation Complete!! 17 | 2020-10-09/16:50:33 Starting Data Transforamtion!! 18 | 2020-10-09/16:50:46 DataTransformation Completed!!! 19 | 2020-10-09/16:50:46 Creating Training_Database and tables on the basis of given schema!!! 20 | 2020-10-09/16:50:47 Table creation Completed!! 21 | 2020-10-09/16:50:47 Insertion of Data into Table started!!!! 22 | 2020-10-09/16:53:26 Insertion in Table completed!!! 23 | 2020-10-09/16:53:26 Deleting Good Data Folder!!! 24 | 2020-10-09/16:53:26 Good_Data folder deleted!!! 25 | 2020-10-09/16:53:26 Moving bad files to Archive and deleting Bad_Data folder!!! 26 | 2020-10-09/16:53:26 Bad files moved to archive!! Bad folder Deleted!! 27 | 2020-10-09/16:53:26 Validation Operation completed!! 28 | 2020-10-09/16:53:26 Extracting csv file from table 29 | 2020-10-15/11:11:34 Start of Validation on files!! 30 | 2020-10-15/11:11:38 Raw Data Validation Complete!! 31 | 2020-10-15/11:11:38 Starting Data Transforamtion!! 32 | 2020-10-15/11:11:44 DataTransformation Completed!!! 33 | 2020-10-15/11:11:44 Creating Training_Database and tables on the basis of given schema!!! 34 | 2020-10-15/11:11:44 Table creation Completed!! 35 | 2020-10-15/11:11:44 Insertion of Data into Table started!!!! 36 | 2020-10-15/11:13:48 Insertion in Table completed!!! 37 | 2020-10-15/11:13:48 Deleting Good Data Folder!!! 38 | 2020-10-15/11:13:48 Good_Data folder deleted!!! 39 | 2020-10-15/11:13:48 Moving bad files to Archive and deleting Bad_Data folder!!! 40 | 2020-10-15/11:13:48 Bad files moved to archive!! Bad folder Deleted!! 41 | 2020-10-15/11:13:48 Validation Operation completed!! 42 | 2020-10-15/11:13:48 Extracting csv file from table 43 | -------------------------------------------------------------------------------- /Training_Logs/columnValidationLog.txt: -------------------------------------------------------------------------------- 1 | 2020-10-09/16:23:20 Column Length Validation Started!! 2 | 2020-10-09/16:23:21 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_16012020_134553.csv 3 | 2020-10-09/16:23:21 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_17012020_125434.csv 4 | 2020-10-09/16:23:22 Column Length Validation Completed!! 5 | 2020-10-09/16:50:25 Column Length Validation Started!! 6 | 2020-10-09/16:50:26 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_16012020_134553.csv 7 | 2020-10-09/16:50:26 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_17012020_125434.csv 8 | 2020-10-09/16:50:28 Column Length Validation Completed!! 9 | 2020-10-15/11:11:35 Column Length Validation Started!! 10 | 2020-10-15/11:11:36 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_16012020_134553.csv 11 | 2020-10-15/11:11:36 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_17012020_125434.csv 12 | 2020-10-15/11:11:36 Column Length Validation Completed!! 13 | -------------------------------------------------------------------------------- /Training_Logs/dataTransformLog.txt: -------------------------------------------------------------------------------- 1 | 2020-10-09/16:23:29 wafer_13012020_090817.csv: File Transformed successfully!! 2 | 2020-10-09/16:23:30 wafer_16012020_051629.csv: File Transformed successfully!! 3 | 2020-10-09/16:23:31 wafer_20022020_090716.csv: File Transformed successfully!! 4 | 2020-10-09/16:23:32 wafer_21012020_080913.csv: File Transformed successfully!! 5 | 2020-10-09/16:23:34 wafer_23012020_041211.csv: File Transformed successfully!! 6 | 2020-10-09/16:23:35 wafer_27012020_080911.csv: File Transformed successfully!! 7 | 2020-10-09/16:23:37 wafer_28012020_090817.csv: File Transformed successfully!! 8 | 2020-10-09/16:23:38 wafer_28042020_031911.csv: File Transformed successfully!! 9 | 2020-10-09/16:23:38 wafer_29012020_050617.csv: File Transformed successfully!! 10 | 2020-10-09/16:50:34 wafer_13012020_090817.csv: File Transformed successfully!! 11 | 2020-10-09/16:50:35 wafer_16012020_051629.csv: File Transformed successfully!! 12 | 2020-10-09/16:50:35 wafer_20022020_090716.csv: File Transformed successfully!! 13 | 2020-10-09/16:50:37 wafer_21012020_080913.csv: File Transformed successfully!! 14 | 2020-10-09/16:50:38 wafer_23012020_041211.csv: File Transformed successfully!! 15 | 2020-10-09/16:50:39 wafer_27012020_080911.csv: File Transformed successfully!! 16 | 2020-10-09/16:50:42 wafer_28012020_090817.csv: File Transformed successfully!! 17 | 2020-10-09/16:50:45 wafer_28042020_031911.csv: File Transformed successfully!! 18 | 2020-10-09/16:50:46 wafer_29012020_050617.csv: File Transformed successfully!! 19 | 2020-10-15/11:11:38 wafer_13012020_090817.csv: File Transformed successfully!! 20 | 2020-10-15/11:11:39 wafer_20022020_090716.csv: File Transformed successfully!! 21 | 2020-10-15/11:11:39 wafer_23012020_041211.csv: File Transformed successfully!! 22 | 2020-10-15/11:11:41 wafer_28042020_031911.csv: File Transformed successfully!! 23 | 2020-10-15/11:11:42 wafer_16012020_051629.csv: File Transformed successfully!! 24 | 2020-10-15/11:11:42 wafer_21012020_080913.csv: File Transformed successfully!! 25 | 2020-10-15/11:11:43 wafer_27012020_080911.csv: File Transformed successfully!! 26 | 2020-10-15/11:11:43 wafer_28012020_090817.csv: File Transformed successfully!! 27 | 2020-10-15/11:11:44 wafer_29012020_050617.csv: File Transformed successfully!! 28 | -------------------------------------------------------------------------------- /Training_Logs/missingValuesInColumn.txt: -------------------------------------------------------------------------------- 1 | 2020-10-09/16:23:22 Missing Values Validation Started!! 2 | 2020-10-09/16:23:22 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_07012020_041011.csv 3 | 2020-10-09/16:23:23 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_20012020_090819.csv 4 | 2020-10-09/16:23:24 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_22022020_041119.csv 5 | 2020-10-09/16:23:24 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_23012020_011008.csv 6 | 2020-10-09/16:23:26 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_28012020_051011.csv 7 | 2020-10-09/16:23:29 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_29012020_060756.csv 8 | 2020-10-09/16:23:29 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_31012020_090811.csv 9 | 2020-10-09/16:50:28 Missing Values Validation Started!! 10 | 2020-10-09/16:50:28 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_07012020_041011.csv 11 | 2020-10-09/16:50:29 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_20012020_090819.csv 12 | 2020-10-09/16:50:30 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_22022020_041119.csv 13 | 2020-10-09/16:50:30 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_23012020_011008.csv 14 | 2020-10-09/16:50:31 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_28012020_051011.csv 15 | 2020-10-09/16:50:33 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_29012020_060756.csv 16 | 2020-10-09/16:50:33 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_31012020_090811.csv 17 | 2020-10-15/11:11:36 Missing Values Validation Started!! 18 | 2020-10-15/11:11:36 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_23012020_011008.csv 19 | 2020-10-15/11:11:36 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_22022020_041119.csv 20 | 2020-10-15/11:11:36 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_29012020_060756.csv 21 | 2020-10-15/11:11:36 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_28012020_051011.csv 22 | 2020-10-15/11:11:37 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_07012020_041011.csv 23 | 2020-10-15/11:11:37 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_20012020_090819.csv 24 | 2020-10-15/11:11:37 Invalid Column Length for the file!! File moved to Bad Raw Folder :: wafer_31012020_090811.csv 25 | -------------------------------------------------------------------------------- /Training_Logs/nameValidationLog.txt: -------------------------------------------------------------------------------- 1 | 2020-10-09/16:23:17 Invalid File Name!! File moved to Bad Raw Folder :: Wafer12_20012.csv 2 | 2020-10-09/16:23:17 Invalid File Name!! File moved to Bad Raw Folder :: Waferdefault_15010_130532.csv 3 | 2020-10-09/16:23:17 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_07012020_000000.csv 4 | 2020-10-09/16:23:17 Valid File name!! File moved to GoodRaw Folder :: wafer_07012020_041011.csv 5 | 2020-10-09/16:23:17 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_07012020_223345.csv 6 | 2020-10-09/16:23:17 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_08012020_120000.csv 7 | 2020-10-09/16:23:17 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_10012020_131534.csv 8 | 2020-10-09/16:23:17 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_11012020_151432.csv 9 | 2020-10-09/16:23:17 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_12012020_111.csv 10 | 2020-10-09/16:23:18 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_12012020_111213.csv 11 | 2020-10-09/16:23:18 Valid File name!! File moved to GoodRaw Folder :: wafer_13012020_090817.csv 12 | 2020-10-09/16:23:18 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_13012020_141000.csv 13 | 2020-10-09/16:23:18 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_14012020_113045.csv 14 | 2020-10-09/16:23:18 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_15010_130532.csv 15 | 2020-10-09/16:23:18 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_15012020_130532.csv 16 | 2020-10-09/16:23:18 Valid File name!! File moved to GoodRaw Folder :: wafer_16012020_051629.csv 17 | 2020-10-09/16:23:18 Valid File name!! File moved to GoodRaw Folder :: wafer_16012020_134553.csv 18 | 2020-10-09/16:23:18 Valid File name!! File moved to GoodRaw Folder :: wafer_17012020_125434.csv 19 | 2020-10-09/16:23:18 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_18012020_121532.csv 20 | 2020-10-09/16:23:19 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_19012020_141432.csv 21 | 2020-10-09/16:23:19 Valid File name!! File moved to GoodRaw Folder :: wafer_20012020_090819.csv 22 | 2020-10-09/16:23:19 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_20012020_135132.csv 23 | 2020-10-09/16:23:19 Valid File name!! File moved to GoodRaw Folder :: wafer_20022020_090716.csv 24 | 2020-10-09/16:23:19 Valid File name!! File moved to GoodRaw Folder :: wafer_21012020_080913.csv 25 | 2020-10-09/16:23:19 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_21012020_143634.csv 26 | 2020-10-09/16:23:19 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_22012020_173245.csv 27 | 2020-10-09/16:23:19 Valid File name!! File moved to GoodRaw Folder :: wafer_22022020_041119.csv 28 | 2020-10-09/16:23:19 Valid File name!! File moved to GoodRaw Folder :: wafer_23012020_011008.csv 29 | 2020-10-09/16:23:19 Valid File name!! File moved to GoodRaw Folder :: wafer_23012020_041211.csv 30 | 2020-10-09/16:23:19 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_23012020_163456.csv 31 | 2020-10-09/16:23:19 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_24012020_150121.csv 32 | 2020-10-09/16:23:19 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_25012020_142112.csv 33 | 2020-10-09/16:23:20 Valid File name!! File moved to GoodRaw Folder :: wafer_27012020_080911.csv 34 | 2020-10-09/16:23:20 Valid File name!! File moved to GoodRaw Folder :: wafer_28012020_051011.csv 35 | 2020-10-09/16:23:20 Valid File name!! File moved to GoodRaw Folder :: wafer_28012020_090817.csv 36 | 2020-10-09/16:23:20 Valid File name!! File moved to GoodRaw Folder :: wafer_28042020_031911.csv 37 | 2020-10-09/16:23:20 Valid File name!! File moved to GoodRaw Folder :: wafer_29012020_050617.csv 38 | 2020-10-09/16:23:20 Valid File name!! File moved to GoodRaw Folder :: wafer_29012020_060756.csv 39 | 2020-10-09/16:23:20 Valid File name!! File moved to GoodRaw Folder :: wafer_31012020_090811.csv 40 | 2020-10-09/16:23:20 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_fault_detection.csv 41 | 2020-10-09/16:23:20 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_text_130532.csv 42 | 2020-10-09/16:50:23 Invalid File Name!! File moved to Bad Raw Folder :: Wafer12_20012.csv 43 | 2020-10-09/16:50:23 Invalid File Name!! File moved to Bad Raw Folder :: Waferdefault_15010_130532.csv 44 | 2020-10-09/16:50:23 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_07012020_000000.csv 45 | 2020-10-09/16:50:23 Valid File name!! File moved to GoodRaw Folder :: wafer_07012020_041011.csv 46 | 2020-10-09/16:50:23 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_07012020_223345.csv 47 | 2020-10-09/16:50:23 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_08012020_120000.csv 48 | 2020-10-09/16:50:24 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_10012020_131534.csv 49 | 2020-10-09/16:50:24 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_11012020_151432.csv 50 | 2020-10-09/16:50:24 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_12012020_111.csv 51 | 2020-10-09/16:50:24 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_12012020_111213.csv 52 | 2020-10-09/16:50:24 Valid File name!! File moved to GoodRaw Folder :: wafer_13012020_090817.csv 53 | 2020-10-09/16:50:24 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_13012020_141000.csv 54 | 2020-10-09/16:50:24 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_14012020_113045.csv 55 | 2020-10-09/16:50:24 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_15010_130532.csv 56 | 2020-10-09/16:50:24 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_15012020_130532.csv 57 | 2020-10-09/16:50:24 Valid File name!! File moved to GoodRaw Folder :: wafer_16012020_051629.csv 58 | 2020-10-09/16:50:24 Valid File name!! File moved to GoodRaw Folder :: wafer_16012020_134553.csv 59 | 2020-10-09/16:50:24 Valid File name!! File moved to GoodRaw Folder :: wafer_17012020_125434.csv 60 | 2020-10-09/16:50:24 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_18012020_121532.csv 61 | 2020-10-09/16:50:24 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_19012020_141432.csv 62 | 2020-10-09/16:50:24 Valid File name!! File moved to GoodRaw Folder :: wafer_20012020_090819.csv 63 | 2020-10-09/16:50:24 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_20012020_135132.csv 64 | 2020-10-09/16:50:24 Valid File name!! File moved to GoodRaw Folder :: wafer_20022020_090716.csv 65 | 2020-10-09/16:50:24 Valid File name!! File moved to GoodRaw Folder :: wafer_21012020_080913.csv 66 | 2020-10-09/16:50:24 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_21012020_143634.csv 67 | 2020-10-09/16:50:25 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_22012020_173245.csv 68 | 2020-10-09/16:50:25 Valid File name!! File moved to GoodRaw Folder :: wafer_22022020_041119.csv 69 | 2020-10-09/16:50:25 Valid File name!! File moved to GoodRaw Folder :: wafer_23012020_011008.csv 70 | 2020-10-09/16:50:25 Valid File name!! File moved to GoodRaw Folder :: wafer_23012020_041211.csv 71 | 2020-10-09/16:50:25 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_23012020_163456.csv 72 | 2020-10-09/16:50:25 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_24012020_150121.csv 73 | 2020-10-09/16:50:25 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_25012020_142112.csv 74 | 2020-10-09/16:50:25 Valid File name!! File moved to GoodRaw Folder :: wafer_27012020_080911.csv 75 | 2020-10-09/16:50:25 Valid File name!! File moved to GoodRaw Folder :: wafer_28012020_051011.csv 76 | 2020-10-09/16:50:25 Valid File name!! File moved to GoodRaw Folder :: wafer_28012020_090817.csv 77 | 2020-10-09/16:50:25 Valid File name!! File moved to GoodRaw Folder :: wafer_28042020_031911.csv 78 | 2020-10-09/16:50:25 Valid File name!! File moved to GoodRaw Folder :: wafer_29012020_050617.csv 79 | 2020-10-09/16:50:25 Valid File name!! File moved to GoodRaw Folder :: wafer_29012020_060756.csv 80 | 2020-10-09/16:50:25 Valid File name!! File moved to GoodRaw Folder :: wafer_31012020_090811.csv 81 | 2020-10-09/16:50:25 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_fault_detection.csv 82 | 2020-10-09/16:50:25 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_text_130532.csv 83 | 2020-10-15/11:11:34 Valid File name!! File moved to GoodRaw Folder :: wafer_13012020_090817.csv 84 | 2020-10-15/11:11:34 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_21012020_143634.csv 85 | 2020-10-15/11:11:34 Valid File name!! File moved to GoodRaw Folder :: wafer_20022020_090716.csv 86 | 2020-10-15/11:11:34 Valid File name!! File moved to GoodRaw Folder :: wafer_23012020_011008.csv 87 | 2020-10-15/11:11:34 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_12012020_111.csv 88 | 2020-10-15/11:11:35 Valid File name!! File moved to GoodRaw Folder :: wafer_22022020_041119.csv 89 | 2020-10-15/11:11:35 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_22012020_173245.csv 90 | 2020-10-15/11:11:35 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_11012020_151432.csv 91 | 2020-10-15/11:11:35 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_13012020_141000.csv 92 | 2020-10-15/11:11:35 Invalid File Name!! File moved to Bad Raw Folder :: Wafer12_20012.csv 93 | 2020-10-15/11:11:35 Valid File name!! File moved to GoodRaw Folder :: wafer_29012020_060756.csv 94 | 2020-10-15/11:11:35 Invalid File Name!! File moved to Bad Raw Folder :: Waferdefault_15010_130532.csv 95 | 2020-10-15/11:11:35 Valid File name!! File moved to GoodRaw Folder :: wafer_28012020_051011.csv 96 | 2020-10-15/11:11:35 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_24012020_150121.csv 97 | 2020-10-15/11:11:35 Valid File name!! File moved to GoodRaw Folder :: wafer_23012020_041211.csv 98 | 2020-10-15/11:11:35 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_text_130532.csv 99 | 2020-10-15/11:11:35 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_25012020_142112.csv 100 | 2020-10-15/11:11:35 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_20012020_135132.csv 101 | 2020-10-15/11:11:35 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_15012020_130532.csv 102 | 2020-10-15/11:11:35 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_19012020_141432.csv 103 | 2020-10-15/11:11:35 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_07012020_223345.csv 104 | 2020-10-15/11:11:35 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_15010_130532.csv 105 | 2020-10-15/11:11:35 Valid File name!! File moved to GoodRaw Folder :: wafer_07012020_041011.csv 106 | 2020-10-15/11:11:35 Valid File name!! File moved to GoodRaw Folder :: wafer_28042020_031911.csv 107 | 2020-10-15/11:11:35 Valid File name!! File moved to GoodRaw Folder :: wafer_20012020_090819.csv 108 | 2020-10-15/11:11:35 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_07012020_000000.csv 109 | 2020-10-15/11:11:35 Valid File name!! File moved to GoodRaw Folder :: wafer_31012020_090811.csv 110 | 2020-10-15/11:11:35 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_14012020_113045.csv 111 | 2020-10-15/11:11:35 Valid File name!! File moved to GoodRaw Folder :: wafer_16012020_051629.csv 112 | 2020-10-15/11:11:35 Valid File name!! File moved to GoodRaw Folder :: wafer_21012020_080913.csv 113 | 2020-10-15/11:11:35 Valid File name!! File moved to GoodRaw Folder :: wafer_27012020_080911.csv 114 | 2020-10-15/11:11:35 Valid File name!! File moved to GoodRaw Folder :: wafer_16012020_134553.csv 115 | 2020-10-15/11:11:35 Valid File name!! File moved to GoodRaw Folder :: wafer_28012020_090817.csv 116 | 2020-10-15/11:11:35 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_23012020_163456.csv 117 | 2020-10-15/11:11:35 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_12012020_111213.csv 118 | 2020-10-15/11:11:35 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_18012020_121532.csv 119 | 2020-10-15/11:11:35 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_10012020_131534.csv 120 | 2020-10-15/11:11:35 Valid File name!! File moved to GoodRaw Folder :: wafer_29012020_050617.csv 121 | 2020-10-15/11:11:35 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_08012020_120000.csv 122 | 2020-10-15/11:11:35 Valid File name!! File moved to GoodRaw Folder :: wafer_17012020_125434.csv 123 | 2020-10-15/11:11:35 Invalid File Name!! File moved to Bad Raw Folder :: Wafer_fault_detection.csv 124 | -------------------------------------------------------------------------------- /Training_Logs/valuesfromSchemaValidationLog.txt: -------------------------------------------------------------------------------- 1 | 2020-10-09/16:14:10 LengthOfDateStampInFile:: 8 LengthOfTimeStampInFile:: 6 NumberofColumns:: 591 2 | 3 | 2020-10-09/16:23:16 LengthOfDateStampInFile:: 8 LengthOfTimeStampInFile:: 6 NumberofColumns:: 592 4 | 5 | 2020-10-09/16:50:23 LengthOfDateStampInFile:: 8 LengthOfTimeStampInFile:: 6 NumberofColumns:: 592 6 | 7 | 2020-10-09/18:19:13 LengthOfDateStampInFile:: 8 LengthOfTimeStampInFile:: 6 NumberofColumns:: 591 8 | 9 | 2020-10-15/10:29:24 LengthOfDateStampInFile:: 8 LengthOfTimeStampInFile:: 6 NumberofColumns:: 591 10 | 11 | 2020-10-15/11:11:34 LengthOfDateStampInFile:: 8 LengthOfTimeStampInFile:: 6 NumberofColumns:: 592 12 | 13 | 2021-02-05/21:26:14 LengthOfDateStampInFile:: 8 LengthOfTimeStampInFile:: 6 NumberofColumns:: 591 14 | 15 | 2021-02-07/22:34:42 LengthOfDateStampInFile:: 8 LengthOfTimeStampInFile:: 6 NumberofColumns:: 591 16 | 17 | 2021-02-07/22:39:47 LengthOfDateStampInFile:: 8 LengthOfTimeStampInFile:: 6 NumberofColumns:: 591 18 | 19 | 2021-02-07/22:40:35 LengthOfDateStampInFile:: 8 LengthOfTimeStampInFile:: 6 NumberofColumns:: 591 20 | 21 | 2021-02-25/11:49:30 LengthOfDateStampInFile:: 8 LengthOfTimeStampInFile:: 6 NumberofColumns:: 591 22 | 23 | 2021-07-12/10:12:16 LengthOfDateStampInFile:: 8 LengthOfTimeStampInFile:: 6 NumberofColumns:: 591 24 | 25 | 2021-07-12/10:15:43 LengthOfDateStampInFile:: 8 LengthOfTimeStampInFile:: 6 NumberofColumns:: 591 26 | 27 | 2021-07-20/10:20:19 LengthOfDateStampInFile:: 8 LengthOfTimeStampInFile:: 6 NumberofColumns:: 591 28 | 29 | 2021-11-20/00:07:19 LengthOfDateStampInFile:: 8 LengthOfTimeStampInFile:: 6 NumberofColumns:: 591 30 | 31 | 2021-11-20/00:10:05 LengthOfDateStampInFile:: 8 LengthOfTimeStampInFile:: 6 NumberofColumns:: 591 32 | 33 | 2021-11-20/00:10:10 LengthOfDateStampInFile:: 8 LengthOfTimeStampInFile:: 6 NumberofColumns:: 591 34 | 35 | 2021-11-20/00:10:11 LengthOfDateStampInFile:: 8 LengthOfTimeStampInFile:: 6 NumberofColumns:: 591 36 | 37 | 2021-11-20/00:10:27 LengthOfDateStampInFile:: 8 LengthOfTimeStampInFile:: 6 NumberofColumns:: 591 38 | 39 | 2021-11-20/00:10:28 LengthOfDateStampInFile:: 8 LengthOfTimeStampInFile:: 6 NumberofColumns:: 591 40 | 41 | 2021-11-20/00:10:28 LengthOfDateStampInFile:: 8 LengthOfTimeStampInFile:: 6 NumberofColumns:: 591 42 | 43 | 2021-11-20/00:10:29 LengthOfDateStampInFile:: 8 LengthOfTimeStampInFile:: 6 NumberofColumns:: 591 44 | 45 | 2021-11-20/00:13:30 LengthOfDateStampInFile:: 8 LengthOfTimeStampInFile:: 6 NumberofColumns:: 591 46 | 47 | -------------------------------------------------------------------------------- /Training_Raw_data_validation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/Training_Raw_data_validation/__init__.py -------------------------------------------------------------------------------- /Training_Raw_data_validation/__pycache__/rawValidation.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/Training_Raw_data_validation/__pycache__/rawValidation.cpython-36.pyc -------------------------------------------------------------------------------- /Training_Raw_data_validation/__pycache__/rawValidation.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/Training_Raw_data_validation/__pycache__/rawValidation.cpython-37.pyc -------------------------------------------------------------------------------- /__pycache__/predictFromModel.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/__pycache__/predictFromModel.cpython-37.pyc -------------------------------------------------------------------------------- /__pycache__/prediction_Validation_Insertion.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/__pycache__/prediction_Validation_Insertion.cpython-37.pyc -------------------------------------------------------------------------------- /__pycache__/trainingModel.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/__pycache__/trainingModel.cpython-37.pyc -------------------------------------------------------------------------------- /__pycache__/training_Validation_Insertion.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/__pycache__/training_Validation_Insertion.cpython-37.pyc -------------------------------------------------------------------------------- /application_logging/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/application_logging/__init__.py -------------------------------------------------------------------------------- /application_logging/__pycache__/logger.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/application_logging/__pycache__/logger.cpython-36.pyc -------------------------------------------------------------------------------- /application_logging/__pycache__/logger.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/application_logging/__pycache__/logger.cpython-37.pyc -------------------------------------------------------------------------------- /application_logging/logger.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | 4 | class App_Logger: 5 | def __init__(self): 6 | pass 7 | 8 | def log(self, file_object, log_message): 9 | self.now = datetime.now() 10 | self.date = self.now.date() 11 | self.current_time = self.now.strftime("%H:%M:%S") 12 | file_object.write( 13 | str(self.date) + "/" + str(self.current_time) + "\t\t" + log_message +"\n") 14 | -------------------------------------------------------------------------------- /best_model_finder/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/best_model_finder/__init__.py -------------------------------------------------------------------------------- /best_model_finder/__pycache__/tuner.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/best_model_finder/__pycache__/tuner.cpython-36.pyc -------------------------------------------------------------------------------- /best_model_finder/__pycache__/tuner.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/best_model_finder/__pycache__/tuner.cpython-37.pyc -------------------------------------------------------------------------------- /best_model_finder/tuner.py: -------------------------------------------------------------------------------- 1 | from sklearn.ensemble import RandomForestClassifier 2 | from sklearn.model_selection import GridSearchCV 3 | from xgboost import XGBClassifier 4 | from sklearn.metrics import roc_auc_score,accuracy_score 5 | 6 | class Model_Finder: 7 | """ 8 | This class shall be used to find the model with best accuracy and AUC score. 9 | Written By: iNeuron Intelligence 10 | Version: 1.0 11 | Revisions: None 12 | 13 | """ 14 | 15 | def __init__(self,file_object,logger_object): 16 | self.file_object = file_object 17 | self.logger_object = logger_object 18 | self.clf = RandomForestClassifier() 19 | self.xgb = XGBClassifier(objective='binary:logistic') 20 | 21 | def get_best_params_for_random_forest(self,train_x,train_y): 22 | """ 23 | Method Name: get_best_params_for_random_forest 24 | Description: get the parameters for Random Forest Algorithm which give the best accuracy. 25 | Use Hyper Parameter Tuning. 26 | Output: The model with the best parameters 27 | On Failure: Raise Exception 28 | 29 | Written By: iNeuron Intelligence 30 | Version: 1.0 31 | Revisions: None 32 | 33 | """ 34 | self.logger_object.log(self.file_object, 'Entered the get_best_params_for_random_forest method of the Model_Finder class') 35 | try: 36 | # initializing with different combination of parameters 37 | self.param_grid = {"n_estimators": [10, 50, 100, 130], "criterion": ['gini', 'entropy'], 38 | "max_depth": range(2, 4, 1), "max_features": ['auto', 'log2']} 39 | 40 | #Creating an object of the Grid Search class 41 | self.grid = GridSearchCV(estimator=self.clf, param_grid=self.param_grid, cv=5, verbose=3) 42 | #finding the best parameters 43 | self.grid.fit(train_x, train_y) 44 | 45 | #extracting the best parameters 46 | self.criterion = self.grid.best_params_['criterion'] 47 | self.max_depth = self.grid.best_params_['max_depth'] 48 | self.max_features = self.grid.best_params_['max_features'] 49 | self.n_estimators = self.grid.best_params_['n_estimators'] 50 | 51 | #creating a new model with the best parameters 52 | self.clf = RandomForestClassifier(n_estimators=self.n_estimators, criterion=self.criterion, 53 | max_depth=self.max_depth, max_features=self.max_features) 54 | # training the mew model 55 | self.clf.fit(train_x, train_y) 56 | self.logger_object.log(self.file_object, 57 | 'Random Forest best params: '+str(self.grid.best_params_)+'. Exited the get_best_params_for_random_forest method of the Model_Finder class') 58 | 59 | return self.clf 60 | except Exception as e: 61 | self.logger_object.log(self.file_object, 62 | 'Exception occured in get_best_params_for_random_forest method of the Model_Finder class. Exception message: ' + str( 63 | e)) 64 | self.logger_object.log(self.file_object, 65 | 'Random Forest Parameter tuning failed. Exited the get_best_params_for_random_forest method of the Model_Finder class') 66 | raise Exception() 67 | 68 | def get_best_params_for_xgboost(self,train_x,train_y): 69 | 70 | """ 71 | Method Name: get_best_params_for_xgboost 72 | Description: get the parameters for XGBoost Algorithm which give the best accuracy. 73 | Use Hyper Parameter Tuning. 74 | Output: The model with the best parameters 75 | On Failure: Raise Exception 76 | 77 | Written By: iNeuron Intelligence 78 | Version: 1.0 79 | Revisions: None 80 | 81 | """ 82 | self.logger_object.log(self.file_object, 83 | 'Entered the get_best_params_for_xgboost method of the Model_Finder class') 84 | try: 85 | # initializing with different combination of parameters 86 | self.param_grid_xgboost = { 87 | 88 | 'learning_rate': [0.5, 0.1, 0.01, 0.001], 89 | 'max_depth': [3, 5, 10, 20], 90 | 'n_estimators': [10, 50, 100, 200] 91 | 92 | } 93 | # Creating an object of the Grid Search class 94 | self.grid= GridSearchCV(XGBClassifier(objective='binary:logistic'),self.param_grid_xgboost, verbose=3,cv=5) 95 | # finding the best parameters 96 | self.grid.fit(train_x, train_y) 97 | 98 | # extracting the best parameters 99 | self.learning_rate = self.grid.best_params_['learning_rate'] 100 | self.max_depth = self.grid.best_params_['max_depth'] 101 | self.n_estimators = self.grid.best_params_['n_estimators'] 102 | 103 | # creating a new model with the best parameters 104 | self.xgb = XGBClassifier(learning_rate=self.learning_rate, max_depth=self.max_depth, n_estimators=self.n_estimators) 105 | # training the mew model 106 | self.xgb.fit(train_x, train_y) 107 | self.logger_object.log(self.file_object, 108 | 'XGBoost best params: ' + str( 109 | self.grid.best_params_) + '. Exited the get_best_params_for_xgboost method of the Model_Finder class') 110 | return self.xgb 111 | except Exception as e: 112 | self.logger_object.log(self.file_object, 113 | 'Exception occured in get_best_params_for_xgboost method of the Model_Finder class. Exception message: ' + str( 114 | e)) 115 | self.logger_object.log(self.file_object, 116 | 'XGBoost Parameter tuning failed. Exited the get_best_params_for_xgboost method of the Model_Finder class') 117 | raise Exception() 118 | 119 | 120 | def get_best_model(self,train_x,train_y,test_x,test_y): 121 | """ 122 | Method Name: get_best_model 123 | Description: Find out the Model which has the best AUC score. 124 | Output: The best model name and the model object 125 | On Failure: Raise Exception 126 | 127 | Written By: iNeuron Intelligence 128 | Version: 1.0 129 | Revisions: None 130 | 131 | """ 132 | self.logger_object.log(self.file_object, 133 | 'Entered the get_best_model method of the Model_Finder class') 134 | # create best model for XGBoost 135 | try: 136 | self.xgboost= self.get_best_params_for_xgboost(train_x,train_y) 137 | self.prediction_xgboost = self.xgboost.predict(test_x) # Predictions using the XGBoost Model 138 | 139 | if len(test_y.unique()) == 1: #if there is only one label in y, then roc_auc_score returns error. We will use accuracy in that case 140 | self.xgboost_score = accuracy_score(test_y, self.prediction_xgboost) 141 | self.logger_object.log(self.file_object, 'Accuracy for XGBoost:' + str(self.xgboost_score)) # Log AUC 142 | else: 143 | self.xgboost_score = roc_auc_score(test_y, self.prediction_xgboost) # AUC for XGBoost 144 | self.logger_object.log(self.file_object, 'AUC for XGBoost:' + str(self.xgboost_score)) # Log AUC 145 | 146 | # create best model for Random Forest 147 | self.random_forest=self.get_best_params_for_random_forest(train_x,train_y) 148 | self.prediction_random_forest=self.random_forest.predict(test_x) # prediction using the Random Forest Algorithm 149 | 150 | if len(test_y.unique()) == 1:#if there is only one label in y, then roc_auc_score returns error. We will use accuracy in that case 151 | self.random_forest_score = accuracy_score(test_y,self.prediction_random_forest) 152 | self.logger_object.log(self.file_object, 'Accuracy for RF:' + str(self.random_forest_score)) 153 | else: 154 | self.random_forest_score = roc_auc_score(test_y, self.prediction_random_forest) # AUC for Random Forest 155 | self.logger_object.log(self.file_object, 'AUC for RF:' + str(self.random_forest_score)) 156 | 157 | #comparing the two models 158 | if(self.random_forest_score < self.xgboost_score): 159 | return 'XGBoost',self.xgboost 160 | else: 161 | return 'RandomForest',self.random_forest 162 | 163 | except Exception as e: 164 | self.logger_object.log(self.file_object, 165 | 'Exception occured in get_best_model method of the Model_Finder class. Exception message: ' + str( 166 | e)) 167 | self.logger_object.log(self.file_object, 168 | 'Model Selection Failed. Exited the get_best_model method of the Model_Finder class') 169 | raise Exception() 170 | 171 | -------------------------------------------------------------------------------- /data_ingestion/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/data_ingestion/__init__.py -------------------------------------------------------------------------------- /data_ingestion/__pycache__/data_loader.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/data_ingestion/__pycache__/data_loader.cpython-36.pyc -------------------------------------------------------------------------------- /data_ingestion/__pycache__/data_loader.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/data_ingestion/__pycache__/data_loader.cpython-37.pyc -------------------------------------------------------------------------------- /data_ingestion/__pycache__/data_loader_prediction.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/data_ingestion/__pycache__/data_loader_prediction.cpython-36.pyc -------------------------------------------------------------------------------- /data_ingestion/__pycache__/data_loader_prediction.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/data_ingestion/__pycache__/data_loader_prediction.cpython-37.pyc -------------------------------------------------------------------------------- /data_ingestion/data_loader.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | class Data_Getter: 4 | """ 5 | This class shall be used for obtaining the data from the source for training. 6 | 7 | Written By: iNeuron Intelligence 8 | Version: 1.0 9 | Revisions: None 10 | 11 | """ 12 | def __init__(self, file_object, logger_object): 13 | self.training_file='Training_FileFromDB/InputFile.csv' 14 | self.file_object=file_object 15 | self.logger_object=logger_object 16 | 17 | def get_data(self): 18 | """ 19 | Method Name: get_data 20 | Description: This method reads the data from source. 21 | Output: A pandas DataFrame. 22 | On Failure: Raise Exception 23 | 24 | Written By: iNeuron Intelligence 25 | Version: 1.0 26 | Revisions: None 27 | 28 | """ 29 | self.logger_object.log(self.file_object,'Entered the get_data method of the Data_Getter class') 30 | try: 31 | self.data= pd.read_csv(self.training_file) # reading the data file 32 | self.logger_object.log(self.file_object,'Data Load Successful.Exited the get_data method of the Data_Getter class') 33 | return self.data 34 | except Exception as e: 35 | self.logger_object.log(self.file_object,'Exception occured in get_data method of the Data_Getter class. Exception message: '+str(e)) 36 | self.logger_object.log(self.file_object, 37 | 'Data Load Unsuccessful.Exited the get_data method of the Data_Getter class') 38 | raise Exception() 39 | 40 | 41 | -------------------------------------------------------------------------------- /data_ingestion/data_loader_prediction.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | class Data_Getter_Pred: 4 | """ 5 | This class shall be used for obtaining the data from the source for prediction. 6 | 7 | Written By: iNeuron Intelligence 8 | Version: 1.0 9 | Revisions: None 10 | 11 | """ 12 | def __init__(self, file_object, logger_object): 13 | self.prediction_file='Prediction_FileFromDB/InputFile.csv' 14 | self.file_object=file_object 15 | self.logger_object=logger_object 16 | 17 | def get_data(self): 18 | """ 19 | Method Name: get_data 20 | Description: This method reads the data from source. 21 | Output: A pandas DataFrame. 22 | On Failure: Raise Exception 23 | 24 | Written By: iNeuron Intelligence 25 | Version: 1.0 26 | Revisions: None 27 | 28 | """ 29 | self.logger_object.log(self.file_object,'Entered the get_data method of the Data_Getter class') 30 | try: 31 | self.data= pd.read_csv(self.prediction_file) # reading the data file 32 | self.logger_object.log(self.file_object,'Data Load Successful.Exited the get_data method of the Data_Getter class') 33 | return self.data 34 | except Exception as e: 35 | self.logger_object.log(self.file_object,'Exception occured in get_data method of the Data_Getter class. Exception message: '+str(e)) 36 | self.logger_object.log(self.file_object, 37 | 'Data Load Unsuccessful.Exited the get_data method of the Data_Getter class') 38 | raise Exception() 39 | 40 | 41 | -------------------------------------------------------------------------------- /data_preprocessing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/data_preprocessing/__init__.py -------------------------------------------------------------------------------- /data_preprocessing/__pycache__/clustering.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/data_preprocessing/__pycache__/clustering.cpython-36.pyc -------------------------------------------------------------------------------- /data_preprocessing/__pycache__/clustering.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/data_preprocessing/__pycache__/clustering.cpython-37.pyc -------------------------------------------------------------------------------- /data_preprocessing/__pycache__/preprocessing.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/data_preprocessing/__pycache__/preprocessing.cpython-36.pyc -------------------------------------------------------------------------------- /data_preprocessing/__pycache__/preprocessing.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/data_preprocessing/__pycache__/preprocessing.cpython-37.pyc -------------------------------------------------------------------------------- /data_preprocessing/__pycache__/preprocessing_pred.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/data_preprocessing/__pycache__/preprocessing_pred.cpython-36.pyc -------------------------------------------------------------------------------- /data_preprocessing/clustering.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | from sklearn.cluster import KMeans 3 | from kneed import KneeLocator 4 | from file_operations import file_methods 5 | 6 | class KMeansClustering: 7 | """ 8 | This class shall be used to divide the data into clusters before training. 9 | 10 | Written By: iNeuron Intelligence 11 | Version: 1.0 12 | Revisions: None 13 | 14 | """ 15 | 16 | def __init__(self, file_object, logger_object): 17 | self.file_object = file_object 18 | self.logger_object = logger_object 19 | 20 | def elbow_plot(self,data): 21 | """ 22 | Method Name: elbow_plot 23 | Description: This method saves the plot to decide the optimum number of clusters to the file. 24 | Output: A picture saved to the directory 25 | On Failure: Raise Exception 26 | 27 | Written By: iNeuron Intelligence 28 | Version: 1.0 29 | Revisions: None 30 | 31 | """ 32 | self.logger_object.log(self.file_object, 'Entered the elbow_plot method of the KMeansClustering class') 33 | wcss=[] # initializing an empty list 34 | try: 35 | for i in range (1,11): 36 | kmeans=KMeans(n_clusters=i,init='k-means++',random_state=42) # initializing the KMeans object 37 | kmeans.fit(data) # fitting the data to the KMeans Algorithm 38 | wcss.append(kmeans.inertia_) 39 | plt.plot(range(1,11),wcss) # creating the graph between WCSS and the number of clusters 40 | plt.title('The Elbow Method') 41 | plt.xlabel('Number of clusters') 42 | plt.ylabel('WCSS') 43 | #plt.show() 44 | plt.savefig('preprocessing_data/K-Means_Elbow.PNG') # saving the elbow plot locally 45 | # finding the value of the optimum cluster programmatically 46 | self.kn = KneeLocator(range(1, 11), wcss, curve='convex', direction='decreasing') 47 | self.logger_object.log(self.file_object, 'The optimum number of clusters is: '+str(self.kn.knee)+' . Exited the elbow_plot method of the KMeansClustering class') 48 | return self.kn.knee 49 | 50 | except Exception as e: 51 | self.logger_object.log(self.file_object,'Exception occured in elbow_plot method of the KMeansClustering class. Exception message: ' + str(e)) 52 | self.logger_object.log(self.file_object,'Finding the number of clusters failed. Exited the elbow_plot method of the KMeansClustering class') 53 | raise Exception() 54 | 55 | def create_clusters(self,data,number_of_clusters): 56 | """ 57 | Method Name: create_clusters 58 | Description: Create a new dataframe consisting of the cluster information. 59 | Output: A datframe with cluster column 60 | On Failure: Raise Exception 61 | 62 | Written By: iNeuron Intelligence 63 | Version: 1.0 64 | Revisions: None 65 | 66 | """ 67 | self.logger_object.log(self.file_object, 'Entered the create_clusters method of the KMeansClustering class') 68 | self.data=data 69 | try: 70 | self.kmeans = KMeans(n_clusters=number_of_clusters, init='k-means++', random_state=42) 71 | #self.data = self.data[~self.data.isin([np.nan, np.inf, -np.inf]).any(1)] 72 | self.y_kmeans=self.kmeans.fit_predict(data) # divide data into clusters 73 | 74 | self.file_op = file_methods.File_Operation(self.file_object,self.logger_object) 75 | self.save_model = self.file_op.save_model(self.kmeans, 'KMeans') # saving the KMeans model to directory 76 | # passing 'Model' as the functions need three parameters 77 | 78 | self.data['Cluster']=self.y_kmeans # create a new column in dataset for storing the cluster information 79 | self.logger_object.log(self.file_object, 'succesfully created '+str(self.kn.knee)+ 'clusters. Exited the create_clusters method of the KMeansClustering class') 80 | return self.data 81 | except Exception as e: 82 | self.logger_object.log(self.file_object,'Exception occured in create_clusters method of the KMeansClustering class. Exception message: ' + str(e)) 83 | self.logger_object.log(self.file_object,'Fitting the data to clusters failed. Exited the create_clusters method of the KMeansClustering class') 84 | raise Exception() -------------------------------------------------------------------------------- /data_preprocessing/preprocessing.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | from sklearn.impute import KNNImputer 4 | class Preprocessor: 5 | """ 6 | This class shall be used to clean and transform the data before training. 7 | 8 | Written By: iNeuron Intelligence 9 | Version: 1.0 10 | Revisions: None 11 | 12 | """ 13 | 14 | def __init__(self, file_object, logger_object): 15 | self.file_object = file_object 16 | self.logger_object = logger_object 17 | 18 | def remove_columns(self,data,columns): 19 | """ 20 | Method Name: remove_columns 21 | Description: This method removes the given columns from a pandas dataframe. 22 | Output: A pandas DataFrame after removing the specified columns. 23 | On Failure: Raise Exception 24 | 25 | Written By: iNeuron Intelligence 26 | Version: 1.0 27 | Revisions: None 28 | 29 | """ 30 | self.logger_object.log(self.file_object, 'Entered the remove_columns method of the Preprocessor class') 31 | self.data=data 32 | self.columns=columns 33 | try: 34 | self.useful_data=self.data.drop(labels=self.columns, axis=1) # drop the labels specified in the columns 35 | self.logger_object.log(self.file_object, 36 | 'Column removal Successful.Exited the remove_columns method of the Preprocessor class') 37 | return self.useful_data 38 | except Exception as e: 39 | self.logger_object.log(self.file_object,'Exception occured in remove_columns method of the Preprocessor class. Exception message: '+str(e)) 40 | self.logger_object.log(self.file_object, 41 | 'Column removal Unsuccessful. Exited the remove_columns method of the Preprocessor class') 42 | raise Exception() 43 | 44 | def separate_label_feature(self, data, label_column_name): 45 | """ 46 | Method Name: separate_label_feature 47 | Description: This method separates the features and a Label Coulmns. 48 | Output: Returns two separate Dataframes, one containing features and the other containing Labels . 49 | On Failure: Raise Exception 50 | 51 | Written By: iNeuron Intelligence 52 | Version: 1.0 53 | Revisions: None 54 | 55 | """ 56 | self.logger_object.log(self.file_object, 'Entered the separate_label_feature method of the Preprocessor class') 57 | try: 58 | self.X=data.drop(labels=label_column_name,axis=1) # drop the columns specified and separate the feature columns 59 | self.Y=data[label_column_name] # Filter the Label columns 60 | self.logger_object.log(self.file_object, 61 | 'Label Separation Successful. Exited the separate_label_feature method of the Preprocessor class') 62 | return self.X,self.Y 63 | except Exception as e: 64 | self.logger_object.log(self.file_object,'Exception occured in separate_label_feature method of the Preprocessor class. Exception message: ' + str(e)) 65 | self.logger_object.log(self.file_object, 'Label Separation Unsuccessful. Exited the separate_label_feature method of the Preprocessor class') 66 | raise Exception() 67 | 68 | def is_null_present(self,data): 69 | """ 70 | Method Name: is_null_present 71 | Description: This method checks whether there are null values present in the pandas Dataframe or not. 72 | Output: Returns a Boolean Value. True if null values are present in the DataFrame, False if they are not present. 73 | On Failure: Raise Exception 74 | 75 | Written By: iNeuron Intelligence 76 | Version: 1.0 77 | Revisions: None 78 | 79 | """ 80 | self.logger_object.log(self.file_object, 'Entered the is_null_present method of the Preprocessor class') 81 | self.null_present = False 82 | try: 83 | self.null_counts=data.isna().sum() # check for the count of null values per column 84 | for i in self.null_counts: 85 | if i>0: 86 | self.null_present=True 87 | break 88 | if(self.null_present): # write the logs to see which columns have null values 89 | dataframe_with_null = pd.DataFrame() 90 | dataframe_with_null['columns'] = data.columns 91 | dataframe_with_null['missing values count'] = np.asarray(data.isna().sum()) 92 | dataframe_with_null.to_csv('preprocessing_data/null_values.csv') # storing the null column information to file 93 | self.logger_object.log(self.file_object,'Finding missing values is a success.Data written to the null values file. Exited the is_null_present method of the Preprocessor class') 94 | return self.null_present 95 | except Exception as e: 96 | self.logger_object.log(self.file_object,'Exception occured in is_null_present method of the Preprocessor class. Exception message: ' + str(e)) 97 | self.logger_object.log(self.file_object,'Finding missing values failed. Exited the is_null_present method of the Preprocessor class') 98 | raise Exception() 99 | 100 | def impute_missing_values(self, data): 101 | """ 102 | Method Name: impute_missing_values 103 | Description: This method replaces all the missing values in the Dataframe using KNN Imputer. 104 | Output: A Dataframe which has all the missing values imputed. 105 | On Failure: Raise Exception 106 | 107 | Written By: iNeuron Intelligence 108 | Version: 1.0 109 | Revisions: None 110 | """ 111 | self.logger_object.log(self.file_object, 'Entered the impute_missing_values method of the Preprocessor class') 112 | self.data= data 113 | try: 114 | imputer=KNNImputer(n_neighbors=3, weights='uniform',missing_values=np.nan) 115 | self.new_array=imputer.fit_transform(self.data) # impute the missing values 116 | # convert the nd-array returned in the step above to a Dataframe 117 | self.new_data=pd.DataFrame(data=self.new_array, columns=self.data.columns) 118 | self.logger_object.log(self.file_object, 'Imputing missing values Successful. Exited the impute_missing_values method of the Preprocessor class') 119 | return self.new_data 120 | except Exception as e: 121 | self.logger_object.log(self.file_object,'Exception occured in impute_missing_values method of the Preprocessor class. Exception message: ' + str(e)) 122 | self.logger_object.log(self.file_object,'Imputing missing values failed. Exited the impute_missing_values method of the Preprocessor class') 123 | raise Exception() 124 | 125 | def get_columns_with_zero_std_deviation(self,data): 126 | """ 127 | Method Name: get_columns_with_zero_std_deviation 128 | Description: This method finds out the columns which have a standard deviation of zero. 129 | Output: List of the columns with standard deviation of zero 130 | On Failure: Raise Exception 131 | 132 | Written By: iNeuron Intelligence 133 | Version: 1.0 134 | Revisions: None 135 | """ 136 | self.logger_object.log(self.file_object, 'Entered the get_columns_with_zero_std_deviation method of the Preprocessor class') 137 | self.columns=data.columns 138 | self.data_n = data.describe() 139 | self.col_to_drop=[] 140 | try: 141 | for x in self.columns: 142 | if (self.data_n[x]['std'] == 0): # check if standard deviation is zero 143 | self.col_to_drop.append(x) # prepare the list of columns with standard deviation zero 144 | self.logger_object.log(self.file_object, 'Column search for Standard Deviation of Zero Successful. Exited the get_columns_with_zero_std_deviation method of the Preprocessor class') 145 | return self.col_to_drop 146 | 147 | except Exception as e: 148 | self.logger_object.log(self.file_object,'Exception occured in get_columns_with_zero_std_deviation method of the Preprocessor class. Exception message: ' + str(e)) 149 | self.logger_object.log(self.file_object, 'Column search for Standard Deviation of Zero Failed. Exited the get_columns_with_zero_std_deviation method of the Preprocessor class') 150 | raise Exception() -------------------------------------------------------------------------------- /file_operations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/file_operations/__init__.py -------------------------------------------------------------------------------- /file_operations/__pycache__/file_methods.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/file_operations/__pycache__/file_methods.cpython-36.pyc -------------------------------------------------------------------------------- /file_operations/__pycache__/file_methods.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/file_operations/__pycache__/file_methods.cpython-37.pyc -------------------------------------------------------------------------------- /file_operations/file_methods.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import os 3 | import shutil 4 | 5 | 6 | class File_Operation: 7 | """ 8 | This class shall be used to save the model after training 9 | and load the saved model for prediction. 10 | 11 | Written By: iNeuron Intelligence 12 | Version: 1.0 13 | Revisions: None 14 | 15 | """ 16 | def __init__(self,file_object,logger_object): 17 | self.file_object = file_object 18 | self.logger_object = logger_object 19 | self.model_directory='models/' 20 | 21 | def save_model(self,model,filename): 22 | """ 23 | Method Name: save_model 24 | Description: Save the model file to directory 25 | Outcome: File gets saved 26 | On Failure: Raise Exception 27 | 28 | Written By: iNeuron Intelligence 29 | Version: 1.0 30 | Revisions: None 31 | """ 32 | self.logger_object.log(self.file_object, 'Entered the save_model method of the File_Operation class') 33 | try: 34 | path = os.path.join(self.model_directory,filename) #create seperate directory for each cluster 35 | if os.path.isdir(path): #remove previously existing models for each clusters 36 | shutil.rmtree(self.model_directory) 37 | os.makedirs(path) 38 | else: 39 | os.makedirs(path) # 40 | with open(path +'/' + filename+'.sav', 41 | 'wb') as f: 42 | pickle.dump(model, f) # save the model to file 43 | self.logger_object.log(self.file_object, 44 | 'Model File '+filename+' saved. Exited the save_model method of the Model_Finder class') 45 | 46 | return 'success' 47 | except Exception as e: 48 | self.logger_object.log(self.file_object,'Exception occured in save_model method of the Model_Finder class. Exception message: ' + str(e)) 49 | self.logger_object.log(self.file_object, 50 | 'Model File '+filename+' could not be saved. Exited the save_model method of the Model_Finder class') 51 | raise Exception() 52 | 53 | def load_model(self,filename): 54 | """ 55 | Method Name: load_model 56 | Description: load the model file to memory 57 | Output: The Model file loaded in memory 58 | On Failure: Raise Exception 59 | 60 | Written By: iNeuron Intelligence 61 | Version: 1.0 62 | Revisions: None 63 | """ 64 | self.logger_object.log(self.file_object, 'Entered the load_model method of the File_Operation class') 65 | try: 66 | with open(self.model_directory + filename + '/' + filename + '.sav', 67 | 'rb') as f: 68 | self.logger_object.log(self.file_object, 69 | 'Model File ' + filename + ' loaded. Exited the load_model method of the Model_Finder class') 70 | return pickle.load(f) 71 | except Exception as e: 72 | self.logger_object.log(self.file_object, 73 | 'Exception occured in load_model method of the Model_Finder class. Exception message: ' + str( 74 | e)) 75 | self.logger_object.log(self.file_object, 76 | 'Model File ' + filename + ' could not be saved. Exited the load_model method of the Model_Finder class') 77 | raise Exception() 78 | 79 | def find_correct_model_file(self,cluster_number): 80 | """ 81 | Method Name: find_correct_model_file 82 | Description: Select the correct model based on cluster number 83 | Output: The Model file 84 | On Failure: Raise Exception 85 | 86 | Written By: iNeuron Intelligence 87 | Version: 1.0 88 | Revisions: None 89 | """ 90 | self.logger_object.log(self.file_object, 'Entered the find_correct_model_file method of the File_Operation class') 91 | try: 92 | self.cluster_number= cluster_number 93 | self.folder_name=self.model_directory 94 | self.list_of_model_files = [] 95 | self.list_of_files = os.listdir(self.folder_name) 96 | for self.file in self.list_of_files: 97 | try: 98 | if (self.file.index(str( self.cluster_number))!=-1): 99 | self.model_name=self.file 100 | except: 101 | continue 102 | self.model_name=self.model_name.split('.')[0] 103 | self.logger_object.log(self.file_object, 104 | 'Exited the find_correct_model_file method of the Model_Finder class.') 105 | return self.model_name 106 | except Exception as e: 107 | self.logger_object.log(self.file_object, 108 | 'Exception occured in find_correct_model_file method of the Model_Finder class. Exception message: ' + str( 109 | e)) 110 | self.logger_object.log(self.file_object, 111 | 'Exited the find_correct_model_file method of the Model_Finder class with Failure') 112 | raise Exception() -------------------------------------------------------------------------------- /flask_monitoringdashboard.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/flask_monitoringdashboard.db -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from wsgiref import simple_server 4 | from flask import Flask, request, render_template 5 | from flask import Response 6 | import os 7 | from flask_cors import CORS, cross_origin 8 | from prediction_Validation_Insertion import pred_validation 9 | from trainingModel import trainModel 10 | from training_Validation_Insertion import train_validation 11 | import flask_monitoringdashboard as dashboard 12 | from predictFromModel import prediction 13 | import json 14 | 15 | os.putenv('LANG', 'en_US.UTF-8') 16 | os.putenv('LC_ALL', 'en_US.UTF-8') 17 | 18 | app = Flask(__name__) 19 | dashboard.bind(app) 20 | CORS(app) 21 | 22 | 23 | @app.route("/", methods=['GET']) 24 | @cross_origin() 25 | def home(): 26 | return render_template('index.html') 27 | 28 | 29 | @app.route("/predict", methods=['POST']) 30 | @cross_origin() 31 | def predictRouteClient(): 32 | try: 33 | if request.json is not None: 34 | path = request.json['filepath'] 35 | 36 | pred_val = pred_validation(path) # object initialization 37 | 38 | pred_val.prediction_validation() # calling the prediction_validation function 39 | 40 | pred = prediction(path) # object initialization 41 | 42 | # predicting for dataset present in database 43 | path, json_predictions = pred.predictionFromModel() 44 | return Response("Prediction File created at !!!" + str(path) + 'and few of the predictions are ' + str( 45 | json.loads(json_predictions))) 46 | elif request.form is not None: 47 | path = request.form['filepath'] 48 | 49 | pred_val = pred_validation(path) # object initialization 50 | 51 | pred_val.prediction_validation() # calling the prediction_validation function 52 | 53 | pred = prediction(path) # object initialization 54 | 55 | # predicting for dataset present in database 56 | path, json_predictions = pred.predictionFromModel() 57 | return Response("Prediction File created at !!!" + str(path) + 'and few of the predictions are ' + str( 58 | json.loads(json_predictions))) 59 | else: 60 | print('Nothing Matched') 61 | except ValueError: 62 | return Response("Error Occurred! %s" % ValueError) 63 | except KeyError: 64 | return Response("Error Occurred! %s" % KeyError) 65 | except Exception as e: 66 | return Response("Error Occurred! %s" % e) 67 | 68 | 69 | @app.route("/train", methods=['GET', 'POST']) 70 | @cross_origin() 71 | def trainRouteClient(): 72 | try: 73 | # if request.json['folderPath'] is not None: 74 | folder_path = "Training_Batch_Files" 75 | # path = request.json['folderPath'] 76 | if folder_path is not None: 77 | path = folder_path 78 | 79 | train_valObj = train_validation(path) # object initialization 80 | 81 | train_valObj.train_validation() # calling the training_validation function 82 | 83 | trainModelObj = trainModel() # object initialization 84 | trainModelObj.trainingModel() # training the model for the files in the table 85 | 86 | 87 | except ValueError: 88 | 89 | return Response("Error Occurred! %s" % ValueError) 90 | 91 | except KeyError: 92 | 93 | return Response("Error Occurred! %s" % KeyError) 94 | 95 | except Exception as e: 96 | 97 | return Response("Error Occurred! %s" % e) 98 | return Response("Training successful!!") 99 | 100 | 101 | port = int(os.getenv("PORT", 5000)) 102 | if __name__ == "__main__": 103 | host = '0.0.0.0' 104 | # port = 5000 105 | httpd = simple_server.make_server(host, port, app) 106 | # print("Serving on %s %d" % (host, port)) 107 | httpd.serve_forever() 108 | -------------------------------------------------------------------------------- /models/KMeans/KMeans.sav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/models/KMeans/KMeans.sav -------------------------------------------------------------------------------- /models/RandomForest2/RandomForest2.sav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/models/RandomForest2/RandomForest2.sav -------------------------------------------------------------------------------- /models/XGBoost0/XGBoost0.sav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/models/XGBoost0/XGBoost0.sav -------------------------------------------------------------------------------- /models/XGBoost1/XGBoost1.sav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/models/XGBoost1/XGBoost1.sav -------------------------------------------------------------------------------- /pattern.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/pattern.py -------------------------------------------------------------------------------- /predictFromModel.py: -------------------------------------------------------------------------------- 1 | import pandas 2 | from file_operations import file_methods 3 | from data_preprocessing import preprocessing 4 | from data_ingestion import data_loader_prediction 5 | from application_logging import logger 6 | from Prediction_Raw_Data_Validation.predictionDataValidation import Prediction_Data_validation 7 | 8 | 9 | class prediction: 10 | 11 | def __init__(self,path): 12 | self.file_object = open("Prediction_Logs/Prediction_Log.txt", 'a+') 13 | self.log_writer = logger.App_Logger() 14 | if path is not None: 15 | self.pred_data_val = Prediction_Data_validation(path) 16 | 17 | def predictionFromModel(self): 18 | 19 | try: 20 | self.pred_data_val.deletePredictionFile() #deletes the existing prediction file from last run! 21 | self.log_writer.log(self.file_object,'Start of Prediction') 22 | data_getter=data_loader_prediction.Data_Getter_Pred(self.file_object,self.log_writer) 23 | data=data_getter.get_data() 24 | 25 | #code change 26 | # wafer_names=data['Wafer'] 27 | # data=data.drop(labels=['Wafer'],axis=1) 28 | 29 | preprocessor=preprocessing.Preprocessor(self.file_object,self.log_writer) 30 | is_null_present=preprocessor.is_null_present(data) 31 | if(is_null_present): 32 | data=preprocessor.impute_missing_values(data) 33 | 34 | cols_to_drop=preprocessor.get_columns_with_zero_std_deviation(data) 35 | data=preprocessor.remove_columns(data,cols_to_drop) 36 | #data=data.to_numpy() 37 | file_loader=file_methods.File_Operation(self.file_object,self.log_writer) 38 | kmeans=file_loader.load_model('KMeans') 39 | 40 | ##Code changed 41 | #pred_data = data.drop(['Wafer'],axis=1) 42 | clusters=kmeans.predict(data.drop(['Wafer'],axis=1))#drops the first column for cluster prediction 43 | data['clusters']=clusters 44 | clusters=data['clusters'].unique() 45 | for i in clusters: 46 | cluster_data= data[data['clusters']==i] 47 | wafer_names = list(cluster_data['Wafer']) 48 | cluster_data=data.drop(labels=['Wafer'],axis=1) 49 | cluster_data = cluster_data.drop(['clusters'],axis=1) 50 | model_name = file_loader.find_correct_model_file(i) 51 | model = file_loader.load_model(model_name) 52 | result=list(model.predict(cluster_data)) 53 | result = pandas.DataFrame(list(zip(wafer_names,result)),columns=['Wafer','Prediction']) 54 | path="Prediction_Output_File/Predictions.csv" 55 | result.to_csv("Prediction_Output_File/Predictions.csv",header=True,mode='a+') #appends result to prediction file 56 | self.log_writer.log(self.file_object,'End of Prediction') 57 | except Exception as ex: 58 | self.log_writer.log(self.file_object, 'Error occured while running the prediction!! Error:: %s' % ex) 59 | raise ex 60 | return path, result.head().to_json(orient="records") 61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /prediction_Validation_Insertion.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from Prediction_Raw_Data_Validation.predictionDataValidation import Prediction_Data_validation 3 | from DataTypeValidation_Insertion_Prediction.DataTypeValidationPrediction import dBOperation 4 | from DataTransformation_Prediction.DataTransformationPrediction import dataTransformPredict 5 | from application_logging import logger 6 | 7 | class pred_validation: 8 | def __init__(self,path): 9 | self.raw_data = Prediction_Data_validation(path) 10 | self.dataTransform = dataTransformPredict() 11 | self.dBOperation = dBOperation() 12 | self.file_object = open("Prediction_Logs/Prediction_Log.txt", 'a+') 13 | self.log_writer = logger.App_Logger() 14 | 15 | def prediction_validation(self): 16 | 17 | try: 18 | 19 | self.log_writer.log(self.file_object,'Start of Validation on files for prediction!!') 20 | #extracting values from prediction schema 21 | LengthOfDateStampInFile,LengthOfTimeStampInFile,column_names,noofcolumns = self.raw_data.valuesFromSchema() 22 | #getting the regex defined to validate filename 23 | regex = self.raw_data.manualRegexCreation() 24 | #validating filename of prediction files 25 | self.raw_data.validationFileNameRaw(regex,LengthOfDateStampInFile,LengthOfTimeStampInFile) 26 | #validating column length in the file 27 | self.raw_data.validateColumnLength(noofcolumns) 28 | #validating if any column has all values missing 29 | self.raw_data.validateMissingValuesInWholeColumn() 30 | self.log_writer.log(self.file_object,"Raw Data Validation Complete!!") 31 | 32 | self.log_writer.log(self.file_object,("Starting Data Transforamtion!!")) 33 | #replacing blanks in the csv file with "Null" values to insert in table 34 | self.dataTransform.replaceMissingWithNull() 35 | 36 | self.log_writer.log(self.file_object,"DataTransformation Completed!!!") 37 | 38 | self.log_writer.log(self.file_object,"Creating Prediction_Database and tables on the basis of given schema!!!") 39 | #create database with given name, if present open the connection! Create table with columns given in schema 40 | self.dBOperation.createTableDb('Prediction',column_names) 41 | self.log_writer.log(self.file_object,"Table creation Completed!!") 42 | self.log_writer.log(self.file_object,"Insertion of Data into Table started!!!!") 43 | #insert csv files in the table 44 | self.dBOperation.insertIntoTableGoodData('Prediction') 45 | self.log_writer.log(self.file_object,"Insertion in Table completed!!!") 46 | self.log_writer.log(self.file_object,"Deleting Good Data Folder!!!") 47 | #Delete the good data folder after loading files in table 48 | self.raw_data.deleteExistingGoodDataTrainingFolder() 49 | self.log_writer.log(self.file_object,"Good_Data folder deleted!!!") 50 | self.log_writer.log(self.file_object,"Moving bad files to Archive and deleting Bad_Data folder!!!") 51 | #Move the bad files to archive folder 52 | self.raw_data.moveBadFilesToArchiveBad() 53 | self.log_writer.log(self.file_object,"Bad files moved to archive!! Bad folder Deleted!!") 54 | self.log_writer.log(self.file_object,"Validation Operation completed!!") 55 | self.log_writer.log(self.file_object,"Extracting csv file from table") 56 | #export data in table to csvfile 57 | self.dBOperation.selectingDatafromtableintocsv('Prediction') 58 | 59 | except Exception as e: 60 | raise e 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | -------------------------------------------------------------------------------- /preprocessing_data/K-Means_Elbow.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/preprocessing_data/K-Means_Elbow.PNG -------------------------------------------------------------------------------- /preprocessing_data/null_values.csv: -------------------------------------------------------------------------------- 1 | ,columns,missing values count 2 | 0,Wafer,0 3 | 1,Sensor - 1,1 4 | 2,Sensor - 2,3 5 | 3,Sensor - 3,13 6 | 4,Sensor - 4,13 7 | 5,Sensor - 5,13 8 | 6,Sensor - 6,13 9 | 7,Sensor - 7,13 10 | 8,Sensor - 8,8 11 | 9,Sensor - 9,2 12 | 10,Sensor - 10,2 13 | 11,Sensor - 11,2 14 | 12,Sensor - 12,2 15 | 13,Sensor - 13,2 16 | 14,Sensor - 14,3 17 | 15,Sensor - 15,3 18 | 16,Sensor - 16,3 19 | 17,Sensor - 17,3 20 | 18,Sensor - 18,3 21 | 19,Sensor - 19,3 22 | 20,Sensor - 20,0 23 | 21,Sensor - 21,0 24 | 22,Sensor - 22,1 25 | 23,Sensor - 23,1 26 | 24,Sensor - 24,1 27 | 25,Sensor - 25,1 28 | 26,Sensor - 26,1 29 | 27,Sensor - 27,1 30 | 28,Sensor - 28,1 31 | 29,Sensor - 29,1 32 | 30,Sensor - 30,1 33 | 31,Sensor - 31,1 34 | 32,Sensor - 32,1 35 | 33,Sensor - 33,0 36 | 34,Sensor - 34,0 37 | 35,Sensor - 35,0 38 | 36,Sensor - 36,0 39 | 37,Sensor - 37,0 40 | 38,Sensor - 38,0 41 | 39,Sensor - 39,0 42 | 40,Sensor - 40,0 43 | 41,Sensor - 41,14 44 | 42,Sensor - 42,14 45 | 43,Sensor - 43,0 46 | 44,Sensor - 44,0 47 | 45,Sensor - 45,0 48 | 46,Sensor - 46,0 49 | 47,Sensor - 47,0 50 | 48,Sensor - 48,0 51 | 49,Sensor - 49,0 52 | 50,Sensor - 50,0 53 | 51,Sensor - 51,0 54 | 52,Sensor - 52,0 55 | 53,Sensor - 53,0 56 | 54,Sensor - 54,1 57 | 55,Sensor - 55,1 58 | 56,Sensor - 56,1 59 | 57,Sensor - 57,1 60 | 58,Sensor - 58,1 61 | 59,Sensor - 59,1 62 | 60,Sensor - 60,5 63 | 61,Sensor - 61,5 64 | 62,Sensor - 62,5 65 | 63,Sensor - 63,5 66 | 64,Sensor - 64,5 67 | 65,Sensor - 65,5 68 | 66,Sensor - 66,5 69 | 67,Sensor - 67,5 70 | 68,Sensor - 68,5 71 | 69,Sensor - 69,5 72 | 70,Sensor - 70,5 73 | 71,Sensor - 71,5 74 | 72,Sensor - 72,5 75 | 73,Sensor - 73,529 76 | 74,Sensor - 74,529 77 | 75,Sensor - 75,5 78 | 76,Sensor - 76,24 79 | 77,Sensor - 77,24 80 | 78,Sensor - 78,24 81 | 79,Sensor - 79,24 82 | 80,Sensor - 80,24 83 | 81,Sensor - 81,24 84 | 82,Sensor - 82,24 85 | 83,Sensor - 83,24 86 | 84,Sensor - 84,1 87 | 85,Sensor - 85,2 88 | 86,Sensor - 86,610 89 | 87,Sensor - 87,0 90 | 88,Sensor - 88,0 91 | 89,Sensor - 89,0 92 | 90,Sensor - 90,51 93 | 91,Sensor - 91,51 94 | 92,Sensor - 92,5 95 | 93,Sensor - 93,2 96 | 94,Sensor - 94,2 97 | 95,Sensor - 95,5 98 | 96,Sensor - 96,5 99 | 97,Sensor - 97,5 100 | 98,Sensor - 98,5 101 | 99,Sensor - 99,5 102 | 100,Sensor - 100,5 103 | 101,Sensor - 101,5 104 | 102,Sensor - 102,5 105 | 103,Sensor - 103,5 106 | 104,Sensor - 104,2 107 | 105,Sensor - 105,2 108 | 106,Sensor - 106,5 109 | 107,Sensor - 107,5 110 | 108,Sensor - 108,5 111 | 109,Sensor - 109,5 112 | 110,Sensor - 110,288 113 | 111,Sensor - 111,288 114 | 112,Sensor - 112,288 115 | 113,Sensor - 113,556 116 | 114,Sensor - 114,0 117 | 115,Sensor - 115,0 118 | 116,Sensor - 116,0 119 | 117,Sensor - 117,0 120 | 118,Sensor - 118,0 121 | 119,Sensor - 119,24 122 | 120,Sensor - 120,0 123 | 121,Sensor - 121,0 124 | 122,Sensor - 122,3 125 | 123,Sensor - 123,3 126 | 124,Sensor - 124,3 127 | 125,Sensor - 125,3 128 | 126,Sensor - 126,3 129 | 127,Sensor - 127,3 130 | 128,Sensor - 128,3 131 | 129,Sensor - 129,3 132 | 130,Sensor - 130,3 133 | 131,Sensor - 131,3 134 | 132,Sensor - 132,3 135 | 133,Sensor - 133,3 136 | 134,Sensor - 134,3 137 | 135,Sensor - 135,3 138 | 136,Sensor - 136,0 139 | 137,Sensor - 137,1 140 | 138,Sensor - 138,3 141 | 139,Sensor - 139,13 142 | 140,Sensor - 140,13 143 | 141,Sensor - 141,13 144 | 142,Sensor - 142,13 145 | 143,Sensor - 143,13 146 | 144,Sensor - 144,8 147 | 145,Sensor - 145,2 148 | 146,Sensor - 146,2 149 | 147,Sensor - 147,2 150 | 148,Sensor - 148,2 151 | 149,Sensor - 149,2 152 | 150,Sensor - 150,3 153 | 151,Sensor - 151,3 154 | 152,Sensor - 152,3 155 | 153,Sensor - 153,3 156 | 154,Sensor - 154,3 157 | 155,Sensor - 155,3 158 | 156,Sensor - 156,0 159 | 157,Sensor - 157,0 160 | 158,Sensor - 158,770 161 | 159,Sensor - 159,770 162 | 160,Sensor - 160,1 163 | 161,Sensor - 161,1 164 | 162,Sensor - 162,1 165 | 163,Sensor - 163,1 166 | 164,Sensor - 164,1 167 | 165,Sensor - 165,1 168 | 166,Sensor - 166,1 169 | 167,Sensor - 167,1 170 | 168,Sensor - 168,1 171 | 169,Sensor - 169,1 172 | 170,Sensor - 170,1 173 | 171,Sensor - 171,0 174 | 172,Sensor - 172,0 175 | 173,Sensor - 173,0 176 | 174,Sensor - 174,0 177 | 175,Sensor - 175,0 178 | 176,Sensor - 176,0 179 | 177,Sensor - 177,0 180 | 178,Sensor - 178,0 181 | 179,Sensor - 179,14 182 | 180,Sensor - 180,0 183 | 181,Sensor - 181,0 184 | 182,Sensor - 182,0 185 | 183,Sensor - 183,0 186 | 184,Sensor - 184,0 187 | 185,Sensor - 185,0 188 | 186,Sensor - 186,0 189 | 187,Sensor - 187,0 190 | 188,Sensor - 188,0 191 | 189,Sensor - 189,0 192 | 190,Sensor - 190,0 193 | 191,Sensor - 191,1 194 | 192,Sensor - 192,1 195 | 193,Sensor - 193,1 196 | 194,Sensor - 194,1 197 | 195,Sensor - 195,1 198 | 196,Sensor - 196,1 199 | 197,Sensor - 197,5 200 | 198,Sensor - 198,5 201 | 199,Sensor - 199,5 202 | 200,Sensor - 200,5 203 | 201,Sensor - 201,5 204 | 202,Sensor - 202,5 205 | 203,Sensor - 203,5 206 | 204,Sensor - 204,5 207 | 205,Sensor - 205,5 208 | 206,Sensor - 206,5 209 | 207,Sensor - 207,5 210 | 208,Sensor - 208,5 211 | 209,Sensor - 209,5 212 | 210,Sensor - 210,5 213 | 211,Sensor - 211,24 214 | 212,Sensor - 212,24 215 | 213,Sensor - 213,24 216 | 214,Sensor - 214,24 217 | 215,Sensor - 215,24 218 | 216,Sensor - 216,24 219 | 217,Sensor - 217,24 220 | 218,Sensor - 218,24 221 | 219,Sensor - 219,1 222 | 220,Sensor - 220,2 223 | 221,Sensor - 221,610 224 | 222,Sensor - 222,0 225 | 223,Sensor - 223,0 226 | 224,Sensor - 224,0 227 | 225,Sensor - 225,51 228 | 226,Sensor - 226,51 229 | 227,Sensor - 227,5 230 | 228,Sensor - 228,2 231 | 229,Sensor - 229,2 232 | 230,Sensor - 230,5 233 | 231,Sensor - 231,5 234 | 232,Sensor - 232,5 235 | 233,Sensor - 233,5 236 | 234,Sensor - 234,5 237 | 235,Sensor - 235,5 238 | 236,Sensor - 236,5 239 | 237,Sensor - 237,5 240 | 238,Sensor - 238,5 241 | 239,Sensor - 239,2 242 | 240,Sensor - 240,2 243 | 241,Sensor - 241,5 244 | 242,Sensor - 242,5 245 | 243,Sensor - 243,5 246 | 244,Sensor - 244,5 247 | 245,Sensor - 245,288 248 | 246,Sensor - 246,288 249 | 247,Sensor - 247,288 250 | 248,Sensor - 248,556 251 | 249,Sensor - 249,0 252 | 250,Sensor - 250,0 253 | 251,Sensor - 251,0 254 | 252,Sensor - 252,0 255 | 253,Sensor - 253,0 256 | 254,Sensor - 254,24 257 | 255,Sensor - 255,0 258 | 256,Sensor - 256,0 259 | 257,Sensor - 257,3 260 | 258,Sensor - 258,3 261 | 259,Sensor - 259,3 262 | 260,Sensor - 260,3 263 | 261,Sensor - 261,3 264 | 262,Sensor - 262,3 265 | 263,Sensor - 263,3 266 | 264,Sensor - 264,3 267 | 265,Sensor - 265,3 268 | 266,Sensor - 266,3 269 | 267,Sensor - 267,3 270 | 268,Sensor - 268,3 271 | 269,Sensor - 269,3 272 | 270,Sensor - 270,3 273 | 271,Sensor - 271,0 274 | 272,Sensor - 272,1 275 | 273,Sensor - 273,3 276 | 274,Sensor - 274,13 277 | 275,Sensor - 275,13 278 | 276,Sensor - 276,13 279 | 277,Sensor - 277,13 280 | 278,Sensor - 278,13 281 | 279,Sensor - 279,8 282 | 280,Sensor - 280,2 283 | 281,Sensor - 281,2 284 | 282,Sensor - 282,2 285 | 283,Sensor - 283,2 286 | 284,Sensor - 284,2 287 | 285,Sensor - 285,3 288 | 286,Sensor - 286,3 289 | 287,Sensor - 287,3 290 | 288,Sensor - 288,3 291 | 289,Sensor - 289,3 292 | 290,Sensor - 290,3 293 | 291,Sensor - 291,0 294 | 292,Sensor - 292,0 295 | 293,Sensor - 293,770 296 | 294,Sensor - 294,770 297 | 295,Sensor - 295,1 298 | 296,Sensor - 296,1 299 | 297,Sensor - 297,1 300 | 298,Sensor - 298,1 301 | 299,Sensor - 299,1 302 | 300,Sensor - 300,1 303 | 301,Sensor - 301,1 304 | 302,Sensor - 302,1 305 | 303,Sensor - 303,1 306 | 304,Sensor - 304,1 307 | 305,Sensor - 305,1 308 | 306,Sensor - 306,0 309 | 307,Sensor - 307,0 310 | 308,Sensor - 308,0 311 | 309,Sensor - 309,0 312 | 310,Sensor - 310,0 313 | 311,Sensor - 311,0 314 | 312,Sensor - 312,0 315 | 313,Sensor - 313,0 316 | 314,Sensor - 314,14 317 | 315,Sensor - 315,14 318 | 316,Sensor - 316,0 319 | 317,Sensor - 317,0 320 | 318,Sensor - 318,0 321 | 319,Sensor - 319,0 322 | 320,Sensor - 320,0 323 | 321,Sensor - 321,0 324 | 322,Sensor - 322,0 325 | 323,Sensor - 323,0 326 | 324,Sensor - 324,0 327 | 325,Sensor - 325,0 328 | 326,Sensor - 326,0 329 | 327,Sensor - 327,1 330 | 328,Sensor - 328,1 331 | 329,Sensor - 329,1 332 | 330,Sensor - 330,1 333 | 331,Sensor - 331,1 334 | 332,Sensor - 332,1 335 | 333,Sensor - 333,5 336 | 334,Sensor - 334,5 337 | 335,Sensor - 335,5 338 | 336,Sensor - 336,5 339 | 337,Sensor - 337,5 340 | 338,Sensor - 338,5 341 | 339,Sensor - 339,5 342 | 340,Sensor - 340,5 343 | 341,Sensor - 341,5 344 | 342,Sensor - 342,5 345 | 343,Sensor - 343,5 346 | 344,Sensor - 344,5 347 | 345,Sensor - 345,5 348 | 346,Sensor - 346,529 349 | 347,Sensor - 347,529 350 | 348,Sensor - 348,5 351 | 349,Sensor - 349,24 352 | 350,Sensor - 350,24 353 | 351,Sensor - 351,24 354 | 352,Sensor - 352,24 355 | 353,Sensor - 353,24 356 | 354,Sensor - 354,24 357 | 355,Sensor - 355,24 358 | 356,Sensor - 356,24 359 | 357,Sensor - 357,1 360 | 358,Sensor - 358,2 361 | 359,Sensor - 359,610 362 | 360,Sensor - 360,0 363 | 361,Sensor - 361,0 364 | 362,Sensor - 362,0 365 | 363,Sensor - 363,51 366 | 364,Sensor - 364,51 367 | 365,Sensor - 365,5 368 | 366,Sensor - 366,2 369 | 367,Sensor - 367,2 370 | 368,Sensor - 368,5 371 | 369,Sensor - 369,5 372 | 370,Sensor - 370,5 373 | 371,Sensor - 371,5 374 | 372,Sensor - 372,5 375 | 373,Sensor - 373,5 376 | 374,Sensor - 374,5 377 | 375,Sensor - 375,5 378 | 376,Sensor - 376,5 379 | 377,Sensor - 377,2 380 | 378,Sensor - 378,2 381 | 379,Sensor - 379,5 382 | 380,Sensor - 380,5 383 | 381,Sensor - 381,5 384 | 382,Sensor - 382,5 385 | 383,Sensor - 383,288 386 | 384,Sensor - 384,288 387 | 385,Sensor - 385,288 388 | 386,Sensor - 386,556 389 | 387,Sensor - 387,0 390 | 388,Sensor - 388,0 391 | 389,Sensor - 389,0 392 | 390,Sensor - 390,0 393 | 391,Sensor - 391,0 394 | 392,Sensor - 392,24 395 | 393,Sensor - 393,0 396 | 394,Sensor - 394,0 397 | 395,Sensor - 395,3 398 | 396,Sensor - 396,3 399 | 397,Sensor - 397,3 400 | 398,Sensor - 398,3 401 | 399,Sensor - 399,3 402 | 400,Sensor - 400,3 403 | 401,Sensor - 401,3 404 | 402,Sensor - 402,3 405 | 403,Sensor - 403,3 406 | 404,Sensor - 404,3 407 | 405,Sensor - 405,3 408 | 406,Sensor - 406,3 409 | 407,Sensor - 407,3 410 | 408,Sensor - 408,3 411 | 409,Sensor - 409,0 412 | 410,Sensor - 410,1 413 | 411,Sensor - 411,3 414 | 412,Sensor - 412,13 415 | 413,Sensor - 413,13 416 | 414,Sensor - 414,13 417 | 415,Sensor - 415,13 418 | 416,Sensor - 416,13 419 | 417,Sensor - 417,8 420 | 418,Sensor - 418,2 421 | 419,Sensor - 419,2 422 | 420,Sensor - 420,2 423 | 421,Sensor - 421,2 424 | 422,Sensor - 422,2 425 | 423,Sensor - 423,3 426 | 424,Sensor - 424,3 427 | 425,Sensor - 425,3 428 | 426,Sensor - 426,3 429 | 427,Sensor - 427,3 430 | 428,Sensor - 428,3 431 | 429,Sensor - 429,0 432 | 430,Sensor - 430,0 433 | 431,Sensor - 431,1 434 | 432,Sensor - 432,1 435 | 433,Sensor - 433,1 436 | 434,Sensor - 434,1 437 | 435,Sensor - 435,1 438 | 436,Sensor - 436,1 439 | 437,Sensor - 437,1 440 | 438,Sensor - 438,1 441 | 439,Sensor - 439,1 442 | 440,Sensor - 440,1 443 | 441,Sensor - 441,1 444 | 442,Sensor - 442,0 445 | 443,Sensor - 443,0 446 | 444,Sensor - 444,0 447 | 445,Sensor - 445,0 448 | 446,Sensor - 446,0 449 | 447,Sensor - 447,0 450 | 448,Sensor - 448,0 451 | 449,Sensor - 449,0 452 | 450,Sensor - 450,14 453 | 451,Sensor - 451,14 454 | 452,Sensor - 452,0 455 | 453,Sensor - 453,0 456 | 454,Sensor - 454,0 457 | 455,Sensor - 455,0 458 | 456,Sensor - 456,0 459 | 457,Sensor - 457,0 460 | 458,Sensor - 458,0 461 | 459,Sensor - 459,0 462 | 460,Sensor - 460,0 463 | 461,Sensor - 461,0 464 | 462,Sensor - 462,0 465 | 463,Sensor - 463,1 466 | 464,Sensor - 464,1 467 | 465,Sensor - 465,1 468 | 466,Sensor - 466,1 469 | 467,Sensor - 467,1 470 | 468,Sensor - 468,1 471 | 469,Sensor - 469,5 472 | 470,Sensor - 470,5 473 | 471,Sensor - 471,5 474 | 472,Sensor - 472,5 475 | 473,Sensor - 473,5 476 | 474,Sensor - 474,5 477 | 475,Sensor - 475,5 478 | 476,Sensor - 476,5 479 | 477,Sensor - 477,5 480 | 478,Sensor - 478,5 481 | 479,Sensor - 479,5 482 | 480,Sensor - 480,5 483 | 481,Sensor - 481,5 484 | 482,Sensor - 482,5 485 | 483,Sensor - 483,24 486 | 484,Sensor - 484,24 487 | 485,Sensor - 485,24 488 | 486,Sensor - 486,24 489 | 487,Sensor - 487,24 490 | 488,Sensor - 488,24 491 | 489,Sensor - 489,24 492 | 490,Sensor - 490,24 493 | 491,Sensor - 491,1 494 | 492,Sensor - 492,2 495 | 493,Sensor - 493,610 496 | 494,Sensor - 494,0 497 | 495,Sensor - 495,0 498 | 496,Sensor - 496,0 499 | 497,Sensor - 497,51 500 | 498,Sensor - 498,51 501 | 499,Sensor - 499,5 502 | 500,Sensor - 500,2 503 | 501,Sensor - 501,2 504 | 502,Sensor - 502,5 505 | 503,Sensor - 503,5 506 | 504,Sensor - 504,5 507 | 505,Sensor - 505,5 508 | 506,Sensor - 506,5 509 | 507,Sensor - 507,5 510 | 508,Sensor - 508,5 511 | 509,Sensor - 509,5 512 | 510,Sensor - 510,5 513 | 511,Sensor - 511,2 514 | 512,Sensor - 512,2 515 | 513,Sensor - 513,5 516 | 514,Sensor - 514,5 517 | 515,Sensor - 515,5 518 | 516,Sensor - 516,5 519 | 517,Sensor - 517,288 520 | 518,Sensor - 518,288 521 | 519,Sensor - 519,288 522 | 520,Sensor - 520,556 523 | 521,Sensor - 521,0 524 | 522,Sensor - 522,0 525 | 523,Sensor - 523,0 526 | 524,Sensor - 524,0 527 | 525,Sensor - 525,0 528 | 526,Sensor - 526,24 529 | 527,Sensor - 527,0 530 | 528,Sensor - 528,0 531 | 529,Sensor - 529,3 532 | 530,Sensor - 530,3 533 | 531,Sensor - 531,3 534 | 532,Sensor - 532,3 535 | 533,Sensor - 533,3 536 | 534,Sensor - 534,3 537 | 535,Sensor - 535,3 538 | 536,Sensor - 536,3 539 | 537,Sensor - 537,3 540 | 538,Sensor - 538,3 541 | 539,Sensor - 539,3 542 | 540,Sensor - 540,3 543 | 541,Sensor - 541,3 544 | 542,Sensor - 542,3 545 | 543,Sensor - 543,2 546 | 544,Sensor - 544,2 547 | 545,Sensor - 545,2 548 | 546,Sensor - 546,2 549 | 547,Sensor - 547,73 550 | 548,Sensor - 548,73 551 | 549,Sensor - 549,73 552 | 550,Sensor - 550,73 553 | 551,Sensor - 551,73 554 | 552,Sensor - 552,73 555 | 553,Sensor - 553,73 556 | 554,Sensor - 554,73 557 | 555,Sensor - 555,73 558 | 556,Sensor - 556,73 559 | 557,Sensor - 557,73 560 | 558,Sensor - 558,73 561 | 559,Sensor - 559,0 562 | 560,Sensor - 560,0 563 | 561,Sensor - 561,0 564 | 562,Sensor - 562,0 565 | 563,Sensor - 563,15 566 | 564,Sensor - 564,15 567 | 565,Sensor - 565,15 568 | 566,Sensor - 566,15 569 | 567,Sensor - 567,15 570 | 568,Sensor - 568,15 571 | 569,Sensor - 569,15 572 | 570,Sensor - 570,15 573 | 571,Sensor - 571,0 574 | 572,Sensor - 572,0 575 | 573,Sensor - 573,0 576 | 574,Sensor - 574,0 577 | 575,Sensor - 575,0 578 | 576,Sensor - 576,0 579 | 577,Sensor - 577,0 580 | 578,Sensor - 578,0 581 | 579,Sensor - 579,502 582 | 580,Sensor - 580,502 583 | 581,Sensor - 581,502 584 | 582,Sensor - 582,502 585 | 583,Sensor - 583,0 586 | 584,Sensor - 584,0 587 | 585,Sensor - 585,0 588 | 586,Sensor - 586,0 589 | 587,Sensor - 587,0 590 | 588,Sensor - 588,0 591 | 589,Sensor - 589,0 592 | 590,Sensor - 590,0 593 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | attrs==19.3.0 2 | certifi==2019.11.28 3 | Click==7.0 4 | cycler==0.10.0 5 | Flask==1.1.1 6 | Flask-Cors==3.0.8 7 | importlib-metadata==1.4.0 8 | Flask-MonitoringDashboard==3.0.6 9 | itsdangerous==1.1.0 10 | Jinja2==2.11.0 11 | joblib==0.14.1 12 | jsonschema==3.2.0 13 | kiwisolver==1.1.0 14 | kneed==0.5.1 15 | MarkupSafe==1.1.1 16 | matplotlib==3.1.2 17 | more-itertools==8.1.0 18 | numpy==1.18.1 19 | pandas==0.25.3 20 | pyparsing==2.4.6 21 | pyrsistent==0.15.7 22 | python-dateutil==2.8.1 23 | pytz==2019.3 24 | PyYAML==5.3 25 | regexp==0.1 26 | scikit-learn==0.22.1 27 | six==1.14.0 28 | sklearn==0.0 29 | Werkzeug==0.16.1 30 | wincertstore==0.2 31 | xgboost==0.90 32 | zipp==2.0.1 33 | pytest==6.2.2 34 | -e . -------------------------------------------------------------------------------- /runtime.txt: -------------------------------------------------------------------------------- 1 | python-3.6.9 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | setup( 4 | name="wafer-fault-detection", 5 | version="0.0.3", 6 | description="ML project", 7 | author="Avnish yadav", 8 | packages=find_packages(), 9 | license="MIT" 10 | ) -------------------------------------------------------------------------------- /templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Prediction Page 5 | 6 | 7 | 9 | 10 | 11 | 110 | 111 | 112 | 113 | 114 | 115 | 120 | 121 | 122 |
123 |
124 |

Wafer Fault Detection

125 |
126 | Train Wafer Fault Detection 127 |
128 |
129 |
130 |
131 |
132 | 133 |
134 |
135 |
Or

136 | 139 |
140 |
141 |
142 |
143 |
Results
144 |
145 | 146 |
147 |
148 |
149 |
150 |
151 |
152 | 153 | 154 | 170 | 171 | 172 |
173 | 174 | 175 | 176 | 177 | 213 | 214 | 215 | 216 | -------------------------------------------------------------------------------- /testing.py: -------------------------------------------------------------------------------- 1 | if __name__ == "__main__": 2 | 3 | for i in range(1, 6): 4 | accept = [4 + 2 * num for num in range(1, i + 1)] 5 | print(accept) 6 | for j in range(i, i + 10): 7 | if j in accept: 8 | print("*", end=" ") 9 | else: 10 | print(" ", end=" ") 11 | print(end="\n") 12 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/tests/__init__.py -------------------------------------------------------------------------------- /tests/__pycache__/test_script.cpython-37-pytest-6.2.5.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avnyadav/wafer_circleci/2d17bd316ea1d78ee4dfd4da700acc3315024d29/tests/__pycache__/test_script.cpython-37-pytest-6.2.5.pyc -------------------------------------------------------------------------------- /tests/test_script.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from main import app 4 | import os 5 | 6 | 7 | class TestToPerform(unittest.TestCase): 8 | def setUp(self): 9 | self.app = app.test_client() 10 | 11 | def tearDown(self): 12 | pass 13 | 14 | def test_page(self): 15 | response = self.app.get('/', follow_redirects=True) 16 | print(response) 17 | self.assertEqual(response.status_code, 200) 18 | 19 | 20 | if __name__ == '__main__': 21 | unittest.main() 22 | -------------------------------------------------------------------------------- /trainingModel.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is the Entry point for Training the Machine Learning Model. 3 | 4 | Written By: iNeuron Intelligence 5 | Version: 1.0 6 | Revisions: None 7 | 8 | """ 9 | 10 | 11 | # Doing the necessary imports 12 | from sklearn.model_selection import train_test_split 13 | from data_ingestion import data_loader 14 | from data_preprocessing import preprocessing 15 | from data_preprocessing import clustering 16 | from best_model_finder import tuner 17 | from file_operations import file_methods 18 | from application_logging import logger 19 | 20 | #Creating the common Logging object 21 | 22 | 23 | class trainModel: 24 | 25 | def __init__(self): 26 | self.log_writer = logger.App_Logger() 27 | self.file_object = open("Training_Logs/ModelTrainingLog.txt", 'a+') 28 | 29 | def trainingModel(self): 30 | # Logging the start of Training 31 | self.log_writer.log(self.file_object, 'Start of Training') 32 | try: 33 | # Getting the data from the source 34 | data_getter=data_loader.Data_Getter(self.file_object,self.log_writer) 35 | data=data_getter.get_data() 36 | 37 | 38 | """doing the data preprocessing""" 39 | 40 | preprocessor=preprocessing.Preprocessor(self.file_object,self.log_writer) 41 | data=preprocessor.remove_columns(data,['Wafer']) # remove the unnamed column as it doesn't contribute to prediction. 42 | 43 | # create separate features and labels 44 | X,Y=preprocessor.separate_label_feature(data,label_column_name='Output') 45 | 46 | # check if missing values are present in the dataset 47 | is_null_present=preprocessor.is_null_present(X) 48 | 49 | # if missing values are there, replace them appropriately. 50 | if(is_null_present): 51 | X=preprocessor.impute_missing_values(X) # missing value imputation 52 | 53 | # check further which columns do not contribute to predictions 54 | # if the standard deviation for a column is zero, it means that the column has constant values 55 | # and they are giving the same output both for good and bad sensors 56 | # prepare the list of such columns to drop 57 | cols_to_drop=preprocessor.get_columns_with_zero_std_deviation(X) 58 | 59 | # drop the columns obtained above 60 | X=preprocessor.remove_columns(X,cols_to_drop) 61 | 62 | """ Applying the clustering approach""" 63 | 64 | kmeans=clustering.KMeansClustering(self.file_object,self.log_writer) # object initialization. 65 | number_of_clusters=kmeans.elbow_plot(X) # using the elbow plot to find the number of optimum clusters 66 | 67 | # Divide the data into clusters 68 | X=kmeans.create_clusters(X,number_of_clusters) 69 | 70 | #create a new column in the dataset consisting of the corresponding cluster assignments. 71 | X['Labels']=Y 72 | 73 | # getting the unique clusters from our dataset 74 | list_of_clusters=X['Cluster'].unique() 75 | 76 | """parsing all the clusters and looking for the best ML algorithm to fit on individual cluster""" 77 | 78 | for i in list_of_clusters: 79 | cluster_data=X[X['Cluster']==i] # filter the data for one cluster 80 | 81 | # Prepare the feature and Label columns 82 | cluster_features=cluster_data.drop(['Labels','Cluster'],axis=1) 83 | cluster_label= cluster_data['Labels'] 84 | 85 | # splitting the data into training and test set for each cluster one by one 86 | x_train, x_test, y_train, y_test = train_test_split(cluster_features, cluster_label, test_size=1 / 3, random_state=355) 87 | 88 | model_finder=tuner.Model_Finder(self.file_object,self.log_writer) # object initialization 89 | 90 | #getting the best model for each of the clusters 91 | best_model_name,best_model=model_finder.get_best_model(x_train,y_train,x_test,y_test) 92 | 93 | #saving the best model to the directory. 94 | file_op = file_methods.File_Operation(self.file_object,self.log_writer) 95 | save_model=file_op.save_model(best_model,best_model_name+str(i)) 96 | 97 | # logging the successful Training 98 | self.log_writer.log(self.file_object, 'Successful End of Training') 99 | self.file_object.close() 100 | 101 | except Exception: 102 | # logging the unsuccessful Training 103 | self.log_writer.log(self.file_object, 'Unsuccessful End of Training') 104 | self.file_object.close() 105 | raise Exception -------------------------------------------------------------------------------- /training_Validation_Insertion.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from Training_Raw_data_validation.rawValidation import Raw_Data_validation 3 | from DataTypeValidation_Insertion_Training.DataTypeValidation import dBOperation 4 | from DataTransform_Training.DataTransformation import dataTransform 5 | from application_logging import logger 6 | 7 | class train_validation: 8 | def __init__(self,path): 9 | self.raw_data = Raw_Data_validation(path) 10 | self.dataTransform = dataTransform() 11 | self.dBOperation = dBOperation() 12 | self.file_object = open("Training_Logs/Training_Main_Log.txt", 'a+') 13 | self.log_writer = logger.App_Logger() 14 | 15 | def train_validation(self): 16 | try: 17 | self.log_writer.log(self.file_object, 'Start of Validation on files!!') 18 | # extracting values from prediction schema 19 | LengthOfDateStampInFile, LengthOfTimeStampInFile, column_names, noofcolumns = self.raw_data.valuesFromSchema() 20 | # getting the regex defined to validate filename 21 | regex = self.raw_data.manualRegexCreation() 22 | # validating filename of prediction files 23 | self.raw_data.validationFileNameRaw(regex, LengthOfDateStampInFile, LengthOfTimeStampInFile) 24 | # validating column length in the file 25 | self.raw_data.validateColumnLength(noofcolumns) 26 | # validating if any column has all values missing 27 | self.raw_data.validateMissingValuesInWholeColumn() 28 | self.log_writer.log(self.file_object, "Raw Data Validation Complete!!") 29 | 30 | self.log_writer.log(self.file_object, "Starting Data Transforamtion!!") 31 | # replacing blanks in the csv file with "Null" values to insert in table 32 | self.dataTransform.replaceMissingWithNull() 33 | 34 | self.log_writer.log(self.file_object, "DataTransformation Completed!!!") 35 | 36 | self.log_writer.log(self.file_object, 37 | "Creating Training_Database and tables on the basis of given schema!!!") 38 | # create database with given name, if present open the connection! Create table with columns given in schema 39 | self.dBOperation.createTableDb('Training', column_names) 40 | self.log_writer.log(self.file_object, "Table creation Completed!!") 41 | self.log_writer.log(self.file_object, "Insertion of Data into Table started!!!!") 42 | # insert csv files in the table 43 | self.dBOperation.insertIntoTableGoodData('Training') 44 | self.log_writer.log(self.file_object, "Insertion in Table completed!!!") 45 | self.log_writer.log(self.file_object, "Deleting Good Data Folder!!!") 46 | # Delete the good data folder after loading files in table 47 | self.raw_data.deleteExistingGoodDataTrainingFolder() 48 | self.log_writer.log(self.file_object, "Good_Data folder deleted!!!") 49 | self.log_writer.log(self.file_object, "Moving bad files to Archive and deleting Bad_Data folder!!!") 50 | # Move the bad files to archive folder 51 | self.raw_data.moveBadFilesToArchiveBad() 52 | self.log_writer.log(self.file_object, "Bad files moved to archive!! Bad folder Deleted!!") 53 | self.log_writer.log(self.file_object, "Validation Operation completed!!") 54 | self.log_writer.log(self.file_object, "Extracting csv file from table") 55 | # export data in table to csvfile 56 | self.dBOperation.selectingDatafromtableintocsv('Training') 57 | self.file_object.close() 58 | 59 | except Exception as e: 60 | raise e 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | --------------------------------------------------------------------------------