├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── config └── gspread │ └── service.json ├── data ├── data.csv └── data.json ├── db ├── __pycache__ │ ├── db.cpython-38.pyc │ └── optasia_db.cpython-38.pyc ├── db.db └── db.py ├── docker-compose.yml ├── features ├── features_customers.csv └── features_loans.csv ├── images └── fastapi.png ├── logs └── logs.log ├── requirements.txt ├── src ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-38.pyc │ ├── config.cpython-38.pyc │ ├── data_loading.cpython-38.pyc │ ├── data_reporting.cpython-38.pyc │ ├── feature_engineering.cpython-38.pyc │ ├── main.cpython-38.pyc │ ├── markI.cpython-38.pyc │ └── models.cpython-38.pyc ├── config.ini ├── config.py ├── data_loading.py ├── data_reporting.py ├── data_visualization.py ├── feature_engineering.py ├── main.py ├── markI.py └── models.py └── tests └── test_api.py /Dockerfile: -------------------------------------------------------------------------------- 1 | # pull the official docker image 2 | FROM python:3.8-slim 3 | #FROM tiangolo/uvicorn-gunicorn-fastapi:python3.9 4 | 5 | # set work directory 6 | WORKDIR /spp 7 | 8 | # set env variables 9 | ENV PYTHONDONTWRITEBYTECODE 1 10 | ENV PYTHONUNBUFFERED 1 11 | 12 | # install dependencies 13 | #COPY ./requirements.txt /src/requirements.txt 14 | COPY requirements.txt ./ 15 | RUN pip install --no-cache-dir --upgrade -r requirements.txt 16 | 17 | # copy project 18 | #COPY ./src /src 19 | COPY . . 20 | EXPOSE 8000 21 | 22 | 23 | #COPY ./requirements.txt /code/requirements.txt 24 | #RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt 25 | #COPY ./src /code/src 26 | CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "80"] 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Dimitrios Georgiou 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ### Compose shortcuts 2 | up: 3 | docker-compose up -d 4 | 5 | down: 6 | docker-compose down 7 | 8 | build: 9 | docker-compose build 10 | 11 | sh: 12 | docker-compose run -p 8000:8000 --rm api bash 13 | 14 | logs: 15 | docker-compose logs -f 16 | 17 | ### Project shortcuts 18 | fast_api: 19 | docker-compose run --rm api python src/main.py 20 | 21 | fast_api_app: 22 | docker-compose run --rm api uvicorn src.main:app --proxy-headers --host 0.0.0.0 --port 8000 23 | 24 | 25 | ### Network shortcuts 26 | fast_api_net_app : 27 | docker run --rm --name app --net=fast_api_net -p 8000:8000 fastapi_example 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Feature Engineering API for ML purposes 2 | The objective of this **project** is to create a simple feature engineering module that generates some features/variables that are useful for further Machine & Deep Learning modeling. The python module (which is dockerized) is called via an API endpoint, returning some features/variables. 11 more endpoints were built to support the functionality 3 | 4 | 5 | 6 | ## 🚀 TECH STACK 7 | 8 | For project the following tech stack, APIs, architecture was used and applied: 9 | 10 | * Python✅ 11 | * FastAPI ✅ 12 | * Docker Compose ✅ 13 | * Logger ✅ 14 | * Mark1 System Design + Configuration ✅ 15 | * Spreadsheet API (through gspread wrapper) ✅ 16 | 17 | 18 | 19 | 20 | ## 📁 FILES HIERARCHY 21 | 22 | ``` 23 | api 24 | │ requirements.txt 25 | │ Dockerfile 26 | │ docker-compose.yml 27 | │ 28 | └───src 29 | │ │ __init__.py 30 | │ │ config.ini 31 | │ │ config.py 32 | │ │ data_loading.py 33 | │ │ feature_engineering.py 34 | │ │ data_reporting.py 35 | │ │ main.py 36 | │ │ models.py 37 | │ └─── markI.py 38 | │ 39 | └───data 40 | │ │ data.csv 41 | │ └─── data.json 42 | │ 43 | └───features 44 | │ │ features_loans.csv 45 | │ └─── features_customers.csv 46 | │ 47 | └───db 48 | │ │ db.db 49 | │ └─── db.py 50 | │ 51 | └───config 52 | └─── gspread 53 | └─── service.json 54 | ``` 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | ## 📌 ENDPOINTS 65 | 66 | Please feel free to hit all our endpoints by visiting http://localhost:8000/docs#/. The list of the endpoints is the following 67 | 68 | ![Alt text](https://github.com/jimmyg1997/ml-feature-engineering-fastapi-docker/blob/main/images/fastapi.png?raw=true "Feature Engineering API") 69 | 70 | 71 | 72 | #### Endpoints for features 73 | * ```[GET] /api/v1/features/{ontology}``` : The basic endpoint which returns all the features generated after applying the feature engineering analysis. The permitted ontologies here are (a) customers (b) loans. In the project requirements we are asked to create the endpoint only for customers, however we expanded our work. This permit us eg. to create a machine learning model that is dedicated to loans analysis, so having features in a loan-based level may be proven useful 74 | * ```[GET] /api/v1/api_status``` : The endpoint which returns {“status” : “UP”} if both our basic endpoints /api/v1/features/customers and /api/v1/features/loans return a status code of 200 after being hit 75 | 76 | #### Endpoints for loans 77 | * ```[GET] /api/v1/loans/``` : This endpoint is used when we need to fetch the list of loans currently existing 78 | * ```[GET] /api/v1/loans/{loand_id}``` : This endpoint is used when we need to fetch a specific loan currently existing 79 | * ```[POST] /api/v1/loans/``` : This endpoint can be used to register a new loan (for a specific customer) 80 | * ```[DELETE] /api/v1/loans/{loan_id}``` : This endpoint can be used to delete a specific loan 81 | 82 | #### Endpoints for customers 83 | * ```[GET] /api/v1/customers/``` : This endpoint is used when we need to fetch the list of customers currently existing 84 | * ```[GET] /api/v1/customers/{customer_id}``` : This endpoint is used when we need to fetch a specific customer currently existing 85 | * ```[POST] /api/v1/customers/``` : This endpoint can be used to register a new customer 86 | * ```[DELETE] /api/v1/customers/{customer_id}``` : This endpoint can be used to delete a specific customer 87 | 88 | 89 | #### Endpoints for the local db 90 | * ```[GET] /api/v1/database``` : Fetches the list of customers defined in the ./db/optasia_db.py file, creates local dbs, serialises the data and pushes them into these 2 new data tables (of the same ./db/optasia_db.db database) 91 | * ```[POST] /api/v1/database/{db_name}``` : This endpoint is used to clear (recreate an empty data table) for a specific datatable options are (customers, loans) 92 | 93 | 94 | 95 | 96 | 97 | ## 1️⃣ UNIT TESTING 98 | 99 | In order for the unit testing to be successful we moved from a 100 | 101 | * *json loading* logic to a dual 102 | * *json loading* and *local db loading* logic, so that the code is not hard coded only for the provided given data. 103 | 104 | 105 | Therefore, during the unit testing (all the tests can be found under **/tests/test_api.py**) we can now create *customer* and *loan* objects that are properly pushed to a local db (2 respective data tables). All the endpoints retrieve the necessary data from the local db (which is smoothly and systematically updated), except for the **[POST] /api/v1/features/{ontology}** endpoint (which works smoothly for both *json* and *local db* cases) 106 | 107 | 108 | 109 | 110 | 111 | ## 📋 SPREADSHEET 112 | 113 | In the spreadsheet [Feature Engineering API Reporter](https://docs.google.com/spreadsheets/d/1iIBuignJj5oPW7NDRlirenACJuEEeV2RZOi5X9NSH7A/edit#gid=822346508) several tabs can be found and all are split into 2 clusters 114 | 115 | * *api analysis tabs* (1) `customers_features` (2) `loans_features` in which the features generated by the feature engineering process are pushed here 116 | * *custom analysis tabs* : Firstly, we can push (manually, or automatically) data to the following tabs : (1) `customers_raw` and (2) `loans_raw` tabs. Then we calculated some statistics (normal distribution) for some of the features at (3) `customers_statistics` and (4) `loans_statistics` tabs and finally we gathered all the visualisations under the tab called `visualisations` for further inspection. This approach is considered scalable and is useful when tech people should provide business insights to their stakeholders. 117 | 118 | 119 | 120 | 121 | ## 👇 RECOMMENDED USAGE 122 | 123 | #### Setup Python Virtual Environment 124 | 1. apt-get update 125 | 2. sudo apt install python3-pip 126 | 3. download the [installer](https://docs.conda.io/en/latest/miniconda.html#linux-installers) for your python version 127 | (run "python -V" or "python3 -V" to see your version) 128 | 4. run the installer and follow instructions 129 | 5. source ~/.bashrc #or restart shell 130 | 6. conda install python=3.8 131 | 7. conda create --name featengapi python=3.8 132 | 8. conda activate featengapi 133 | 134 | #### Export PYTHONPATH 135 | export PYTHONPATH="${PYTHONPATH}:/path/to/your/project/" 136 | 137 | #### Deploy with docker compose 138 | Execute the command 139 | 140 | ```$ docker-compose up --build``` 141 | 142 | After the application starts, navigate to http://localhost:8000 in your web browser and you should see the following json response: 143 | 144 | ``` 145 | { 146 | "Hello": "World" 147 | } 148 | ``` 149 | 150 | 151 | #### Hit the Endpoints 152 | Visit http://localhost:8000/docs#/. 153 | 154 | 155 | 1. Hit endpoint ```[POST] /api/v1/database/customers``` with arguments (customers, customer_id) to flush the local db “customers” 156 | 2. Hit endpoint ```[POST] /api/v1/database/loans``` with arguments (loans, loan_id) to flush the local db “loans” 157 | 3. Hit endpoint ```[GET] /api/v1/api_status``` to validate that both [GET] /api/v1/features/customers and [GET] /api/v1/features/loans are running smoothly 158 | 4. Hit endpoint ```[GET] /api/v1/features/customers``` to retrieve the features of customers in json format 159 | 5. Hit endpoint ```[GET] /api/v1/features/loans``` to retrieve the features in json format 160 | 8. Hit endpoint ```[POST] /api/v1/features/customers``` to upload the customers features on the Optasia API Reporter 161 | 9. Hit endpoint ```[POST] /api/v1/features/loans``` to upload the loans features on the Optasia API Reporter 162 | 163 | 164 | #### Stop and remove the containers 165 | 166 | ```$ docker-compose down``` 167 | 168 | 169 | ## 🔮 FUTURE WORK 170 | 171 | Feature Work would be 172 | 173 | * To build more endpoints 174 | * To extract more **features** manually 175 | * To build **baseline machine learning models** and extract **feature importances* (eg. through the use of the LIME algorithm) 176 | * To optimise **machine learning models* by experimenting (by properly storing the experiment configurations and results eg. Spreadsheet) 177 | * To optimise **deep learning models** 178 | -------------------------------------------------------------------------------- /config/gspread/service.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "service_account", 3 | "project_id": "feature-engineering-api", 4 | "private_key_id": "88f1cc121cf6690beba04ab9edff9e0e677c9acd", 5 | "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCbM/ogt3RTgQsN\nJYvqgh5db4UPQHDydTfh0dbdJm2+8vcVFQqosnNy7DRRlnANRNhqkxDhiYN7igVb\n3U+Gm6pOkFN++wxL3qMAscz5WNZh22d9/EgGnJl7T5pa9AHzBDKmb4OQv3fUizxS\n4ovpS2F0RYikxCfnUT3MtoNz7coKZDhtz0Sj5MRVTvAReAZJlcUWknGvtAvbLzNv\nNCsAm0/Xs0OqZzAHFJFWMpSedWiYp9z4EaV0vTA+mk2rzy7GMW4sf5VLeNwety4C\niiRY9mgbSLprwvP01kEM8dPsJs83wkiEqJtTEorCdOWkAGQ3+Vs1NNS+2rzgd2Rb\n7SXU+huvAgMBAAECggEAG3Dzu0AkkbUhwpfOZs7N1qBmqt0uezgNguBquJxYUCg+\nVpxrT4uJzRqFX52VP0taZnGNTQtW04uCSg1Vx8uTL6PmF3AE6w84y+EkCDxUX2X/\nJO/rvjuHCZ34ATfedlzaJcJWxUuNQOfDaTZEHaIl33yB3fwOwCAFqMHmeSNkkzSB\npg5na2QB7I95YJlQaevhW4MPpBWq1ob6YgIamwUozILtmG3ygJVEnv9+PpU42WBt\n4MpMskR30voAzTnH2EPAZcw1YFPVk0sCg0lKPXPBTYe+jvtgVAx3dizBLq7/KrOD\n6pzCeeWaDN+fnS/7f6GYtm1MnKEcDHNOFsnLDI9PjQKBgQDJV0omOVmX23BoeVHi\nmqYRRwdQbPlVkqSStRA4Eo3fDGEBuXKQbHd6faTOVA0+Pn2a1ld4Enm5s8eKFFKe\nR/YrBf3ds+JEV9UqISCiIvAG/FEh0kb5lk1P6/3Wme1nYLs7tS27Z6+NpR7UZomP\nbYA/1v0rEWkcd78nPKH9daUqEwKBgQDFVjZY+kURAI9iOQkN0T1rKThHVZb6i7Ul\nudTX3+KbUaxqc5QAkYcMFLLoPAlKHpMgiOst6Z9Pzcd9lfMGOXcySBAzXbj+VCc5\nnVlCQhZyPI80rhOBZubNhemK9ZXVg+CbI+z7XcUxOI/LyNs9Hxu/DYjMMvTl3g7A\n0LCqona7dQKBgF3TxOVNQphHFfpJ10Qfx5YzoI/1SLh9Lx6DLHzFAMJaFXblQiNV\nuNQnaz8P4mAIZSZcDsUQ7xMAWdPaJqpGZMyl7Sl9I8uvltIcQAXu/CZ1PSXixVqY\nlYzSMCfq8byEUpy/kQFNj30V3ukio6ZoZCTb/n0sjuxKTZVuZxsj0QVrAoGAJ+G1\nmCP0Y2xGjiZSwx0fmZG6EwikO7ySfq5g6sQPqy5FdzomljrTHUjto2vGwNkPCQtC\n5IOLs/g1shblB8QDaAESbR8q7ROa3dUEHhT3Ll043/+yZ3uhd7XxrijWAFHnydLn\n6SFZYgo5XYkZbe/kjqcwl3xQ7EgFmUhT/BhbgKUCgYEAvEuDtJ55mwSOlaTo0IFw\nvF0HoEzKjR9+t3RQZ0ebiLMm6QILyPH8Y0Otw12cWv84EXbbSNJUpXV9sP2F3Jag\nF6vXgXenlrKD/ao/OXmoINsTC+VzI/m10Cp1tQ1/O8sk5g8hiMEV4g/euI7Hp4yY\nZWONR+c6LIV2M5tP502ozLs=\n-----END PRIVATE KEY-----\n", 6 | "client_email": "feature-engineering-api@feature-engineering-api.iam.gserviceaccount.com", 7 | "client_id": "105352353176093867534", 8 | "auth_uri": "https://accounts.google.com/o/oauth2/auth", 9 | "token_uri": "https://oauth2.googleapis.com/token", 10 | "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", 11 | "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/feature-engineering-api%40feature-engineering-api.iam.gserviceaccount.com" 12 | } 13 | -------------------------------------------------------------------------------- /data/data.csv: -------------------------------------------------------------------------------- 1 | customer_ID,loan_date,amount,fee,loan_status,term,annual_income 2 | 124,23/06/2021,785,173,0,short,69246 3 | 125,2/3/2020,2626,124,0,short,77110 4 | 296,17/08/2020,2003,24,0,short,41557 5 | 447,18/12/2019,438,168,1,long,62868 6 | 451,9/1/2020,284,169,1,short,51834 7 | 467,9/10/2020,2995,140,0,long,69972 8 | 467,26/04/2019,1124,192,0,short,69972 9 | 467,14/10/2021,1871,189,1,short,69972 10 | 666,18/07/2021,1687,189,1,long,42456 11 | 675,15/09/2019,826,53,0,long,32965 12 | 675,31/08/2021,1425,199,1,long,32965 13 | 675,27/01/2021,2678,100,0,long,32965 14 | 691,9/6/2020,188,80,0,long,44557 15 | 800,8/4/2019,1691,22,0,long,49583 16 | 829,29/07/2019,2905,54,0,short,39613 17 | 853,23/12/2021,1603,139,0,long,79823 18 | 870,3/9/2021,2892,46,0,long,54335 19 | 919,18/10/2019,453,62,1,long,79307 20 | 919,11/1/2019,1306,24,1,long,79307 21 | 926,20/01/2022,926,35,1,short,56935 22 | 944,26/05/2021,1039,104,1,long,31991 23 | 1090,15/11/2021,2426,199,0,long,41333 24 | 1159,20/12/2021,2581,117,1,short,46330 25 | 1241,1/9/2021,2888,75,0,short,36275 26 | 1265,31/01/2019,2080,108,0,short,43882 27 | 1265,31/08/2020,2064,114,1,short,43882 28 | 1265,23/09/2020,424,67,0,long,43882 29 | 1280,20/05/2020,2261,180,1,long,75523 30 | 1280,1/5/2019,385,72,1,long,75523 31 | 1361,28/09/2020,1840,176,0,short,47342 32 | 1361,4/1/2022,2821,154,1,short,47342 33 | 1372,30/11/2021,2096,158,0,short,74938 34 | 1394,14/12/2020,2308,65,0,short,31704 35 | 1436,16/02/2020,1145,182,1,long,58256 36 | 1436,18/11/2019,2700,186,0,short,58256 37 | 1512,29/03/2021,2114,75,1,short,67215 38 | 1512,24/10/2019,210,78,0,long,67215 39 | 1528,7/2/2021,1032,156,0,short,34023 40 | 1572,4/5/2021,2569,108,1,short,74816 41 | 1669,16/07/2021,458,173,1,short,72881 42 | 1746,30/06/2021,1659,33,1,short,59171 43 | 1808,29/08/2021,114,96,1,long,62299 44 | 1808,26/12/2021,107,28,1,long,62299 45 | 1870,28/06/2020,643,96,0,short,59798 46 | 1903,23/10/2021,1581,180,0,long,50325 47 | 1953,27/06/2021,2204,113,1,long,78787 48 | 2034,31/05/2020,105,29,1,short,74897 49 | 2034,22/09/2021,1929,162,1,short,74897 50 | 2034,4/1/2022,1444,146,0,short,74897 51 | 2034,4/12/2019,1718,124,1,short,74897 52 | 2069,2/1/2021,615,131,0,short,58477 53 | 2088,18/07/2020,860,176,1,short,31645 54 | 2088,28/04/2020,1945,33,1,short,31645 55 | 2110,3/5/2021,404,52,0,short,72789 56 | 2136,29/04/2019,2573,81,1,long,62157 57 | 2136,30/04/2019,475,160,0,long,62157 58 | 2162,18/11/2021,174,26,0,short,74546 59 | 2162,5/1/2022,2344,127,0,long,74546 60 | 2162,6/7/2019,1738,188,0,long,74546 61 | 2275,5/7/2019,256,50,1,long,62422 62 | 2294,4/1/2020,2304,125,1,short,45001 63 | 2393,28/01/2020,2291,73,0,long,71526 64 | 2393,12/4/2021,2861,89,0,long,71526 65 | 2393,13/11/2019,2514,22,0,short,71526 66 | 2464,15/05/2021,2478,171,1,long,44628 67 | 2530,17/06/2019,2222,106,1,long,69700 68 | 2550,3/6/2019,2783,36,0,long,57268 69 | 2696,17/05/2021,2490,78,0,short,35578 70 | 2696,27/02/2021,1169,92,0,long,35578 71 | 2696,11/2/2020,1504,179,1,short,35578 72 | 2696,4/7/2020,2551,79,0,long,35578 73 | 2696,23/04/2021,608,188,1,short,35578 74 | 2700,28/05/2021,599,187,0,long,76092 75 | 2725,26/05/2019,2186,135,1,short,50873 76 | 2756,11/7/2020,397,157,0,short,46157 77 | 2756,25/09/2020,326,65,0,long,46157 78 | 2888,15/04/2021,906,20,1,short,68731 79 | 2888,24/07/2021,877,159,0,short,68731 80 | 2895,2/4/2019,2174,165,1,short,57689 81 | 2898,12/2/2020,2270,85,0,long,56076 82 | 2904,14/01/2022,2396,108,0,short,63914 83 | 2904,21/03/2021,2553,124,0,long,63914 84 | 2917,22/10/2021,994,125,1,short,54949 85 | 2930,21/05/2020,2995,80,1,short,62154 86 | 3015,4/7/2019,417,48,1,long,62823 87 | 3121,13/06/2019,1868,49,1,long,48384 88 | 3200,20/02/2020,615,42,0,long,73510 89 | 3210,10/11/2019,2961,66,1,long,60852 90 | 3249,5/4/2019,2905,182,0,long,62431 91 | 3266,19/03/2021,1694,177,1,short,51619 92 | 3281,2/3/2019,309,89,1,long,75974 93 | 3312,29/05/2020,543,66,1,long,44831 94 | 3368,3/7/2019,1067,55,0,short,35537 95 | 3414,25/02/2019,2644,106,0,short,52362 96 | 3441,3/12/2019,794,163,0,short,37213 97 | 3517,2/2/2019,1716,76,0,long,55092 98 | 3526,17/02/2019,842,135,1,long,46000 99 | 3565,7/3/2021,2153,53,0,short,76498 100 | 3565,6/8/2021,1538,89,1,long,76498 101 | 3565,30/09/2021,2997,24,0,short,76498 102 | 3565,9/3/2021,2184,38,0,long,76498 103 | 3565,7/7/2020,2707,42,1,short,76498 104 | 3565,17/08/2019,2219,104,0,short,76498 105 | 3565,26/10/2021,207,146,1,short,76498 106 | 3606,9/3/2020,2832,37,0,short,63707 107 | 3606,11/7/2019,189,30,1,short,63707 108 | 3606,4/6/2020,481,174,0,short,63707 109 | 3650,16/09/2019,2972,52,1,short,35611 110 | 3652,15/03/2020,1148,66,1,long,79510 111 | 3826,1/1/2022,1628,96,1,long,63146 112 | 3836,4/10/2019,1488,44,1,short,53107 113 | 3838,19/02/2021,2856,109,0,long,66824 114 | 3858,24/01/2022,671,154,0,long,39885 115 | 3949,21/08/2019,613,149,0,long,53961 116 | 3949,6/5/2021,2692,144,0,long,53961 117 | 3959,26/11/2021,1076,125,1,short,52358 118 | 3959,27/09/2019,2941,178,0,short,52358 119 | 3959,17/05/2021,2955,163,0,short,52358 120 | 4032,27/11/2019,2361,144,0,short,63118 121 | 4127,31/01/2020,2949,200,1,short,71869 122 | 4236,19/10/2020,445,108,0,long,66364 123 | 4249,3/10/2020,396,137,0,short,70433 124 | 4276,18/03/2021,1898,39,0,long,50915 125 | 4276,5/6/2019,2243,161,1,short,50915 126 | 4276,30/06/2020,1899,116,0,short,50915 127 | 4276,20/09/2020,1601,93,0,long,50915 128 | 4276,13/06/2020,1858,122,0,long,50915 129 | 4276,9/4/2019,884,125,0,short,50915 130 | 4583,6/6/2020,1672,73,1,short,64470 131 | 4665,26/09/2019,2555,26,1,long,33923 132 | 4821,24/02/2021,1968,92,0,short,46451 133 | 4821,4/12/2021,1625,157,1,short,46451 134 | 4821,23/10/2021,1349,129,0,short,46451 135 | 4821,12/10/2019,885,129,1,short,46451 136 | 4838,8/7/2021,626,27,0,long,47472 137 | 4850,17/03/2021,2653,177,1,short,34077 138 | 4850,22/06/2020,842,167,1,short,34077 139 | 4850,10/4/2020,1147,122,1,short,34077 140 | 4850,18/04/2021,1746,122,1,long,34077 141 | 4894,12/7/2020,2167,97,0,short,66009 142 | 4937,22/11/2019,2774,74,1,short,60727 143 | 4975,18/05/2019,2168,27,0,long,49801 144 | 4988,7/11/2021,1586,200,1,long,55708 145 | 5023,30/10/2021,1570,157,1,short,60842 146 | 5044,9/10/2021,1135,152,1,long,73236 147 | 5067,26/05/2021,1430,125,1,long,67733 148 | 5149,23/08/2019,2593,98,1,short,78806 149 | 5149,16/04/2020,1776,66,0,long,78806 150 | 5181,31/12/2020,2474,121,1,long,48624 151 | 5316,11/2/2021,1254,112,0,long,78780 152 | 5325,27/07/2020,1000,69,1,long,31528 153 | -------------------------------------------------------------------------------- /data/data.json: -------------------------------------------------------------------------------- 1 | {"data": 2 | [{"customer_ID":"1090","loans":[{"customer_ID":"1090","loan_date":"15/11/2021","amount":"2426","fee":"199","loan_status":"0","term":"long","annual_income":"41333"}]}, 3 | {"customer_ID":"3565","loans":[{"customer_ID":"3565","loan_date":"07/03/2021","amount":"2153","fee":"53","loan_status":"0","term":"short","annual_income":"76498"},{"customer_ID":"3565","loan_date":"06/08/2021","amount":"1538","fee":"89","loan_status":"1","term":"long","annual_income":"76498"},{"customer_ID":"3565","loan_date":"30/09/2021","amount":"2997","fee":"24","loan_status":"0","term":"short","annual_income":"76498"},{"customer_ID":"3565","loan_date":"09/03/2021","amount":"2184","fee":"38","loan_status":"0","term":"long","annual_income":"76498"},{"customer_ID":"3565","loan_date":"07/07/2020","amount":"2707","fee":"42","loan_status":"1","term":"short","annual_income":"76498"},{"customer_ID":"3565","loan_date":"17/08/2019","amount":"2219","fee":"104","loan_status":"0","term":"short","annual_income":"76498"},{"customer_ID":"3565","loan_date":"26/10/2021","amount":"207","fee":"146","loan_status":"1","term":"short","annual_income":"76498"}]}, 4 | {"customer_ID":"1159","loans":[{"customer_ID":"1159","loan_date":"20/12/2021","amount":"2581","fee":"117","loan_status":"1","term":"short","annual_income":"46330"}]}, 5 | {"customer_ID":"1436","loans":[{"customer_ID":"1436","loan_date":"16/02/2020","amount":"1145","fee":"182","loan_status":"1","term":"long","annual_income":"58256"},{"customer_ID":"1436","loan_date":"18/11/2019","amount":"2700","fee":"186","loan_status":"0","term":"short","annual_income":"58256"}]}, 6 | {"customer_ID":"1512","loans":[{"customer_ID":"1512","loan_date":"29/03/2021","amount":"2114","fee":"75","loan_status":"1","term":"short","annual_income":"67215"},{"customer_ID":"1512","loan_date":"24/10/2019","amount":"210","fee":"78","loan_status":"0","term":"long","annual_income":"67215"}]}, 7 | {"customer_ID":"1572","loans":[{"customer_ID":"1572","loan_date":"04/05/2021","amount":"2569","fee":"108","loan_status":"1","term":"short","annual_income":"74816"}]}, 8 | {"customer_ID":"2069","loans":[{"customer_ID":"2069","loan_date":"02/01/2021","amount":"615","fee":"131","loan_status":"0","term":"short","annual_income":"58477"}]}, 9 | {"customer_ID":"2088","loans":[{"customer_ID":"2088","loan_date":"18/07/2020","amount":"860","fee":"176","loan_status":"1","term":"short","annual_income":"31645"},{"customer_ID":"2088","loan_date":"28/04/2020","amount":"1945","fee":"33","loan_status":"1","term":"short","annual_income":"31645"}]}, 10 | {"customer_ID":"2136","loans":[{"customer_ID":"2136","loan_date":"29/04/2019","amount":"2573","fee":"81","loan_status":"1","term":"long","annual_income":"62157"},{"customer_ID":"2136","loan_date":"30/04/2019","amount":"475","fee":"160","loan_status":"0","term":"long","annual_income":"62157"}]}, 11 | {"customer_ID":"2162","loans":[{"customer_ID":"2162","loan_date":"18/11/2021","amount":"174","fee":"26","loan_status":"0","term":"short","annual_income":"74546"},{"customer_ID":"2162","loan_date":"05/01/2022","amount":"2344","fee":"127","loan_status":"0","term":"long","annual_income":"74546"},{"customer_ID":"2162","loan_date":"06/07/2019","amount":"1738","fee":"188","loan_status":"0","term":"long","annual_income":"74546"}]}, 12 | {"customer_ID":"2294","loans":[{"customer_ID":"2294","loan_date":"04/01/2020","amount":"2304","fee":"125","loan_status":"1","term":"short","annual_income":"45001"}]}, 13 | {"customer_ID":"2904","loans":[{"customer_ID":"2904","loan_date":"14/01/2022","amount":"2396","fee":"108","loan_status":"0","term":"short","annual_income":"63914"},{"customer_ID":"2904","loan_date":"21/03/2021","amount":"2553","fee":"124","loan_status":"0","term":"long","annual_income":"63914"}]}, 14 | {"customer_ID":"296","loans":[{"customer_ID":"296","loan_date":"17/08/2020","amount":"2003","fee":"24","loan_status":"0","term":"short","annual_income":"41557"}]}, 15 | {"customer_ID":"3210","loans":[{"customer_ID":"3210","loan_date":"10/11/2019","amount":"2961","fee":"66","loan_status":"1","term":"long","annual_income":"60852"}]}, 16 | {"customer_ID":"3414","loans":[{"customer_ID":"3414","loan_date":"25/02/2019","amount":"2644","fee":"106","loan_status":"0","term":"short","annual_income":"52362"}]}, 17 | {"customer_ID":"3606","loans":[{"customer_ID":"3606","loan_date":"09/03/2020","amount":"2832","fee":"37","loan_status":"0","term":"short","annual_income":"63707"},{"customer_ID":"3606","loan_date":"11/07/2019","amount":"189","fee":"30","loan_status":"1","term":"short","annual_income":"63707"},{"customer_ID":"3606","loan_date":"04/06/2020","amount":"481","fee":"174","loan_status":"0","term":"short","annual_income":"63707"}]}, 18 | {"customer_ID":"3959","loans":[{"customer_ID":"3959","loan_date":"26/11/2021","amount":"1076","fee":"125","loan_status":"1","term":"short","annual_income":"52358"},{"customer_ID":"3959","loan_date":"27/09/2019","amount":"2941","fee":"178","loan_status":"0","term":"short","annual_income":"52358"},{"customer_ID":"3959","loan_date":"17/05/2021","amount":"2955","fee":"163","loan_status":"0","term":"short","annual_income":"52358"}]}, 19 | {"customer_ID":"4032","loans":[{"customer_ID":"4032","loan_date":"27/11/2019","amount":"2361","fee":"144","loan_status":"0","term":"short","annual_income":"63118"}]}, 20 | {"customer_ID":"467","loans":[{"customer_ID":"467","loan_date":"09/10/2020","amount":"2995","fee":"140","loan_status":"0","term":"long","annual_income":"69972"},{"customer_ID":"467","loan_date":"26/04/2019","amount":"1124","fee":"192","loan_status":"0","term":"short","annual_income":"69972"},{"customer_ID":"467","loan_date":"14/10/2021","amount":"1871","fee":"189","loan_status":"1","term":"short","annual_income":"69972"}]}, 21 | {"customer_ID":"4821","loans":[{"customer_ID":"4821","loan_date":"24/02/2021","amount":"1968","fee":"92","loan_status":"0","term":"short","annual_income":"46451"},{"customer_ID":"4821","loan_date":"04/12/2021","amount":"1625","fee":"157","loan_status":"1","term":"short","annual_income":"46451"},{"customer_ID":"4821","loan_date":"23/10/2021","amount":"1349","fee":"129","loan_status":"0","term":"short","annual_income":"46451"},{"customer_ID":"4821","loan_date":"12/10/2019","amount":"885","fee":"129","loan_status":"1","term":"short","annual_income":"46451"}]}, 22 | {"customer_ID":"4937","loans":[{"customer_ID":"4937","loan_date":"22/11/2019","amount":"2774","fee":"74","loan_status":"1","term":"short","annual_income":"60727"}]}, 23 | {"customer_ID":"5325","loans":[{"customer_ID":"5325","loan_date":"27/07/2020","amount":"1000","fee":"69","loan_status":"1","term":"long","annual_income":"31528"}]}, 24 | {"customer_ID":"675","loans":[{"customer_ID":"675","loan_date":"15/09/2019","amount":"826","fee":"53","loan_status":"0","term":"long","annual_income":"32965"},{"customer_ID":"675","loan_date":"31/08/2021","amount":"1425","fee":"199","loan_status":"1","term":"long","annual_income":"32965"},{"customer_ID":"675","loan_date":"27/01/2021","amount":"2678","fee":"100","loan_status":"0","term":"long","annual_income":"32965"}]}, 25 | {"customer_ID":"691","loans":[{"customer_ID":"691","loan_date":"09/06/2020","amount":"188","fee":"80","loan_status":"0","term":"long","annual_income":"44557"}]}, 26 | {"customer_ID":"829","loans":[{"customer_ID":"829","loan_date":"29/07/2019","amount":"2905","fee":"54","loan_status":"0","term":"short","annual_income":"39613"}]}, 27 | {"customer_ID":"125","loans":[{"customer_ID":"125","loan_date":"02/03/2020","amount":"2626","fee":"124","loan_status":"0","term":"short","annual_income":"77110"}]}, 28 | {"customer_ID":"1372","loans":[{"customer_ID":"1372","loan_date":"30/11/2021","amount":"2096","fee":"158","loan_status":"0","term":"short","annual_income":"74938"}]}, 29 | {"customer_ID":"1394","loans":[{"customer_ID":"1394","loan_date":"14/12/2020","amount":"2308","fee":"65","loan_status":"0","term":"short","annual_income":"31704"}]}, 30 | {"customer_ID":"1669","loans":[{"customer_ID":"1669","loan_date":"16/07/2021","amount":"458","fee":"173","loan_status":"1","term":"short","annual_income":"72881"}]}, 31 | {"customer_ID":"2110","loans":[{"customer_ID":"2110","loan_date":"03/05/2021","amount":"404","fee":"52","loan_status":"0","term":"short","annual_income":"72789"}]}, 32 | {"customer_ID":"2275","loans":[{"customer_ID":"2275","loan_date":"05/07/2019","amount":"256","fee":"50","loan_status":"1","term":"long","annual_income":"62422"}]}, 33 | {"customer_ID":"2393","loans":[{"customer_ID":"2393","loan_date":"28/01/2020","amount":"2291","fee":"73","loan_status":"0","term":"long","annual_income":"71526"},{"customer_ID":"2393","loan_date":"12/04/2021","amount":"2861","fee":"89","loan_status":"0","term":"long","annual_income":"71526"},{"customer_ID":"2393","loan_date":"13/11/2019","amount":"2514","fee":"22","loan_status":"0","term":"short","annual_income":"71526"}]}, 34 | {"customer_ID":"2464","loans":[{"customer_ID":"2464","loan_date":"15/05/2021","amount":"2478","fee":"171","loan_status":"1","term":"long","annual_income":"44628"}]}, 35 | {"customer_ID":"2756","loans":[{"customer_ID":"2756","loan_date":"11/07/2020","amount":"397","fee":"157","loan_status":"0","term":"short","annual_income":"46157"},{"customer_ID":"2756","loan_date":"25/09/2020","amount":"326","fee":"65","loan_status":"0","term":"long","annual_income":"46157"}]}, 36 | {"customer_ID":"3015","loans":[{"customer_ID":"3015","loan_date":"04/07/2019","amount":"417","fee":"48","loan_status":"1","term":"long","annual_income":"62823"}]}, 37 | {"customer_ID":"3281","loans":[{"customer_ID":"3281","loan_date":"02/03/2019","amount":"309","fee":"89","loan_status":"1","term":"long","annual_income":"75974"}]}, 38 | {"customer_ID":"3368","loans":[{"customer_ID":"3368","loan_date":"03/07/2019","amount":"1067","fee":"55","loan_status":"0","term":"short","annual_income":"35537"}]}, 39 | {"customer_ID":"3441","loans":[{"customer_ID":"3441","loan_date":"03/12/2019","amount":"794","fee":"163","loan_status":"0","term":"short","annual_income":"37213"}]}, 40 | {"customer_ID":"3517","loans":[{"customer_ID":"3517","loan_date":"02/02/2019","amount":"1716","fee":"76","loan_status":"0","term":"long","annual_income":"55092"}]}, 41 | {"customer_ID":"3650","loans":[{"customer_ID":"3650","loan_date":"16/09/2019","amount":"2972","fee":"52","loan_status":"1","term":"short","annual_income":"35611"}]}, 42 | {"customer_ID":"3826","loans":[{"customer_ID":"3826","loan_date":"01/01/2022","amount":"1628","fee":"96","loan_status":"1","term":"long","annual_income":"63146"}]}, 43 | {"customer_ID":"3858","loans":[{"customer_ID":"3858","loan_date":"24/01/2022","amount":"671","fee":"154","loan_status":"0","term":"long","annual_income":"39885"}]}, 44 | {"customer_ID":"451","loans":[{"customer_ID":"451","loan_date":"09/01/2020","amount":"284","fee":"169","loan_status":"1","term":"short","annual_income":"51834"}]}, 45 | {"customer_ID":"4838","loans":[{"customer_ID":"4838","loan_date":"08/07/2021","amount":"626","fee":"27","loan_status":"0","term":"long","annual_income":"47472"}]}, 46 | {"customer_ID":"4894","loans":[{"customer_ID":"4894","loan_date":"12/07/2020","amount":"2167","fee":"97","loan_status":"0","term":"short","annual_income":"66009"}]}, 47 | {"customer_ID":"4975","loans":[{"customer_ID":"4975","loan_date":"18/05/2019","amount":"2168","fee":"27","loan_status":"0","term":"long","annual_income":"49801"}]}, 48 | {"customer_ID":"5023","loans":[{"customer_ID":"5023","loan_date":"30/10/2021","amount":"1570","fee":"157","loan_status":"1","term":"short","annual_income":"60842"}]}, 49 | {"customer_ID":"5067","loans":[{"customer_ID":"5067","loan_date":"26/05/2021","amount":"1430","fee":"125","loan_status":"1","term":"long","annual_income":"67733"}]}, 50 | {"customer_ID":"5149","loans":[{"customer_ID":"5149","loan_date":"23/08/2019","amount":"2593","fee":"98","loan_status":"1","term":"short","annual_income":"78806"},{"customer_ID":"5149","loan_date":"16/04/2020","amount":"1776","fee":"66","loan_status":"0","term":"long","annual_income":"78806"}]}, 51 | {"customer_ID":"800","loans":[{"customer_ID":"800","loan_date":"08/04/2019","amount":"1691","fee":"22","loan_status":"0","term":"long","annual_income":"49583"}]}, 52 | {"customer_ID":"853","loans":[{"customer_ID":"853","loan_date":"23/12/2021","amount":"1603","fee":"139","loan_status":"0","term":"long","annual_income":"79823"}]}, 53 | {"customer_ID":"944","loans":[{"customer_ID":"944","loan_date":"26/05/2021","amount":"1039","fee":"104","loan_status":"1","term":"long","annual_income":"31991"}]}, 54 | {"customer_ID":"1241","loans":[{"customer_ID":"1241","loan_date":"01/09/2021","amount":"2888","fee":"75","loan_status":"0","term":"short","annual_income":"36275"}]}, 55 | {"customer_ID":"1265","loans":[{"customer_ID":"1265","loan_date":"31/01/2019","amount":"2080","fee":"108","loan_status":"0","term":"short","annual_income":"43882"},{"customer_ID":"1265","loan_date":"31/08/2020","amount":"2064","fee":"114","loan_status":"1","term":"short","annual_income":"43882"},{"customer_ID":"1265","loan_date":"23/09/2020","amount":"424","fee":"67","loan_status":"0","term":"long","annual_income":"43882"}]}, 56 | {"customer_ID":"1280","loans":[{"customer_ID":"1280","loan_date":"20/05/2020","amount":"2261","fee":"180","loan_status":"1","term":"long","annual_income":"75523"},{"customer_ID":"1280","loan_date":"01/05/2019","amount":"385","fee":"72","loan_status":"1","term":"long","annual_income":"75523"}]}, 57 | {"customer_ID":"1361","loans":[{"customer_ID":"1361","loan_date":"28/09/2020","amount":"1840","fee":"176","loan_status":"0","term":"short","annual_income":"47342"},{"customer_ID":"1361","loan_date":"04/01/2022","amount":"2821","fee":"154","loan_status":"1","term":"short","annual_income":"47342"}]}, 58 | {"customer_ID":"1746","loans":[{"customer_ID":"1746","loan_date":"30/06/2021","amount":"1659","fee":"33","loan_status":"1","term":"short","annual_income":"59171"}]}, 59 | {"customer_ID":"1808","loans":[{"customer_ID":"1808","loan_date":"29/08/2021","amount":"114","fee":"96","loan_status":"1","term":"long","annual_income":"62299"},{"customer_ID":"1808","loan_date":"26/12/2021","amount":"107","fee":"28","loan_status":"1","term":"long","annual_income":"62299"}]}, 60 | {"customer_ID":"1870","loans":[{"customer_ID":"1870","loan_date":"28/06/2020","amount":"643","fee":"96","loan_status":"0","term":"short","annual_income":"59798"}]}, 61 | {"customer_ID":"2530","loans":[{"customer_ID":"2530","loan_date":"17/06/2019","amount":"2222","fee":"106","loan_status":"1","term":"long","annual_income":"69700"}]}, 62 | {"customer_ID":"2696","loans":[{"customer_ID":"2696","loan_date":"17/05/2021","amount":"2490","fee":"78","loan_status":"0","term":"short","annual_income":"35578"},{"customer_ID":"2696","loan_date":"27/02/2021","amount":"1169","fee":"92","loan_status":"0","term":"long","annual_income":"35578"},{"customer_ID":"2696","loan_date":"11/02/2020","amount":"1504","fee":"179","loan_status":"1","term":"short","annual_income":"35578"},{"customer_ID":"2696","loan_date":"04/07/2020","amount":"2551","fee":"79","loan_status":"0","term":"long","annual_income":"35578"},{"customer_ID":"2696","loan_date":"23/04/2021","amount":"608","fee":"188","loan_status":"1","term":"short","annual_income":"35578"}]}, 63 | {"customer_ID":"2700","loans":[{"customer_ID":"2700","loan_date":"28/05/2021","amount":"599","fee":"187","loan_status":"0","term":"long","annual_income":"76092"}]}, 64 | {"customer_ID":"2725","loans":[{"customer_ID":"2725","loan_date":"26/05/2019","amount":"2186","fee":"135","loan_status":"1","term":"short","annual_income":"50873"}]}, 65 | {"customer_ID":"2898","loans":[{"customer_ID":"2898","loan_date":"12/02/2020","amount":"2270","fee":"85","loan_status":"0","term":"long","annual_income":"56076"}]}, 66 | {"customer_ID":"2917","loans":[{"customer_ID":"2917","loan_date":"22/10/2021","amount":"994","fee":"125","loan_status":"1","term":"short","annual_income":"54949"}]}, 67 | {"customer_ID":"3200","loans":[{"customer_ID":"3200","loan_date":"20/02/2020","amount":"615","fee":"42","loan_status":"0","term":"long","annual_income":"73510"}]}, 68 | {"customer_ID":"3526","loans":[{"customer_ID":"3526","loan_date":"17/02/2019","amount":"842","fee":"135","loan_status":"1","term":"long","annual_income":"46000"}]}, 69 | {"customer_ID":"3836","loans":[{"customer_ID":"3836","loan_date":"04/10/2019","amount":"1488","fee":"44","loan_status":"1","term":"short","annual_income":"53107"}]}, 70 | {"customer_ID":"3838","loans":[{"customer_ID":"3838","loan_date":"19/02/2021","amount":"2856","fee":"109","loan_status":"0","term":"long","annual_income":"66824"}]}, 71 | {"customer_ID":"3949","loans":[{"customer_ID":"3949","loan_date":"21/08/2019","amount":"613","fee":"149","loan_status":"0","term":"long","annual_income":"53961"},{"customer_ID":"3949","loan_date":"06/05/2021","amount":"2692","fee":"144","loan_status":"0","term":"long","annual_income":"53961"}]}, 72 | {"customer_ID":"4127","loans":[{"customer_ID":"4127","loan_date":"31/01/2020","amount":"2949","fee":"200","loan_status":"1","term":"short","annual_income":"71869"}]}, 73 | {"customer_ID":"4850","loans":[{"customer_ID":"4850","loan_date":"17/03/2021","amount":"2653","fee":"177","loan_status":"1","term":"short","annual_income":"34077"},{"customer_ID":"4850","loan_date":"22/06/2020","amount":"842","fee":"167","loan_status":"1","term":"short","annual_income":"34077"},{"customer_ID":"4850","loan_date":"10/04/2020","amount":"1147","fee":"122","loan_status":"1","term":"short","annual_income":"34077"},{"customer_ID":"4850","loan_date":"18/04/2021","amount":"1746","fee":"122","loan_status":"1","term":"long","annual_income":"34077"}]}, 74 | {"customer_ID":"5044","loans":[{"customer_ID":"5044","loan_date":"09/10/2021","amount":"1135","fee":"152","loan_status":"1","term":"long","annual_income":"73236"}]}, 75 | {"customer_ID":"5181","loans":[{"customer_ID":"5181","loan_date":"31/12/2020","amount":"2474","fee":"121","loan_status":"1","term":"long","annual_income":"48624"}]}, 76 | {"customer_ID":"5316","loans":[{"customer_ID":"5316","loan_date":"11/02/2021","amount":"1254","fee":"112","loan_status":"0","term":"long","annual_income":"78780"}]}, 77 | {"customer_ID":"666","loans":[{"customer_ID":"666","loan_date":"18/07/2021","amount":"1687","fee":"189","loan_status":"1","term":"long","annual_income":"42456"}]}, 78 | {"customer_ID":"870","loans":[{"customer_ID":"870","loan_date":"03/09/2021","amount":"2892","fee":"46","loan_status":"0","term":"long","annual_income":"54335"}]}, 79 | {"customer_ID":"919","loans":[{"customer_ID":"919","loan_date":"18/10/2019","amount":"453","fee":"62","loan_status":"1","term":"long","annual_income":"79307"},{"customer_ID":"919","loan_date":"11/01/2019","amount":"1306","fee":"24","loan_status":"1","term":"long","annual_income":"79307"}]}, 80 | {"customer_ID":"926","loans":[{"customer_ID":"926","loan_date":"20/01/2022","amount":"926","fee":"35","loan_status":"1","term":"short","annual_income":"56935"}]}, 81 | {"customer_ID":"124","loans":[{"customer_ID":"124","loan_date":"23/06/2021","amount":"785","fee":"173","loan_status":"0","term":"short","annual_income":"69246"}]}, 82 | {"customer_ID":"1528","loans":[{"customer_ID":"1528","loan_date":"07/02/2021","amount":"1032","fee":"156","loan_status":"0","term":"short","annual_income":"34023"}]}, 83 | {"customer_ID":"1903","loans":[{"customer_ID":"1903","loan_date":"23/10/2021","amount":"1581","fee":"180","loan_status":"0","term":"long","annual_income":"50325"}]}, 84 | {"customer_ID":"1953","loans":[{"customer_ID":"1953","loan_date":"27/06/2021","amount":"2204","fee":"113","loan_status":"1","term":"long","annual_income":"78787"}]}, 85 | {"customer_ID":"2034","loans":[{"customer_ID":"2034","loan_date":"31/05/2020","amount":"105","fee":"29","loan_status":"1","term":"short","annual_income":"74897"},{"customer_ID":"2034","loan_date":"22/09/2021","amount":"1929","fee":"162","loan_status":"1","term":"short","annual_income":"74897"},{"customer_ID":"2034","loan_date":"04/01/2022","amount":"1444","fee":"146","loan_status":"0","term":"short","annual_income":"74897"},{"customer_ID":"2034","loan_date":"04/12/2019","amount":"1718","fee":"124","loan_status":"1","term":"short","annual_income":"74897"}]}, 86 | {"customer_ID":"2550","loans":[{"customer_ID":"2550","loan_date":"03/06/2019","amount":"2783","fee":"36","loan_status":"0","term":"long","annual_income":"57268"}]}, 87 | {"customer_ID":"2888","loans":[{"customer_ID":"2888","loan_date":"15/04/2021","amount":"906","fee":"20","loan_status":"1","term":"short","annual_income":"68731"},{"customer_ID":"2888","loan_date":"24/07/2021","amount":"877","fee":"159","loan_status":"0","term":"short","annual_income":"68731"}]}, 88 | {"customer_ID":"2895","loans":[{"customer_ID":"2895","loan_date":"02/04/2019","amount":"2174","fee":"165","loan_status":"1","term":"short","annual_income":"57689"}]}, 89 | {"customer_ID":"2930","loans":[{"customer_ID":"2930","loan_date":"21/05/2020","amount":"2995","fee":"80","loan_status":"1","term":"short","annual_income":"62154"}]}, 90 | {"customer_ID":"3121","loans":[{"customer_ID":"3121","loan_date":"13/06/2019","amount":"1868","fee":"49","loan_status":"1","term":"long","annual_income":"48384"}]}, 91 | {"customer_ID":"3249","loans":[{"customer_ID":"3249","loan_date":"05/04/2019","amount":"2905","fee":"182","loan_status":"0","term":"long","annual_income":"62431"}]}, 92 | {"customer_ID":"3266","loans":[{"customer_ID":"3266","loan_date":"19/03/2021","amount":"1694","fee":"177","loan_status":"1","term":"short","annual_income":"51619"}]}, 93 | {"customer_ID":"3312","loans":[{"customer_ID":"3312","loan_date":"29/05/2020","amount":"543","fee":"66","loan_status":"1","term":"long","annual_income":"44831"}]}, 94 | {"customer_ID":"3652","loans":[{"customer_ID":"3652","loan_date":"15/03/2020","amount":"1148","fee":"66","loan_status":"1","term":"long","annual_income":"79510"}]}, 95 | {"customer_ID":"4236","loans":[{"customer_ID":"4236","loan_date":"19/10/2020","amount":"445","fee":"108","loan_status":"0","term":"long","annual_income":"66364"}]}, 96 | {"customer_ID":"4249","loans":[{"customer_ID":"4249","loan_date":"03/10/2020","amount":"396","fee":"137","loan_status":"0","term":"short","annual_income":"70433"}]}, 97 | {"customer_ID":"4276","loans":[{"customer_ID":"4276","loan_date":"18/03/2021","amount":"1898","fee":"39","loan_status":"0","term":"long","annual_income":"50915"},{"customer_ID":"4276","loan_date":"05/06/2019","amount":"2243","fee":"161","loan_status":"1","term":"short","annual_income":"50915"},{"customer_ID":"4276","loan_date":"30/06/2020","amount":"1899","fee":"116","loan_status":"0","term":"short","annual_income":"50915"},{"customer_ID":"4276","loan_date":"20/09/2020","amount":"1601","fee":"93","loan_status":"0","term":"long","annual_income":"50915"},{"customer_ID":"4276","loan_date":"13/06/2020","amount":"1858","fee":"122","loan_status":"0","term":"long","annual_income":"50915"},{"customer_ID":"4276","loan_date":"09/04/2019","amount":"884","fee":"125","loan_status":"0","term":"short","annual_income":"50915"}]}, 98 | {"customer_ID":"447","loans":[{"customer_ID":"447","loan_date":"18/12/2019","amount":"438","fee":"168","loan_status":"1","term":"long","annual_income":"62868"}]}, 99 | {"customer_ID":"4583","loans":[{"customer_ID":"4583","loan_date":"06/06/2020","amount":"1672","fee":"73","loan_status":"1","term":"short","annual_income":"64470"}]}, 100 | {"customer_ID":"4665","loans":[{"customer_ID":"4665","loan_date":"26/09/2019","amount":"2555","fee":"26","loan_status":"1","term":"long","annual_income":"33923"}]}, 101 | {"customer_ID":"4988","loans":[{"customer_ID":"4988","loan_date":"07/11/2021","amount":"1586","fee":"200","loan_status":"1","term":"long","annual_income":"55708"}]}] 102 | } -------------------------------------------------------------------------------- /db/__pycache__/db.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jimmyg1997/ml-feature-engineering-fastapi-docker/504ad7653fcc1e1a6b0dab91894ce952836e54cb/db/__pycache__/db.cpython-38.pyc -------------------------------------------------------------------------------- /db/__pycache__/optasia_db.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jimmyg1997/ml-feature-engineering-fastapi-docker/504ad7653fcc1e1a6b0dab91894ce952836e54cb/db/__pycache__/optasia_db.cpython-38.pyc -------------------------------------------------------------------------------- /db/db.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jimmyg1997/ml-feature-engineering-fastapi-docker/504ad7653fcc1e1a6b0dab91894ce952836e54cb/db/db.db -------------------------------------------------------------------------------- /db/db.py: -------------------------------------------------------------------------------- 1 | from dateutil import parser 2 | from typing import Optional, List, Dict, Any 3 | from src.models import Customer, LoanStatus, Term, Customer, Loan 4 | 5 | 6 | db : List[Customer] = [ 7 | Customer(id = 1090, 8 | annual_income = 41333, 9 | loans = [ Loan(id = 1, 10 | loan_date = parser.parse("15/11/2021"), 11 | amount = 2426, 12 | term = Term.long, 13 | fee = 199, 14 | loan_status = LoanStatus.not_paid 15 | ) 16 | ] 17 | ), 18 | 19 | Customer(id = 3565, 20 | annual_income = 76498, 21 | loans = [ Loan(id = 2, 22 | loan_date = parser.parse("07/03/2021"), 23 | amount = 2153, 24 | term = Term.short, 25 | fee = 53, 26 | loan_status = LoanStatus.not_paid 27 | ), 28 | 29 | Loan(id = 3, 30 | loan_date = parser.parse("06/08/2021"), 31 | amount = 1538, 32 | term = Term.long, 33 | fee = 89, 34 | loan_status = LoanStatus.paid 35 | ) 36 | 37 | ] 38 | ), 39 | ] -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | # version: "3" 2 | # services: 3 | # app_launch: 4 | # build: . 5 | # working_dir: /main 6 | # command: uvicorn main:app --host 0.0.0.0 --reload 7 | # #image: fastapi-service:dev 8 | # environment: 9 | # DEBUG: 1 10 | # ports: 11 | # - "8000:8000" 12 | # volumes: 13 | # - .:/app 14 | 15 | 16 | 17 | version: "3" 18 | services: 19 | api: 20 | # Specifies thse path to Docker file &&the tag name respectively. 21 | build: ./ 22 | image: api/demo-docker:v1 23 | # Specifies the set of commands we want to run inside the container. 24 | command: uvicorn src.main:app --reload --workers 1 --host 0.0.0.0 --port 8000 25 | # Specifies port mappings : 26 | ports: 27 | - 8000:8000 28 | 29 | -------------------------------------------------------------------------------- /features/features_customers.csv: -------------------------------------------------------------------------------- 1 | customer_id,annual_income,annual_income_bins,COUNT(loans),MAX(loans.amount),MAX(loans.days),MAX(loans.fee),MAX(loans.fee_pct),MAX(loans.total_amount),MEAN(loans.amount),MEAN(loans.days),MEAN(loans.fee),MEAN(loans.fee_pct),MEAN(loans.total_amount),MIN(loans.amount),MIN(loans.days),MIN(loans.fee),MIN(loans.fee_pct),MIN(loans.total_amount),SKEW(loans.amount),SKEW(loans.days),SKEW(loans.fee),SKEW(loans.fee_pct),SKEW(loans.total_amount),STD(loans.amount),STD(loans.days),STD(loans.fee),STD(loans.fee_pct),STD(loans.total_amount),SUM(loans.amount),SUM(loans.days),SUM(loans.fee),SUM(loans.fee_pct),SUM(loans.total_amount),MODE(loans.DAY(loan_date)),MODE(loans.MONTH(loan_date)),MODE(loans.WEEKDAY(loan_date)),MODE(loans.YEAR(loan_date)),NUM_UNIQUE(loans.DAY(loan_date)),NUM_UNIQUE(loans.MONTH(loan_date)),NUM_UNIQUE(loans.WEEKDAY(loan_date)),NUM_UNIQUE(loans.YEAR(loan_date)) 2 | 3565,76498.0,very high,2,2153.0,533.0,89.0,0.05786736020806242,2206.0,1845.5,457.0,71.0,0.041242086978160335,1916.5,1538.0,381.0,53.0,0.024616813748258245,1627.0,,,,,,434.8706704297267,107.48023074035522,25.45584412271571,0.023511686879885887,409.414826307011,3691.0,914.0,142.0,0.08248417395632067,3833.0,6,3,4,2021,2,2,2,1 3 | 1423,34513.0,very low,0,,,,,,,,,,,,,,,,,,,,,,,,,,0.0,0.0,0.0,0.0,0.0,,,,,,,, 4 | -------------------------------------------------------------------------------- /features/features_loans.csv: -------------------------------------------------------------------------------- 1 | loan_id,amount,fee,fee_pct,total_amount,days,DAY(loan_date),MONTH(loan_date),WEEKDAY(loan_date),YEAR(loan_date),customers.annual_income,customers.annual_income_bins,customers.COUNT(loans),customers.MAX(loans.amount),customers.MAX(loans.days),customers.MAX(loans.fee),customers.MAX(loans.fee_pct),customers.MAX(loans.total_amount),customers.MEAN(loans.amount),customers.MEAN(loans.days),customers.MEAN(loans.fee),customers.MEAN(loans.fee_pct),customers.MEAN(loans.total_amount),customers.MIN(loans.amount),customers.MIN(loans.days),customers.MIN(loans.fee),customers.MIN(loans.fee_pct),customers.MIN(loans.total_amount),customers.SKEW(loans.amount),customers.SKEW(loans.days),customers.SKEW(loans.fee),customers.SKEW(loans.fee_pct),customers.SKEW(loans.total_amount),customers.STD(loans.amount),customers.STD(loans.days),customers.STD(loans.fee),customers.STD(loans.fee_pct),customers.STD(loans.total_amount),customers.SUM(loans.amount),customers.SUM(loans.days),customers.SUM(loans.fee),customers.SUM(loans.fee_pct),customers.SUM(loans.total_amount) 2 | 2,2153.0,53.0,0.024616813748258245,2206.0,533,7,3,6,2021,76498.0,very high,2,2153.0,533.0,89.0,0.05786736020806242,2206.0,1845.5,457.0,71.0,0.041242086978160335,1916.5,1538.0,381.0,53.0,0.024616813748258245,1627.0,,,,,,434.8706704297267,107.48023074035522,25.45584412271571,0.023511686879885887,409.414826307011,3691.0,914.0,142.0,0.08248417395632067,3833.0 3 | 3,1538.0,89.0,0.05786736020806242,1627.0,381,6,8,4,2021,76498.0,very high,2,2153.0,533.0,89.0,0.05786736020806242,2206.0,1845.5,457.0,71.0,0.041242086978160335,1916.5,1538.0,381.0,53.0,0.024616813748258245,1627.0,,,,,,434.8706704297267,107.48023074035522,25.45584412271571,0.023511686879885887,409.414826307011,3691.0,914.0,142.0,0.08248417395632067,3833.0 4 | 15,1000.0,100.0,0.1,1100.0,2,20,8,5,2022,,,0,,,,,,,,,,,,,,,,,,,,,,,,,,0.0,0.0,0.0,0.0,0.0 5 | -------------------------------------------------------------------------------- /images/fastapi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jimmyg1997/ml-feature-engineering-fastapi-docker/504ad7653fcc1e1a6b0dab91894ce952836e54cb/images/fastapi.png -------------------------------------------------------------------------------- /logs/logs.log: -------------------------------------------------------------------------------- 1 | 2022-08-20 13:34:39 - INFO - ---------------------------------- Optasia API Started 2 | 2022-08-20 13:34:40 - ERROR - (test_api.test_fetch_customer) Hitting endpoint /api/v1/customers/1090 failed (status code 200) : 3 | 2022-08-20 13:35:33 - INFO - ---------------------------------- Optasia API Started 4 | 2022-08-20 13:35:33 - INFO - (test_api.test_fetch_customer) Endpoint /api/v1/customers/1090 runs sucessfully ✅ 5 | 2022-08-20 13:35:33 - INFO - (test_api.test_register_customer) Endpoint /api/v1/customers/ runs sucessfully ✅ 6 | 2022-08-20 13:35:33 - INFO - (test_api.test_fetch_loan) Endpoint /api/v1/customers/1090 runs sucessfully ✅ 7 | 2022-08-20 13:35:33 - INFO - (test_api.test_register_loan) Endpoint /api/v1/loans for the loan registration was sucessful ✅ 8 | 2022-08-20 13:35:33 - INFO - (test_api.test_delete_customer) Endpoint /api/v1/customers/1090 runs sucessfully ✅ 9 | 2022-08-20 13:35:33 - INFO - (test_api.test_delete_loan) Endpoint /api/v1/loans/15 runs sucessfully ✅ 10 | 2022-08-20 13:35:33 - INFO - (DataLoader.postprocess_loans_dataframe) Loans Data are postprocessed sucessfully ✅ 11 | 2022-08-20 13:35:33 - INFO - (FeatureEngineer.extract_features_loans) Manual extraction of features for the loans dataframe was successful ✅ 12 | 2022-08-20 13:35:33 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 13 | 2022-08-20 13:35:33 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 14 | 2022-08-20 13:35:33 - INFO - (FeatureEngineer.add_relationship) The tuple indicating the relationship between 2 dataframes was sucessfully added to the list of relationshipss ✅ 15 | 2022-08-20 13:35:33 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 16 | 2022-08-20 13:35:33 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 17 | 2022-08-20 13:35:33 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 18 | 2022-08-20 13:35:33 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 19 | 2022-08-20 13:35:33 - INFO - (test_api.test_fetch_customers_features) Endpoint /api/v1/features/customers runs sucessfully ✅ 20 | 2022-08-20 13:35:33 - INFO - (DataLoader.postprocess_loans_dataframe) Loans Data are postprocessed sucessfully ✅ 21 | 2022-08-20 13:35:33 - INFO - (FeatureEngineer.extract_features_loans) Manual extraction of features for the loans dataframe was successful ✅ 22 | 2022-08-20 13:35:33 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 23 | 2022-08-20 13:35:33 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 24 | 2022-08-20 13:35:33 - INFO - (FeatureEngineer.add_relationship) The tuple indicating the relationship between 2 dataframes was sucessfully added to the list of relationshipss ✅ 25 | 2022-08-20 13:35:33 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 26 | 2022-08-20 13:35:34 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 27 | 2022-08-20 13:35:34 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 28 | 2022-08-20 13:35:34 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 29 | 2022-08-20 13:35:34 - INFO - (test_api.test_fetch_customers_features) Endpoint /api/v1/features/customers runs sucessfully ✅ 30 | 2022-08-20 13:35:34 - INFO - (DataReporter.cast_to_spreadsheet_friendly_format) Columns casted to string for compatability with Spreadsheets API ✅ 31 | 2022-08-20 13:35:35 - INFO - (DataReporter.write_to_spreadsheet) Data uploaded to Spreadsheet successfully ✅ 32 | 2022-08-20 13:35:35 - INFO - (test_api.test_upload_features_customers) Endpoint /api/v1/features/customers for features uploading runs sucessfully ✅ 33 | 2022-08-20 13:35:35 - INFO - (DataReporter.cast_to_spreadsheet_friendly_format) Columns casted to string for compatability with Spreadsheets API ✅ 34 | 2022-08-20 13:35:37 - INFO - (DataReporter.write_to_spreadsheet) Data uploaded to Spreadsheet successfully ✅ 35 | 2022-08-20 13:35:37 - INFO - (test_api.test_upload_features_loans) Endpoint /api/v1/features/loans for features uploading runs sucessfully ✅ 36 | 2022-08-20 13:36:01 - INFO - ---------------------------------- Optasia API Started 37 | 2022-08-20 13:36:01 - INFO - (test_api.test_fetch_customer) Endpoint /api/v1/customers/1090 runs sucessfully ✅ 38 | 2022-08-20 13:36:01 - INFO - (test_api.test_register_customer) Endpoint /api/v1/customers/ runs sucessfully ✅ 39 | 2022-08-20 13:36:01 - INFO - (test_api.test_fetch_loan) Endpoint /api/v1/customers/1090 runs sucessfully ✅ 40 | 2022-08-20 13:36:01 - INFO - (test_api.test_register_loan) Endpoint /api/v1/loans for the loan registration was sucessful ✅ 41 | 2022-08-20 13:36:01 - INFO - (test_api.test_delete_customer) Endpoint /api/v1/customers/1090 runs sucessfully ✅ 42 | 2022-08-20 13:36:01 - INFO - (test_api.test_delete_loan) Endpoint /api/v1/loans/15 runs sucessfully ✅ 43 | 2022-08-20 13:36:01 - INFO - (DataLoader.postprocess_loans_dataframe) Loans Data are postprocessed sucessfully ✅ 44 | 2022-08-20 13:36:01 - INFO - (FeatureEngineer.extract_features_loans) Manual extraction of features for the loans dataframe was successful ✅ 45 | 2022-08-20 13:36:01 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 46 | 2022-08-20 13:36:01 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 47 | 2022-08-20 13:36:01 - INFO - (FeatureEngineer.add_relationship) The tuple indicating the relationship between 2 dataframes was sucessfully added to the list of relationshipss ✅ 48 | 2022-08-20 13:36:01 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 49 | 2022-08-20 13:36:01 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 50 | 2022-08-20 13:36:01 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 51 | 2022-08-20 13:36:01 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 52 | 2022-08-20 13:36:01 - INFO - (test_api.test_fetch_customers_features) Endpoint /api/v1/features/customers runs sucessfully ✅ 53 | 2022-08-20 13:36:01 - INFO - (DataLoader.postprocess_loans_dataframe) Loans Data are postprocessed sucessfully ✅ 54 | 2022-08-20 13:36:01 - INFO - (FeatureEngineer.extract_features_loans) Manual extraction of features for the loans dataframe was successful ✅ 55 | 2022-08-20 13:36:01 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 56 | 2022-08-20 13:36:01 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 57 | 2022-08-20 13:36:01 - INFO - (FeatureEngineer.add_relationship) The tuple indicating the relationship between 2 dataframes was sucessfully added to the list of relationshipss ✅ 58 | 2022-08-20 13:36:01 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 59 | 2022-08-20 13:36:01 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 60 | 2022-08-20 13:36:01 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 61 | 2022-08-20 13:36:01 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 62 | 2022-08-20 13:36:01 - INFO - (test_api.test_fetch_customers_features) Endpoint /api/v1/features/customers runs sucessfully ✅ 63 | 2022-08-20 13:36:01 - INFO - (DataReporter.cast_to_spreadsheet_friendly_format) Columns casted to string for compatability with Spreadsheets API ✅ 64 | 2022-08-20 13:36:03 - INFO - (DataReporter.write_to_spreadsheet) Data uploaded to Spreadsheet successfully ✅ 65 | 2022-08-20 13:36:03 - INFO - (test_api.test_upload_features_customers) Endpoint /api/v1/features/customers for features uploading runs sucessfully ✅ 66 | 2022-08-20 13:36:03 - INFO - (DataReporter.cast_to_spreadsheet_friendly_format) Columns casted to string for compatability with Spreadsheets API ✅ 67 | 2022-08-20 13:36:04 - INFO - (DataReporter.write_to_spreadsheet) Data uploaded to Spreadsheet successfully ✅ 68 | 2022-08-20 13:36:04 - INFO - (test_api.test_upload_features_loans) Endpoint /api/v1/features/loans for features uploading runs sucessfully ✅ 69 | 2022-08-20 13:36:57 - INFO - ---------------------------------- Optasia API Started 70 | 2022-08-20 13:36:57 - INFO - (test_api.test_fetch_customer) Endpoint /api/v1/customers/1090 runs sucessfully ✅ 71 | 2022-08-20 13:36:57 - INFO - (test_api.test_register_customer) Endpoint /api/v1/customers/ runs sucessfully ✅ 72 | 2022-08-20 13:36:57 - INFO - (test_api.test_fetch_loan) Endpoint /api/v1/customers/1090 runs sucessfully ✅ 73 | 2022-08-20 13:36:57 - INFO - (test_api.test_register_loan) Endpoint /api/v1/loans for the loan registration was sucessful ✅ 74 | 2022-08-20 13:36:57 - INFO - (test_api.test_delete_customer) Endpoint /api/v1/customers/1090 runs sucessfully ✅ 75 | 2022-08-20 13:36:57 - INFO - (test_api.test_delete_loan) Endpoint /api/v1/loans/15 runs sucessfully ✅ 76 | 2022-08-20 13:36:57 - INFO - (DataLoader.postprocess_loans_dataframe) Loans Data are postprocessed sucessfully ✅ 77 | 2022-08-20 13:36:57 - INFO - (FeatureEngineer.extract_features_loans) Manual extraction of features for the loans dataframe was successful ✅ 78 | 2022-08-20 13:36:57 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 79 | 2022-08-20 13:36:57 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 80 | 2022-08-20 13:36:57 - INFO - (FeatureEngineer.add_relationship) The tuple indicating the relationship between 2 dataframes was sucessfully added to the list of relationshipss ✅ 81 | 2022-08-20 13:36:58 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 82 | 2022-08-20 13:36:58 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 83 | 2022-08-20 13:36:58 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 84 | 2022-08-20 13:36:58 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 85 | 2022-08-20 13:36:58 - INFO - (test_api.test_fetch_customers_features) Endpoint /api/v1/features/customers runs sucessfully ✅ 86 | 2022-08-20 13:36:58 - INFO - (DataLoader.postprocess_loans_dataframe) Loans Data are postprocessed sucessfully ✅ 87 | 2022-08-20 13:36:58 - INFO - (FeatureEngineer.extract_features_loans) Manual extraction of features for the loans dataframe was successful ✅ 88 | 2022-08-20 13:36:58 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 89 | 2022-08-20 13:36:58 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 90 | 2022-08-20 13:36:58 - INFO - (FeatureEngineer.add_relationship) The tuple indicating the relationship between 2 dataframes was sucessfully added to the list of relationshipss ✅ 91 | 2022-08-20 13:36:58 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 92 | 2022-08-20 13:36:58 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 93 | 2022-08-20 13:36:58 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 94 | 2022-08-20 13:36:58 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 95 | 2022-08-20 13:36:58 - INFO - (test_api.test_fetch_customers_features) Endpoint /api/v1/features/customers runs sucessfully ✅ 96 | 2022-08-20 13:36:58 - INFO - (DataReporter.cast_to_spreadsheet_friendly_format) Columns casted to string for compatability with Spreadsheets API ✅ 97 | 2022-08-20 13:36:59 - INFO - (DataReporter.write_to_spreadsheet) Data uploaded to Spreadsheet successfully ✅ 98 | 2022-08-20 13:36:59 - INFO - (test_api.test_upload_features_customers) Endpoint /api/v1/features/customers for features uploading runs sucessfully ✅ 99 | 2022-08-20 13:36:59 - INFO - (DataReporter.cast_to_spreadsheet_friendly_format) Columns casted to string for compatability with Spreadsheets API ✅ 100 | 2022-08-20 13:37:01 - INFO - (DataReporter.write_to_spreadsheet) Data uploaded to Spreadsheet successfully ✅ 101 | 2022-08-20 13:37:01 - INFO - (test_api.test_upload_features_loans) Endpoint /api/v1/features/loans for features uploading runs sucessfully ✅ 102 | 2022-08-22 14:59:35 - INFO - ---------------------------------- Feature Engineering API Started 103 | 2022-08-22 14:59:35 - ERROR - (test_api.test_fetch_customer) Hitting endpoint /api/v1/customers/1090 failed (status code 200) : 104 | 2022-08-22 15:00:09 - INFO - ---------------------------------- Feature Engineering API Started 105 | 2022-08-22 15:00:09 - INFO - (test_api.test_fetch_customer) Endpoint /api/v1/customers/1090 runs sucessfully ✅ 106 | 2022-08-22 15:00:09 - INFO - (test_api.test_register_customer) Endpoint /api/v1/customers/ runs sucessfully ✅ 107 | 2022-08-22 15:00:09 - INFO - (test_api.test_fetch_loan) Endpoint /api/v1/customers/1090 runs sucessfully ✅ 108 | 2022-08-22 15:00:09 - INFO - (test_api.test_register_loan) Endpoint /api/v1/loans for the loan registration was sucessful ✅ 109 | 2022-08-22 15:00:09 - INFO - (test_api.test_delete_customer) Endpoint /api/v1/customers/1090 runs sucessfully ✅ 110 | 2022-08-22 15:00:09 - INFO - (test_api.test_delete_loan) Endpoint /api/v1/loans/15 runs sucessfully ✅ 111 | 2022-08-22 15:00:10 - INFO - (DataLoader.postprocess_loans_dataframe) Loans Data are postprocessed sucessfully ✅ 112 | 2022-08-22 15:00:10 - INFO - (FeatureEngineer.extract_features_loans) Manual extraction of features for the loans dataframe was successful ✅ 113 | 2022-08-22 15:00:10 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 114 | 2022-08-22 15:00:10 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 115 | 2022-08-22 15:00:10 - INFO - (FeatureEngineer.add_relationship) The tuple indicating the relationship between 2 dataframes was sucessfully added to the list of relationshipss ✅ 116 | 2022-08-22 15:00:10 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 117 | 2022-08-22 15:00:10 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 118 | 2022-08-22 15:00:10 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 119 | 2022-08-22 15:00:10 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 120 | 2022-08-22 15:00:10 - INFO - (test_api.test_fetch_customers_features) Endpoint /api/v1/features/customers runs sucessfully ✅ 121 | 2022-08-22 15:00:10 - INFO - (DataLoader.postprocess_loans_dataframe) Loans Data are postprocessed sucessfully ✅ 122 | 2022-08-22 15:00:10 - INFO - (FeatureEngineer.extract_features_loans) Manual extraction of features for the loans dataframe was successful ✅ 123 | 2022-08-22 15:00:10 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 124 | 2022-08-22 15:00:10 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 125 | 2022-08-22 15:00:10 - INFO - (FeatureEngineer.add_relationship) The tuple indicating the relationship between 2 dataframes was sucessfully added to the list of relationshipss ✅ 126 | 2022-08-22 15:00:10 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 127 | 2022-08-22 15:00:10 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 128 | 2022-08-22 15:00:10 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 129 | 2022-08-22 15:00:10 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 130 | 2022-08-22 15:00:10 - INFO - (test_api.test_fetch_customers_features) Endpoint /api/v1/features/customers runs sucessfully ✅ 131 | 2022-08-22 15:00:23 - INFO - ---------------------------------- Feature Engineering API Started 132 | 2022-08-22 15:00:23 - INFO - (test_api.test_fetch_customer) Endpoint /api/v1/customers/1090 runs sucessfully ✅ 133 | 2022-08-22 15:00:23 - INFO - (test_api.test_register_customer) Endpoint /api/v1/customers/ runs sucessfully ✅ 134 | 2022-08-22 15:00:23 - INFO - (test_api.test_fetch_loan) Endpoint /api/v1/customers/1090 runs sucessfully ✅ 135 | 2022-08-22 15:00:23 - INFO - (test_api.test_register_loan) Endpoint /api/v1/loans for the loan registration was sucessful ✅ 136 | 2022-08-22 15:00:23 - INFO - (test_api.test_delete_customer) Endpoint /api/v1/customers/1090 runs sucessfully ✅ 137 | 2022-08-22 15:00:23 - INFO - (test_api.test_delete_loan) Endpoint /api/v1/loans/15 runs sucessfully ✅ 138 | 2022-08-22 15:00:23 - INFO - (DataLoader.postprocess_loans_dataframe) Loans Data are postprocessed sucessfully ✅ 139 | 2022-08-22 15:00:23 - INFO - (FeatureEngineer.extract_features_loans) Manual extraction of features for the loans dataframe was successful ✅ 140 | 2022-08-22 15:00:23 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 141 | 2022-08-22 15:00:23 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 142 | 2022-08-22 15:00:23 - INFO - (FeatureEngineer.add_relationship) The tuple indicating the relationship between 2 dataframes was sucessfully added to the list of relationshipss ✅ 143 | 2022-08-22 15:00:23 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 144 | 2022-08-22 15:00:24 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 145 | 2022-08-22 15:00:24 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 146 | 2022-08-22 15:00:24 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 147 | 2022-08-22 15:00:24 - INFO - (test_api.test_fetch_customers_features) Endpoint /api/v1/features/customers runs sucessfully ✅ 148 | 2022-08-22 15:00:24 - INFO - (DataLoader.postprocess_loans_dataframe) Loans Data are postprocessed sucessfully ✅ 149 | 2022-08-22 15:00:24 - INFO - (FeatureEngineer.extract_features_loans) Manual extraction of features for the loans dataframe was successful ✅ 150 | 2022-08-22 15:00:24 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 151 | 2022-08-22 15:00:24 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 152 | 2022-08-22 15:00:24 - INFO - (FeatureEngineer.add_relationship) The tuple indicating the relationship between 2 dataframes was sucessfully added to the list of relationshipss ✅ 153 | 2022-08-22 15:00:24 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 154 | 2022-08-22 15:00:24 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 155 | 2022-08-22 15:00:24 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 156 | 2022-08-22 15:00:24 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 157 | 2022-08-22 15:00:24 - INFO - (test_api.test_fetch_customers_features) Endpoint /api/v1/features/customers runs sucessfully ✅ 158 | 2022-08-22 15:00:54 - INFO - ---------------------------------- Feature Engineering API Started 159 | 2022-08-22 15:00:54 - INFO - (test_api.test_fetch_customer) Endpoint /api/v1/customers/1090 runs sucessfully ✅ 160 | 2022-08-22 15:00:54 - INFO - (test_api.test_register_customer) Endpoint /api/v1/customers/ runs sucessfully ✅ 161 | 2022-08-22 15:00:54 - INFO - (test_api.test_fetch_loan) Endpoint /api/v1/customers/1090 runs sucessfully ✅ 162 | 2022-08-22 15:00:54 - INFO - (test_api.test_register_loan) Endpoint /api/v1/loans for the loan registration was sucessful ✅ 163 | 2022-08-22 15:00:54 - INFO - (test_api.test_delete_customer) Endpoint /api/v1/customers/1090 runs sucessfully ✅ 164 | 2022-08-22 15:00:54 - INFO - (test_api.test_delete_loan) Endpoint /api/v1/loans/15 runs sucessfully ✅ 165 | 2022-08-22 15:00:54 - INFO - (DataLoader.postprocess_loans_dataframe) Loans Data are postprocessed sucessfully ✅ 166 | 2022-08-22 15:00:54 - INFO - (FeatureEngineer.extract_features_loans) Manual extraction of features for the loans dataframe was successful ✅ 167 | 2022-08-22 15:00:54 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 168 | 2022-08-22 15:00:54 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 169 | 2022-08-22 15:00:54 - INFO - (FeatureEngineer.add_relationship) The tuple indicating the relationship between 2 dataframes was sucessfully added to the list of relationshipss ✅ 170 | 2022-08-22 15:00:54 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 171 | 2022-08-22 15:00:54 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 172 | 2022-08-22 15:00:54 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 173 | 2022-08-22 15:00:54 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 174 | 2022-08-22 15:00:54 - INFO - (test_api.test_fetch_customers_features) Endpoint /api/v1/features/customers runs sucessfully ✅ 175 | 2022-08-22 15:00:54 - INFO - (DataLoader.postprocess_loans_dataframe) Loans Data are postprocessed sucessfully ✅ 176 | 2022-08-22 15:00:54 - INFO - (FeatureEngineer.extract_features_loans) Manual extraction of features for the loans dataframe was successful ✅ 177 | 2022-08-22 15:00:54 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 178 | 2022-08-22 15:00:54 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 179 | 2022-08-22 15:00:54 - INFO - (FeatureEngineer.add_relationship) The tuple indicating the relationship between 2 dataframes was sucessfully added to the list of relationshipss ✅ 180 | 2022-08-22 15:00:55 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 181 | 2022-08-22 15:00:55 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 182 | 2022-08-22 15:00:55 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 183 | 2022-08-22 15:00:55 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 184 | 2022-08-22 15:00:55 - INFO - (test_api.test_fetch_customers_features) Endpoint /api/v1/features/customers runs sucessfully ✅ 185 | 2022-08-22 15:01:13 - INFO - ---------------------------------- Feature Engineering API Started 186 | 2022-08-22 15:01:13 - INFO - (test_api.test_fetch_customer) Endpoint /api/v1/customers/1090 runs sucessfully ✅ 187 | 2022-08-22 15:01:13 - INFO - (test_api.test_register_customer) Endpoint /api/v1/customers/ runs sucessfully ✅ 188 | 2022-08-22 15:01:13 - INFO - (test_api.test_fetch_loan) Endpoint /api/v1/customers/1090 runs sucessfully ✅ 189 | 2022-08-22 15:01:13 - INFO - (test_api.test_register_loan) Endpoint /api/v1/loans for the loan registration was sucessful ✅ 190 | 2022-08-22 15:01:13 - INFO - (test_api.test_delete_customer) Endpoint /api/v1/customers/1090 runs sucessfully ✅ 191 | 2022-08-22 15:01:13 - INFO - (test_api.test_delete_loan) Endpoint /api/v1/loans/15 runs sucessfully ✅ 192 | 2022-08-22 15:01:13 - INFO - (DataLoader.postprocess_loans_dataframe) Loans Data are postprocessed sucessfully ✅ 193 | 2022-08-22 15:01:13 - INFO - (FeatureEngineer.extract_features_loans) Manual extraction of features for the loans dataframe was successful ✅ 194 | 2022-08-22 15:01:13 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 195 | 2022-08-22 15:01:13 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 196 | 2022-08-22 15:01:13 - INFO - (FeatureEngineer.add_relationship) The tuple indicating the relationship between 2 dataframes was sucessfully added to the list of relationshipss ✅ 197 | 2022-08-22 15:01:13 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 198 | 2022-08-22 15:01:13 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 199 | 2022-08-22 15:01:13 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 200 | 2022-08-22 15:01:13 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 201 | 2022-08-22 15:01:13 - INFO - (test_api.test_fetch_customers_features) Endpoint /api/v1/features/customers runs sucessfully ✅ 202 | 2022-08-22 15:01:13 - INFO - (DataLoader.postprocess_loans_dataframe) Loans Data are postprocessed sucessfully ✅ 203 | 2022-08-22 15:01:13 - INFO - (FeatureEngineer.extract_features_loans) Manual extraction of features for the loans dataframe was successful ✅ 204 | 2022-08-22 15:01:13 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 205 | 2022-08-22 15:01:13 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 206 | 2022-08-22 15:01:13 - INFO - (FeatureEngineer.add_relationship) The tuple indicating the relationship between 2 dataframes was sucessfully added to the list of relationshipss ✅ 207 | 2022-08-22 15:01:13 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 208 | 2022-08-22 15:01:13 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 209 | 2022-08-22 15:01:13 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 210 | 2022-08-22 15:01:13 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 211 | 2022-08-22 15:01:13 - INFO - (test_api.test_fetch_customers_features) Endpoint /api/v1/features/customers runs sucessfully ✅ 212 | 2022-08-22 15:02:36 - INFO - ---------------------------------- Feature Engineering API Started 213 | 2022-08-22 15:02:36 - INFO - (test_api.test_fetch_customer) Endpoint /api/v1/customers/1090 runs sucessfully ✅ 214 | 2022-08-22 15:02:36 - INFO - (test_api.test_register_customer) Endpoint /api/v1/customers/ runs sucessfully ✅ 215 | 2022-08-22 15:02:37 - INFO - (test_api.test_fetch_loan) Endpoint /api/v1/customers/1090 runs sucessfully ✅ 216 | 2022-08-22 15:02:37 - INFO - (test_api.test_register_loan) Endpoint /api/v1/loans for the loan registration was sucessful ✅ 217 | 2022-08-22 15:02:37 - INFO - (test_api.test_delete_customer) Endpoint /api/v1/customers/1090 runs sucessfully ✅ 218 | 2022-08-22 15:02:37 - INFO - (test_api.test_delete_loan) Endpoint /api/v1/loans/15 runs sucessfully ✅ 219 | 2022-08-22 15:02:37 - INFO - (DataLoader.postprocess_loans_dataframe) Loans Data are postprocessed sucessfully ✅ 220 | 2022-08-22 15:02:37 - INFO - (FeatureEngineer.extract_features_loans) Manual extraction of features for the loans dataframe was successful ✅ 221 | 2022-08-22 15:02:37 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 222 | 2022-08-22 15:02:37 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 223 | 2022-08-22 15:02:37 - INFO - (FeatureEngineer.add_relationship) The tuple indicating the relationship between 2 dataframes was sucessfully added to the list of relationshipss ✅ 224 | 2022-08-22 15:02:37 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 225 | 2022-08-22 15:02:37 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 226 | 2022-08-22 15:02:37 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 227 | 2022-08-22 15:02:37 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 228 | 2022-08-22 15:02:37 - INFO - (test_api.test_fetch_customers_features) Endpoint /api/v1/features/customers runs sucessfully ✅ 229 | 2022-08-22 15:02:37 - INFO - (DataLoader.postprocess_loans_dataframe) Loans Data are postprocessed sucessfully ✅ 230 | 2022-08-22 15:02:37 - INFO - (FeatureEngineer.extract_features_loans) Manual extraction of features for the loans dataframe was successful ✅ 231 | 2022-08-22 15:02:37 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 232 | 2022-08-22 15:02:37 - INFO - (FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅ 233 | 2022-08-22 15:02:37 - INFO - (FeatureEngineer.add_relationship) The tuple indicating the relationship between 2 dataframes was sucessfully added to the list of relationshipss ✅ 234 | 2022-08-22 15:02:37 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 235 | 2022-08-22 15:02:37 - INFO - (FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅ 236 | 2022-08-22 15:02:37 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 237 | 2022-08-22 15:02:37 - INFO - (FeatureEngineer.store_features) Features were sucessfully saved as a csv file ✅ 238 | 2022-08-22 15:02:37 - INFO - (test_api.test_fetch_customers_features) Endpoint /api/v1/features/customers runs sucessfully ✅ 239 | 2022-08-22 15:02:37 - INFO - (DataReporter.cast_to_spreadsheet_friendly_format) Columns casted to string for compatability with Spreadsheets API ✅ 240 | 2022-08-22 15:02:42 - INFO - (DataReporter.write_to_spreadsheet) Data uploaded to Spreadsheet successfully ✅ 241 | 2022-08-22 15:02:42 - INFO - (test_api.test_upload_features_customers) Endpoint /api/v1/features/customers for features uploading runs sucessfully ✅ 242 | 2022-08-22 15:02:42 - INFO - (DataReporter.cast_to_spreadsheet_friendly_format) Columns casted to string for compatability with Spreadsheets API ✅ 243 | 2022-08-22 15:02:50 - INFO - (DataReporter.write_to_spreadsheet) Data uploaded to Spreadsheet successfully ✅ 244 | 2022-08-22 15:02:50 - INFO - (test_api.test_upload_features_loans) Endpoint /api/v1/features/loans for features uploading runs sucessfully ✅ 245 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | textblob==0.17.1 2 | diophila==0.3.0 3 | ujson==5.1.0 4 | tqdm==4.63.0 5 | pytest==7.1.2 6 | python-louvain 7 | pydot 8 | featuretools 9 | fastapi 10 | docker 11 | IPython 12 | gspread 13 | matplotlib 14 | seaborn 15 | graphviz 16 | dataset 17 | uvicorn 18 | -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jimmyg1997/ml-feature-engineering-fastapi-docker/504ad7653fcc1e1a6b0dab91894ce952836e54cb/src/__init__.py -------------------------------------------------------------------------------- /src/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jimmyg1997/ml-feature-engineering-fastapi-docker/504ad7653fcc1e1a6b0dab91894ce952836e54cb/src/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /src/__pycache__/config.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jimmyg1997/ml-feature-engineering-fastapi-docker/504ad7653fcc1e1a6b0dab91894ce952836e54cb/src/__pycache__/config.cpython-38.pyc -------------------------------------------------------------------------------- /src/__pycache__/data_loading.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jimmyg1997/ml-feature-engineering-fastapi-docker/504ad7653fcc1e1a6b0dab91894ce952836e54cb/src/__pycache__/data_loading.cpython-38.pyc -------------------------------------------------------------------------------- /src/__pycache__/data_reporting.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jimmyg1997/ml-feature-engineering-fastapi-docker/504ad7653fcc1e1a6b0dab91894ce952836e54cb/src/__pycache__/data_reporting.cpython-38.pyc -------------------------------------------------------------------------------- /src/__pycache__/feature_engineering.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jimmyg1997/ml-feature-engineering-fastapi-docker/504ad7653fcc1e1a6b0dab91894ce952836e54cb/src/__pycache__/feature_engineering.cpython-38.pyc -------------------------------------------------------------------------------- /src/__pycache__/main.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jimmyg1997/ml-feature-engineering-fastapi-docker/504ad7653fcc1e1a6b0dab91894ce952836e54cb/src/__pycache__/main.cpython-38.pyc -------------------------------------------------------------------------------- /src/__pycache__/markI.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jimmyg1997/ml-feature-engineering-fastapi-docker/504ad7653fcc1e1a6b0dab91894ce952836e54cb/src/__pycache__/markI.cpython-38.pyc -------------------------------------------------------------------------------- /src/__pycache__/models.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jimmyg1997/ml-feature-engineering-fastapi-docker/504ad7653fcc1e1a6b0dab91894ce952836e54cb/src/__pycache__/models.cpython-38.pyc -------------------------------------------------------------------------------- /src/config.ini: -------------------------------------------------------------------------------- 1 | [author] 2 | name = Dimitrios Georgiou 3 | email = dgeorgiou3@gmail.com 4 | 5 | [app] 6 | app_name = Feature Engineering API 7 | 8 | [data] 9 | data_dir = ./data 10 | data_json_path = ./data/data.json 11 | data_csv_path = ./data/data.csv 12 | 13 | 14 | [db] 15 | db_path = ./db 16 | db_file = db.db 17 | 18 | [logger] 19 | log_path = ./logs 20 | log_file = logs.log 21 | log_name = logger 22 | level = INFO 23 | format = %(asctime)s - %(levelname)s - %(message)s 24 | asctime = %Y-%m-%d %H:%M:%S 25 | 26 | [features] 27 | features_dir = ./features 28 | features_customers_path = ./features/features_customers.csv 29 | features_loans_path = ./features/features_loans.csv 30 | 31 | [visualization] 32 | visualization_dir = ./visualization 33 | 34 | [google_sheets_api] 35 | service_token_path = ./config/gspread/service.json 36 | oauth_token_path = ./config/gspread/oauth.json 37 | 38 | [google_sheets] 39 | api_reporter_spreadsheet_id = 1iIBuignJj5oPW7NDRlirenACJuEEeV2RZOi5X9NSH7A 40 | api_reporter_tab_overview = overview 41 | api_reporter_tab_customers = customers_raw 42 | api_reporter_tab_loans = loans_raw 43 | api_reporter_tab_customers_features = customers_features 44 | api_reporter_tab_loans_features = loans_features 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /src/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | from configparser import RawConfigParser 3 | 4 | class Config(object): 5 | """ ConfigParser provides a basic configuration language which provides a structure similar to 6 | Microsoft Windows .INI files 7 | """ 8 | def __init__(self): 9 | self.parser = self.build_parser() 10 | 11 | def build_parser(self, config_path : str = "/src/config.ini"): 12 | """Creates the "parser" object from the "config.ini" file 13 | 14 | :param: `config_path` - path to the "config.ini" file (default: file's root directory) 15 | :returns: ConfigParser object (with write/read on "config.ini") 16 | """ 17 | parser = RawConfigParser() 18 | #path = os.path.join(os.path.dirname(__file__), config_path) 19 | path = os.getcwd() + config_path 20 | #print(path) 21 | parser.read(path) 22 | 23 | 24 | return parser -------------------------------------------------------------------------------- /src/data_loading.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import ujson 4 | import numpy as np 5 | import pandas as pd 6 | import datetime as dt 7 | from IPython.display import display 8 | from collections import defaultdict 9 | from typing import Callable, Dict, Generic, Optional, Set, Tuple, TypeVar, Deque, List, Any 10 | 11 | 12 | from src.models import Customer, LoanStatus, Term, Customer, Loan, CustomerUpdateRequest 13 | from src.config import Config 14 | from src.markI import MkI 15 | 16 | 17 | class DataLoader(object) : 18 | def __init__(self, mk1 : MkI, config : Config): 19 | 20 | # system design 21 | self.mk1 = mk1 22 | self.config = config 23 | 24 | # self.customer_features = ["customer_ID", "annual_income", "loans"] 25 | # self.loans_features = ["customer_ID", "loan_date", "amount", "term", "fee", "loan_status"] 26 | 27 | # data 28 | self.data_dir = str(config.get("data","data_dir")) 29 | self.data_json_path = str(config.get("data","data_json_path")) 30 | self.data_csv_path = str(config.get("data","data_csv_path")) 31 | 32 | 33 | self.customers_df = None 34 | self.customers = [] 35 | self.loans_df = None 36 | self.loans = [] 37 | self.data_df = None 38 | pass 39 | 40 | 41 | #*-*-*-*-*-*-*-*-*-*-*-*# 42 | # Dataframes # 43 | #*-*-*-*-*-*-*-*-*-*-*-*# 44 | 45 | def preprocess_customers_dataframe(self, customers_df : pd.DataFrame) : 46 | try : 47 | 48 | # 1. Extract `annual_income` feature & deletion 49 | customers_df["annual_income"] = customers_df.apply(lambda x : float(x["loans"][0]["annual_income"]), axis = 1) 50 | 51 | # 2. Deletions 52 | customers_df = customers_df.drop(["loans"], axis = 1, errors = 'ignore') 53 | 54 | # 3. Renamings 55 | customers_df = customers_df.rename(columns = {"customer_ID" : "customer_id"}) 56 | 57 | # logger 58 | self.mk1.logging.logger.info("(DataLoader.preprocess_customers_dataframe) Customer Data are preprocessed sucessfully ✅") 59 | return customers_df 60 | 61 | except Exception as e: 62 | self.mk1.logging.logger.error("(DataLoader.preprocess_customers_dataframe) Customer Data preprocessing failed : {}".format(e)) 63 | raise e 64 | 65 | def preprocess_loans_dataframe(self, loans_df : pd.DataFrame) -> pd.DataFrame: 66 | 67 | try : 68 | 69 | # 1. Explode loans into multiple rows & normalize 70 | loans_df = loans_df.explode() 71 | loans_df = pd.json_normalize(loans_df) 72 | 73 | # 2. Deletions 74 | loans_df = loans_df.drop(["annual_income"], axis = 1, errors = 'ignore') 75 | 76 | # 2. Renamings & index 77 | loans_df = loans_df.rename(columns = {"customer_ID" : "customer_id"}) 78 | loans_df = loans_df.rename_axis("loan_id").reset_index().astype(str) 79 | 80 | # logger 81 | self.mk1.logging.logger.info("(DataLoader.preprocess_loans_dataframe) Loans Data are preprocessed sucessfully ✅") 82 | return loans_df 83 | 84 | except Exception as e: 85 | self.mk1.logging.logger.error("(DataLoader.preprocess_loans_dataframe) Loans Data preprocessing failed : {}".format(e)) 86 | raise e 87 | 88 | def postprocess_loans_dataframe(self, loans_df : pd.DataFrame) -> pd.DataFrame : 89 | try : 90 | # Fix Datatypes 91 | loans_df["loan_date"] = pd.to_datetime(loans_df["loan_date"], dayfirst = True) 92 | loans_df["amount"] = loans_df["amount"].apply(pd.to_numeric) 93 | loans_df["fee"] = loans_df["fee"].apply(pd.to_numeric) 94 | 95 | # logger 96 | self.mk1.logging.logger.info("(DataLoader.postprocess_loans_dataframe) Loans Data are postprocessed sucessfully ✅") 97 | return loans_df 98 | 99 | except Exception as e: 100 | self.mk1.logging.logger.error("(DataLoader.postprocess_loans_dataframe) Loans Data postprocessing failed : {}".format(e)) 101 | raise e 102 | 103 | 104 | def split_dataframes(self, data_df : pd.DataFrame) : 105 | 106 | customers_df = data_df.copy() 107 | loans_df = data_df["loans"] 108 | 109 | return (customers_df, loans_df) 110 | 111 | 112 | 113 | 114 | # def get_customers_dataframe(self, data_df : pd.DataFrame) -> pd.DataFrame : 115 | 116 | # try : 117 | 118 | # self.customers_df = data_df.copy() 119 | # self.customers_df["annual_income"] = self.customers_df.apply(lambda x : float(x["loans"][0]["annual_income"]), axis = 1) 120 | # del self.customers_df["loans"] 121 | # self.customers_df = self.customers_df.rename(columns = {"customer_ID" : "customer_id"}) 122 | # #self.customers_df = self.customers_df.reset_index(drop = True) 123 | # # logger 124 | # self.mk1.logging.logger.info("(DataLoader.get_customers_dataframe) Customer Data are retrieved sucessfully ✅") 125 | # return self.customers_df 126 | 127 | # except Exception as e: 128 | # self.mk1.logging.logger.error("(DataLoader.get_customers_dataframe) Customer Data retrieval failed : {}".format(e)) 129 | # raise e 130 | 131 | 132 | 133 | # def get_customers_objects(self): 134 | # """ Iterate over all customer key, values and create a list of customer objects""" 135 | # for _, row in self.customers_df.iterrows(): 136 | 137 | # try : 138 | 139 | # customer = Customer( 140 | # id = row["customer_id"], 141 | # annual_income = row["annual_income"], 142 | # ) 143 | # self.customers.append(customer) 144 | # # logger 145 | # self.mk1.logging.logger.info("(DataLoader.get_customers_objects) Customer object was created sucessfully ✅") 146 | 147 | # except Exception as e: 148 | # self.mk1.logging.logger.error("(DataLoader.get_customers_objects) Customer object creation failed: {}".format(e)) 149 | # raise e 150 | 151 | 152 | 153 | # def get_loans_dataframe(self, data_df : pd.DataFrame) -> pd.DataFrame: 154 | 155 | # try : 156 | 157 | # # 1. Explore loans into multiple rows & normalize 158 | # self.loans_df = data_df["loans"].explode() 159 | # self.loans_df = pd.json_normalize(self.loans_df) 160 | 161 | # del self.loans_df["annual_income"] 162 | 163 | # # 2. Renamings & index 164 | # self.loans_df = self.loans_df.rename(columns = {"customer_ID" : "customer_id"}) 165 | # self.loans_df = self.loans_df.rename_axis("loan_id").reset_index().astype(str) 166 | 167 | # # 3. Fix Datatypes 168 | # self.loans_df["loan_date"] = pd.to_datetime(self.loans_df["loan_date"], dayfirst = True) 169 | # self.loans_df["amount"] = self.loans_df["amount"].apply(pd.to_numeric) 170 | # self.loans_df["fee"] = self.loans_df["fee"].apply(pd.to_numeric) 171 | 172 | # # logger 173 | # self.mk1.logging.logger.info("(DataLoader.get_loans_dataframe) Loans Data are retrieved sucessfully ✅") 174 | # return self.loans_df 175 | 176 | # except Exception as e: 177 | # self.mk1.logging.logger.error("(DataLoader.get_loans_dataframe) Loans Data retrieval failed : {}".format(e)) 178 | # raise e 179 | 180 | 181 | 182 | # def get_loans_objects(self): 183 | # """ Iterate over all customer key, values and create a list of loans objects""" 184 | # for _, row in self.loans_df.iterrows(): 185 | 186 | 187 | # try : 188 | # loan = Loan( 189 | # id = row["loan_id"], 190 | # date = row["loan_date"], 191 | # amount = float(row["amount"]), 192 | # term = Term(row["term"]), 193 | # fee = float(row["fee"]), 194 | # status = LoanStatus(row["loan_status"]) 195 | # ) 196 | # self.loans.append(loan) 197 | 198 | # # logger 199 | # self.mk1.logging.logger.info("(DataLoader.get_loans_objects) Loan object was created sucessfully ✅") 200 | 201 | # except Exception as e: 202 | # self.mk1.logging.logger.error("(DataLoader.get_loans_objects) Loan object creation failed: {}".format(e)) 203 | # raise e 204 | 205 | 206 | #*-*-*-*-*-*-*-*-*-*-*-*# 207 | # json # 208 | #*-*-*-*-*-*-*-*-*-*-*-*# 209 | 210 | def load_data_from_json(self) -> pd.DataFrame: 211 | 212 | 213 | try: 214 | with open(self.data_json_path, "r") as fn: 215 | entries = fn.read() 216 | entries = ujson.loads(entries)["data"] 217 | data_df = pd.DataFrame.from_dict(entries) 218 | # logger 219 | self.mk1.logging.logger.info("(DataLoader.load_data) Data loaded sucessfully ✅") 220 | return data_df 221 | except Exception as e: 222 | self.mk1.logging.logger.error("(DataLoader.load_data) Data loading failed. File name was mistype : {}".format(e)) 223 | raise e 224 | 225 | 226 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*# 227 | # Dataset : Local DB # 228 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*# 229 | 230 | 231 | def serialize_data(self, db : List[Customer]) -> Dict[str, Any] : 232 | customers_l = [] 233 | loans_l = [] 234 | 235 | for customer in db : 236 | loans_l += self.serialize_loans(customer) 237 | customers_l += [ self.serialize_customer(customer) ] 238 | 239 | 240 | return (customers_l, loans_l) 241 | 242 | 243 | def serialize_customer(self, customer : Customer) -> Dict[str, Any] : 244 | customer_dict = vars(customer) 245 | customer_dict["customer_id"] = customer_dict.pop("id") 246 | del customer_dict["loans"] 247 | 248 | return customer_dict 249 | 250 | 251 | def serialize_loans(self, customer : Customer) : 252 | loans_list = [] 253 | customer_dict = vars(customer) 254 | for loan_id, loan in enumerate(customer_dict["loans"]) : 255 | loan_dict = self.serialize_loan(loan, customer_dict["id"], customer_dict["annual_income"]) 256 | loan_dict["loan_id"] = loan_dict.pop("id") 257 | loans_list.append(loan_dict) 258 | 259 | return loans_list 260 | 261 | def serialize_loan(self, loan : Loan, customer_id : str, annual_income : float) : 262 | loan_dict = vars(loan) 263 | loan_dict["customer_id"] = str(customer_id) 264 | loan_dict["loan_date"] = loan_dict["loan_date"].strftime('%m/%d/%Y') 265 | loan_dict["term"] = loan_dict["term"].value 266 | loan_dict["loan_status"] = loan_dict["loan_status"].value 267 | 268 | return loan_dict 269 | 270 | 271 | def clear_local_db(self, db_name : str, pk : str ) : 272 | 273 | ids = self.mk1.dataset.db_query(query_str = f"SELECT {pk} FROM {db_name}")[pk].values 274 | 275 | for id in ids : 276 | 277 | try : 278 | self.mk1.dataset.db_delete(table_name = f"{db_name}", filters_dict = {f"{pk}": id}) 279 | self.mk1.logging.logger.info("(test_api.clear_local_db) Object (id = {}) deleted from db named '{}' sucessfully ✅".format(id, db_name)) 280 | 281 | except Exception as e: 282 | self.mk1.logging.logger.error("(test_api.clear_local_db) Object (id = {}) deletion from local db named '{}' failed {}".format(id, db_name, e)) 283 | raise e 284 | 285 | 286 | def push_data_to_local_db(self, db_name : str, data : List[Dict[str, Any]]) : 287 | 288 | for d_dict in data : 289 | self.mk1.dataset.db_append_row(table_name = db_name, input_dict = d_dict) 290 | 291 | 292 | def load_data_from_local_db(self, db_name : str ) -> pd.DataFrame: 293 | return self.mk1.dataset.db_query(query_str = f"SELECT * FROM {db_name}") 294 | 295 | 296 | def check_if_local_db_empty(self, db_name : str) -> bool: 297 | 298 | if db_name in self.mk1.dataset.get_tables() : 299 | 300 | query_response = self.mk1.dataset.db_query(query_str = f"SELECT count(*) FROM {db_name}") 301 | num_rows = query_response.iloc[0].values[0] 302 | 303 | if num_rows > 0 : 304 | return False 305 | 306 | return True 307 | 308 | def create_local_db(self, db_name : str, pk_name : str, pk_str : str) : 309 | self.mk1.dataset.db_create_table(table_name = db_name, pk_name = pk_name, pk_str = pk_str) 310 | 311 | def delete_local_db(self, db_name : str ) : 312 | self.mk1.dataset.db_delete_table(table_name = db_name) 313 | 314 | 315 | 316 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*# 317 | # General # 318 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*# 319 | 320 | def load_data(self) -> Tuple[pd.DataFrame, pd.DataFrame] : 321 | 322 | if not self.check_if_local_db_empty("customers") : 323 | customers_df = self.load_data_from_local_db(db_name = "customers") 324 | loans_df = self.load_data_from_local_db(db_name = "loans") 325 | 326 | else : 327 | data_df = self.load_data_from_json() 328 | customers_df, loans_df = self.split_dataframes(data_df) 329 | 330 | return (customers_df, loans_df) 331 | 332 | 333 | def preprocess_data(self, customers_df : pd.DataFrame, loans_df : pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame] : 334 | 335 | if self.check_if_local_db_empty("customers") : 336 | customers_df = self.preprocess_customers_dataframe(customers_df) 337 | loans_df = self.preprocess_loans_dataframe(loans_df) 338 | 339 | return (customers_df, loans_df) 340 | 341 | 342 | -------------------------------------------------------------------------------- /src/data_reporting.py: -------------------------------------------------------------------------------- 1 | """ 2 | [*] Description : Google Sheets API reporting 3 | [*] Author : dgeorgiou3@gmail.com 4 | [*] Date : Aug, 2022 5 | [*] Links : 6 | """ 7 | 8 | import os.path 9 | import warnings 10 | import gspread 11 | import ujson 12 | import argparse 13 | import numpy as np 14 | import pandas as pd 15 | import datetime as dt 16 | import matplotlib.pyplot as plt 17 | import matplotlib.cm as cm 18 | import seaborn as sns 19 | 20 | from IPython.display import display 21 | from tqdm import tqdm 22 | from collections import defaultdict 23 | from typing import Any, List, Dict 24 | 25 | # Project modules 26 | from src.config import Config 27 | from src.markI import MkI 28 | from src.data_loading import DataLoader 29 | 30 | class DataReporter(object) : 31 | 32 | def __init__(self, mk1 : MkI, config : Config): 33 | """ 34 | * overview 35 | * customers 36 | * loans 37 | * visualizations (basics from customers, loans) 38 | """ 39 | # system design 40 | self.mk1 = mk1 41 | self.config = config 42 | 43 | # google sheets api 44 | self.service_token_path = str(config.get("google_sheets_api","service_token_path")) 45 | self.google_sheets_api = gspread.service_account(filename = self.service_token_path) 46 | 47 | 48 | # google sheets 49 | self.optasia_reporter_spreadsheet_id = str(config.get("google_sheets","api_reporter_spreadsheet_id")) 50 | self.tabs = { 51 | "customers" : str(config.get("google_sheets","api_reporter_tab_customers")), 52 | "loans" : str(config.get("google_sheets","api_reporter_tab_loans")), 53 | "overview" : str(config.get("google_sheets","api_reporter_tab_overview")), 54 | "customers_features" : str(config.get("google_sheets","api_reporter_tab_customers_features")), 55 | "loans_features" : str(config.get("google_sheets","api_reporter_tab_loans_features")) 56 | 57 | } 58 | 59 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*# 60 | # Utilities # 61 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*# 62 | 63 | def df_to_list(self, 64 | df : pd.DataFrame, 65 | has_index : bool = True, 66 | has_headers : bool = True) -> List[Any]: 67 | 68 | if has_index : 69 | index_name = df.index.name 70 | l = df.reset_index().values.tolist() 71 | 72 | if has_headers: 73 | l.insert(0, [index_name] + list(df.columns)) 74 | 75 | else : 76 | l = df.values.tolist() 77 | 78 | if has_headers: 79 | l.insert(0, list(df.columns)) 80 | 81 | return l 82 | 83 | 84 | 85 | 86 | def cast_to_spreadsheet_friendly_format(self, df : pd.DataFrame) : 87 | 88 | datatype_map = df.dtypes.to_dict() 89 | 90 | 91 | try : 92 | for col in df.columns : 93 | df[col] = df[col].astype(str) 94 | # logger 95 | self.mk1.logging.logger.info("(DataReporter.cast_to_spreadsheet_friendly_format) Columns casted to string for compatability with Spreadsheets API ✅") 96 | 97 | except Exception as e: 98 | self.mk1.logging.logger.error("(DataReporter.cast_to_spreadsheet_friendly_format) Columns casting to string failed : {}".format(e)) 99 | raise e 100 | 101 | return df 102 | 103 | 104 | 105 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*# 106 | # HTTP Requests # 107 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*# 108 | 109 | def write_data_overview(self, features): 110 | """ 111 | Analytis (per customer) 112 | * avg loan number 113 | * avg 114 | """ 115 | 116 | pass 117 | 118 | def write_to_spreadsheet(self, 119 | df : pd.DataFrame, 120 | tab_name : str = "customers", 121 | has_index : bool = False, 122 | has_headers : bool = False ) : 123 | l = self.df_to_list(df, has_index, has_headers) 124 | 125 | try : 126 | 127 | spreadsheet = self.google_sheets_api.open_by_key(self.optasia_reporter_spreadsheet_id) 128 | tab = spreadsheet.worksheet(self.tabs[tab_name]) 129 | tab.clear() 130 | tab.append_rows(l, table_range = 'A1') 131 | 132 | #logger 133 | self.mk1.logging.logger.info("(DataReporter.write_to_spreadsheet) Data uploaded to Spreadsheet successfully ✅") 134 | 135 | except Exception as e: 136 | self.mk1.logging.logger.error("(DataReporter.write_to_spreadsheet) Data uploading to spreadsheet failed : {}".format(e)) 137 | raise e 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | -------------------------------------------------------------------------------- /src/data_visualization.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import numpy as np 4 | import pandas as pd 5 | import featuretools as ft 6 | import datetime as dt 7 | from typing import Callable, Dict, Generic, Optional, Set, Tuple, TypeVar, Deque, List 8 | from IPython.display import display 9 | 10 | 11 | # Project modules 12 | from config import Config 13 | from data_loading import DataLoader 14 | from data_reporting import DataReporter 15 | 16 | 17 | 18 | 19 | 20 | class Visualization(object) : 21 | def __init__(self, ): 22 | """ 23 | Notes 24 | ------ 25 | [1] First, we specify a dictionary with all the DataFrames in our dataset. 26 | The DataFrames are passed in with their index column and time index column 27 | if one exists for the DataFrame. 28 | 29 | [2] Second, we specify how the DataFrames are related. 30 | """ 31 | 32 | # visualizations 33 | self.visualization_dir = str(config.get("google_sheets","optasia_reporter_spreadsheet_id")) 34 | 35 | 36 | def barplot(self, ) : 37 | """ 38 | * Status <> Amount (is it unpaid because it is too much?) 39 | * Status <> Term (is it unpaid because it is long ?) 40 | * Days unpaid <> Amount (how many days have passed being unpaid? Is it unpaid because it is hig amount) 41 | * Days unpaid <> Term (how many days have passed being unpaid? Is it unpaid because it is long) 42 | * Fee percetange <> days 43 | 44 | """ 45 | 46 | 47 | -------------------------------------------------------------------------------- /src/feature_engineering.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import numpy as np 4 | import pandas as pd 5 | import featuretools as ft 6 | import datetime as dt 7 | from typing import Callable, Dict, Generic, Optional, Set, Tuple, TypeVar, Deque, List 8 | from IPython.display import display 9 | from collections import defaultdict 10 | 11 | 12 | # Project modules 13 | from src.markI import MkI 14 | from src.config import Config 15 | from src.data_loading import DataLoader 16 | from src.data_reporting import DataReporter 17 | 18 | 19 | 20 | 21 | class FeatureEngineer(object) : 22 | def __init__(self, mk1 : MkI, config : Config): 23 | # system design 24 | self.mk1 = mk1 25 | self.config = config 26 | 27 | # optasia features 28 | self.features_dir = str(config.get("features","features_dir")) 29 | self.features_paths = { 30 | "features_customers" : str(config.get("features","features_customers_path")), 31 | "features_loans" : str(config.get("features","features_loans_path")), 32 | } 33 | 34 | self.dataframes = defaultdict(set) 35 | self.relationships = [] 36 | self.feature_matrix = None 37 | self.features_defs = None 38 | 39 | 40 | 41 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*# 42 | # Utilities # 43 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*# 44 | 45 | def store_features(self, 46 | features_df : pd.DataFrame, 47 | fn_path : str = "./features/features.csv" 48 | ) : 49 | """ 50 | :param: `type` - It can be either (json, csv) 51 | """ 52 | try : 53 | 54 | fn_type = fn_path.split(".")[-1] 55 | 56 | if fn_type == "json" : 57 | features_df.to_json(fn_path) 58 | 59 | elif fn_type == "csv" : 60 | features_df.to_csv(fn_path) 61 | 62 | #logger 63 | self.mk1.logging.logger.info("(FeatureEngineer.store_features) Features were sucessfully saved as a {} file ✅".format(fn_type)) 64 | 65 | except Exception as e: 66 | self.mk1.logging.logger.error("(FeatureEngineer.store_features) Features file saving failed : {}".format(e)) 67 | raise e 68 | 69 | return 70 | 71 | 72 | #*-*-*-*-*-*-*-*-*-*-*-*# 73 | # featuretools # 74 | #*-*-*-*-*-*-*-*-*-*-*-*# 75 | 76 | def add_dataframe(self, 77 | df_name : str, 78 | df : pd.DataFrame, 79 | pk : str) : 80 | """ 81 | First, we specify a dictionary with all the DataFrames in our dataset. 82 | The DataFrames are passed in with their index column and time index column 83 | if one exists for the DataFrame. 84 | 85 | :param: `pk` - primary key 86 | """ 87 | try : 88 | 89 | self.dataframes[df_name] = (df, pk) 90 | #logger 91 | self.mk1.logging.logger.info("(FeatureEngineer.add_dataframe) Dataframe was sucessfully added to the dictionary of dataframes ✅") 92 | 93 | except Exception as e: 94 | self.mk1.logging.logger.error("(FeatureEngineer.add_dataframe) Dataframe adding to the dict of dataframess failed : {}".format(e)) 95 | raise e 96 | 97 | 98 | def add_relationship(self, 99 | parent_dataframe : str, 100 | parent_column : str, 101 | child_dataframe : str, 102 | child_column : str) -> None : 103 | 104 | try : 105 | 106 | rel = (parent_dataframe, parent_column, child_dataframe, child_column) 107 | self.relationships.append(rel) 108 | #logger 109 | self.mk1.logging.logger.info("(FeatureEngineer.add_relationship) The tuple indicating the relationship between 2 dataframes was sucessfully added to the list of relationshipss ✅") 110 | 111 | except Exception as e: 112 | self.mk1.logging.logger.error("(FeatureEngineer.add_relationship) The relationship tuple was not added. Failure : {}".format(e)) 113 | raise e 114 | 115 | 116 | # def clean_feature_matrix(self, feature_matrix): 117 | # feature_matrix = feature_matrix.fillna(0.0) 118 | # return feature_matrix 119 | 120 | 121 | def run_dfs(self, target_name : str = "customers") : 122 | """ 123 | A minimal input to DFS is a dictionary of DataFrames, a list of relationships, and 124 | the name of the target DataFrame whose features we want to calculate. The ouput of 125 | DFS is a feature matrix and the corresponding list of feature definitions. 126 | """ 127 | 128 | try : 129 | 130 | 131 | feature_matrix, features_defs = ft.dfs( 132 | dataframes = self.dataframes, 133 | relationships = self.relationships, 134 | target_dataframe_name = target_name, 135 | ) 136 | 137 | #logger 138 | self.mk1.logging.logger.info("(FeatureEngineer.run_dfs) DFS was sucessfully executed for all dataframes and relationships ✅") 139 | return (feature_matrix, features_defs) 140 | 141 | except Exception as e: 142 | self.mk1.logging.logger.error("(FeatureEngineer.run_dfs) DFS execution failed : {}".format(e)) 143 | raise e 144 | 145 | 146 | def describe_feature(self, feature_idx : int) : 147 | 148 | feature = self.features_defs[feature_idx] 149 | display(ft.describe_feature(feature)) 150 | ft.graph_feature(feature) 151 | 152 | 153 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*# 154 | # Manual Extraction # 155 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*# 156 | 157 | def extract_days(self, row) : 158 | 159 | return (dt.datetime.now() - row["loan_date"]).days 160 | 161 | 162 | 163 | def extract_features_loans(self, loans_df : pd.DataFrame): 164 | 165 | try : 166 | 167 | loans_df["fee_pct"] = loans_df["fee"] / loans_df["amount"] 168 | loans_df["total_amount"] = loans_df["amount"] + loans_df["fee"] 169 | loans_df["days"] = loans_df.apply(lambda x : self.extract_days(x), axis = 1) 170 | #logger 171 | self.mk1.logging.logger.info("(FeatureEngineer.extract_features_loans) Manual extraction of features for the loans dataframe was successful ✅") 172 | return loans_df 173 | 174 | except Exception as e: 175 | self.mk1.logging.logger.error("(FeatureEngineer.extract_features_loans) Manual extraction of features for the loans dataframe failed : {}".format(e)) 176 | raise e 177 | 178 | 179 | def extract_features_customers(self, customers_df : pd.DataFrame) : 180 | 181 | # 1. `annual income` binning 182 | min_value = float(customers_df["annual_income"].min()) 183 | max_value = float(customers_df["annual_income"].max()) 184 | bins = np.linspace(min_value, max_value, 6) 185 | labels = ['very low', 'low', 'middle', 'high', 'very high'] 186 | customers_df['annual_income_bins'] = pd.cut(customers_df['annual_income'], bins = bins, labels = labels, include_lowest = True) 187 | 188 | 189 | return customers_df 190 | 191 | 192 | 193 | 194 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*# 195 | # Combine Features # 196 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*# 197 | 198 | def combine_features(self, 199 | df1 : pd.DataFrame, 200 | df2 : pd.DataFrame, 201 | pk : str, 202 | ) : 203 | 204 | """Merge Features from both processes (featuretools, manual extraction)""" 205 | try : 206 | cols = df2.columns.difference(df1.columns) 207 | features_df = pd.merge(df1, df2[cols], on = [pk]) 208 | #logger 209 | self.mk1.logging.logger.info("(FeatureEngineer.combine_features) Merging of 2 different features dataframes was successful ✅") 210 | return features_df 211 | 212 | except Exception as e: 213 | self.mk1.logging.logger.error("(FeatureEngineer.combine_features) Merging of 2 different features dataframes failed : {}".format(e)) 214 | raise e 215 | 216 | return features_df 217 | 218 | 219 | def delete_features(self, 220 | features_df : pd.DataFrame, 221 | fts : List[str]) : 222 | """Delete Features that are not useful""" 223 | try : 224 | features_df = features_df.drop(fts, axis = 1) 225 | self.mk1.logging.logger.info("(FeatureEngineer.delete_features) # {} features were deleted successfully ✅".format(len(fts))) 226 | return features_df 227 | 228 | except Exception as e: 229 | self.mk1.logging.logger.error("(FeatureEngineer.delete_features) Deleting # {} features failed : {}".format(len(fts), e)) 230 | raise e 231 | 232 | 233 | 234 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*# 235 | # Feature Importance # 236 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*# 237 | 238 | 239 | 240 | 241 | 242 | if __name__ == "__main__": 243 | 244 | mk1 = MkI.get_instance(_logging = True) 245 | config = Config().parser 246 | 247 | 248 | # Data Loading 249 | data_loader = DataLoader(mk1, config) 250 | data_loader.load_data() 251 | customers_df = data_loader.get_customers_dataframe() 252 | loans_df = data_loader.get_loans_dataframe() 253 | 254 | # ------------------- # 255 | 256 | 257 | # Feature Engineering 258 | ## 1. Extract features for both loans and customers 259 | feature_engineer = FeatureEngineer(mk1, config) 260 | loans_df = feature_engineer.extract_features_loans(loans_df) 261 | customers_df = feature_engineer.extract_features_customers(customers_df) 262 | 263 | 264 | ## 2. Create feature tools (dataframes, relationships) 265 | feature_engineer.add_dataframe("customers", customers_df, "customer_id") 266 | feature_engineer.add_dataframe("loans", loans_df, "loan_id") 267 | feature_engineer.add_relationship("customers", "customer_id", "loans", "customer_id") 268 | 269 | 270 | ## 3. Use "featuretools" to extract extra feature 271 | loans_features_df, _ = feature_engineer.run_dfs(target_name = "loans") 272 | customers_features_df, _ = feature_engineer.run_dfs(target_name = "customers") 273 | 274 | ## 4. Store features as ".csv" file 275 | feature_engineer.store_features( 276 | features_df = loans_features_df, 277 | fn_path = self.features_paths["features_loans"] 278 | ) 279 | 280 | feature_engineer.store_features( 281 | features_df = customers_features_df, 282 | fn_path = self.features_paths["features_customers"] 283 | ) 284 | 285 | 286 | # ------------------- # 287 | 288 | # Data Reporting 289 | data_reporter = DataReporter(mk1, config) 290 | loans_features_df = data_reporter.cast_to_spreadsheet_friendly_format(loans_features_df) 291 | customers_features_df = data_reporter.cast_to_spreadsheet_friendly_format(customers_features_df) 292 | 293 | 294 | data_reporter.write_to_spreadsheet( 295 | df = loans_features_df, 296 | tab_name = data_reporter.optiasia_reporter_tabs["loans_features"], 297 | has_index = False, 298 | has_headers = True 299 | ) 300 | 301 | 302 | data_reporter.write_to_spreadsheet( 303 | df = customers_features_df, 304 | tab_name = data_reporter.optiasia_reporter_tabs["customers_features"], 305 | has_index = False, 306 | has_headers = True 307 | ) 308 | 309 | 310 | 311 | 312 | -------------------------------------------------------------------------------- /src/main.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import numpy as np 4 | import pandas as pd 5 | import datetime as dt 6 | from uuid import UUID, uuid4 7 | from dateutil import parser 8 | from typing import Optional, List, Dict, Any 9 | from IPython.display import display 10 | 11 | ## API modules 12 | from fastapi import FastAPI, HTTPException, status 13 | from fastapi.testclient import TestClient 14 | import docker 15 | 16 | ## Project modules 17 | 18 | from src.models import Customer, LoanStatus, Term, Customer, Loan, CustomerUpdateRequest 19 | from src.markI import MkI 20 | from src.config import Config 21 | from src.data_loading import DataLoader 22 | from src.data_reporting import DataReporter 23 | from src.feature_engineering import FeatureEngineer 24 | 25 | ## Testing db 26 | from db.db import db 27 | 28 | 29 | # client = docker.from_env() 30 | # container = client.containers.run("bfirsh/reticulate-splines", detach=True) 31 | # print(container.id) 32 | 33 | 34 | app = FastAPI() 35 | client = TestClient(app) 36 | mk1 = MkI.get_instance(_logging = True, _dataset = True) 37 | config = Config().parser 38 | 39 | 40 | # db : List[Customer] = [] 41 | # @staticmethod 42 | # def insert(db : Session, item : Item) -> None: 43 | # db.add(item) 44 | # db.commit() 45 | 46 | 47 | # @staticmethod 48 | # def update(db : Session, item : Item) -> None: 49 | # db.commit() 50 | 51 | 52 | @app.get("/") 53 | async def root(): 54 | return {"msg" : "Hello World"} 55 | 56 | 57 | 58 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*# 59 | # Endpoint : Feature Engineering # 60 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*# 61 | 62 | 63 | @app.get("/api/v1/api_status") 64 | async def fetch_api_status(): 65 | 66 | # Check endpoints : "Customers Features", "Loans Features" 67 | response_customers = client.get("/api/v1/features/customers") 68 | response_loans = client.get("/api/v1/features/loans") 69 | 70 | try : 71 | assert response_customers.status_code == 200 72 | assert response_loans.status_code == 200 73 | return json.dumps({"status" : "UP"}) 74 | except Exception as e: 75 | return json.dumps({"status" : "DOWN"}) 76 | 77 | 78 | 79 | @app.get("/api/v1/features/{ontology}") 80 | async def fetch_features(ontology : str): 81 | """Choose the ontology for which features will be fetched {customers, loans}""" 82 | 83 | # Data Loading & Preprocessing 84 | data_loader = DataLoader(mk1, config) 85 | customers_df, loans_df = data_loader.load_data() 86 | customers_df, loans_df = data_loader.preprocess_data(customers_df, loans_df) 87 | loans_df = data_loader.postprocess_loans_dataframe(loans_df) 88 | 89 | 90 | # ------------------- # 91 | 92 | # Feature Engineering 93 | ## 1. Extract features for both loans and customers 94 | feature_engineer = FeatureEngineer(mk1, config) 95 | loans_df = feature_engineer.extract_features_loans(loans_df) 96 | customers_df = feature_engineer.extract_features_customers(customers_df) 97 | 98 | 99 | 100 | ## 2. Create feature tools (dataframes, relationships) 101 | feature_engineer.add_dataframe("customers", customers_df, "customer_id") 102 | feature_engineer.add_dataframe("loans", loans_df, "loan_id") 103 | feature_engineer.add_relationship("customers", "customer_id", "loans", "customer_id") 104 | 105 | 106 | ## 3. Use "featuretools" to extract extra feature 107 | loans_features_df, _ = feature_engineer.run_dfs(target_name = "loans") 108 | customers_features_df, _ = feature_engineer.run_dfs(target_name = "customers") 109 | 110 | ## 4. Store features as ".csv" file 111 | feature_engineer.store_features( 112 | features_df = loans_features_df, 113 | fn_path = feature_engineer.features_paths["features_loans"] 114 | ) 115 | 116 | feature_engineer.store_features( 117 | features_df = customers_features_df, 118 | fn_path = feature_engineer.features_paths["features_customers"] 119 | ) 120 | 121 | ## 5. Convert to json format 122 | customers_features_json = customers_features_df.to_json(orient = "records", indent = 2) 123 | customers_features_dict = json.loads(customers_features_json) 124 | 125 | loans_features_json = loans_features_df.to_json(orient = "records", indent = 2) 126 | loans_features_dict = json.loads(loans_features_json) 127 | 128 | 129 | jsons = { 130 | "customers" : customers_features_json, 131 | "loans" : loans_features_json 132 | } 133 | 134 | ## 6. Return 135 | return jsons[ontology] 136 | 137 | @app.post("/api/v1/features/{ontology}") 138 | async def upload_features(ontology : str): 139 | 140 | data_reporter = DataReporter(mk1, config) 141 | features_path = str(config.get("features",f"features_{ontology}_path")) 142 | features_tab = str(config.get("google_sheets", f"api_reporter_tab_{ontology}_features")) 143 | features_df = pd.read_csv(features_path, index_col = 0) 144 | features_df = data_reporter.cast_to_spreadsheet_friendly_format(features_df) 145 | 146 | data_reporter.write_to_spreadsheet( 147 | df = features_df, 148 | tab_name = features_tab, 149 | has_index = False, 150 | has_headers = True 151 | ) 152 | 153 | 154 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*# 155 | # Endpoint : Local DB # 156 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*# 157 | 158 | 159 | 160 | @app.post("/api/v1/database/{db_name}") 161 | async def clear_local_db(db_name : str, pk_name : str): 162 | data_loader = DataLoader(mk1, config) 163 | 164 | # 1. Delete if existing 165 | data_loader.delete_local_db(db_name = db_name) 166 | 167 | # 2. Create Local DB 168 | data_loader.create_local_db(db_name = db_name, pk_name = pk_name, pk_str = "str") 169 | 170 | 171 | 172 | @app.get("/api/v1/database") 173 | async def create_local_dbs(): 174 | data_loader = DataLoader(mk1, config) 175 | 176 | # 1. Delete if existing 177 | data_loader.delete_local_db(db_name = "customers") 178 | data_loader.delete_local_db(db_name = "loans") 179 | 180 | # 2. Create 181 | data_loader.create_local_db(db_name = "customers", pk_name = "customer_id", pk_str = "str") 182 | data_loader.create_local_db(db_name = "loans", pk_name = "loan_id", pk_str = "str") 183 | 184 | # 3. Clear if not empty 185 | if not data_loader.check_if_local_db_empty(db_name = "customers") : 186 | data_loader.clear_local_db(db_name = "customers", pk = "customer_id") 187 | 188 | if not data_loader.check_if_local_db_empty(db_name = "loans") : 189 | data_loader.clear_local_db(db_name = "loans", pk = "loan_id") 190 | 191 | 192 | # 4. Serialize & Push data to both data tables 193 | customers_l, loans_l = data_loader.serialize_data(db) 194 | data_loader.push_data_to_local_db(db_name = "customers", data = customers_l) 195 | data_loader.push_data_to_local_db(db_name = "loans", data = loans_l) 196 | 197 | 198 | # 5. Check rows of local dbs 199 | display(mk1.dataset.db_query(query_str = "SELECT * FROM loans")) 200 | display(mk1.dataset.db_query(query_str = "SELECT * FROM customers")) 201 | 202 | 203 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*# 204 | # Endpoint : Customers # 205 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*# 206 | 207 | @app.get("/api/v1/customers") 208 | async def fetch_customers(): 209 | return mk1.dataset.db_query(query_str = "SELECT * FROM customers").to_json(orient = "records", indent = 2) 210 | 211 | 212 | 213 | @app.get("/api/v1/customers/{customer_id}") 214 | async def fetch_customer(customer_id : int): 215 | return mk1.dataset.db_query(query_str = f"SELECT * FROM customers WHERE customer_id == {customer_id}").to_json(orient = "records", indent = 2) 216 | 217 | 218 | 219 | @app.post("/api/v1/customers") 220 | async def register_customer(customer_dict : Dict[str, Any]): 221 | mk1.dataset.db_append_row(table_name = "customers", input_dict = customer_dict) 222 | customer_id = customer_dict["customer_id"] 223 | return mk1.dataset.db_query(query_str = f"SELECT * FROM customers WHERE customer_id == {customer_id}").to_json(orient = "records", indent = 2) 224 | 225 | 226 | @app.delete("/api/v1/customers/{customer_id}") 227 | async def delete_customer(customer_id : int): 228 | 229 | try : 230 | mk1.dataset.db_delete(table_name = "customers", filters_dict = {"customer_id": customer_id}) 231 | return mk1.dataset.db_query(query_str = f"SELECT * FROM customers WHERE customer_id == {customer_id}").to_json(orient = "records", indent = 2) 232 | 233 | 234 | except HTTPException : 235 | raise HTTPException( 236 | status_code = 404, 237 | detail = f"Customer with id : {customer_id} does not exist" 238 | ) 239 | 240 | 241 | 242 | # @app.put("/api/v1/customers/{customer_id}") 243 | # async def update_customer(customer_update : CustomerUpdateRequest, customer_id : int): 244 | # for customer in db : 245 | # if customer.id == customer_id: 246 | 247 | # if customer_update.annual_income is not None : 248 | # customer.annual_income = customer_update.annual_income 249 | 250 | # if customer_update.loans is not None : 251 | # customer.loans = customer_update.loans 252 | 253 | # return 254 | 255 | # raise HTTPException( 256 | # status_code = 404, 257 | # detail = f"Customer with id : {customer_id} does not exist" 258 | # ) 259 | 260 | 261 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*# 262 | # Endpoint : Loans # 263 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*# 264 | 265 | @app.get("/api/v1/loans") 266 | async def fetch_loans(): 267 | return mk1.dataset.db_query(query_str = "SELECT * FROM loans").to_json(orient = "records", indent = 2) 268 | 269 | 270 | @app.get("/api/v1/loans/{loan_id}") 271 | async def fetch_loan(loan_id : int): 272 | return mk1.dataset.db_query(query_str = f"SELECT * FROM loans WHERE loan_id == {loan_id}").to_json(orient = "records", indent = 2) 273 | 274 | 275 | @app.post("/api/v1/loans") 276 | async def register_loan(loan_dict : Dict[str, Any]): 277 | mk1.dataset.db_append_row(table_name = "loans", input_dict = loan_dict) 278 | loan_id = loan_dict["loan_id"] 279 | return mk1.dataset.db_query(query_str = f"SELECT * FROM loans WHERE loan_id == {loan_id}").to_json(orient = "records", indent = 2) 280 | 281 | 282 | 283 | 284 | @app.delete("/api/v1/loans/{loan_id}") 285 | async def delete_loan(loan_id : int): 286 | 287 | try : 288 | mk1.dataset.db_delete(table_name = "loans", filters_dict = {"loan_id": loan_id}) 289 | return mk1.dataset.db_query(query_str = f"SELECT * FROM loans WHERE loan_id == {loan_id}").to_json(orient = "records") 290 | 291 | 292 | except HTTPException : 293 | raise HTTPException( 294 | status_code = 404, 295 | detail = f"Loan with id : {loan_id} does not exist" 296 | ) 297 | 298 | 299 | 300 | 301 | 302 | 303 | 304 | 305 | 306 | -------------------------------------------------------------------------------- /src/markI.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import dataset 4 | import numpy as np 5 | import pandas as pd 6 | from datetime import datetime 7 | 8 | import logging 9 | import logging.handlers as handlers 10 | from typing import Callable, Dict, Generic, Optional, Set, Tuple, TypeVar, Deque, List 11 | 12 | 13 | from src.config import Config 14 | 15 | 16 | 17 | ###################################################################################################### 18 | 19 | class MkI(object): 20 | """Builds the Singleton interface for all the contemplated features (treated as attributes)""" 21 | instance = None 22 | 23 | def __init__(self): 24 | pass 25 | 26 | @staticmethod 27 | def get_instance(**kwargs): 28 | if not MkI.instance: 29 | MkI.instance = MkI.__MkI(**kwargs) 30 | return MkI.instance 31 | 32 | class __MkI: 33 | def __init__(self, **kwargs): 34 | self.config = self.get_config() # default: config_path="./config.ini" 35 | self.dataset = self.get_dataset() if kwargs.get("_dataset", False) else None 36 | self.logging = self.get_logging() if kwargs.get("_logging", False) else None 37 | 38 | # Initialize Methods 39 | def get_config(self): 40 | return Config().parser 41 | def get_dataset(self): 42 | return DataSet(self.config) 43 | def get_logging(self): 44 | return Logging(self.config) 45 | 46 | 47 | ###################################################################################################### 48 | 49 | class Logging(object): 50 | """Logging provides a flexible event logging system for applications and libraries""" 51 | def __init__(self, config_obj): 52 | self.config = config_obj 53 | self.level = self.config.get("logger","level") 54 | self.formatter = self.set_formatter() 55 | self.handler = self.set_handler() 56 | self.logger = self.start_logger() 57 | 58 | def set_formatter(self): 59 | """Instantiates the Formatter class and sets the messages/dates formats 60 | 61 | :param: None 62 | :returns: Formatter class instance with format from "self.format" 63 | """ 64 | return logging.Formatter(fmt = self.config.get("logger","format"), 65 | datefmt = self.config.get("logger","asctime")) 66 | 67 | def set_handler(self): 68 | """Instantiates the FileHandler class, sets it as a handler, sets its level and receives the 69 | Formatter instance ("self.formatter") 70 | 71 | :param: None 72 | :returns: FileHandler class instance with "self.formatter" as formatter 73 | """ 74 | # Creating a handler 75 | handler = logging.FileHandler(os.path.join(self.config.get("logger","log_path"), 76 | self.config.get("logger","log_file"))) 77 | # Adding the formatter to the handler 78 | handler.setFormatter(self.formatter) 79 | return handler 80 | 81 | def start_logger(self): 82 | """Instantiates a logger and receives a handler("self.handler") 83 | 84 | :param: None 85 | :returns: Customized logger class with a INFO message to states the beginning of a session 86 | """ 87 | # Creating and storing a new logger 88 | logger = logging.getLogger(self.config.get("logger","log_name")) 89 | # Setting the level on the handler 90 | logger.setLevel(self.level) 91 | # Adding the handler to "my_logger" 92 | logger.addHandler(self.handler) 93 | # Starting the session 94 | logger.info("---------------------------------- {} Started".format(self.config.get("app","app_name"))) 95 | return logger 96 | 97 | 98 | ###################################################################################################### 99 | 100 | class DataSet(object): 101 | """Dataset provides a simple abstraction layer that removes most direct SQL statements without the 102 | necessity for a full ORM model - essentially, databases can be used like a JSON file 103 | """ 104 | def __init__(self, config_obj): 105 | self.config = config_obj 106 | self.db = self.db_connect() 107 | 108 | def auto_search(self): 109 | """Searches for ".db" files within folders in this file's root directory 110 | 111 | :param: None 112 | :returns: dictionary with database's path/name (.db extension) 113 | """ 114 | db_dict = {"path": None, "name": None} 115 | # "os.walk" on this file's root folder (./) 116 | for root, dirs, files in os.walk("./"): 117 | # Loop files 118 | for file in files: 119 | # Check for ".db" files 120 | if file.endswith(".db"): 121 | db_dict["path"], db_dict["name"] = root, file 122 | return db_dict 123 | 124 | def auto_update(self, db_dict): 125 | """Updates the database's path/name, found with "auto_search()", in the "config.ini" params 126 | 127 | :param: dict db_dict: dictionary with database path/name (.db extension) 128 | :returns: None 129 | """ 130 | # Updating the "path"/"name" in "config.ini" file 131 | self.config.set('db', 'db_path', db_dict["path"]) 132 | self.config.set('db', 'db_file', db_dict["name"]) 133 | # # Writing our configuration file to 'example.ini' 134 | # with open("./config.ini", 'w') as config_file: 135 | # self.config.write(config_file) 136 | return None 137 | 138 | def db_connect(self): 139 | """Connects to an existing database or creates a new database from "config.ini" params 140 | 141 | :param: None 142 | :returns: dataset database object 143 | """ 144 | # Searching an existing database 145 | db_info = self.auto_search() 146 | # If database already exists... 147 | if db_info["name"] is not None: 148 | # Connect to existing database 149 | db_obj = dataset.connect(os.path.join("sqlite:///", db_info["path"], db_info["name"])) 150 | 151 | # Updating the "config.ini" file 152 | self.auto_update(db_info) 153 | else: 154 | # Create new database 155 | db_obj = dataset.connect(os.path.join("sqlite:///", 156 | self.config.get("db","db_path"), 157 | self.config.get("db","db_file"))) 158 | 159 | return db_obj 160 | 161 | def db_disconnect(self): 162 | """Disconnects from the database object stored in "self.db" 163 | 164 | :param: None 165 | :returns: None 166 | """ 167 | # Disconnecting from the database 168 | self.db.executable.close() 169 | return None 170 | 171 | def db_create_table(self, table_name = None, pk_name = None, pk_str = None): 172 | """Creates a table with name and primary key (with type) in the "self.db" database object 173 | 174 | :param str table_name: name of the table being created 175 | :param str pk_name: name of the column to be used as primary key 176 | 177 | :returns: None 178 | """ 179 | try: 180 | # Creating the table and commiting changes 181 | self.db.create_table(table_name, primary_id = pk_name, primary_type = self.get_pk_type(pk_str)) 182 | self.db.commit() 183 | except: 184 | # Rolling changes back 185 | self.db.rollback() 186 | return None 187 | 188 | def db_delete_table(self, table_name = None): 189 | """Deletes a table (by its name) in the "self.db" database object 190 | 191 | :param str table_name: name of the table being deleted 192 | :returns: None 193 | """ 194 | try: 195 | # Deleting the table and commiting changes 196 | self.db[table_name].drop() 197 | self.db.commit() 198 | except: 199 | # Rolling changes back 200 | self.db.rollback() 201 | return None 202 | 203 | def db_append_row(self, table_name = None, input_dict = None): 204 | """Appends a single row (through a dictionary) to the "self.db" database object 205 | 206 | :param str table_name: name of the table receiving the row 207 | :param dict input_dict: dictionary holding data to be appended (keys as columns, values as values) 208 | :returns: None 209 | """ 210 | try: 211 | # Inserting a row (through a dictionary) and commiting changes 212 | self.db[table_name].insert(input_dict) 213 | self.db.commit() 214 | except: 215 | # Rolling changes back 216 | self.db.rollback() 217 | return None 218 | 219 | def db_append_df(self, table_name = None, input_df = None): 220 | """Appends multiple rows (through a dataframe) to the "self.db" database object 221 | 222 | :param str table_name: name of the table receiving the rows 223 | :param df input_df: dataframe holding data to be appended (headers as columns, values as values) 224 | :returns: None 225 | """ 226 | # Preparing the Dataframe 227 | df = input_df.to_dict(orient = "records") 228 | try: 229 | # Inserting a row (through df) and commiting changes 230 | self.db[table_name].insert_many(df) 231 | self.db.commit() 232 | except: 233 | # Rolling changes back 234 | self.db.rollback() 235 | return None 236 | 237 | def db_update(self, table_name=None, values_dict=None, col_filter=None): 238 | """Updates all rows filtered by the "col_filter" list with key/values specified by "values_dict" 239 | 240 | :param str table_name: name of the table receiving the data 241 | :param dict values_dict: dictionary with values for "col_filter" and additional columns to be updated 242 | :param lst col_filter: list with columns' names used to filter rows to be updated (value must be inputed in "values_dict") 243 | :returns: None 244 | """ 245 | try: 246 | # Updating rows (based on "col_filter" and "values_dict") and commiting changes 247 | self.db[table_name].update(row = values_dict, keys = col_filter) 248 | self.db.commit() 249 | except: 250 | # Rolling changes back 251 | self.db.rollback() 252 | return None 253 | 254 | def db_upsert(self, table_name=None, values_dict=None, col_filter=None): 255 | """Updates all rows (present in "table_name") filtered by "col_filter" with key/values specified by "values_dict" 256 | Inserts "values_dict" as a new row, otherwise (columns not mentioned in "values_dict" get None as value) 257 | 258 | :param str table_name: name of the table receiving the data 259 | :param dict values_dict: dictionary with values for "col_filter" and additional columns to be updated 260 | :param lst col_filter: list with columns' names used to filter rows to be updated (value must be inputed in "values_dict") 261 | :returns: None 262 | """ 263 | try: 264 | # Updating rows (based on "col_filter" and "values_dict") and commiting changes 265 | self.db[table_name].upsert(row=values_dict, keys=col_filter) 266 | self.db.commit() 267 | except: 268 | # Rolling changes back 269 | self.db.rollback() 270 | return None 271 | 272 | def db_delete(self, table_name = None, filters_dict = None): 273 | """Deletes rows by filters (conditions are joined with ANDs statements) 274 | 275 | :param str table_name: name of the table losing the data 276 | :param dict values_dict: dictionary with values for "col_filter" and additional columns to be updated 277 | :returns: None 278 | """ 279 | try: 280 | # Deleting rows (based on "filters_dict") and commiting changes 281 | self.db[table_name].delete(**filters_dict) 282 | self.db.commit() 283 | except: 284 | # Rolling changes back 285 | self.db.rollback() 286 | return None 287 | 288 | def db_query(self, query_str=None): 289 | """Queries against the "self.db" database object 290 | 291 | :param str query_str: complete query string 292 | :returns: dataframe with query results 293 | """ 294 | try: 295 | # Querying the db and commiting changes 296 | result = self.db.query(query_str) 297 | self.db.commit() 298 | # Pushing "result" to a Dataframe 299 | df = pd.DataFrame(data=list(result)) 300 | return df 301 | except: 302 | # Rolling changes back 303 | self.db.rollback() 304 | return None 305 | 306 | def get_pk_type(self, pk_str): 307 | """Translates pre-defined strings to SQLite data types, used on "db_create_table"s "primary_type" parameter 308 | 309 | :param str pk_type: String representation of data type. Any of: 310 | - "b_int": for big integers (returns db.types.biginteger) 311 | - "int": for integers (returns db.types.integer) 312 | - "s_int": for small integers (returns db.types.smallinteger) 313 | - "float": for floats (returns db.types.float) 314 | - "str": for fixed-sized strings (returns db.types.string) 315 | - "txt": for variable-sized strings (returns db.types.text) 316 | - "bool": for booleans (returns db.types.boolean) 317 | - "date": for datetime.date() objects (returns db.types.date) 318 | - "datetime": for datetime.datetime() objects (returns db.types.datetime) 319 | :returns: SQLite data type obj 320 | """ 321 | # Translating "pk_type" into a SQLite data type object 322 | if pk_str.lower() == "b_int": 323 | return self.db.types.biginteger 324 | elif pk_str.lower() == "int": 325 | return self.db.types.integer 326 | elif pk_str.lower() == "s_int": 327 | return self.db.types.smallinteger 328 | elif pk_str.lower() == "float": 329 | return self.db.types.float 330 | elif pk_str.lower() == "str": 331 | return self.db.types.string 332 | elif pk_str.lower() == "txt": 333 | return self.db.types.string 334 | elif pk_str.lower() == "bool": 335 | return self.db.types.boolean 336 | elif pk_str.lower() == "date": 337 | return self.db.types.date 338 | elif pk_str.lower() == "datetime": 339 | return self.db.types.datetime 340 | else: 341 | return None 342 | 343 | def get_tables(self): 344 | """Lists all existing tables in the database 345 | 346 | :param: None 347 | :returns: list with existing tables' names in the database 348 | """ 349 | return self.db.tables 350 | 351 | def get_cols(self, table_name = None): 352 | """Lists all existing columns in a table 353 | 354 | :param str table_name: name of the table containing the columns 355 | :returns: list with existing columns in "table_name" 356 | """ 357 | return self.db[table_name].columns 358 | 359 | def get_rows(self, table_name=None): 360 | """Gets the total rows in a table 361 | 362 | :param str table_name: name of the table containing the rows 363 | :returns: total rows (integer) in the "table_name" 364 | """ 365 | return len(self.db[table_name]) 366 | 367 | def get_unique(self, table_name=None, col_name=None): 368 | """Gets unique values for a column in a table 369 | 370 | :param str table_name: name of the table containing the column 371 | :param str col_name: name of the column to be analyzed 372 | :returns: list with unique values in "col_name" 373 | """ 374 | return [list(each.values())[0] for each in self.db[table_name].distinct(col_name)] -------------------------------------------------------------------------------- /src/models.py: -------------------------------------------------------------------------------- 1 | import datetime as dt 2 | 3 | from pydantic import BaseModel 4 | from typing import Optional, List, Dict, Any 5 | from uuid import UUID, uuid4 6 | from enum import Enum 7 | 8 | 9 | class LoanStatus(str, Enum) : 10 | paid = "0" 11 | not_paid = "1" 12 | 13 | class Term(str, Enum) : 14 | long = "long" 15 | short = "short" 16 | 17 | 18 | 19 | class Loan(BaseModel) : 20 | id : int 21 | loan_date : dt.datetime 22 | amount : float 23 | term : Term 24 | fee : float 25 | loan_status : LoanStatus 26 | 27 | 28 | class Customer(BaseModel): 29 | id : int 30 | annual_income : float 31 | loans : Optional[List[Loan]] 32 | 33 | 34 | class CustomerUpdateRequest(BaseModel): 35 | annual_income : Optional[float] 36 | loans : Optional[List[Loan]] 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /tests/test_api.py: -------------------------------------------------------------------------------- 1 | import json 2 | from dateutil import parser 3 | from typing import Optional, List, Dict, Any 4 | from fastapi.testclient import TestClient 5 | from IPython.display import display 6 | 7 | 8 | from src.main import app 9 | from src.markI import MkI 10 | from src.config import Config 11 | from src.data_loading import DataLoader 12 | from src.models import Customer, LoanStatus, Term, Customer, Loan, CustomerUpdateRequest 13 | 14 | 15 | client = TestClient(app) 16 | mk1 = MkI.get_instance(_logging = True, _dataset = True) 17 | config = Config().parser 18 | data_loader = DataLoader(mk1, config) 19 | 20 | 21 | def test_create_local_dbs() : 22 | response = client.get("/api/v1/database") 23 | 24 | 25 | 26 | def test_fetch_customer(): 27 | response = client.get("/api/v1/customers/1090") 28 | response_dict = json.loads(response.json()) 29 | #response_dict = response.json() 30 | 31 | try : 32 | assert response.status_code == 200 33 | assert response_dict[0]["customer_id"] == '1090' 34 | assert response_dict[0]["annual_income"] == 41333 35 | # logger 36 | mk1.logging.logger.info("(test_api.test_fetch_customer) Endpoint /api/v1/customers/1090 runs sucessfully ✅") 37 | 38 | except Exception as e: 39 | mk1.logging.logger.error("(test_api.test_fetch_customer) Hitting endpoint /api/v1/customers/1090 failed (status code {}) : {}".format(response.status_code, e)) 40 | raise e 41 | 42 | 43 | 44 | def test_register_customer(): 45 | data = { "customer_id" : 1423, 46 | "annual_income" : 34513 47 | } 48 | 49 | response = client.post("/api/v1/customers", json.dumps(data)) 50 | response_dict = json.loads(response.json()) 51 | #response_dict = response.json() 52 | 53 | 54 | try : 55 | assert response.status_code == 200 56 | assert response_dict[0]["customer_id"] == "1423" 57 | assert response_dict[0]["annual_income"] == 34513 58 | # logger 59 | mk1.logging.logger.info("(test_api.test_register_customer) Endpoint /api/v1/customers/ runs sucessfully ✅") 60 | 61 | except Exception as e: 62 | mk1.logging.logger.error("(test_api.test_register_customer) Hitting endpoint /api/v1/customers/ failed (status code {}) : {}".format(response.status_code, e)) 63 | raise e 64 | 65 | 66 | def test_delete_customer() : 67 | 68 | response = client.delete("/api/v1/customers/1090") 69 | #response_dict = response.json() 70 | response_dict = json.loads(response.json()) 71 | 72 | try : 73 | assert response.status_code == 200 74 | assert response_dict == [] 75 | # logger 76 | mk1.logging.logger.info("(test_api.test_delete_customer) Endpoint /api/v1/customers/1090 runs sucessfully ✅") 77 | 78 | except Exception as e: 79 | mk1.logging.logger.error("(test_api.test_delete_customer) Hitting endpoint /api/v1/customers/1090 failed (status code {}) : {}".format(response.status_code, e)) 80 | raise e 81 | 82 | 83 | 84 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*# 85 | # Endpoint : Loans # 86 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*# 87 | 88 | def test_fetch_loan(): 89 | response = client.get("/api/v1/loans/1") 90 | #response_dict = response.json() 91 | response_dict = json.loads(response.json()) 92 | 93 | try : 94 | assert response.status_code == 200 95 | assert response_dict[0]["loan_id"] == "1" 96 | assert response_dict[0]["loan_date"] == "11/15/2021" 97 | assert response_dict[0]["amount"] == 2426.0 98 | assert response_dict[0]["term"] == "long" 99 | assert response_dict[0]["fee"] == 199.0 100 | assert response_dict[0]["loan_status"] == "1" 101 | assert response_dict[0]["customer_id"] == "1090" 102 | # logger 103 | mk1.logging.logger.info("(test_api.test_fetch_loan) Endpoint /api/v1/customers/1090 runs sucessfully ✅") 104 | 105 | except Exception as e: 106 | mk1.logging.logger.error("(test_api.test_fetch_loan) Hitting endpoint /api/v1/customers/1090 failed (status code {}) : {}".format(response.status_code, e)) 107 | raise e 108 | 109 | 110 | 111 | def test_register_loan(): 112 | data = { 113 | "loan_id" : 15, 114 | "loan_date" : "08/20/2022", 115 | "amount" : 1000, 116 | "term" : "long", 117 | "fee" : 100, 118 | "loan_status" : "0", 119 | "customer_id" : 1090 120 | } 121 | 122 | response = client.post("/api/v1/loans", json.dumps(data)) 123 | #response_dict = response.json() 124 | response_dict = json.loads(response.json()) 125 | 126 | 127 | try : 128 | assert response.status_code == 200 129 | assert response_dict[0]["loan_id"] == "15" 130 | assert response_dict[0]["loan_date"] == "08/20/2022" 131 | assert response_dict[0]["amount"] == 1000.0 132 | assert response_dict[0]["term"] == "long" 133 | assert response_dict[0]["fee"] == 100.0 134 | assert response_dict[0]["loan_status"] == "0" 135 | assert response_dict[0]["customer_id"] == "1090" 136 | 137 | # logger 138 | mk1.logging.logger.info("(test_api.test_register_loan) Endpoint /api/v1/loans for the loan registration was sucessful ✅") 139 | 140 | except Exception as e: 141 | mk1.logging.logger.error("(test_api.test_register_loan) Hitting endpoint /api/v1/loans for loan registration failed (status code {}) : {}".format(response.status_code, e)) 142 | raise e 143 | 144 | 145 | def test_delete_loan() : 146 | 147 | response = client.delete("/api/v1/loans/1") 148 | #response_dict = response.json() 149 | response_dict = json.loads(response.json()) 150 | 151 | 152 | try : 153 | assert response.status_code == 200 154 | assert response_dict == [] 155 | # logger 156 | mk1.logging.logger.info("(test_api.test_delete_loan) Endpoint /api/v1/loans/15 runs sucessfully ✅") 157 | 158 | except Exception as e: 159 | mk1.logging.logger.error("(test_api.test_delete_loan) Hitting endpoint /api/v1/loans/15 failed (status code {}) : {}".format(response.status_code, e)) 160 | raise e 161 | 162 | 163 | 164 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*# 165 | # Endpoint : Feature Engineering # 166 | #*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*# 167 | 168 | def test_fetch_features_customers(): 169 | response = client.get("/api/v1/features/customers") 170 | print(json.dumps(json.loads(response.json()), indent = 4, sort_keys = True)) 171 | 172 | try : 173 | assert response.status_code == 200 174 | mk1.logging.logger.info("(test_api.test_fetch_customers_features) Endpoint /api/v1/features/customers runs sucessfully ✅") 175 | 176 | except Exception as e: 177 | mk1.logging.logger.error("(test_api.test_delete_loan) Hitting endpoint /api/v1/features/customers failed (status code {}) : {}".format(response.status_code, e)) 178 | raise e 179 | 180 | def test_fetch_features_loans(): 181 | response = client.get("/api/v1/features/loans") 182 | print(json.dumps(json.loads(response.json()), indent = 4, sort_keys = True)) 183 | 184 | try : 185 | assert response.status_code == 200 186 | mk1.logging.logger.info("(test_api.test_fetch_customers_features) Endpoint /api/v1/features/customers runs sucessfully ✅") 187 | 188 | except Exception as e: 189 | mk1.logging.logger.error("(test_api.test_delete_loan) Hitting endpoint /api/v1/features/customers failed (status code {}) : {}".format(response.status_code, e)) 190 | raise e 191 | 192 | def test_upload_features_customers(): 193 | response = client.post("/api/v1/features/customers", "customers") 194 | 195 | try : 196 | assert response.status_code == 200 197 | mk1.logging.logger.info("(test_api.test_upload_features_customers) Endpoint /api/v1/features/customers for features uploading runs sucessfully ✅") 198 | 199 | except Exception as e: 200 | mk1.logging.logger.error("(test_api.test_upload_features_customers) Hitting endpoint /api/v1/features/customers for features uploading failed (status code {}) : {}".format(response.status_code, e)) 201 | raise e 202 | 203 | 204 | def test_upload_features_loans(): 205 | response = client.post("/api/v1/features/loans", "loans") 206 | 207 | try : 208 | assert response.status_code == 200 209 | mk1.logging.logger.info("(test_api.test_upload_features_loans) Endpoint /api/v1/features/loans for features uploading runs sucessfully ✅") 210 | 211 | except Exception as e: 212 | mk1.logging.logger.error("(test_api.test_upload_features_loans) Hitting endpoint /api/v1/features/loans for features uploading failed (status code {}) : {}".format(response.status_code, e)) 213 | raise e 214 | 215 | 216 | def test_api_status() : 217 | response = client.get("/api/v1/api_status") 218 | response_dict = json.loads(response.json()) 219 | print(response, response_dict, response_dict["status"]) 220 | 221 | try : 222 | assert response.status_code == 200 223 | assert response_dict["status"] == "UP" 224 | mk1.logging.logger.info("(test_api.test_api_status) Both endpoints /api/v1/features/loans and /api/v1/features/customers runs sucessfully ✅") 225 | 226 | except Exception as e: 227 | mk1.logging.logger.error("(test_api.test_api_status) 1 and/or 2 of the endpoints /api/v1/features/loans and /api/v1/features/customers failed (status code {}) : {}".format(response.status_code, e)) 228 | raise e 229 | 230 | 231 | 232 | 233 | if __name__ == '__main__': 234 | print("----- Initial Databases") 235 | test_create_local_dbs() 236 | test_fetch_customer() 237 | print("----- Registering Customer") 238 | test_register_customer() 239 | display(mk1.dataset.db_query(query_str = "SELECT * FROM customers")) 240 | test_fetch_loan() 241 | print("----- Registering Loan") 242 | test_register_loan() 243 | display(mk1.dataset.db_query(query_str = "SELECT * FROM loans")) 244 | print("----- Deleting Customer & Loan") 245 | test_delete_customer() 246 | test_delete_loan() 247 | display(mk1.dataset.db_query(query_str = "SELECT * FROM customers")) 248 | display(mk1.dataset.db_query(query_str = "SELECT * FROM loans")) 249 | 250 | print("----- Final Merged Datatable") 251 | display(mk1.dataset.db_query(query_str = "SELECT * FROM customers c INNER JOIN loans l ON l.customer_id = c.customer_id")) 252 | 253 | 254 | print("----- Feature Engineering") 255 | test_fetch_features_customers() 256 | test_fetch_features_loans() 257 | test_upload_features_customers() 258 | test_upload_features_loans() 259 | #test_api_status() 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | --------------------------------------------------------------------------------