├── .gitignore ├── LICENSE ├── README.md ├── automl-predict.ipynb ├── media ├── demo.gif ├── demo.mp4 ├── notebook.gif ├── pipeline_AutoML.png ├── run-automl.png ├── web-app-advanced.gif ├── web-app-download.gif ├── web-app-notebooks.gif ├── web-app-online.png ├── web-app-predictions.png ├── web-app-xgboost.gif ├── web-app.gif └── what_is_supervised.png ├── requirements.txt ├── train-automl-advanced.ipynb └── train-automl.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 MLJAR 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # New way for building ML pipelines! 3 | 4 | We are working on new way for Python visual programming. We developed desktop application called [MLJAR Studio](https://mljar.com). 5 | It is a notebook based development environment with interactive code recipes and managed Python environment. All running locally on your machine. We are waiting for your feedback. 6 | 7 | It has [code recipes](https://mljar.com/docs/python-automl/) to build ML pipelines with MLJAR AutoML. 8 | 9 |

10 | mljar AutoML 13 |

14 | 15 | --- 16 | 17 | # AutoML Web App 🤖 18 | 19 |

20 | 🚀 AutoML 21 |   •   22 | 📓 Mercury 23 |   •   24 | 🤝 Issues 25 |   •   26 | 🐦 Twitter 27 |   •   28 | 👩‍💼 LinkedIn 29 |   •   30 | 🌐 MLJAR Website 31 |

32 | 33 | 34 | This is a Web Application designed to train Machine Learning pipelines using MLJAR AutoML, specifically tailored for tabular data. All the generated models are compressed into an archive format, allowing their reuse to compute predictions in batch mode. 35 | 36 | This repo consists of three notebooks: 37 | - [notebook](https://github.com/mljar/automl-app/blob/main/train-automl.ipynb) for training AutoML with simple UI, 38 | - [advanced notebook](https://github.com/mljar/automl-app/blob/main/train-automl-advanced.ipynb) for training AutoML with more advanced UI (you can select feature engineering methods, algorithms, validation strategy, and evaluation metric), 39 | - [notebook](https://github.com/mljar/automl-app/blob/main/automl-predict.ipynb) for computing predictions. 40 | 41 | 42 | The Web App harnesses the capabilities of [mljar-supervised](https://github.com/mljar/mercury) to construct the Machine Learning pipeline with AutoML. This involves the automation of several key tasks: 43 | - data preprocessing, 44 | - features engineering, 45 | - algorithm selection & tuning, 46 | - ML models explanations, 47 | - automatic documentation. 48 |

49 | Supervised learning 50 |

51 | 52 | The Web App is created directly from Jupyter Notebooks with [Mercury](https://github.com/mljar/mercury) framework. 53 | 54 | ### Demo 55 | 56 | 57 | 58 | https://github.com/mljar/automl-app/assets/6959032/3363631a-2187-44cd-94a8-3cbd5418de98 59 | 60 | 61 | 62 | ### Online demo 63 | 64 | The Web App is available online at [automl.runmercury.com](https://automl.runmercury.com). Input data upload is limited to 1MB. 65 | 66 | 67 | AutoML Web App online 68 | 69 | 70 | ### Run locally 🖥️ 71 | 72 | Please run the below commands to run Web App locally. It requires Python >= 3.8. 73 | 74 | ```bash 75 | pip install -r requirements.txt 76 | mercury run 77 | ``` 78 | 79 | ### Training Notebook 📓 80 | 81 | If you would like to increase the input file limit, please change the cell: 82 | 83 | ```python 84 | data_file = mr.File(label="Upload CSV with training data", max_file_size="1MB") 85 | ``` 86 | 87 | and set your `max_file_size`. 88 | 89 | Please change the following cell to increase training time: 90 | 91 | ```python 92 | time_limit = mr.Select(label="Time limit (seconds)", value="60", choices=["60", "120", "240", "300"]) 93 | ``` 94 | 95 | Times are in seconds. Please just increase the values. 96 | 97 | 98 | AutoML training notebook 99 | 100 | 101 | ### Training models in Web App 102 | 103 | Please upload a CSV file with training data, select input features & target, and click `Start training`. 104 | 105 | 106 | AutoML training in Web App 107 | 108 | 109 | All models created during the training are available for download as a zip file: 110 | 111 | 112 | AutoML models available for download 113 | 114 | 115 | ### Advanced Training Notebook 💪 116 | 117 | Please use advanced mode if you would like to tweak AutoML parameters: 118 | 119 | 120 | Advanced AutoML training notebook 121 | 122 | 123 | ## 👩‍💼🐦 Connect with Us on LinkedIn & Twitter 124 | 125 | Stay up-to-date with the latest updates about MLJAR 🎨🤖 by following us on Twitter ([MLJAR Twitter](https://twitter.com/MLJAROfficial)) and LinkedIn ([Aleksandra LinkedIn](https://www.linkedin.com/in/aleksandra-p%C5%82o%C5%84ska-42047432/) & [Piotr LinkedIn](https://www.linkedin.com/in/piotr-plonski-mljar/)). We look forward to connecting with you and hearing your thoughts, ideas, and experiences. 126 | 127 | 128 | ### Good luck with ML training! 129 | -------------------------------------------------------------------------------- /automl-predict.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "cd859f09", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import os\n", 11 | "import shutil\n", 12 | "import pandas as pd\n", 13 | "import mercury as mr\n", 14 | "from supervised.automl import AutoML " 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 2, 20 | "id": "d5e4f553", 21 | "metadata": {}, 22 | "outputs": [], 23 | "source": [ 24 | "import warnings\n", 25 | "warnings.filterwarnings(\"ignore\")" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 3, 31 | "id": "e4a86241", 32 | "metadata": {}, 33 | "outputs": [ 34 | { 35 | "data": { 36 | "application/mercury+json": "{\n \"widget\": \"App\",\n \"title\": \"Predict with AutoML \\ud83c\\udfaf\",\n \"description\": \"Compute predictions on test data with AutoML\",\n \"show_code\": false,\n \"show_prompt\": false,\n \"output\": \"app\",\n \"schedule\": \"\",\n \"notify\": \"{}\",\n \"continuous_update\": true,\n \"static_notebook\": false,\n \"show_sidebar\": true,\n \"full_screen\": true,\n \"allow_download\": true,\n \"stop_on_error\": false,\n \"model_id\": \"mercury-app\",\n \"code_uid\": \"App.0.40.25.1-randc8be43bc\"\n}", 37 | "text/html": [ 38 | "

Mercury Application

This output won't appear in the web app." 39 | ], 40 | "text/plain": [ 41 | "mercury.App" 42 | ] 43 | }, 44 | "metadata": {}, 45 | "output_type": "display_data" 46 | } 47 | ], 48 | "source": [ 49 | "app = mr.App(title=\"Predict with AutoML 🎯\", description=\"Compute predictions on test data with AutoML\")" 50 | ] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "id": "9ab46d42", 55 | "metadata": {}, 56 | "source": [ 57 | "# Compute predictions\n", 58 | "\n", 59 | "Please follow the steps:\n", 60 | "1. Upload zip file with AutoML.\n", 61 | "2. Upload test data as CSV file.\n", 62 | "3. Download predictions." 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": 4, 68 | "id": "f2acef34", 69 | "metadata": {}, 70 | "outputs": [ 71 | { 72 | "data": { 73 | "application/mercury+json": "{\n \"widget\": \"File\",\n \"max_file_size\": \"100MB\",\n \"label\": \"Upload zip with AutoML\",\n \"model_id\": \"43b96d54f6c24b73949563e6d4b933b8\",\n \"code_uid\": \"File.0.40.18.1-rand3b8fa11b\",\n \"disabled\": false,\n \"hidden\": false\n}", 74 | "application/vnd.jupyter.widget-view+json": { 75 | "model_id": "43b96d54f6c24b73949563e6d4b933b8", 76 | "version_major": 2, 77 | "version_minor": 0 78 | }, 79 | "text/plain": [ 80 | "mercury.File" 81 | ] 82 | }, 83 | "metadata": {}, 84 | "output_type": "display_data" 85 | } 86 | ], 87 | "source": [ 88 | "automl_zip = mr.File(label=\"Upload zip with AutoML\")" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": 5, 94 | "id": "c4bef80f", 95 | "metadata": {}, 96 | "outputs": [], 97 | "source": [ 98 | "if automl_zip.filepath is None:\n", 99 | " mr.Stop()" 100 | ] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "execution_count": null, 105 | "id": "172440f1", 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "extract_dir = \"my_automl\"" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": null, 115 | "id": "00c491c1", 116 | "metadata": {}, 117 | "outputs": [], 118 | "source": [ 119 | "shutil.unpack_archive(automl_zip.filepath, extract_dir, \"zip\")" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": null, 125 | "id": "144d9d52", 126 | "metadata": {}, 127 | "outputs": [], 128 | "source": [ 129 | "automl = AutoML(results_path=extract_dir)" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": null, 135 | "id": "8a070136", 136 | "metadata": {}, 137 | "outputs": [], 138 | "source": [] 139 | }, 140 | { 141 | "cell_type": "code", 142 | "execution_count": null, 143 | "id": "ab4d18b1", 144 | "metadata": {}, 145 | "outputs": [], 146 | "source": [ 147 | "test_data = mr.File(label=\"Upload test data\")" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": null, 153 | "id": "69c8bfc6", 154 | "metadata": {}, 155 | "outputs": [], 156 | "source": [ 157 | "if test_data.filepath is None:\n", 158 | " mr.Stop()" 159 | ] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": null, 164 | "id": "45d2a28e", 165 | "metadata": {}, 166 | "outputs": [], 167 | "source": [ 168 | "df = pd.read_csv(test_data.filepath)" 169 | ] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "execution_count": null, 174 | "id": "559e0c12", 175 | "metadata": {}, 176 | "outputs": [], 177 | "source": [ 178 | "df" 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": null, 184 | "id": "99dfd387", 185 | "metadata": {}, 186 | "outputs": [], 187 | "source": [] 188 | }, 189 | { 190 | "cell_type": "code", 191 | "execution_count": null, 192 | "id": "d67b4d0e", 193 | "metadata": {}, 194 | "outputs": [], 195 | "source": [ 196 | "predictions = automl.predict_all(df)" 197 | ] 198 | }, 199 | { 200 | "cell_type": "code", 201 | "execution_count": null, 202 | "id": "a8a5d19e", 203 | "metadata": {}, 204 | "outputs": [], 205 | "source": [ 206 | "predictions" 207 | ] 208 | }, 209 | { 210 | "cell_type": "code", 211 | "execution_count": null, 212 | "id": "c3dc1562", 213 | "metadata": {}, 214 | "outputs": [], 215 | "source": [ 216 | "output_dir = mr.OutputDir()" 217 | ] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "execution_count": null, 222 | "id": "b995ba03", 223 | "metadata": {}, 224 | "outputs": [], 225 | "source": [ 226 | "predictions.to_csv(os.path.join(output_dir.path, \"predictions.csv\"), index=False)" 227 | ] 228 | }, 229 | { 230 | "cell_type": "code", 231 | "execution_count": null, 232 | "id": "ca7e1737", 233 | "metadata": {}, 234 | "outputs": [], 235 | "source": [] 236 | } 237 | ], 238 | "metadata": { 239 | "kernelspec": { 240 | "display_name": "appenv", 241 | "language": "python", 242 | "name": "appenv" 243 | }, 244 | "language_info": { 245 | "codemirror_mode": { 246 | "name": "ipython", 247 | "version": 3 248 | }, 249 | "file_extension": ".py", 250 | "mimetype": "text/x-python", 251 | "name": "python", 252 | "nbconvert_exporter": "python", 253 | "pygments_lexer": "ipython3", 254 | "version": "3.8.10" 255 | } 256 | }, 257 | "nbformat": 4, 258 | "nbformat_minor": 5 259 | } 260 | -------------------------------------------------------------------------------- /media/demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mljar/automl-app/047a6903d3276b85643af0b7e69a0b594898ea3d/media/demo.gif -------------------------------------------------------------------------------- /media/demo.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mljar/automl-app/047a6903d3276b85643af0b7e69a0b594898ea3d/media/demo.mp4 -------------------------------------------------------------------------------- /media/notebook.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mljar/automl-app/047a6903d3276b85643af0b7e69a0b594898ea3d/media/notebook.gif -------------------------------------------------------------------------------- /media/pipeline_AutoML.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mljar/automl-app/047a6903d3276b85643af0b7e69a0b594898ea3d/media/pipeline_AutoML.png -------------------------------------------------------------------------------- /media/run-automl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mljar/automl-app/047a6903d3276b85643af0b7e69a0b594898ea3d/media/run-automl.png -------------------------------------------------------------------------------- /media/web-app-advanced.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mljar/automl-app/047a6903d3276b85643af0b7e69a0b594898ea3d/media/web-app-advanced.gif -------------------------------------------------------------------------------- /media/web-app-download.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mljar/automl-app/047a6903d3276b85643af0b7e69a0b594898ea3d/media/web-app-download.gif -------------------------------------------------------------------------------- /media/web-app-notebooks.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mljar/automl-app/047a6903d3276b85643af0b7e69a0b594898ea3d/media/web-app-notebooks.gif -------------------------------------------------------------------------------- /media/web-app-online.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mljar/automl-app/047a6903d3276b85643af0b7e69a0b594898ea3d/media/web-app-online.png -------------------------------------------------------------------------------- /media/web-app-predictions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mljar/automl-app/047a6903d3276b85643af0b7e69a0b594898ea3d/media/web-app-predictions.png -------------------------------------------------------------------------------- /media/web-app-xgboost.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mljar/automl-app/047a6903d3276b85643af0b7e69a0b594898ea3d/media/web-app-xgboost.gif -------------------------------------------------------------------------------- /media/web-app.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mljar/automl-app/047a6903d3276b85643af0b7e69a0b594898ea3d/media/web-app.gif -------------------------------------------------------------------------------- /media/what_is_supervised.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mljar/automl-app/047a6903d3276b85643af0b7e69a0b594898ea3d/media/what_is_supervised.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | mljar-supervised 2 | mercury 3 | -------------------------------------------------------------------------------- /train-automl-advanced.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "9aab5e1e", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import os\n", 11 | "import shutil\n", 12 | "import pandas as pd\n", 13 | "import mercury as mr\n", 14 | "from supervised.automl import AutoML " 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 2, 20 | "id": "5b576b07", 21 | "metadata": {}, 22 | "outputs": [], 23 | "source": [ 24 | "import warnings\n", 25 | "warnings.filterwarnings(\"ignore\")" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 3, 31 | "id": "6313fc0d", 32 | "metadata": {}, 33 | "outputs": [ 34 | { 35 | "data": { 36 | "application/mercury+json": "{\n \"widget\": \"App\",\n \"title\": \"Train AutoML (advanced) \\ud83e\\udd13\",\n \"description\": \"Train ML pipeline with MLJAR AutoML with more params\",\n \"show_code\": false,\n \"show_prompt\": false,\n \"output\": \"app\",\n \"schedule\": \"\",\n \"notify\": \"{}\",\n \"continuous_update\": true,\n \"static_notebook\": false,\n \"show_sidebar\": true,\n \"full_screen\": true,\n \"allow_download\": true,\n \"stop_on_error\": false,\n \"model_id\": \"mercury-app\",\n \"code_uid\": \"App.0.40.25.1-randb32964f3\"\n}", 37 | "text/html": [ 38 | "

Mercury Application

This output won't appear in the web app." 39 | ], 40 | "text/plain": [ 41 | "mercury.App" 42 | ] 43 | }, 44 | "metadata": {}, 45 | "output_type": "display_data" 46 | } 47 | ], 48 | "source": [ 49 | "app = mr.App(title=\"Train AutoML (advanced) 🤓\", \n", 50 | " description=\"Train ML pipeline with MLJAR AutoML with more params\")" 51 | ] 52 | }, 53 | { 54 | "cell_type": "markdown", 55 | "id": "6d62fe00", 56 | "metadata": {}, 57 | "source": [ 58 | "# Train Machine Learning Pipeline with MLJAR AutoML\n", 59 | "\n", 60 | "You can control AutoML behavior with more parameters. This notebook is running autoML in the `Compete` mode. \n", 61 | "\n", 62 | "You can choose:\n", 63 | "- feature preprocessing parameters: golden features and features selection\n", 64 | "- select algorithms, stack, and ensemble them,\n", 65 | "- set cross-validation strategy (number of folds, stratify and shuffle),\n", 66 | "- choose evaluation metric.\n", 67 | "\n", 68 | "### Steps\n", 69 | "1. Upload CSV file with data. Data should have column names in the first line.\n", 70 | "2. Select input features and target column.\n", 71 | "3. Select AutoML training mode, algorithms, and training time limit.\n", 72 | "4. Directory with all ML models will be zipped and available to download." 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": 4, 78 | "id": "aafac626", 79 | "metadata": {}, 80 | "outputs": [ 81 | { 82 | "data": { 83 | "application/mercury+json": "{\n \"widget\": \"File\",\n \"max_file_size\": \"1MB\",\n \"label\": \"Upload CSV with training data\",\n \"model_id\": \"6594fd64518f4a9b974e68aaec3702fc\",\n \"code_uid\": \"File.0.40.18.1-rand943e01dd\",\n \"disabled\": false,\n \"hidden\": false\n}", 84 | "application/vnd.jupyter.widget-view+json": { 85 | "model_id": "6594fd64518f4a9b974e68aaec3702fc", 86 | "version_major": 2, 87 | "version_minor": 0 88 | }, 89 | "text/plain": [ 90 | "mercury.File" 91 | ] 92 | }, 93 | "metadata": {}, 94 | "output_type": "display_data" 95 | } 96 | ], 97 | "source": [ 98 | "data_file = mr.File(label=\"Upload CSV with training data\", max_file_size=\"1MB\")" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": 5, 104 | "id": "cb3cfa8c", 105 | "metadata": {}, 106 | "outputs": [], 107 | "source": [ 108 | "if data_file.filepath is None:\n", 109 | " mr.Stop()" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": null, 115 | "id": "02c8b639", 116 | "metadata": {}, 117 | "outputs": [], 118 | "source": [ 119 | "df = pd.read_csv(data_file.filepath)" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": null, 125 | "id": "7557b68a", 126 | "metadata": {}, 127 | "outputs": [], 128 | "source": [ 129 | "mr.Markdown(\"### Training data\")" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": null, 135 | "id": "83921208", 136 | "metadata": {}, 137 | "outputs": [], 138 | "source": [ 139 | "df" 140 | ] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "execution_count": null, 145 | "id": "44fbf6e2", 146 | "metadata": {}, 147 | "outputs": [], 148 | "source": [ 149 | "x_columns = mr.MultiSelect(label=\"Input features\", value=list(df.columns)[:-1], \n", 150 | " choices=list(df.columns))" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": null, 156 | "id": "81d64072", 157 | "metadata": {}, 158 | "outputs": [], 159 | "source": [ 160 | "y_column = mr.Select(label=\"Target\", value=list(df.columns)[-1], choices=list(df.columns))" 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": null, 166 | "id": "465f07df", 167 | "metadata": {}, 168 | "outputs": [], 169 | "source": [ 170 | "if x_columns.value is None or len(x_columns.value) == 0 or y_column.value is None:\n", 171 | " print(\"Please select input features and target column\")\n", 172 | " mr.Stop()" 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": null, 178 | "id": "c91f5ef3", 179 | "metadata": {}, 180 | "outputs": [], 181 | "source": [ 182 | "_ = mr.Note(\"#### Prepare data\")" 183 | ] 184 | }, 185 | { 186 | "cell_type": "code", 187 | "execution_count": null, 188 | "id": "5adbefad", 189 | "metadata": {}, 190 | "outputs": [], 191 | "source": [ 192 | "golden_features = mr.Checkbox(label=\"Construct Golden Features\")" 193 | ] 194 | }, 195 | { 196 | "cell_type": "code", 197 | "execution_count": null, 198 | "id": "c829fd4c", 199 | "metadata": {}, 200 | "outputs": [], 201 | "source": [ 202 | "features_selection = mr.Checkbox(label=\"Features Selection\")" 203 | ] 204 | }, 205 | { 206 | "cell_type": "code", 207 | "execution_count": null, 208 | "id": "6e4cc96d", 209 | "metadata": {}, 210 | "outputs": [], 211 | "source": [ 212 | "_ = mr.Note(\"#### Algorithms\")" 213 | ] 214 | }, 215 | { 216 | "cell_type": "code", 217 | "execution_count": null, 218 | "id": "4ad21129", 219 | "metadata": {}, 220 | "outputs": [], 221 | "source": [ 222 | "algos = [\"Decision Tree\", \"Linear\", \"Random Forest\", \"Extra Trees\", \"LightGBM\", \n", 223 | " \"Xgboost\", \"CatBoost\", \"Neural Network\", \"Nearest Neighbors\"]\n" 224 | ] 225 | }, 226 | { 227 | "cell_type": "code", 228 | "execution_count": null, 229 | "id": "e957c5a2", 230 | "metadata": {}, 231 | "outputs": [], 232 | "source": [ 233 | "algorithms = mr.MultiSelect(label=\"Algorithms\", value=algos, choices=algos)" 234 | ] 235 | }, 236 | { 237 | "cell_type": "code", 238 | "execution_count": null, 239 | "id": "b85b4124", 240 | "metadata": {}, 241 | "outputs": [], 242 | "source": [] 243 | }, 244 | { 245 | "cell_type": "code", 246 | "execution_count": null, 247 | "id": "9cd4a21b", 248 | "metadata": {}, 249 | "outputs": [], 250 | "source": [] 251 | }, 252 | { 253 | "cell_type": "code", 254 | "execution_count": null, 255 | "id": "78506f25", 256 | "metadata": {}, 257 | "outputs": [], 258 | "source": [ 259 | "stack_models = mr.Checkbox(label=\"Stack Models\")" 260 | ] 261 | }, 262 | { 263 | "cell_type": "code", 264 | "execution_count": null, 265 | "id": "5bd6c864", 266 | "metadata": {}, 267 | "outputs": [], 268 | "source": [ 269 | "train_ensemble = mr.Checkbox(label=\"Train Ensemble\")" 270 | ] 271 | }, 272 | { 273 | "cell_type": "code", 274 | "execution_count": null, 275 | "id": "052df3f7", 276 | "metadata": {}, 277 | "outputs": [], 278 | "source": [] 279 | }, 280 | { 281 | "cell_type": "code", 282 | "execution_count": null, 283 | "id": "532ff2e5", 284 | "metadata": {}, 285 | "outputs": [], 286 | "source": [ 287 | "_ = mr.Note(\"#### Validation\")" 288 | ] 289 | }, 290 | { 291 | "cell_type": "code", 292 | "execution_count": null, 293 | "id": "d1d8d9ab", 294 | "metadata": {}, 295 | "outputs": [], 296 | "source": [ 297 | "folds = mr.Numeric(label=\"Number of Folds\", value=5, min=2, max=100)" 298 | ] 299 | }, 300 | { 301 | "cell_type": "code", 302 | "execution_count": null, 303 | "id": "9f103cf9", 304 | "metadata": {}, 305 | "outputs": [], 306 | "source": [ 307 | "shuffle = mr.Checkbox(label=\"Suffle Samples\")" 308 | ] 309 | }, 310 | { 311 | "cell_type": "code", 312 | "execution_count": null, 313 | "id": "2d234a75", 314 | "metadata": {}, 315 | "outputs": [], 316 | "source": [ 317 | "stratify = mr.Checkbox(label=\"Stratify Samples\")" 318 | ] 319 | }, 320 | { 321 | "cell_type": "code", 322 | "execution_count": null, 323 | "id": "21de4493", 324 | "metadata": {}, 325 | "outputs": [], 326 | "source": [ 327 | "eval_metric = mr.Select(label=\"Evaluation Metric\", value=\"auto\", \n", 328 | " choices=[\"auto\", \"logloss\", \"f1\", \"average_precision\",\n", 329 | " \"accuracy\", \"rmse\", \"mse\", \"mae\", \"r2\",\n", 330 | " \"mape\", \"spearman\", \"pearson\"])" 331 | ] 332 | }, 333 | { 334 | "cell_type": "code", 335 | "execution_count": null, 336 | "id": "bc779217", 337 | "metadata": {}, 338 | "outputs": [], 339 | "source": [ 340 | "time_limit = mr.Select(label=\"Time Limit (seconds)\", value=\"60\", choices=[\"60\", \"120\", \"240\", \"300\"])" 341 | ] 342 | }, 343 | { 344 | "cell_type": "code", 345 | "execution_count": null, 346 | "id": "b3e6ba6c", 347 | "metadata": {}, 348 | "outputs": [], 349 | "source": [ 350 | "start_training = mr.Button(label=\"Start Training\", style=\"success\")" 351 | ] 352 | }, 353 | { 354 | "cell_type": "code", 355 | "execution_count": null, 356 | "id": "a735b3a7", 357 | "metadata": {}, 358 | "outputs": [], 359 | "source": [ 360 | "output_dir = mr.OutputDir()" 361 | ] 362 | }, 363 | { 364 | "cell_type": "code", 365 | "execution_count": null, 366 | "id": "022b6eec", 367 | "metadata": {}, 368 | "outputs": [], 369 | "source": [ 370 | "automl = AutoML(mode=\"Compete\", \n", 371 | " algorithms=algorithms.value,\n", 372 | " train_ensemble=train_ensemble.value,\n", 373 | " stack_models=stack_models.value,\n", 374 | " golden_features=golden_features.value,\n", 375 | " features_selection=features_selection.value,\n", 376 | " validation_strategy={\n", 377 | " \"validation_type\": \"kfold\",\n", 378 | " \"k_folds\": int(folds.value),\n", 379 | " \"shuffle\": shuffle.value,\n", 380 | " \"stratify\": stratify.value,\n", 381 | " \"random_seed\": 123\n", 382 | " },\n", 383 | " eval_metric=eval_metric.value,\n", 384 | " total_time_limit=int(time_limit.value))" 385 | ] 386 | }, 387 | { 388 | "cell_type": "code", 389 | "execution_count": null, 390 | "id": "f07b6a70", 391 | "metadata": {}, 392 | "outputs": [], 393 | "source": [] 394 | }, 395 | { 396 | "cell_type": "code", 397 | "execution_count": null, 398 | "id": "79246d1b", 399 | "metadata": {}, 400 | "outputs": [], 401 | "source": [ 402 | "if start_training.clicked:\n", 403 | " mr.Markdown(\"### AutoML training logs\")\n", 404 | " automl.fit(df[x_columns.value], df[y_column.value])\n", 405 | " \n", 406 | " output_filename = os.path.join(output_dir.path, automl._results_path)\n", 407 | " shutil.make_archive(output_filename, 'zip', automl._results_path)" 408 | ] 409 | }, 410 | { 411 | "cell_type": "code", 412 | "execution_count": null, 413 | "id": "30eefe3c", 414 | "metadata": {}, 415 | "outputs": [], 416 | "source": [ 417 | "if automl._best_model is None:\n", 418 | " mr.Stop()" 419 | ] 420 | }, 421 | { 422 | "cell_type": "code", 423 | "execution_count": null, 424 | "id": "8fcfbc1a", 425 | "metadata": {}, 426 | "outputs": [], 427 | "source": [ 428 | "automl.report()" 429 | ] 430 | }, 431 | { 432 | "cell_type": "code", 433 | "execution_count": null, 434 | "id": "202bccd9", 435 | "metadata": {}, 436 | "outputs": [], 437 | "source": [] 438 | } 439 | ], 440 | "metadata": { 441 | "kernelspec": { 442 | "display_name": "appenv", 443 | "language": "python", 444 | "name": "appenv" 445 | }, 446 | "language_info": { 447 | "codemirror_mode": { 448 | "name": "ipython", 449 | "version": 3 450 | }, 451 | "file_extension": ".py", 452 | "mimetype": "text/x-python", 453 | "name": "python", 454 | "nbconvert_exporter": "python", 455 | "pygments_lexer": "ipython3", 456 | "version": "3.8.10" 457 | } 458 | }, 459 | "nbformat": 4, 460 | "nbformat_minor": 5 461 | } 462 | -------------------------------------------------------------------------------- /train-automl.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "9aab5e1e", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import os\n", 11 | "import shutil\n", 12 | "import pandas as pd\n", 13 | "import mercury as mr\n", 14 | "from supervised.automl import AutoML " 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 2, 20 | "id": "d7009f5f", 21 | "metadata": {}, 22 | "outputs": [], 23 | "source": [ 24 | "import warnings\n", 25 | "warnings.filterwarnings(\"ignore\")" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 3, 31 | "id": "6313fc0d", 32 | "metadata": {}, 33 | "outputs": [ 34 | { 35 | "data": { 36 | "application/mercury+json": "{\n \"widget\": \"App\",\n \"title\": \"Train AutoML \\ud83e\\uddd1\\u200d\\ud83d\\udcbb\",\n \"description\": \"Train ML pipeline with MLJAR AutoML\",\n \"show_code\": false,\n \"show_prompt\": false,\n \"output\": \"app\",\n \"schedule\": \"\",\n \"notify\": \"{}\",\n \"continuous_update\": true,\n \"static_notebook\": false,\n \"show_sidebar\": true,\n \"full_screen\": true,\n \"allow_download\": true,\n \"stop_on_error\": false,\n \"model_id\": \"mercury-app\",\n \"code_uid\": \"App.0.40.25.1-randfee23f4b\"\n}", 37 | "text/html": [ 38 | "

Mercury Application

This output won't appear in the web app." 39 | ], 40 | "text/plain": [ 41 | "mercury.App" 42 | ] 43 | }, 44 | "metadata": {}, 45 | "output_type": "display_data" 46 | } 47 | ], 48 | "source": [ 49 | "app = mr.App(title=\"Train AutoML 🧑‍💻\", description=\"Train ML pipeline with MLJAR AutoML\")" 50 | ] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "id": "6d62fe00", 55 | "metadata": {}, 56 | "source": [ 57 | "# Train Machine Learning Pipeline with MLJAR AutoML\n", 58 | "Please follow the steps:\n", 59 | "1. Upload CSV file with data. Data should heave column names in the first line.\n", 60 | "2. Select input features and target column.\n", 61 | "3. Select AutoML training mode, algorithms and training time limit.\n", 62 | "4. Directory with all ML models will be zipped and available to download." 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": 4, 68 | "id": "aafac626", 69 | "metadata": {}, 70 | "outputs": [ 71 | { 72 | "data": { 73 | "application/mercury+json": "{\n \"widget\": \"File\",\n \"max_file_size\": \"1MB\",\n \"label\": \"Upload CSV with training data\",\n \"model_id\": \"0d0815292051454cb67184fa96a7d7d5\",\n \"code_uid\": \"File.0.40.18.1-rand74e7cef0\",\n \"disabled\": false,\n \"hidden\": false\n}", 74 | "application/vnd.jupyter.widget-view+json": { 75 | "model_id": "0d0815292051454cb67184fa96a7d7d5", 76 | "version_major": 2, 77 | "version_minor": 0 78 | }, 79 | "text/plain": [ 80 | "mercury.File" 81 | ] 82 | }, 83 | "metadata": {}, 84 | "output_type": "display_data" 85 | } 86 | ], 87 | "source": [ 88 | "data_file = mr.File(label=\"Upload CSV with training data\", max_file_size=\"1MB\")" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": 5, 94 | "id": "cb3cfa8c", 95 | "metadata": {}, 96 | "outputs": [], 97 | "source": [ 98 | "if data_file.filepath is None:\n", 99 | " mr.Stop()" 100 | ] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "execution_count": null, 105 | "id": "02c8b639", 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "df = pd.read_csv(data_file.filepath)" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": null, 115 | "id": "7557b68a", 116 | "metadata": {}, 117 | "outputs": [], 118 | "source": [ 119 | "mr.Markdown(\"### Training data\")" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": null, 125 | "id": "83921208", 126 | "metadata": {}, 127 | "outputs": [], 128 | "source": [ 129 | "df" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": null, 135 | "id": "44fbf6e2", 136 | "metadata": {}, 137 | "outputs": [], 138 | "source": [ 139 | "x_columns = mr.MultiSelect(label=\"Input features\", value=list(df.columns)[:-1], \n", 140 | " choices=list(df.columns))" 141 | ] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "execution_count": null, 146 | "id": "81d64072", 147 | "metadata": {}, 148 | "outputs": [], 149 | "source": [ 150 | "y_column = mr.Select(label=\"Target\", value=list(df.columns)[-1], choices=list(df.columns))" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": null, 156 | "id": "465f07df", 157 | "metadata": {}, 158 | "outputs": [], 159 | "source": [ 160 | "if x_columns.value is None or len(x_columns.value) == 0 or y_column.value is None:\n", 161 | " print(\"Please select input features and target column\")\n", 162 | " mr.Stop()" 163 | ] 164 | }, 165 | { 166 | "cell_type": "code", 167 | "execution_count": null, 168 | "id": "c91f5ef3", 169 | "metadata": {}, 170 | "outputs": [], 171 | "source": [] 172 | }, 173 | { 174 | "cell_type": "code", 175 | "execution_count": null, 176 | "id": "6e4cc96d", 177 | "metadata": {}, 178 | "outputs": [], 179 | "source": [ 180 | "mode = mr.Select(label=\"AutoML Mode\", value=\"Explain\", choices=[\"Explain\", \"Perform\", \"Compete\"])" 181 | ] 182 | }, 183 | { 184 | "cell_type": "code", 185 | "execution_count": null, 186 | "id": "4ad21129", 187 | "metadata": {}, 188 | "outputs": [], 189 | "source": [ 190 | "algos = {\n", 191 | " \"Explain\": [\"Baseline\", \"Linear\", \"Decision Tree\", \"Random Forest\", \"Xgboost\", \"Neural Network\"],\n", 192 | " \"Perform\": [\"Linear\", \"Random Forest\", \"LightGBM\", \"Xgboost\", \"CatBoost\", \"Neural Network\"],\n", 193 | " \"Compete\": [\"Decision Tree\", \"Random Forest\", \"Extra Trees\", \"LightGBM\", \n", 194 | " \"Xgboost\", \"CatBoost\", \"Neural Network\", \"Nearest Neighbors\"]\n", 195 | "}" 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": null, 201 | "id": "e957c5a2", 202 | "metadata": {}, 203 | "outputs": [], 204 | "source": [ 205 | "algorithms = mr.MultiSelect(label=\"Algorithms\", value=algos[mode.value], choices=algos[mode.value])" 206 | ] 207 | }, 208 | { 209 | "cell_type": "code", 210 | "execution_count": null, 211 | "id": "b85b4124", 212 | "metadata": {}, 213 | "outputs": [], 214 | "source": [ 215 | "time_limit = mr.Select(label=\"Time limit (seconds)\", value=\"60\", choices=[\"60\", \"120\", \"240\", \"300\"])" 216 | ] 217 | }, 218 | { 219 | "cell_type": "code", 220 | "execution_count": null, 221 | "id": "b3e6ba6c", 222 | "metadata": {}, 223 | "outputs": [], 224 | "source": [ 225 | "start_training = mr.Button(label=\"Start training\", style=\"success\")" 226 | ] 227 | }, 228 | { 229 | "cell_type": "code", 230 | "execution_count": null, 231 | "id": "a735b3a7", 232 | "metadata": {}, 233 | "outputs": [], 234 | "source": [ 235 | "output_dir = mr.OutputDir()" 236 | ] 237 | }, 238 | { 239 | "cell_type": "code", 240 | "execution_count": null, 241 | "id": "022b6eec", 242 | "metadata": {}, 243 | "outputs": [], 244 | "source": [ 245 | "automl = AutoML(mode=mode.value, algorithms=algorithms.value,\n", 246 | " total_time_limit=int(time_limit.value))" 247 | ] 248 | }, 249 | { 250 | "cell_type": "code", 251 | "execution_count": null, 252 | "id": "79246d1b", 253 | "metadata": {}, 254 | "outputs": [], 255 | "source": [ 256 | "if start_training.clicked:\n", 257 | " mr.Markdown(\"### AutoML training logs\")\n", 258 | " automl.fit(df[x_columns.value], df[y_column.value])\n", 259 | " \n", 260 | " output_filename = os.path.join(output_dir.path, automl._results_path)\n", 261 | " shutil.make_archive(output_filename, 'zip', automl._results_path)" 262 | ] 263 | }, 264 | { 265 | "cell_type": "code", 266 | "execution_count": null, 267 | "id": "30eefe3c", 268 | "metadata": {}, 269 | "outputs": [], 270 | "source": [ 271 | "if automl._best_model is None:\n", 272 | " mr.Stop()" 273 | ] 274 | }, 275 | { 276 | "cell_type": "code", 277 | "execution_count": null, 278 | "id": "8fcfbc1a", 279 | "metadata": {}, 280 | "outputs": [], 281 | "source": [ 282 | "automl.report()" 283 | ] 284 | }, 285 | { 286 | "cell_type": "code", 287 | "execution_count": null, 288 | "id": "202bccd9", 289 | "metadata": {}, 290 | "outputs": [], 291 | "source": [] 292 | } 293 | ], 294 | "metadata": { 295 | "kernelspec": { 296 | "display_name": "appenv", 297 | "language": "python", 298 | "name": "appenv" 299 | }, 300 | "language_info": { 301 | "codemirror_mode": { 302 | "name": "ipython", 303 | "version": 3 304 | }, 305 | "file_extension": ".py", 306 | "mimetype": "text/x-python", 307 | "name": "python", 308 | "nbconvert_exporter": "python", 309 | "pygments_lexer": "ipython3", 310 | "version": "3.8.10" 311 | } 312 | }, 313 | "nbformat": 4, 314 | "nbformat_minor": 5 315 | } 316 | --------------------------------------------------------------------------------