├── tests ├── config.py └── test_datasets.py ├── miguellib ├── models │ └── __init__.py ├── tuning │ └── __init__.py ├── utils │ └── __init__.py ├── datasets │ ├── __init__.py │ └── dummy.py ├── evaluation │ └── __init__.py └── __init__.py ├── docs └── README.md ├── examples ├── README.md ├── template.ipynb └── models │ └── model1.ipynb ├── setup.py ├── LICENSE ├── .github └── workflows │ └── pr_gate.yml ├── .gitignore └── README.md /tests/config.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /miguellib/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /miguellib/tuning/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /miguellib/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Documentation -------------------------------------------------------------------------------- /miguellib/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /miguellib/evaluation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # Jupyter notebook examples -------------------------------------------------------------------------------- /miguellib/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.1.2" 2 | VERSION = __version__ 3 | -------------------------------------------------------------------------------- /miguellib/datasets/dummy.py: -------------------------------------------------------------------------------- 1 | def dummy_dataset(): 2 | return [1, 2, 3, 4] 3 | -------------------------------------------------------------------------------- /tests/test_datasets.py: -------------------------------------------------------------------------------- 1 | from miguellib.datasets.dummy import dummy_dataset 2 | 3 | 4 | def test_dummy_dataset(): 5 | assert len(dummy_dataset()) == 4 6 | assert dummy_dataset() == [1, 2, 3, 4] 7 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | from pathlib import Path 3 | 4 | # version 5 | here = Path(__file__).absolute().parent 6 | version_data = {} 7 | with open(here.joinpath("miguellib", "__init__.py"), "r") as f: 8 | exec(f.read(), version_data) 9 | version = version_data.get("__version__", "0.0") 10 | 11 | install_requires = [ 12 | "numpy>=1.19", # 1.19 required by tensorflow 2.6 13 | "pandas>2,<3", 14 | "ipykernel>=4.6.1,<7", 15 | "jupyter>=1,<2", 16 | "black>=18.6b4,<21", 17 | "pytest>=3.6.4", 18 | ] 19 | 20 | setup( 21 | name="miguellib", 22 | version=version, 23 | install_requires=install_requires, 24 | package_dir={"miguellib": "miguellib"}, 25 | python_requires=">=3.6", 26 | packages=find_packages(where=".", exclude=["docs", "examples", "tests"]), 27 | ) 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Project Contributors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /examples/template.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "56b9508c", 6 | "metadata": {}, 7 | "source": [ 8 | "# Python project\n", 9 | "Description" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 1, 15 | "id": "44b1db21", 16 | "metadata": {}, 17 | "outputs": [ 18 | { 19 | "name": "stdout", 20 | "output_type": "stream", 21 | "text": [ 22 | "Numpy version: 2.0.2\n", 23 | "Pandas version: 2.2.3\n", 24 | "Miguellib version: 0.1.1\n" 25 | ] 26 | } 27 | ], 28 | "source": [ 29 | "import numpy as np\n", 30 | "import pandas as pd\n", 31 | "import miguellib\n", 32 | "\n", 33 | "print(f\"Numpy version: {np.__version__}\")\n", 34 | "print(f\"Pandas version: {pd.__version__}\")\n", 35 | "print(f\"Miguellib version: {miguellib.__version__}\")\n", 36 | " " 37 | ] 38 | } 39 | ], 40 | "metadata": { 41 | "kernelspec": { 42 | "display_name": "Python 3 (ipykernel)", 43 | "language": "python", 44 | "name": "python3" 45 | }, 46 | "language_info": { 47 | "codemirror_mode": { 48 | "name": "ipython", 49 | "version": 3 50 | }, 51 | "file_extension": ".py", 52 | "mimetype": "text/x-python", 53 | "name": "python", 54 | "nbconvert_exporter": "python", 55 | "pygments_lexer": "ipython3", 56 | "version": "3.9.21" 57 | } 58 | }, 59 | "nbformat": 4, 60 | "nbformat_minor": 5 61 | } 62 | -------------------------------------------------------------------------------- /.github/workflows/pr_gate.yml: -------------------------------------------------------------------------------- 1 | name: PR gate 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | pull_request: 9 | # Only trigger the workflow with a PR to main 10 | branches: 11 | - main 12 | 13 | # Enable manual trigger 14 | workflow_dispatch: 15 | input: 16 | tags: 17 | description: 'Tags to label this manual run (optional)' 18 | default: 'Manual run' 19 | 20 | # Automatically cancel previous workflows if a new one is executed 21 | concurrency: 22 | group: ${{ github.workflow }}-${{ github.ref }} 23 | cancel-in-progress: true 24 | 25 | jobs: 26 | test: 27 | name: Test Python ${{ matrix.python }} on ${{ matrix.os }} 28 | runs-on: ${{ matrix.os }} 29 | timeout-minutes: 40 30 | strategy: 31 | fail-fast: false 32 | max-parallel: 20 # Usage limits: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration 33 | matrix: 34 | os: [ubuntu-24.04] # Available images: https://github.com/actions/runner-images/#available-images 35 | python: ["3.9", "3.10", "3.11"] 36 | 37 | steps: 38 | - name: Checkout 39 | uses: actions/checkout@v4 # Info: https://github.com/actions/checkout 40 | 41 | - name: Use Python ${{ matrix.python }} 42 | uses: actions/setup-python@v5 # Info: https://github.com/actions/setup-python 43 | with: 44 | python-version: ${{ matrix.python }} 45 | 46 | - name: Install dependencies 47 | run: | 48 | pip install . 49 | 50 | - name: Python version and dependency list 51 | run: | 52 | echo "Python version expected: ${{ matrix.python }}" 53 | python --version 54 | which python 55 | pip list 56 | 57 | - name: Run tests 58 | run: | 59 | pytest tests --disable-warnings --durations 0 60 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Machine Learning project template 2 | 3 | The objective of this repository is to serve as a template for machine learning projects. 4 | 5 | ## Getting Started 6 | 7 | To set up the project, the easiest way is to copy the all the folders into an empty GitHub repository. Make sure you change the name of the library `miguellib`. 8 | 9 | ## Structure 10 | 11 | - .github: CI/CD with GitHub Actions. It runs the tests every time there is a pull request to the repository. 12 | - docs: Documentation of the project. 13 | - examples: Jupyter notebooks with machine learning experiments. Here is where you would do data exploration, try different machine learning models, etc. 14 | - miguellib: Libraries with common functions that you use in the project. 15 | - tests: Python tests of the libraries. 16 | 17 | ## Setup 18 | 19 | pip install -e . 20 | python -c "import miguellib; print(miguellib.__version__)" 21 | 22 | ## Coding Principles 23 | 24 | Next there are a few coding principles that I follow when working on machine learning projects. 25 | 26 | ### Start from something that works 27 | 28 | Here is one of the most practical tips I know about working on machine learning. **Instead of starting from scratch, start with something that works and adapt it to your problem.** 29 | 30 | For example, let's say you want to build a recommendation system with data from your company. What I would do is something as simple as this: 31 | 32 | 1. Go to [Recommenders](https://github.com/recommenders-team/recommenders) and look at an example that a similar dataset structure and compute. For example, if your data is text-based and you want to use GPU, explore the examples of LSTUR or NPA. 33 | 2. Install the dependencies and run the example. Make sure that it works. 34 | 3. Change the data of the example to your data. If your data is different or more extensive, just forget about it and use the part of your data that is similar to the example. Make sure that it works. 35 | 4. Change the code to adapt it to your specific data and problem. 36 | 37 | ### Notebooks that call a library 38 | 39 | One of the main differences between a professional and an amateur machine learning project is this. Don't put your functions and classes in the notebooks, instead, create libraries and call them from the notebooks. This is the only way to reuse your code and make it scalable. 40 | 41 | Most of the time, notebooks are not deployed, they are used for experimentation and visualization. You deploy the libraries. In addition, if you create libraries, you can test them. 42 | 43 | ### Why tests are important? 44 | 45 | Tests solve one of the most expensive problems in development: maintenance. The way I see testing is like the immune system of your project. It protects your project from bugs and errors and makes sure your project is healthy. 46 | 47 | A strong test pipeline minimizes maintenance. It is one of the best investments you can do in your project, because it will avoid new buggy code in the project, and it will detect breaking changes when using dependencies. 48 | -------------------------------------------------------------------------------- /examples/models/model1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "56b9508c", 6 | "metadata": {}, 7 | "source": [ 8 | "# Experiment using model 1\n", 9 | "Description" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 7, 15 | "id": "44b1db21", 16 | "metadata": {}, 17 | "outputs": [ 18 | { 19 | "name": "stdout", 20 | "output_type": "stream", 21 | "text": [ 22 | "Numpy version: 1.21.5\n", 23 | "Pandas version: 1.3.5\n", 24 | "Miguellib version: 0.1.0\n" 25 | ] 26 | } 27 | ], 28 | "source": [ 29 | "import numpy as np\n", 30 | "import pandas as pd\n", 31 | "import miguellib\n", 32 | "from miguellib.datasets.dummy import dummy_dataset\n", 33 | "\n", 34 | "print(f\"Numpy version: {np.__version__}\")\n", 35 | "print(f\"Pandas version: {pd.__version__}\")\n", 36 | "print(f\"Miguellib version: {miguellib.__version__}\")\n", 37 | " " 38 | ] 39 | }, 40 | { 41 | "cell_type": "markdown", 42 | "id": "bd715d4a", 43 | "metadata": {}, 44 | "source": [ 45 | "## Model theoretical explanation\n", 46 | "\n", 47 | "This model is based in ..." 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "id": "920d8ef5", 54 | "metadata": {}, 55 | "outputs": [], 56 | "source": [ 57 | "# Parameters\n", 58 | "batch_size = 32\n", 59 | "n_layers = 5\n", 60 | "#epochs = 10\n", 61 | "EPOCHS = 10" 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "id": "7f047d6c", 67 | "metadata": {}, 68 | "source": [ 69 | "## Data prep\n" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": 8, 75 | "id": "2231c92a", 76 | "metadata": {}, 77 | "outputs": [ 78 | { 79 | "name": "stdout", 80 | "output_type": "stream", 81 | "text": [ 82 | "[1, 2, 3, 4]\n" 83 | ] 84 | } 85 | ], 86 | "source": [ 87 | "dataset = dummy_dataset()\n", 88 | "print(dataset)" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": null, 94 | "id": "5fa36aa6", 95 | "metadata": {}, 96 | "outputs": [], 97 | "source": [] 98 | }, 99 | { 100 | "cell_type": "markdown", 101 | "id": "c61a0ab5", 102 | "metadata": {}, 103 | "source": [ 104 | "## Model training" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": null, 110 | "id": "19a0b498", 111 | "metadata": {}, 112 | "outputs": [], 113 | "source": [ 114 | "# model.fit" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": null, 120 | "id": "2b044274", 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [ 124 | "# preds = model.test" 125 | ] 126 | }, 127 | { 128 | "cell_type": "markdown", 129 | "id": "e8511d1b", 130 | "metadata": {}, 131 | "source": [ 132 | "## Evaluation" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": null, 138 | "id": "4208dda2", 139 | "metadata": {}, 140 | "outputs": [], 141 | "source": [ 142 | "# model_auc = auc(true, preds)" 143 | ] 144 | }, 145 | { 146 | "cell_type": "markdown", 147 | "id": "0ed2afd4", 148 | "metadata": {}, 149 | "source": [ 150 | "## Summary" 151 | ] 152 | } 153 | ], 154 | "metadata": { 155 | "kernelspec": { 156 | "display_name": "Python 3 (ipykernel)", 157 | "language": "python", 158 | "name": "python3" 159 | }, 160 | "language_info": { 161 | "codemirror_mode": { 162 | "name": "ipython", 163 | "version": 3 164 | }, 165 | "file_extension": ".py", 166 | "mimetype": "text/x-python", 167 | "name": "python", 168 | "nbconvert_exporter": "python", 169 | "pygments_lexer": "ipython3", 170 | "version": "3.7.11" 171 | } 172 | }, 173 | "nbformat": 4, 174 | "nbformat_minor": 5 175 | } 176 | --------------------------------------------------------------------------------