├── .coveragerc ├── .github ├── ISSUE_TEMPLATE │ ├── bug-report.md │ ├── feature-request.md │ └── support-request.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── ci.yml │ └── deploy.yml ├── .gitignore ├── .pep8speaks.yml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── docs ├── .nojekyll ├── Analyze.md ├── Asking_Help.md ├── AutoModel.md ├── AutoParams.md ├── AutoPredict.md ├── AutoScan.md ├── Backends.md ├── Custom_Reducers.md ├── Deploy.md ├── Energy_Draw.md ├── Evaluate.md ├── Examples_AutoML.md ├── Examples_AutoML_Code.md ├── Examples_Generator.md ├── Examples_Generator_Code.md ├── Examples_Multiple_Inputs.md ├── Examples_Multiple_Inputs_Code.md ├── Examples_Multiple_Outputs.md ├── Examples_Multiple_Outputs_Code.md ├── Examples_PyTorch.md ├── Examples_PyTorch_Code.md ├── Examples_Typical.md ├── Examples_Typical_Code.md ├── Gamify.md ├── Generator.md ├── Hidden_Layers.md ├── Install_Options.md ├── Learning_Rate_Normalizer.md ├── Local_Strategy.md ├── Metrics.md ├── Monitoring.md ├── Optimization_Strategies.md ├── Overview.md ├── Parallelism.md ├── Predict.md ├── Probabilistic_Reduction.md ├── README.md ├── Restore.md ├── Roadmap.md ├── Scan.md ├── Templates.md ├── Workflow.md ├── _coverpage.md ├── _media │ ├── talos_deep_learning_workflow.png │ ├── talos_logo_bg.png │ └── talos_logo_icon.png ├── _sidebar.md └── index.html ├── examples ├── A Very Short Introduction to Hyperparameter Optimization of Keras Models with Talos.ipynb ├── Functional Model Hyperparameter Optimization.ipynb ├── Hyperparameter Optimization on Keras with Breast Cancer Data.ipynb ├── Hyperparameter Optimization with Keras for the Iris Prediction.ipynb ├── Recover Best Models from Experiment Log.ipynb ├── iris.py └── version-check.py ├── logo.png ├── pyproject.toml ├── requirements.txt ├── setup.cfg ├── setup.py ├── talos ├── __init__.py ├── autom8 │ ├── __init__.py │ ├── automodel.py │ ├── autoparams.py │ ├── autopredict.py │ └── autoscan.py ├── callbacks │ ├── __init__.py │ ├── experiment_log.py │ └── power_draw.py ├── commands │ ├── __init__.py │ ├── analyze.py │ ├── deploy.py │ ├── evaluate.py │ ├── predict.py │ └── restore.py ├── logging │ ├── __init__.py │ ├── logging_finish.py │ ├── logging_run.py │ └── results.py ├── metrics │ ├── __init__.py │ ├── entropy.py │ └── keras_metrics.py ├── model │ ├── __init__.py │ ├── early_stopper.py │ ├── hidden_layers.py │ ├── ingest_model.py │ ├── network_shape.py │ ├── normalizers.py │ └── output_layer.py ├── parameters │ ├── DistributeParamSpace.py │ ├── ParamSpace.py │ └── __init__.py ├── reducers │ ├── GamifyMap.py │ ├── __init__.py │ ├── correlation.py │ ├── forrest.py │ ├── gamify.py │ ├── limit_by_metric.py │ ├── local_strategy.py │ ├── reduce_run.py │ ├── reduce_utils.py │ ├── sample_reducer.py │ └── trees.py ├── scan │ ├── Scan.py │ ├── __init__.py │ ├── scan_addon.py │ ├── scan_finish.py │ ├── scan_prepare.py │ ├── scan_round.py │ ├── scan_run.py │ └── scan_utils.py ├── templates │ ├── __init__.py │ ├── datasets.py │ ├── models.py │ ├── params.py │ └── pipelines.py └── utils │ ├── __init__.py │ ├── best_model.py │ ├── exceptions.py │ ├── generator.py │ ├── gpu_utils.py │ ├── load_model.py │ ├── power_draw_append.py │ ├── recover_best_model.py │ ├── rescale_meanzero.py │ ├── sequence_generator.py │ ├── test_utils.py │ ├── torch_history.py │ └── validation_split.py ├── test-ci.py ├── test-local.sh └── tests ├── __init__.py ├── __main__.py ├── commands ├── __init__.py ├── recover_best_model.py ├── test_analyze.py ├── test_autom8.py ├── test_latest.py ├── test_lr_normalizer.py ├── test_predict.py ├── test_random_methods.py ├── test_reducers.py ├── test_rest.py ├── test_scan.py └── test_templates.py └── performance ├── memory_pressure.py └── memory_pressure_check.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | 3 | omit = 4 | talos/samplers/*/* 5 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug Report 3 | about: I want to report something that is broken 4 | 5 | --- 6 | 7 | Thank you very much for reporting a bug on Talos. Before you do, please go through the below checklist carefully and make sure to prepare your bug report in a way that facilitates effective handling of the matter. 8 | 9 | #### 1) Confirm the below 10 | 11 | - [ ] My Python version is 3.5 or higher 12 | - [ ] I have searched through the issues [Issues](https://github.com/autonomio/talos/issues) for a duplicate 13 | - [ ] I've tested that my Keras model works as a stand-alone 14 | 15 | #### 2) Include the output of: 16 | 17 | `talos.__version__` 18 | 19 | #### 3) Explain clearly what you expect to happen 20 | 21 | *A description of what you tried to do and what you thought should happen.* 22 | 23 | #### 4) Explain what actually happened 24 | 25 | *A description of the issue in Talos that you had identified* 26 | 27 | #### 5) Provide a code-complete reference 28 | 29 | - [ ] My bug report includes an input model 30 | - [ ] My bug report includes a parameter dictionary 31 | - [ ] My bug report includes a `Scan()` command 32 | - [ ] My bug report question includes a link to a sample of the data 33 | 34 | NOTE: If the data is sensitive and can't be shared, [create dummy data](https://scikit-learn.org/stable/modules/classes.html#samples-generator) that mimics it or provide a command for generating it. 35 | 36 | **A self-contained Jupyter Notebook, Google Colab, or similar is highly preferred and will speed up helping you with your issue.** 37 | 38 | --- 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature Request 3 | about: I want to suggest a new feature 4 | 5 | --- 6 | 7 | Thanks a lot for suggesting a feature to Talos. Please take a moment to go through the below checklist to provide context in a way that makes it easy to take your request forward. 8 | 9 | #### 1) I think Talos should add 10 | 11 | *A description of the feature with as much detail as you believe is valuable* 12 | 13 | #### 2) Once implemented, I can see how this feature will 14 | 15 | *Explain how researchers will benefit from having this feature in Talos** 16 | 17 | #### 3) I believe this feature is 18 | 19 | - [ ] critically important 20 | - [ ] must have 21 | - [ ] nice to have 22 | 23 | #### 4) Given the chance, I'd be happy to make a PR for this feature 24 | 25 | - [ ] definitely 26 | - [ ] possibly 27 | - [ ] unlikely 28 | 29 | --- 30 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/support-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Support 3 | about: I want to ask for support 4 | 5 | --- 6 | 7 | First off, make sure to check your [support options](https://github.com/autonomio/talos#-how-to-get-support). 8 | 9 | The preferred way to resolve usage related matters is through the [docs](https://autonomio.github.io/talos/#/) which are maintained up-to-date with the latest version of Talos. 10 | 11 | If you do end up asking for support in a new issue, make sure to follow the below steps carefully. 12 | 13 | #### 1) Confirm the below 14 | 15 | - [ ] I have looked for an answer in the [Docs](https://autonomio.github.io/talos) 16 | - [ ] My Python version is 3.5 or higher 17 | - [ ] I have searched through the issues [Issues](https://github.com/autonomio/talos/issues) for a duplicate 18 | - [ ] I've tested that my Keras model works as a stand-alone 19 | 20 | #### 2) Include the output of: 21 | 22 | `talos.__version__` 23 | 24 | #### 3) Explain clearly what you are trying to achieve 25 | 26 | *A description of your specific use-case and what you hope to achieve with it* 27 | 28 | #### 4) Explain what you have already tried 29 | 30 | *An outline of the steps that you have already taken so far* 31 | 32 | #### 5) Provide a code-complete reference 33 | 34 | - [ ] My support question includes an input model 35 | - [ ] My support question includes a parameter dictionary 36 | - [ ] My support question includes a `Scan()` command 37 | - [ ] My support question includes a link to a sample of the data 38 | 39 | NOTE: If the data is sensitive and can't be shared, [create dummy data](https://scikit-learn.org/stable/modules/classes.html#samples-generator) that mimics it. 40 | 41 | **A self-contained Jupyter Notebook, Google Colab, or similar is highly preferred and will speed up helping you with your issue.** 42 | 43 | --- 44 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## You want to make a PR to Talos 2 | 3 | Thanks so much :) First, please take a moment to carefully check through 4 | the below items: 5 | 6 | #### Sanity 7 | 8 | - [ ] I'm aware of the implications of the proposed changes 9 | - [ ] Code is [PEP8](https://www.python.org/dev/peps/pep-0008/) 10 | - [ ] I'm making the PR to `master` 11 | - [ ] I've updated the versions based on [Semantic Versioning](https://semver.org/) 12 | - [ ] `talos/__init__.py` 13 | - [ ] `docs/index.html` 14 | - [ ] `docs/_coverpage.md` 15 | 16 | #### Docs 17 | 18 | - [ ] [Docs](https://autonomio.github.io/talos) are updated 19 | 20 | #### Tests 21 | 22 | - [ ] Changes have gone through actual use testing 23 | - [ ] All local tests have passed (run ./test.sh in /talos) 24 | - [ ] Tests have been updated to reflect the changes 25 | 26 |
27 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | name: Python ${{ matrix.python-version }} and ${{ matrix.os }} 8 | runs-on: ${{ matrix.os }} 9 | strategy: 10 | max-parallel: 9 11 | matrix: 12 | python-version: ['3.9', '3.10', '3.11'] 13 | os: [ubuntu-latest, macos-latest] 14 | 15 | steps: 16 | - uses: actions/checkout@v1 17 | - name: Python ${{ matrix.python-version }} 18 | uses: actions/setup-python@v1 19 | with: 20 | python-version: ${{ matrix.python-version }} 21 | - name: Dependencies 22 | run: | 23 | export MPLBACKEND=agg 24 | python -m pip install --upgrade pip 25 | pip install -r requirements.txt 26 | - name: Style 27 | run: | 28 | pip install flake8 29 | # stop the build if there are Python syntax errors or undefined names 30 | flake8 . --count --select=E9,F63,F7 --show-source --statistics 31 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 32 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 33 | - name: Tests 34 | run: | 35 | export MPLBACKEND=agg 36 | pip install tensorflow==2.14.1 37 | pip install coveralls 38 | coverage run --source=talos ./test-ci.py 39 | - name: Coverage 40 | env: 41 | COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} 42 | run: | 43 | coveralls 44 | -------------------------------------------------------------------------------- /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: Upload Python Package 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | jobs: 8 | deploy: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v1 12 | - name: Set up Python 13 | uses: actions/setup-python@v1 14 | with: 15 | python-version: '3.x' 16 | - name: Install dependencies 17 | run: | 18 | python -m pip install --upgrade pip 19 | pip install hatch 20 | - name: Build and publish 21 | env: 22 | PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} 23 | run: | 24 | hatch build 25 | hatch publish --user __token__ --auth $PYPI_API_TOKEN 26 | 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.pyc 3 | __pycache__ 4 | *.csv 5 | *.log 6 | dist/ 7 | talos.egg-info 8 | .idea/ 9 | *.zip 10 | *_.csv 11 | *.txt 12 | *.h5 13 | *.json 14 | *.npy 15 | test.sh 16 | .vscode 17 | test_*/ 18 | test/ -------------------------------------------------------------------------------- /.pep8speaks.yml: -------------------------------------------------------------------------------- 1 | # File : .pep8speaks.yml 2 | 3 | scanner: 4 | diff_only: False 5 | linter: flake8 6 | 7 | flake8: # Same as scanner.linter value. Other option is flake8 8 | max-line-length: 88 # Default is 79 in PEP 8 9 | ignore: # Errors and warnings to ignore 10 | - W504 # line break after binary operator 11 | - E402 # module level import not at top of file 12 | - E731 # do not assign a lambda expression, use a def 13 | - C406 # Unnecessary list literal - rewrite as a dict literal. 14 | 15 | no_blank_comment: True # If True, no comment is made on PR without any errors. 16 | descending_issues_order: False # If True, PEP 8 issues in message will be displayed in descending order of line numbers in the file 17 | 18 | message: 19 | opened: 20 | header: "Hello @{name}! Thanks for opening this PR." 21 | footer: "Do see the [Hitchhiker's guide to code style](https://goo.gl/hqbW4r)" 22 | updated: 23 | header: "Hello @{name}! Thanks for updating this PR." 24 | footer: "" 25 | no_errors: "There are currently no PEP 8 issues detected in this PR. Great work! :heart:" -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. 6 | 7 | ## Our Standards 8 | 9 | Examples of behavior that contributes to creating a positive environment include: 10 | 11 | * Using welcoming and inclusive language 12 | * Being respectful of differing viewpoints and experiences 13 | * Gracefully accepting constructive criticism 14 | * Focusing on what is best for the community 15 | * Showing empathy towards other community members 16 | 17 | Examples of unacceptable behavior by participants include: 18 | 19 | * The use of sexualized language or imagery and unwelcome sexual attention or advances 20 | * Trolling, insulting/derogatory comments, and personal or political attacks 21 | * Public or private harassment 22 | * Publishing others' private information, such as a physical or electronic address, without explicit permission 23 | * Other conduct which could reasonably be considered inappropriate in a professional setting 24 | 25 | ## Our Responsibilities 26 | 27 | Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. 28 | 29 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. 30 | 31 | ## Scope 32 | 33 | This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 34 | 35 | ## Enforcement 36 | 37 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at mailme@mikkokotila.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 38 | 39 | Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. 40 | 41 | ## Attribution 42 | 43 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] 44 | 45 | [homepage]: http://contributor-covenant.org 46 | [version]: http://contributor-covenant.org/version/1/4/ 47 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Mikko Kotila 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /docs/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomio/talos/9cbbc42a69396aaeaf25d68af51d52e6a575e3d9/docs/.nojekyll -------------------------------------------------------------------------------- /docs/Analyze.md: -------------------------------------------------------------------------------- 1 | # Analyze (previously Reporting) 2 | 3 | The experiment results can be analyzed through the [Analyze()](https://github.com/autonomio/talos/blob/master/talos/utils/reporting.py) utility. `Analyze()` may be used after Scan completes, or during an experiment (from a different shell / kernel). 4 | 5 | ## Analyze Use 6 | 7 | ```python 8 | r = Reporting('experiment_log.csv') 9 | 10 | # returns the results dataframe 11 | r.data 12 | 13 | # returns the highest value for 'val_fmeasure' 14 | r.high('val_fmeasure') 15 | 16 | # returns the number of rounds it took to find best model 17 | r.rounds2high() 18 | 19 | # draws a histogram for 'val_acc' 20 | r.plot_hist() 21 | ``` 22 | 23 | Reporting works by loading the experiment log .csv file which is saved locally as part of the experiment. The filename can be changed through dataset_name and experiment_no Scan arguments. 24 | 25 | ## Analyze Arguments 26 | 27 | `Analyze()` has only a single argument `source`. This can be either a .csv file which results `Scan()` or the class object which also results from `Scan()`. 28 | 29 | The `Analyze` class object contains several useful properties. 30 | 31 | ## Analyze Properties 32 | 33 | See docstrings for each function for a more detailed description. 34 | 35 | **`high`** The highest result for a given metric 36 | 37 | **`rounds`** The number of rounds in the experiment 38 | 39 | **`rounds2high`** The number of rounds it took to get highest result 40 | 41 | **`low`** The lowest result for a given metric 42 | 43 | **`correlate`** A dataframe with Spearman correlation against a given metric 44 | 45 | **`plot_line`** A round-by-round line graph for a given metric 46 | 47 | **`plot_hist`** A histogram for a given metric where each observation is a permutation 48 | 49 | **`plot_corr`** A correlation heatmap where a single metric is compared against hyperparameters 50 | 51 | **`plot_regs`** A regression plot with data on two axis 52 | 53 | **`plot_box`** A box plot with data on two axis 54 | 55 | **`plot_bars`** A bar chart that allows up to 4 axis of data to be shown at once 56 | 57 | **`plot_kde`** Kernel Destiny Estimation type histogram with support for 1 or 2 axis of data 58 | 59 | **`table`** A sortable dataframe with a given metric and hyperparameters 60 | 61 | **`best_params`** A dictionary of parameters from the best model 62 | -------------------------------------------------------------------------------- /docs/Asking_Help.md: -------------------------------------------------------------------------------- 1 | # 💬 How to get Support 2 | 3 | | I want to... | Go to... | 4 | | -------------------------------- | ---------------------------------------------------------- | 5 | | **...troubleshoot** | [Docs] · [Wiki] · [GitHub Issue Tracker] | 6 | | **...report a bug** | [GitHub Issue Tracker] | 7 | | **...suggest a new feature** | [GitHub Issue Tracker] | 8 | | **...get support** | [Stack Overflow] · [Spectrum Chat] | 9 | | **...have a discussion** | [Spectrum Chat] | 10 | 11 | [github issue tracker]: https://github.com/automio/talos/issues 12 | [docs]: https://autonomio.github.io/docs_talos 13 | [wiki]: https://github.com/autonomio/talos/wiki 14 | [stack overflow]: https://stackoverflow.com/questions/tagged/talos 15 | [spectrum chat]: https://spectrum.chat/talos 16 | -------------------------------------------------------------------------------- /docs/AutoModel.md: -------------------------------------------------------------------------------- 1 | # AutoModel 2 | 3 | `AutoModel` provides a meaningful way to test several network architectures in an automated manner. Currently there are five supported architectures: 4 | 5 | - conv1d 6 | - lstm 7 | - bidirectional_lstm 8 | - simplernn 9 | - dense 10 | 11 | `AutoModel` creates an input model for Scan(). Optimized for being used together with `AutoParams()` and expects one or more of the above architectures to be included in params dictionary, for example: 12 | 13 | ```python 14 | 15 | p = {... 16 | 'networks': ['dense', 'conv1d', 'lstm'] 17 | ...} 18 | 19 | ``` 20 | 21 | ## AutoModel Arguments 22 | 23 | Argument | Input | Description 24 | --------- | ------- | ----------- 25 | `task` | str or None | `binary`, `multi_label`, `multi_class`, or `continuous` 26 | `metric` | None or list | One or more Keras metric (functions) to be used in the model 27 | 28 | Setting `task` effects which various aspects of the model and should be set according to the specific prediction task, or set to `None` in which case `metric` input is required. 29 | -------------------------------------------------------------------------------- /docs/AutoParams.md: -------------------------------------------------------------------------------- 1 | # AutoParams 2 | 3 | `AutoParams()` allows automated generation of comprehensive parameter dictionary to be used as input for `Scan()` experiments as well as a streamlined way to manipulate parameter dictionaries. 4 | 5 | #### to automatically create a params dictionary 6 | 7 | ```python 8 | p = talos.autom8.AutoParams().params 9 | 10 | ``` 11 | NOTE: The above example yields a very large permutation space so configure `Scan()` accordingly with `fraction_limit`. 12 | 13 | #### an alternative way where a class object is returned 14 | 15 | ```python 16 | param_object = talos.autom8.AutoParams() 17 | 18 | ``` 19 | 20 | Now various properties can be accessed through `param_object`, these are detailed below. For example: 21 | 22 | #### modifying a single parameter in the params dictionary 23 | 24 | ```python 25 | param_object.batch_size(bottom_value=20, max_value=100, steps=10) 26 | ``` 27 | 28 | Now the modified params dictionary can be accessed through `params_object.params` 29 | 30 | #### to append a current parameter dictionary 31 | 32 | ```python 33 | params_dict = talos.autom8.AutoParams(p, task='multi_label').params 34 | 35 | ``` 36 | NOTE: Note, when the dictionary is created for a prediction task other than 'binary', the `task` argument has to be declared accordingly (`binary`, `multi_label`, `multi_class`, or `continuous`). 37 | 38 | ## AutoParams Arguments 39 | 40 | Argument | Input | Description 41 | --------- | ------- | ----------- 42 | `params` | dict or None | If `None` then a new parameter dictionary is created 43 | `task` | str | 'binary', 'multi_class', 'multi_label', or 'continuous' 44 | `replace` | bool | Replace current dictionary entries with new ones. 45 | `auto` | bool | automatically generate or append params dictionary with all available parameters. 46 | `network` | network | If `True` several model architectures will be added 47 | `resample_params` | int or False | The number of values per parameter 48 | 49 | ## AutoParams Properties 50 | 51 | The **`params`** property returns the parameter dictionary which can be used as an input to `Scan()`. 52 | 53 | The **`resample_params`** accepts `n` as input and resamples the params dictionary so that n values remain for each parameter. 54 | 55 | All other properties relate with manipulating individual parameters in the parameter dictionary. 56 | 57 | **`activations`** For controlling the corresponding parameter in the parameters dictionary. 58 | 59 | **`batch_size`** For controlling the corresponding parameter in the parameters dictionary. 60 | 61 | **`dropout`** For controlling the corresponding parameter in the parameters dictionary. 62 | 63 | **`epochs`** For controlling the corresponding parameter in the parameters dictionary. 64 | 65 | **`kernel_initializer`** For controlling the corresponding parameter in the parameters dictionary. 66 | 67 | **`last_activation`** For controlling the corresponding parameter in the parameters dictionary. 68 | 69 | **`layers`** For controlling the corresponding parameter (i.e. `hidden_layers`) in the parameters dictionary. 70 | 71 | **`losses`** For controlling the corresponding parameter in the parameters dictionary. 72 | 73 | **`lr`** For controlling the corresponding parameter in the parameters dictionary. 74 | 75 | **`networks`** For controlling the Talos present network architectures (`dense`, `lstm`, `bidirectional_lstm`, `conv1d`, and `simplernn`). NOTE: the use of preset networks requires the use of the input model from `AutoModel()` for `Scan()`. 76 | 77 | **`neurons`** For controlling the corresponding parameter (i.e. `first_neuron`) in the parameters dictionary. 78 | 79 | **`optimizers`** For controlling the corresponding parameter in the parameters dictionary. 80 | 81 | **`shapes`** For controlling the Talos preset network shapes (`brick`, `funnel`, and `triangle`). 82 | 83 | **`shapes_slope`** For controlling the shape parameter with a floating point value to set the slope of the network from input layer to output layer. 84 | -------------------------------------------------------------------------------- /docs/AutoPredict.md: -------------------------------------------------------------------------------- 1 | # AutoPredict 2 | 3 | `AutoPredict()` automatically handles the process of finding the best models from a completed `Scan()` experiment, evaluates those models, and uses the winning model to make predictions on input data. 4 | 5 | ```python 6 | scan_object = talos.autom8.AutoPredict(scan_object, x_val=x, y_val=y, x_pred=x) 7 | ``` 8 | 9 | NOTE: the input data must be in same format as 'x' that was used in `Scan()`. 10 | Also, `x_val` and `y_val` should not have been exposed to the model during the 11 | `Scan()` experiment. 12 | 13 | `AutoPredict()` will add four new properties to `Scan()`: 14 | 15 | **`preds_model`** contains the winning Keras model (function) 16 | **`preds_parameters`** contains the hyperparameters for the selected model 17 | **`preds_probabilities`** contains the prediction probabilities for `x_pred` 18 | **`predict_classes`** contains the predicted classes for `x_pred`. 19 | 20 | ## AutoPredict Arguments 21 | 22 | Argument | Input | Description 23 | --------- | ------- | ----------- 24 | `scan_object` | class object | the class object returned from `Scan()` 25 | `x_val` | array or list of arrays | validation data features 26 | `y_val` | array or list of arrays | validation data labels 27 | `y_pred` | array or list of arrays | prediction data features 28 | `task` | string | 'binary', 'multi_class', 'multi_label', or 'continuous' 29 | `metric` | None | the metric against which the validation is performed 30 | `n_models` | int | number of promising models to be included in the evaluation process 31 | `folds` | None | number of folds to be used for cross-validation 32 | `shuffle` | None | if data is shuffled before splitting 33 | `asc` | None | should be True if metric is a loss 34 | -------------------------------------------------------------------------------- /docs/AutoScan.md: -------------------------------------------------------------------------------- 1 | # AutoScan 2 | 3 | `AutoScan()` provides a streamlined way for conducting a hyperparameter search experiment with any dataset. It is particularly useful for early exploration as with default settings `AutoScan()` casts a very broad parameter space including all common hyperparameters, network shapes, sizes, as well as architectures 4 | 5 | Configure the `AutoScan()` experiment and then use the property `start` in the returned class object to start the actual experiment. 6 | 7 | ```python 8 | auto = talos.autom8.AutoScan(task='binary', max_param_values=2) 9 | auto.start(x, y, experiment_name='testing.new', fraction_limit=0.001) 10 | ``` 11 | 12 | NOTE: `auto.start()` accepts all `Scan()` arguments. 13 | 14 | ## AutoScan Arguments 15 | 16 | Argument | Input | Description 17 | --------- | ------- | ----------- 18 | `task` | str or None | `binary`, `multi_label`, `multi_class`, or `continuous` 19 | `max_param_values` | int | Number of parameter values to be included 20 | 21 | Setting `task` effects which various aspects of the model and should be set according to the specific prediction task, or set to `None` in which case `metric` input is required. 22 | 23 | ## AutoScan Properties 24 | 25 | The only property **`start`** starts the actual experiment. `AutoScan.start()` accepts the following arguments: 26 | 27 | Argument | Input | Description 28 | --------- | ------- | ----------- 29 | `x` | array or list of arrays | prediction features 30 | `y` | array or list of arrays | prediction outcome variable 31 | `kwargs` | arguments | any `Scan()` argument can be passed into `AutoScan.start()` 32 | -------------------------------------------------------------------------------- /docs/Backends.md: -------------------------------------------------------------------------------- 1 | # TensorFlow 2 | 3 | Talos versions 1.0 and higher support TensorFlow 2.0 and higher. If you want to use older version of TensorFlow, Talos versions until 0.6.x support TensorFlow 1.4. 4 | 5 | # PyTorch 6 | 7 | Only Talos versions 1.0 and higher have full support for PyTorch. 8 | 9 | For instrucions on how to use Talos with PyTorch, see [this example](Examples_PyTorch). 10 | 11 | 12 | # Legacy Keras 13 | 14 | Talos versions 0.6.x and lower support the multi-backend Keras. -------------------------------------------------------------------------------- /docs/Custom_Reducers.md: -------------------------------------------------------------------------------- 1 | # Custom Reducer 2 | 3 | A custom reduction strategy can be created and dropped into Talos. Read more about the reduction principle 4 | 5 | There are only two criteria to meet: 6 | 7 | - The input of the custom strategy is 2-dimensional 8 | - The output of the custom strategy is in the form: 9 | 10 | ```python 11 | return label, value 12 | ``` 13 | Here `value` is any hyperparameter value, and `label` is the name of any hyperparameter. Any arbitrary strategy can be implemented, as long as the input and output criteria are met. 14 | 15 | With these in place, one then proceeds to apply the reduction to the current parameter space, with one of the supported functions: 16 | 17 | - `remove_is_not` 18 | - `remove_is` 19 | - `remove_le` 20 | - `remove_ge` 21 | - `remove_lambda` 22 | 23 | See [a working example](https://github.com/autonomio/talos/blob/master/talos/reducers/correlation.py) to make sure you understand the expected structure of a custom reducer. 24 | 25 | The file containing the custom strategy can then be placed in `/reducers` in Talos package, and corresponding changes made into `/reducers/reduce_run.py` to make the strategy available in `Scan()`. Having done this, the reduction strategy is now available as per the example [above](#probabilistic-reduction). 26 | 27 | A [pull request](https://github.com/autonomio/talos/pulls) is highly encouraged once a beneficial reduction strategy has been successfully added. 28 | -------------------------------------------------------------------------------- /docs/Deploy.md: -------------------------------------------------------------------------------- 1 | # Deploy() 2 | 3 | A successful experiment can be deployed easily. Deploy() takes in the object from Scan() and creates a package locally that can be later activated with Restore(). 4 | 5 | ```python 6 | from talos import Deploy 7 | 8 | Deploy(scan_object, 'experiment_name') 9 | ``` 10 | 11 | When you've achieved a successful result, you can use `Deploy()` to prepare a production ready package that can be easily transferred to another environment or system, or sent or uploaded. The deployment package will consists of the best performing model, which is picked base on the `metric` argument. 12 | 13 | NOTE: for a metric that is to be minimized, set `asc=True` or otherwise 14 | you will end up with the model that has the highest loss. 15 | 16 | ## Deploy Arguments 17 | 18 | Parameter | type | Description 19 | --------- | ------- | ----------- 20 | `scan_object` | class object | a `Scan` object 21 | `model_name` | str | Name for the .zip file to be created. 22 | `metric` | str | The metric to be used for picking the best model. 23 | `asc` | bool | Make this True for metrics that are to be minimized (e.g. loss) 24 | `saved` | bool | if a model saved on local machine should be used 25 | `custom_objects` | dict | if the model has a custom object, pass it here 26 | 27 | ## Deploy Package Contents 28 | 29 | The deploy package consists of: 30 | 31 | - details of the scan (details.txt) 32 | - model weights (model.h5) 33 | - model json (model.json) 34 | - results of the experiment (results.csv) 35 | - sample of x data (x.csv) 36 | - sample of y data (y.csv) 37 | 38 | The package can be restored into a copy of the original Scan object using the `Restore()` command. 39 | -------------------------------------------------------------------------------- /docs/Energy_Draw.md: -------------------------------------------------------------------------------- 1 | # Energy Draw Callback 2 | 3 | A callback for recording GPU power draw (watts) on epoch begin and end. The callback allows: 4 | 5 | - record and analyze model energy consumption 6 | - optimize towards energy efficient models 7 | 8 | ### how-to-use 9 | 10 | Before `model.fit()` in the input model: 11 | 12 | `power_draw = PowerDraw()` 13 | 14 | Then use `power_draw` as you would callbacks in general: 15 | 16 | `model.fit(...callbacks=[power_draw]...)` 17 | 18 | To get the energy draw data into the experiment log: 19 | 20 | `history = talos.utils.power_draw_append(history, power_draw)` 21 | 22 | NOTE: this line has to be after `model.fit()`. 23 | 24 | -------------------------------------------------------------------------------- /docs/Evaluate.md: -------------------------------------------------------------------------------- 1 | # Evaluate() 2 | 3 | Once the `Scan()` experiment procedures have been completed, the resulting class object can be used as input for `Evaluate()` in order to evaluate one or more models. 4 | 5 | ```python 6 | from talos import Evaluate 7 | 8 | # create the evaluate object 9 | e = Evaluate(scan_object) 10 | 11 | # perform the evaluation 12 | e.evaluate(x, y, average='macro') 13 | ``` 14 | 15 | NOTE: It's very important to save part of your data for evaluation, and keep it completely separated from the data you use for the actual experiment. A good approach would be where 50% of the data is saved for evaluation. 16 | 17 | ### Evaluate Properties 18 | 19 | `Evaluate()` has just one property, **`evaluate`**, which is used for evaluating one or more models. 20 | 21 | ### Evaluate.evaluate Arguments 22 | 23 | Parameter | Default | Description 24 | --------- | ------- | ----------- 25 | `x` | NA | the predictor data x 26 | `y` | NA | the prediction data y (truth) 27 | `task`| NA | One of the following strings: 'binary', 'multi_class', 'multi_label', or 'continuous'. 28 | `model_id` | None | the model_id to be used 29 | `folds` | None | number of folds to be used for cross-validation 30 | `shuffle` | None | if data is shuffled before splitting 31 | `average` | 'binary' | 'binary', 'micro', 'macro', 'samples', or 'weighted' 32 | `metric` | None | the metric against which the validation is performed 33 | `asc` | None | should be True if metric is a loss 34 | `saved` | bool | if a model saved on local machine should be used 35 | `custom_objects` | dict | if the model has a custom object, pass it here 36 | 37 | The above arguments are for the evaluate attribute of the Evaluate object. 38 | -------------------------------------------------------------------------------- /docs/Examples_AutoML.md: -------------------------------------------------------------------------------- 1 | # AutoML 2 | 3 | Performing an AutoML style hyperparameter search experiment with Talos could not be any easier. 4 | 5 | The single-file code example can be found [here](Examples_AutoML_Code.md). 6 | 7 | ### Imports 8 | 9 | ```python 10 | import talos 11 | import wrangle 12 | ``` 13 | 14 | ### Loading Data 15 | ```python 16 | x, y = talos.templates.datasets.cervical_cancer() 17 | 18 | # we spare 10% of data for testing later 19 | x, y, x_test, y_test = wrangle.array_split(x, y, .1) 20 | 21 | # then validation split 22 | x_train, y_train, x_val, y_val = wrangle.array_split(x, y, .2) 23 | ``` 24 | 25 | `x` and `y` are expected to be either numpy arrays or lists of numpy arrays and same applies for the case where `x_train`, `y_train`, `x_val`, `y_val` is used instead. 26 | 27 | ### Defining the Model 28 | 29 | In this case there is no need to define the model. `talos.autom8.AutoModel()` is used behind the scenes, where several model architectures fully wired for Talos are found. We simply initiate the `AutoScan()` object first: 30 | 31 | ```python 32 | autom8 = talos.autom8.AutoScan('binary', 5) 33 | ``` 34 | 35 | ### Parameter Dictionary 36 | 37 | There is also no need to worry about the parameter dictionary. This is handled in the background with `AutoParams()`. 38 | 39 | 40 | ### Scan() 41 | 42 | The `Scan()` itself is started through the **`start`** property of the `AutoScan()` class object. 43 | 44 | ```python 45 | autom8.start(x=x_train, 46 | y=y_train, 47 | x_val=x_val, 48 | y_val=y_val, 49 | fraction_limit=0.000001) 50 | ``` 51 | We pass data here just like we would do it in `Scan()` normally. Also, you are free to use any of the `Scan()` arguments here to configure the experiment. Find the description for all `Scan()` arguments [here](Scan.md#scan-arguments). 52 | -------------------------------------------------------------------------------- /docs/Examples_AutoML_Code.md: -------------------------------------------------------------------------------- 1 | [BACK](Examples_AutoML.md) 2 | 3 | # AutoML 4 | 5 | ```python 6 | 7 | x, y = talos.templates.datasets.cervical_cancer() 8 | 9 | # we spare 10% of data for testing later 10 | x, y, x_test, y_test = wrangle.array_split(x, y, .1) 11 | 12 | # then validation split 13 | x_train, y_train, x_val, y_val = wrangle.array_split(x, y, .2) 14 | 15 | autom8 = talos.autom8.AutoScan('binary', 5) 16 | 17 | autom8.start(x=x_train, 18 | y=y_train, 19 | x_val=x_val, 20 | y_val=y_val, 21 | fraction_limit=0.000001) 22 | ``` 23 | -------------------------------------------------------------------------------- /docs/Examples_Generator.md: -------------------------------------------------------------------------------- 1 | # Generator 2 | 3 | This example highlights a typical and rather simple example of Talos experiment, and is a good starting point for those new to Talos. 4 | 5 | The single-file code example can be found [here](Examples_Generator_Code.md). 6 | 7 | ### Imports 8 | 9 | ```python 10 | import talos 11 | from talos.utils import SequenceGenerator 12 | 13 | from tensorflow.keras.models import Sequential 14 | from tensorflow.keras.layers import Dense, Conv2D, Flatten, Dropout 15 | ``` 16 | NOTE: In this example we will be using the `SequenceGenerator()` available in Talos. 17 | 18 | 19 | ### Loading Data 20 | ```python 21 | x_train, y_train, x_val, y_val = talos.templates.datasets.mnist() 22 | ``` 23 | `x` and `y` are expected to be either numpy arrays or lists of numpy arrays. 24 | 25 | ### Defining the Model 26 | ```python 27 | 28 | def mnist_model(x_train, y_train, x_val, y_val, params): 29 | 30 | model = Sequential() 31 | model.add(Conv2D(32, kernel_size=(3, 3), activation=params['activation'], input_shape=(28, 28, 1))) 32 | model.add(Flatten()) 33 | model.add(Dense(128, activation=params['activation'])) 34 | model.add(Dropout(params['dropout'])) 35 | model.add(Dense(10, activation='softmax')) 36 | 37 | model.compile(optimizer=params['optimizer'], 38 | loss=params['losses'], 39 | metrics=['acc', talos.utils.metrics.f1score]) 40 | 41 | out = model.fit_generator(SequenceGenerator(x_train, 42 | y_train, 43 | batch_size=params['batch_size']), 44 | epochs=params['epochs'], 45 | validation_data=[x_val, y_val], 46 | callbacks=[], 47 | workers=4, 48 | verbose=0) 49 | 50 | return out, model 51 | ``` 52 | 53 | First, the input model must accept arguments exactly as in the example: 54 | 55 | ```python 56 | def iris_model(x_train, y_train, x_val, y_val, params): 57 | ``` 58 | 59 | Second, the model must explicitly declare `validation_data` in `model.fit`: 60 | 61 | ```python 62 | out = model.fit_generator( ... validation_data=[x_val, y_val] ... ) 63 | ``` 64 | 65 | Third, the model must reference a data generator in `model.fit_generator` exactly as it would be done in stand-alone Keras: 66 | 67 | ```python 68 | model.fit_generator(SequenceGenerator(x_train, 69 | y_train, 70 | batch_size=params['batch_size'], 71 | ...) 72 | ``` 73 | 74 | NOTE: Set the number of `workers` in `fit_generator()` based on your system. 75 | 76 | Finally, the model must `return` the `model.fit` object as well as the model itself in the order of the of the example: 77 | 78 | ```python 79 | return out, model 80 | ``` 81 | 82 | 83 | ### Parameter Dictionary 84 | ```python 85 | p = {'activation':['relu', 'elu'], 86 | 'optimizer': ['Adam'], 87 | 'losses': ['LogCosh'], 88 | 'shapes': ['brick'], 89 | 'first_neuron': [32], 90 | 'dropout': [.2, .3], 91 | 'batch_size': [64, 128, 256], 92 | 'epochs': [1]} 93 | ``` 94 | 95 | Note that the parameter dictionary allows either list of values, or tuples with range in the form `(min, max, step)` 96 | 97 | 98 | ### Scan() 99 | ```python 100 | scan_object = talos.Scan(x=x_train, 101 | y=y_train, 102 | x_val=x_val, 103 | y_val=y_val, 104 | params=p, 105 | model=mnist_model) 106 | ``` 107 | 108 | `Scan()` always needs to have `x`, `y`, `model`, and `params` arguments declared. In the case of `fit_generator()` use, we also have to explicitly declare `val_x` and `val_y`. 109 | 110 | Find the description for all `Scan()` arguments [here](Scan.md#scan-arguments). 111 | -------------------------------------------------------------------------------- /docs/Examples_Generator_Code.md: -------------------------------------------------------------------------------- 1 | [BACK](Examples_Generator.md) 2 | 3 | # Generator 4 | 5 | ```python 6 | import talos 7 | from talos.utils import SequenceGenerator 8 | 9 | from tensorflow.keras.models import Sequential 10 | from tensorflow.keras.layers import Dense, Conv2D, Flatten, Dropout 11 | 12 | x, y = ta.templates.datasets.iris() 13 | 14 | def mnist_model(x_train, y_train, x_val, y_val, params): 15 | 16 | model = Sequential() 17 | model.add(Conv2D(32, kernel_size=(3, 3), activation=params['activation'], input_shape=(28, 28, 1))) 18 | model.add(Flatten()) 19 | model.add(Dense(128, activation=params['activation'])) 20 | model.add(Dropout(params['dropout'])) 21 | model.add(Dense(10, activation='softmax')) 22 | 23 | model.compile(optimizer=params['optimizer'], 24 | loss=params['losses'], 25 | metrics=['acc', talos.utils.metrics.f1score]) 26 | 27 | out = model.fit_generator(SequenceGenerator(x_train, 28 | y_train, 29 | batch_size=params['batch_size']), 30 | epochs=params['epochs'], 31 | validation_data=[x_val, y_val], 32 | callbacks=[], 33 | workers=4, 34 | verbose=0) 35 | 36 | return out, model 37 | 38 | p = {'activation':['relu', 'elu'], 39 | 'optimizer': ['Adam'], 40 | 'losses': ['LogCosh'], 41 | 'shapes': ['brick'], 42 | 'first_neuron': [32], 43 | 'dropout': [.2, .3], 44 | 'batch_size': [64, 128, 256], 45 | 'epochs': [1]} 46 | 47 | scan_object = talos.Scan(x=x_train, 48 | y=y_train, 49 | x_val=x_val, 50 | y_val=y_val, 51 | params=p, 52 | model=mnist_model) 53 | ``` 54 | -------------------------------------------------------------------------------- /docs/Examples_Multiple_Inputs.md: -------------------------------------------------------------------------------- 1 | # Multiple Inputs 2 | 3 | This example highlights a toyish example on using Keras functional API for multi-input model. One would approach building an experiment around a more elaborate multi-input model exactly in the same manner as is highlighted below. 4 | 5 | The single-file code example can be found [here](Examples_Multiple_Inputs_Code.md). 6 | 7 | ### Imports 8 | 9 | ```python 10 | import talos 11 | import wrangle 12 | 13 | from tensorflow.keras.layers import Input, Dense, Dropout 14 | from tensorflow.keras.models import Model 15 | from tensorflow.keras.utils import plot_model 16 | from tensorflow.keras.layers.merge import concatenate 17 | ``` 18 | NOTE: In this example we use another Autonomio package 'wrangle' for splitting the data. 19 | 20 | ### Loading Data 21 | ```python 22 | x, y = talos.templates.datasets.iris() 23 | x_train, y_train, x_val, y_val = wrangle.array_split(x, y, .5) 24 | ``` 25 | In the case of multi-input models, the data must be split into training and validation datasets before using it in `Scan()`. `x` is expected to be a list of numpy arrays and `y` a numpy array. 26 | 27 | **NOTE:** For full support of Talos features for multi-input models, set `Scan(...multi_input=True...)`. 28 | 29 | ### Defining the Model 30 | ```python 31 | 32 | def iris_multi(x_train, y_train, x_val, y_val, params): 33 | 34 | # the first side of the network 35 | first_input = Input(shape=(4,)) 36 | first_hidden1 = Dense(params['left_neurons'], activation=params['activation'])(first_input) 37 | first_hidden2 = Dense(params['left_neurons'], activation=params['activation'])(first_hidden1) 38 | 39 | # the second side of the network 40 | second_input = Input(shape=(4,)) 41 | second_hidden1 = Dense(params['right_neurons'], activation=params['activation'])(second_input) 42 | second_hidden2 = Dense(params['right_neurons'], activation=params['activation'])(second_hidden1) 43 | third_hidden2 = Dense(params['right_neurons'], activation=params['activation'])(second_hidden2) 44 | 45 | # merging the two networks 46 | merged = concatenate([first_hidden2, first_hidden2]) 47 | 48 | # creating the output 49 | output = Dense(3, activation='softmax')(merged) 50 | 51 | # put the model together, compile and fit 52 | model = Model(inputs=[first_input, second_input], outputs=output) 53 | model.compile('adam', 54 | 'binary_crossentropy', 55 | metrics=['acc', talos.utils.metrics.f1score]) 56 | 57 | out = model.fit(x=x_train, 58 | y=y_train, 59 | validation_data=[x_val, y_val], 60 | epochs=150, 61 | batch_size=params['batch_size'], 62 | verbose=0) 63 | 64 | return out, model 65 | ``` 66 | 67 | First, the input model must accept arguments exactly as in the example: 68 | 69 | ```python 70 | def iris_multi(x_train, y_train, x_val, y_val, params): 71 | ``` 72 | 73 | Even though it is a multi-output model, data can be inputted to `model.fit()` as you would otherwise do it. The multi-input part will be handled later in `Scan()` as shown below. 74 | 75 | ```python 76 | out = model.fit(x=x_train, 77 | y=y_train, 78 | ...) 79 | ``` 80 | 81 | The model must explicitly declare `validation_data` in `model.fit` because it is a multi-input model. Talos data splitting is not available for multi-input or multi-output models, or other cases where either `x` or `y` is more than 2d. 82 | 83 | ```python 84 | out = model.fit(... 85 | validation_data=[x_val, y_val] 86 | ...) 87 | ``` 88 | 89 | Finally, the model must `return` the `model.fit` object as well as the model itself in the order of the of the example: 90 | 91 | ```python 92 | return out, model 93 | ``` 94 | 95 | 96 | ### Parameter Dictionary 97 | 98 | ```python 99 | p = {'activation':['relu', 'elu'], 100 | 'left_neurons': [10, 20, 30], 101 | 'right_neurons': [10, 20, 30], 102 | 'batch_size': [15, 20, 25]} 103 | ``` 104 | 105 | Note that the parameter dictionary allows either list of values, or tuples with range in the form `(min, max, step)` 106 | 107 | 108 | ### Scan() 109 | ```python 110 | scan_object = talos.Scan(x=[x_train, x_train], 111 | y=y_train, 112 | x_val=[x_val, x_val], 113 | y_val=y_val, 114 | params=p, 115 | model=iris_multi) 116 | ``` 117 | 118 | `Scan()` always needs to have `x`, `y`, `model`, and `params` arguments declared. In the case of multi-output model, we also have to explicitly declare `val_x` and `val_y`. 119 | 120 | The important thing to note here is that how `y` and `y_val` are handled: 121 | 122 | ```python 123 | scan_object = talos.Scan(x=[x_train, x_train], 124 | ... 125 | x_val=[x_val, x_val], 126 | ...) 127 | ``` 128 | 129 | Find the description for all `Scan()` arguments [here](Scan.md#scan-arguments). 130 | -------------------------------------------------------------------------------- /docs/Examples_Multiple_Inputs_Code.md: -------------------------------------------------------------------------------- 1 | [BACK](Examples_Multiple_Inputs.md) 2 | 3 | # Multiple Inputs 4 | 5 | ```python 6 | import talos 7 | import wrangle 8 | 9 | from tensorflow.keras.layers import Input, Dense, Dropout 10 | from tensorflow.keras.models import Model 11 | from tensorflow.keras.utils import plot_model 12 | from tensorflow.keras.layers.merge import concatenate 13 | 14 | x, y = talos.templates.datasets.iris() 15 | x_train, y_train, x_val, y_val = wrangle.array_split(x, y, .5) 16 | 17 | def iris_multi(x_train, y_train, x_val, y_val, params): 18 | 19 | # the first side of the network 20 | first_input = Input(shape=(4,)) 21 | first_hidden1 = Dense(params['left_neurons'], activation=params['activation'])(first_input) 22 | first_hidden2 = Dense(params['left_neurons'], activation=params['activation'])(first_hidden1) 23 | 24 | # the second side of the network 25 | second_input = Input(shape=(4,)) 26 | second_hidden1 = Dense(params['right_neurons'], activation=params['activation'])(second_input) 27 | second_hidden2 = Dense(params['right_neurons'], activation=params['activation'])(second_hidden1) 28 | third_hidden2 = Dense(params['right_neurons'], activation=params['activation'])(second_hidden2) 29 | 30 | # merging the two networks 31 | merged = concatenate([first_hidden2, first_hidden2]) 32 | 33 | # creating the output 34 | output = Dense(3, activation='softmax')(merged) 35 | 36 | # put the model together, compile and fit 37 | model = Model(inputs=[first_input, second_input], outputs=output) 38 | model.compile('adam', 39 | 'binary_crossentropy', 40 | metrics=['acc', talos.utils.metrics.f1score]) 41 | 42 | out = model.fit(x=[x_train, x_train], 43 | y=y_train, 44 | validation_data=[[x_val, x_val], y_val], 45 | epochs=150, 46 | batch_size=params['batch_size'], 47 | verbose=0) 48 | 49 | return out, model 50 | 51 | 52 | p = {'activation':['relu', 'elu'], 53 | 'left_neurons': [10, 20, 30], 54 | 'right_neurons': [10, 20, 30], 55 | 'batch_size': [15, 20, 25]} 56 | 57 | scan_object = talos.Scan(x=x_train, 58 | y=y_train, 59 | x_val=x_val, 60 | y_val=y_val, 61 | params=p, 62 | model=iris_multi) 63 | ``` 64 | -------------------------------------------------------------------------------- /docs/Examples_Multiple_Outputs_Code.md: -------------------------------------------------------------------------------- 1 | [BACK](Examples_Multiple_Outputs.md) 2 | 3 | # Multiple Outputs 4 | 5 | ```python 6 | import talos 7 | import wrangle 8 | 9 | from tensorflow.keras.layers import Input, Dense, Dropout 10 | from tensorflow.keras.models import Model 11 | 12 | x, y = talos.templates.datasets.telco_churn() 13 | 14 | x_train, y1_train, x_val, y1_val = wrangle.array_split(x, y[0], 0.3) 15 | x_train, y2_train, x_val, y2_val = wrangle.array_split(x, y[1], 0.3) 16 | 17 | def telco_churn(x_train, y_train, x_val, y_val, params): 18 | 19 | # the second side of the network 20 | input_layer = Input(shape=(42,)) 21 | hidden_layer1 = Dense(params['neurons'], activation=params['activation'])(input_layer) 22 | hidden_layer2 = Dense(params['neurons'], activation=params['activation'])(hidden_layer1) 23 | hidden_layer3 = Dense(params['neurons'], activation=params['activation'])(hidden_layer2) 24 | 25 | # creating the outputs 26 | output1 = Dense(1, activation='sigmoid', name='loss_function')(hidden_layer3) 27 | output2 = Dense(1, activation='sigmoid', name='f1_metric')(hidden_layer3) 28 | 29 | losses = {"loss_function": "binary_crossentropy", 30 | "f1_metric": "binary_crossentropy"} 31 | 32 | loss_weights = {"loss_function": 1.0, "f1_metric": 1.0} 33 | 34 | # put the model together, compile and fit 35 | model = Model(inputs=input_layer, outputs=[output1, output2]) 36 | 37 | model.compile('adam', loss=losses, loss_weights=loss_weights, 38 | metrics=['acc', talos.utils.metrics.f1score]) 39 | 40 | out = model.fit(x=x_train, 41 | y=y_train, 42 | validation_data=[x_val, y_val], 43 | epochs=150, 44 | batch_size=params['batch_size'], 45 | verbose=0) 46 | 47 | return out, model 48 | 49 | p = {'activation':['relu', 'elu'], 50 | 'neurons': [10, 20, 30], 51 | 'batch_size': [15, 20, 25]} 52 | 53 | scan_object = talos.Scan(x=x_train, 54 | y=[y1_train, y2_train], 55 | x_val=x_val, 56 | y_val=[y1_val, y2_val], 57 | params=p, 58 | model=telco_churn) 59 | ``` 60 | -------------------------------------------------------------------------------- /docs/Examples_PyTorch_Code.md: -------------------------------------------------------------------------------- 1 | [BACK](Examples_PyTorch.md) 2 | 3 | # PyTorch Example 4 | 5 | ```python 6 | import talos 7 | import numpy as np 8 | 9 | import torch 10 | import torch.nn as nn 11 | import torch.nn.functional as F 12 | from torch_optimizer import torch_optimizer 13 | 14 | from sklearn.metrics import f1_score 15 | 16 | # load the data 17 | x, y = talos.templates.datasets.breast_cancer() 18 | x = talos.utils.rescale_meanzero(x) 19 | x_train, y_train, x_val, y_val = talos.utils.val_split(x, y, .2) 20 | 21 | # convert arrays to tensors 22 | x_train = torch.from_numpy(x_train).float() 23 | y_train = torch.from_numpy(y_train).long() 24 | x_val = torch.from_numpy(x_val).float() 25 | y_val = torch.from_numpy(y_val).long() 26 | 27 | def breast_cancer(x_train, y_train, x_val, y_val, params): 28 | 29 | # takes in a module and applies the specified weight initialization 30 | def weights_init_uniform_rule(m): 31 | classname = m.__class__.__name__ 32 | # for every Linear layer in a model.. 33 | if classname.find('Linear') != -1: 34 | # get the number of the inputs 35 | n = m.in_features 36 | y = 1.0 / np.sqrt(n) 37 | m.weight.data.uniform_(-y, y) 38 | m.bias.data.fill_(0) 39 | 40 | class BreastCancerNet(nn.Module, talos.utils.TorchHistory): 41 | 42 | def __init__(self, n_feature): 43 | 44 | super(BreastCancerNet, self).__init__() 45 | self.hidden = torch.nn.Linear(n_feature, params['first_neuron']) 46 | torch.nn.init.normal_(self.hidden.weight) 47 | self.hidden1 = torch.nn.Linear(params['first_neuron'], params['second_neuron']) 48 | self.dropout = torch.nn.Dropout(params['dropout']) 49 | self.out = torch.nn.Linear(params['second_neuron'], 2) 50 | 51 | def forward(self, x): 52 | 53 | x = F.relu(self.hidden(x)) 54 | x = self.dropout(x) 55 | x = torch.sigmoid(self.hidden1(x)) 56 | x = self.out(x) 57 | return x 58 | 59 | 60 | net = BreastCancerNet(x_train.shape[1]) 61 | net.apply(weights_init_uniform_rule) 62 | loss_func = nn.CrossEntropyLoss() 63 | optimizer = torch_optimizer(net, 64 | params['optimizer'], 65 | lr=params['lr'], 66 | momentum=params['momentum'], 67 | weight_decay=params['weight_decay']) 68 | 69 | # Initialize history of net 70 | net.init_history() 71 | 72 | for epoch in range(params['epochs']): 73 | 74 | # zero the parameter gradients 75 | optimizer.zero_grad() 76 | 77 | # forward 78 | outputs = net(x_train) 79 | 80 | # calculate accuracy 81 | prediction = torch.max(outputs, 1)[1] 82 | metric = f1_score(y_train.data, prediction.data) 83 | 84 | # calculate loss + backward + optimize 85 | loss = loss_func(outputs, y_train) 86 | loss.backward() 87 | optimizer.step() 88 | 89 | # calculate accuracy for validation data 90 | output_val = net(x_val) 91 | prediction = torch.max(output_val, 1)[1] 92 | val_metric = f1_score(y_val.data, prediction.data) 93 | 94 | # calculate loss for validation data 95 | val_loss = loss_func(output_val, y_val) 96 | 97 | # append history 98 | net.append_loss(loss.item()) 99 | net.append_metric(metric) 100 | net.append_val_loss(val_loss.item()) 101 | net.append_val_metric(val_metric) 102 | 103 | 104 | # Get history object 105 | return net, net.parameters() 106 | 107 | 108 | p = {'activation':['relu', 'elu'], 109 | 'optimizer': ['Adagrad', 'Adam'], 110 | 'losses': ['LogCosh'], 111 | 'hidden_layers':[0, 1, 2], 112 | 'batch_size': (20, 50, 5), 113 | 'epochs': [10, 20]} 114 | 115 | scan_object = talos.Scan(x=x_train, 116 | y=y_train, 117 | x_val=x_val, 118 | y_val=y_val, 119 | params=p, 120 | model=breast_cancer, 121 | experiment_name='breast_cancer', 122 | round_limit=100) 123 | ``` 124 | -------------------------------------------------------------------------------- /docs/Examples_Typical.md: -------------------------------------------------------------------------------- 1 | # Typical 2 | 3 | This example highlights a typical and rather simple example of Talos experiment, and is a good starting point for those new to Talos. The single-file example can be found [here](Examples_Typical_Code.md). 4 | 5 | ### Imports 6 | 7 | ```python 8 | import talos 9 | from tensorflow.keras.models import Sequential 10 | from tensorflow.keras.layers import Dense 11 | ``` 12 | 13 | ### Loading Data 14 | ```python 15 | x, y = talos.templates.datasets.iris() 16 | ``` 17 | `x` and `y` are expected to be either numpy arrays or lists of numpy arrays. 18 | 19 | ### Defining the Model 20 | ```python 21 | def iris_model(x_train, y_train, x_val, y_val, params): 22 | 23 | model = Sequential() 24 | model.add(Dense(32, input_dim=4, activation=params['activation'])) 25 | model.add(Dense(3, activation='softmax')) 26 | model.compile(optimizer=params['optimizer'], loss=params['losses']) 27 | 28 | out = model.fit(x_train, y_train, 29 | batch_size=params['batch_size'], 30 | epochs=params['epochs'], 31 | validation_data=[x_val, y_val], 32 | verbose=0) 33 | 34 | return out, model 35 | ``` 36 | 37 | First, the input model must accept arguments exactly as in the example: 38 | 39 | ```python 40 | def iris_model(x_train, y_train, x_val, y_val, params): 41 | ``` 42 | 43 | Second, the model must explicitly declare `validation_data` in `model.fit`: 44 | 45 | ```python 46 | out = model.fit( ... validation_data=[x_val, y_val] ... ) 47 | ``` 48 | Finally, the model must `return` the `model.fit` object as well as the model itself in the order of the of the example: 49 | 50 | ```python 51 | return out, model 52 | ``` 53 | 54 | 55 | ### Parameter Dictionary 56 | ```python 57 | p = {'activation':['relu', 'elu'], 58 | 'optimizer': ['Adagrad', 'Adam'], 59 | 'losses': ['LogCosh'], 60 | 'hidden_layers':[0, 1, 2], 61 | 'batch_size': (20, 50, 5), 62 | 'epochs': [10, 20]} 63 | ``` 64 | 65 | Note that the parameter dictionary allows either list of values, or tuples with range in the form `(min, max, number_of_values)` 66 | 67 | 68 | ### Scan() 69 | ```python 70 | scan_object = talos.Scan(x, y, model=iris_model, params=p, fraction_limit=0.1) 71 | ``` 72 | 73 | `Scan()` always needs to have `x`, `y`, `model`, and `params` arguments declared. Find the description for all `Scan()` arguments [here](Scan.md#scan-arguments). 74 | -------------------------------------------------------------------------------- /docs/Examples_Typical_Code.md: -------------------------------------------------------------------------------- 1 | [BACK](Examples_Typical.md) 2 | 3 | # Typical Case Example 4 | 5 | ```python 6 | import talos as talos 7 | from tensorflow.keras.models import Sequential 8 | from tensorflow.keras.layers import Dense 9 | 10 | x, y = talos.templates.datasets.iris() 11 | 12 | # define the model 13 | def iris_model(x_train, y_train, x_val, y_val, params): 14 | 15 | model = Sequential() 16 | 17 | model.add(Dense(32, input_dim=4, activation=params['activation'])) 18 | model.add(Dense(3, activation='softmax')) 19 | 20 | model.compile(optimizer=params['optimizer'], 21 | loss=params['losses'], 22 | metrics=[talos.utils.metrics.f1score]) 23 | 24 | out = model.fit(x_train, y_train, 25 | batch_size=params['batch_size'], 26 | epochs=params['epochs'], 27 | validation_data=[x_val, y_val], 28 | verbose=0) 29 | 30 | return out, model 31 | 32 | # set the parameter space boundaries 33 | p = {'activation':['relu', 'elu'], 34 | 'optimizer': ['Adagrad', 'Adam'], 35 | 'losses': ['categorical_crossentropy'], 36 | 'epochs': [100, 200], 37 | 'batch_size': [4, 6, 8]} 38 | 39 | # start the experiment 40 | scan_object = talos.Scan(x=x, 41 | y=y, 42 | model=iris_model, 43 | params=p, 44 | experiment_name='iris', 45 | round_limit=20) 46 | ``` 47 | 48 | `Scan()` always needs to have `x`, `y`, `model`, and `params` arguments declared. Find the description for all `Scan()` arguments [here](Scan.md#scan-arguments). 49 | -------------------------------------------------------------------------------- /docs/Gamify.md: -------------------------------------------------------------------------------- 1 | # Gamify 2 | 3 | When `Scan(...reduction_method='gamify'...)` between each permutation, a json file is updated on the local machine in the experiment folder. Gamify is an effort to bring the human back on the drive's seat, and is the work of early computer scientists in the late 1950s around the topic of Man-Machine symbiosis. 4 | 5 | Gamify allows visualization and two way interaction through two components: 6 | 7 | - A browser-based live dashboard 8 | - a round-by-round updating log of each parater value 9 | 10 | ### Gamify Dasbhoard 11 | 12 | First install the add-on package: 13 | 14 | `pip install gamify` 15 | 16 | Then add `gamify` to your `PATH`: 17 | 18 | `export PATH=$PATH:/Users/mikko/miniconda3/envs/wip_conda/lib/python3.6/site-packages/gamify.py` 19 | 20 | Note that your path will be unique to your machine, you can see it on the terminal output once you install the package. 21 | 22 | Now you can start the dashboard by referencing an experiment folder. 23 | 24 | `python gamify /path/to/talos/experiment` 25 | 26 | See more information [here](https://github.com/autonomio/gamify) 27 | 28 | ### Gamify JSON 29 | 30 | The JSON file stores the current activity status of each parameter value, and if the status is `active` then nothing will be changed. If the status is `disabled`, then all permutations with that parameter value will be removed from the parameter space. 31 | 32 | There is also a numeric value for each parameter value, which is a placeholder for storing an arbitrary value associated with the performance of the parameter value. 33 | -------------------------------------------------------------------------------- /docs/Generator.md: -------------------------------------------------------------------------------- 1 | # Generator 2 | 3 | Talos provides two data generator to be used with `model.fit_generator()`. You can of course use your own generator as you would use it otherwise with stand-alone Keras. 4 | 5 | #### basic data generator 6 | 7 | ```python 8 | talos.utils.generator(x=x, y=y, batch_size=20) 9 | 10 | ``` 11 | 12 | #### sequence generator 13 | ```python 14 | talos.utils.SequenceGenerator(x=x, y=y, batch_size=20) 15 | 16 | ``` 17 | 18 | 19 | NOTE: There are many performance considerations that come with using data generators in Keras. If you run in to performance issues, learn more about the experiences of other Keras users online. 20 | 21 | You can also read the Talos thread on [using fit_generator](https://github.com/autonomio/talos/issues/11). 22 | -------------------------------------------------------------------------------- /docs/Hidden_Layers.md: -------------------------------------------------------------------------------- 1 | # Hidden Layers 2 | 3 | Including `hidden_layers` in a model allows the use of number of hidden Dense layers as an optimization parameter. 4 | 5 | Each hidden layer is followed by a Dropout regularizer. If this is undesired, set dropout to 0 with ```dropout: [0]``` in the parameter dictionary. 6 | 7 | ```python 8 | from talos.utils import hidden_layers 9 | 10 | def input_model(): 11 | # model prep and input layer... 12 | hidden_layers(model, params, 1) 13 | # rest of the model... 14 | ``` 15 | 16 | When hidden layers are used, `dropout`, `shapes`, `hidden_layers`, and `first_neuron` parameters must be included in the parameter dictionary. For example: 17 | 18 | ```python 19 | 20 | p = {'activation':['relu', 'elu'], 21 | 'optimizer': ['Adagrad', 'Adam'], 22 | 'losses': ['LogCosh'], 23 | 'shapes': ['brick'], # <<< required 24 | 'first_neuron': [32, 64], # <<< required 25 | 'hidden_layers':[0, 1, 2], # <<< required 26 | 'dropout': [.2, .3], # <<< required 27 | 'batch_size': [20,30,40], 28 | 'epochs': [200]} 29 | 30 | ``` 31 | 32 | ## hidden_layers Arguments 33 | 34 | Parameter | type | Description 35 | --------- | ------- | ----------- 36 | `model` | class object | a `Scan` object 37 | `params` | dict | The input model parameters dictionary 38 | `output_dims` | int | Number of dimensions on the output layer 39 | 40 | NOTE: `params` here refers to the dictionary where parameters of a single permutation are contained. 41 | 42 | # Shapes 43 | 44 | Talos allows several options for testing network architectures as a parameter. `shapes` is invoked by including it in the parameter dictionary: 45 | 46 | ```python 47 | 48 | # alternates between two preset shapes 49 | p = {... 50 | 'shapes': ['brick', 'triangle', 'funnel'], 51 | ...} 52 | 53 | # a linear contraction of set value from input layer to output layer 54 | p = {... 55 | 'shapes': [.1, .15, .2, .25], 56 | ...} 57 | 58 | ``` 59 | NOTE: You must use `hidden_layers` as per described above in order to leverage `shapes`. 60 | -------------------------------------------------------------------------------- /docs/Install_Options.md: -------------------------------------------------------------------------------- 1 | # Install Options 2 | 3 | Before installing Talos, it is recommended to first setup and start either a python or conda environment. 4 | 5 | #### Creating a python virtual environment 6 | ```python 7 | virtualenv -p python3 talos_env 8 | source talos_env/bin/activate 9 | ``` 10 | 11 | #### Creating a conda virtual environment 12 | ```python 13 | conda create --name talos_env 14 | conda activate talos_env 15 | ``` 16 | 17 | #### Install latest from PyPi 18 | ```python 19 | pip install talos 20 | ``` 21 | 22 | #### Install a specific version from PyPi 23 | ```python 24 | pip install talos==0.6.1 25 | ``` 26 | 27 | #### Upgrade installation from PyPi 28 | ```python 29 | pip install -U --no-deps talos 30 | ``` 31 | 32 | #### Install from monthly 33 | ```python 34 | pip install --upgrade --no-deps --force-reinstall git+https://github.com/autonomio/talos 35 | ``` 36 | 37 | #### Install from weekly 38 | ```python 39 | pip install --upgrade --no-deps --force-reinstall git+https://github.com/autonomio/talos@dev 40 | ``` 41 | 42 | #### Install from daily 43 | ```python 44 | pip install --upgrade --no-deps --force-reinstall git+https://github.com/autonomio/talos@daily-dev 45 | ``` 46 | -------------------------------------------------------------------------------- /docs/Learning_Rate_Normalizer.md: -------------------------------------------------------------------------------- 1 | ## LR Normalizer 2 | 3 | As one experiment may include more than one optimizer, and optimizers generally have default learning rates in different order of magnitudes, lr_normalizer can be used to allow simultanously including different optimizers and different degrees of learning rates into the Talos experiment. 4 | 5 | ```python 6 | from talos.model.normalizers import lr_normalizer 7 | # model first part is here ... 8 | model.compile(loss='binary_crossentropy', 9 | optimizer=params['optimizer'](lr_normalizer(params['lr'], params['optimizer'])), 10 | metrics=['accuracy']) 11 | # model ending part is here ... 12 | ``` 13 | 14 | 17 | -------------------------------------------------------------------------------- /docs/Local_Strategy.md: -------------------------------------------------------------------------------- 1 | # Local Strategy 2 | 3 | With `Scan(...reduction_method='local_strategy'...)` it's possible to control the experiment between each permutation without interrupting the experiment. `local_strategy` is similar to [custom strategies](#custom-reducers) with the difference that the optimization strategy can be changed during the experiment, as the function resides on the local machine. 4 | 5 | The `talos_strategy` should be a python function stored in a file `talos_strategy.py` which resides in the present working directory of the experiment. It can be changed anytime. 6 | -------------------------------------------------------------------------------- /docs/Metrics.md: -------------------------------------------------------------------------------- 1 | # Metrics 2 | 3 | Several common and useful performance metrics are available through `talos.utils.metrics`: 4 | 5 | - matthews 6 | - precision 7 | - recall 8 | - fbeta 9 | - f1score 10 | - mae 11 | - mse 12 | - rmae 13 | - rmse 14 | - mape 15 | - msle 16 | - rmsle 17 | 18 | You can use these metrics as you would Keras metrics: 19 | 20 | ```python 21 | metrics=['acc', talos.utils.metrics.f1score] 22 | ``` 23 | If you would like to add new metrics to Talos, make a [feature request](https://github.com/autonomio/talos/issues/new) or create a [pull request](https://github.com/autonomio/talos/compare). 24 | -------------------------------------------------------------------------------- /docs/Monitoring.md: -------------------------------------------------------------------------------- 1 | # Monitoring 2 | 3 | There are several options for monitoring the experiment. 4 | 5 | ```python 6 | # turn off progress bar 7 | Scan(disable_progress_bar=True) 8 | 9 | # enable live training plot 10 | from talos.callbacks import TrainingPlot 11 | 12 | out = model.fit(X, 13 | Y, 14 | epochs=20, 15 | callbacks=[TrainingPlot()]) 16 | 17 | # turn on parameter printing 18 | Scan(print_params=True) 19 | ``` 20 | 21 | **Progress Bar :** A round-by-round updating progress bar that shows the remaining rounds, together with a time estimate to completion. Progress bar is on by default. 22 | 23 | **Live Monitoring :** Live monitoring provides an epoch-by-epoch updating line graph that is enabled through the `TrainingPlot()` custom callback. 24 | 25 | **Round Hyperparameters :** Displays the hyperparameters for each permutation. Does not work together with live monitoring. 26 | 27 | ### Local Monitoring 28 | 29 | Epoch-by-epoch training data is available during the experiment using the `ExperimentLog`: 30 | 31 | ```python 32 | model.fit(... 33 | callbacks=[talos.callbacks.ExperimentLog('experiment_name', params)]) 34 | ``` 35 | Here `params` is the params dictionary in the `Scan()` input model. Both 36 | `experiment_name` and `experiment_id` should match with the current experiment, 37 | as otherwise 38 | -------------------------------------------------------------------------------- /docs/Optimization_Strategies.md: -------------------------------------------------------------------------------- 1 | # Optimization Strategies 2 | 3 | Talos supports several common optimization strategies: 4 | 5 | - Grid search 6 | - Random search 7 | - Probabilistic reduction 8 | - Custom Strategies (arbitrary single python file optimizer) 9 | - Local Stragies (can change anytime during experiment) 10 | - Gamify (man-machine cooperation) 11 | 12 | The object of abstraction is the keras model configuration, of which n number of permutations is tried in a Talos experiment. 13 | 14 | As opposed to adding more complex optimization strategies, which are widely available in various solutions, Talos focus is on: 15 | 16 | - adding variations of random variable picking 17 | - reducing the workload of random variable picking 18 | 19 | As it stands, both of these approaches are currently under leveraged by other solutions, and under represented in the literature. 20 | 21 | # Random Search 22 | 23 | A key focus in Talos develoment is to provide gold standard random search capabilities. Talos implements three kinds of random generation methods: 24 | 25 | - True / Quantum randomness 26 | - Pseudo randomness 27 | - Quasi randomness 28 | 29 | Random methods are selected in `Scan(...random_method...)`. For example, to use `quantum` randomness: 30 | 31 | ```python 32 | talos.Scan(x=x, y=y, model=model, params=p, random_method='quantum') 33 | ``` 34 | 35 | ## Random Options 36 | 37 | PARAMETER | DESCRIPTION 38 | --------- | ----------- 39 | `ambience` | Ambient Sound based randomness 40 | `halton` | Halton sequences 41 | `korobov_matrix` | Korobob matrix based sequence 42 | `latin_matrix` | Latin hypercube 43 | `latin_improved` | Improved Latin hypercube 44 | `latin_sudoku` | Latin hypercube with a Sudoku-style constraint 45 | `quantum` | Quantum randomness (vacuum based) 46 | `sobol` | Sobol sequences 47 | `uniform_crypto` | Cryptographically sound uniform 48 | `uniform_mersenne` | Uniform Mersenne twister 49 | 50 | Each method differs in discrepancy and other observable aspects. Scientific evidence suggest that low discrepancy methods outperform plain pseudo-random methods. 51 | 52 | # Grid Search 53 | 54 | To perform a conventional grid search, simply leave the `Scan(...fraction_limit...)` argument undeclared, that way all possible permutations will be processed in a sequential order. 55 | 56 | # Early Stopping 57 | 58 | Use of early stopper, when set appropriate, can help reduce experiment time by preventing time waste on unproductive permutations. Once a monitored metric is no longer improving, Talos moves to the next permutation. Talos provides three presets - `lazy`, `moderate` and `strict` - in addition to completely custom settings. 59 | 60 | `early_stopper` is invoked in the input model, in `model.fit()`. 61 | 62 | ```python 63 | 64 | out = model.fit(x_train, 65 | y_train, 66 | batch_size=params['batch_size'], 67 | epochs=params['epochs'], 68 | validation_data=[x_val, y_val], 69 | verbose=0, 70 | callbacks=[talos.utils.early_stopper(params['epochs'])]) 71 | 72 | ``` 73 | 74 | The minimum input to Talos `early_stopper` is the `epochs` hyperparameter. This is used with the automated settings. For custom settings, this can be left as `None`. 75 | 76 | Argument | Input | Description 77 | -------- | ----- | ----------- 78 | `epochs` | int | The number of epochs for the permutation e.g. params['epochs'] 79 | `monitor` | int | The metric to monitor for change 80 | `mode` | str | One of the presets `lazy`, `moderate`, `strict` or `None` 81 | `min_delta` | float | The limit for change at which point flag is raised 82 | `patience` | str | the number of epochs before termination from flag 83 | -------------------------------------------------------------------------------- /docs/Overview.md: -------------------------------------------------------------------------------- 1 | # Key Features 2 | 3 | Talos features focus on easy-of-use, intuitive workflow, do not introduce any new syntax, or require reading through complex technical documentations. 4 | 5 | - perform a hyperparameter scan 6 | - perform a model architecture search 7 | - analyze and visualize results 8 | - evaluate results to find best model candidates 9 | - make predictions with selected models 10 | - deploy models into self-contained zip files 11 | - restore models onto a production system 12 | - create AutoML pipelines 13 | 14 | If you have an idea for a new feature, don't hesitate to [suggest it](https://github.com/autonomio/talos/issues/new). 15 | 16 | # How to Use 17 | 18 | Talos provides single-line commands for conducting and analyzing experiments, for evaluating models from the experiment, and making predictions with models. In addition, Talos provides s streamlined way to deploy and restore models across systems. 19 | 20 | All primary commands except `Restore()` and `Deploy()` return a class object specific to the command, with various properties. These are outlined in the corresponding sections of the documentation. 21 | 22 | All primary commands except `Restore()` accept the class object resulting from `Scan()` as input. 23 | 24 | 25 | #### conduct an experiment 26 | 27 | ```python 28 | scan_object = talos.Scan(x, y, model, params) 29 | ``` 30 | 31 | #### analyze the results of an experiment 32 | 33 | ```python 34 | talos.Analyze(scan_object) 35 | ``` 36 | 37 | #### evaluate one or more models from the experiment 38 | 39 | ```python 40 | talos.Evaluate(scan_object) 41 | ``` 42 | 43 | #### make predictions with a model from the experiment 44 | 45 | ```python 46 | talos.Predict(scan_object) 47 | ``` 48 | 49 | #### create a deploy package 50 | 51 | ```python 52 | talos.Deploy(scan_object, model_name='deployed_package.zip', metric='f1score') 53 | ``` 54 | 55 | #### deploy a model 56 | 57 | ```python 58 | talos.Restore('deployed_package.zip') 59 | ``` 60 | 61 | In addition to the primary commands, various utilities can be accessed through `talos.utils`, datasets, parameters, and models from `talos.templates`, and AutoML features through `talos.autom8`. 62 | 63 | 64 | # Creating an Experiment 65 | 66 | To get started with your first experiment is easy. You need to have three things: 67 | 68 | a hyperparameter dictionary 69 | a working Keras model 70 | a Talos experiment 71 | 72 | ### STEP 1 : Prepare the input model 73 | 74 | In order to prepare a Keras model for a Talos experiment, you simply replace parameters you want to include in the scan, with references to the parameter dictionary. 75 | 76 | ### STEP 2 : Define the parameter space 77 | 78 | In a regular Python dictionary, you declare the hyperparameters and the boundaries you want to include in the experiment. 79 | 80 | ### STEP 3 : Configure the experiment 81 | 82 | To start the experiment, you input the parameter dictionary and the Keras model into Talos with the option for Grid, Random, or Probabilistic optimization strategy. 83 | 84 | ### STEP 4 : Let the good times roll in 85 | 86 | Congratulations, you've just freed yourself from the manual, laborous, and somewhat mind-numbing task of finding the right hyperparameters for your deep learning models. 87 | -------------------------------------------------------------------------------- /docs/Parallelism.md: -------------------------------------------------------------------------------- 1 | # GPU Support 2 | 3 | Talos supports scenarios where on a single system one or more GPUs are handling one or more simultaneous jobs. The base GPU support is handled by TensorFlow, so make sure that you have the GPU version of TensorFlow installed. 4 | 5 | You can watch system GPU utilization anytime with: 6 | 7 | `watch -n0.5 nvidia-smi` 8 | 9 | ## Single GPU, Single Job 10 | 11 | Single GPU works out-of-the-box, as long as you have the GPU version of TensorFlow installed. In case you already have the CPU version installed, you have to uninstall TensorFlow and install the GPU version. 12 | 13 | ## Single GPU, Multiple Jobs 14 | 15 | ```python 16 | 17 | from talos.utils.gpu_utils import parallel_gpu_jobs 18 | 19 | # split GPU memory in two for two parallel jobs 20 | parallel_gpu_jobs(0.5) 21 | talos.Scan(...) 22 | 23 | ``` 24 | NOTE: The above lines must be run before the Scan() command: 25 | 26 | A single GPU can be split to simultaneously perform several experiments. This is useful you want to work on more than one scope at one time, or when you're analyzing the results of an ongoing experiment with `Reporting()` and are ready to start the next experiment while keeping the first one running. 27 | 28 | **NOTE:** GPU memory needs to be reserved pro-actively i.e. once the experiment is already running with full GPU memory, part of the memory can no longer be allocated to a new experiment. 29 | 30 | ## Multi-GPU, Single Job 31 | 32 | ```python 33 | from talos.utils.gpu_utils import multi_gpu 34 | 35 | # split a single job to multiple GPUs 36 | model = multi_gpu(model) 37 | model.compile(...) 38 | ``` 39 | NOTE: Include the above line in the input model before model.compile() 40 | 41 | Multiple GPUs on a single machine can be assigned to work on a single machine in a parallelism fashion. This is useful when you have more than one GPU on a single machine, and want to speed up the experiment. Each GPU roughly speaking reduces compute time linearly. 42 | 43 | ## Force CPU 44 | 45 | ```python 46 | from talos.utils.gpu_utils import force_cpu 47 | 48 | # Force CPU use on a GPU system 49 | force_cpu() 50 | talos.Scan(...) 51 | ``` 52 | NOTE: Run the above lines before the Scan() command 53 | 54 | Sometimes it's useful (for example when `batch_size` tends to be very small) to disable GPU and use CPU instead. This can be done simply by invoking `force_cpu()`. 55 | -------------------------------------------------------------------------------- /docs/Predict.md: -------------------------------------------------------------------------------- 1 | # Predict() 2 | 3 | In order to identify the best model from a given experiment, or to perform predictions with model/s, the [Predict()]([Reporting()](https://github.com/autonomio/talos/blob/master/talos/utils/predict.py)) command can be used. 4 | 5 | ```python 6 | p = Predict('scan_object') 7 | 8 | p.predict(x) 9 | ``` 10 | 11 | ### Predict Properties 12 | 13 | **`predict`** makes probability predictions on `x` which has to be in the same form as the input data used in the `Scan()` experiment. 14 | 15 | ```python 16 | scan_object.data 17 | ``` 18 | 19 |
20 | 21 | **`predict_classes`** makes class predictions on `x` which has to be in the same form as the input data used in the `Scan()` experiment. 22 | 23 | ```python 24 | scan_object.data 25 | ``` 26 | 27 | ### Predict Arguments 28 | 29 | ### Predict.predict Arguments 30 | 31 | Parameter | Default | Description 32 | --------- | ------- | ----------- 33 | `x` | NA | the predictor data x 34 | `model_id` | None | the model_id to be used 35 | `metric` | None | the metric against which the validation is performed 36 | `asc` | None | should be True if metric is a loss 37 | `task`| NA | One of the following strings: 'binary' or 'multi_class' 38 | `saved` | bool | if a model saved on local machine should be used 39 | `custom_objects` | dict | if the model has a custom object, pass it here 40 | -------------------------------------------------------------------------------- /docs/Probabilistic_Reduction.md: -------------------------------------------------------------------------------- 1 | # Probabilistic Reduction 2 | 3 | Based on `Scan()` arguments (listed below), probabilistic reducers drop permutations from the remaining parameter space based on model performance. The process mimics the human process where between experiments the researcher uses various probabilistic methods, such as correlations, to identify how to change the parameter dictionary. When set correctly, reducers can dramatically reduce experiment time, with equal or superior results in comparison to random search. 4 | 5 | Argument | Input | Description 6 | -------- | ----- | ----------- 7 | `reduction_method` | str | Type of reduction optimizer to be used used 8 | `reduction_interval` | int | Number of permutations after which reduction is applied 9 | `reduction_window` | int | the look-back window for reduction process 10 | `reduction_threshold` | float | The threshold at which reduction is applied 11 | `reduction_metric` | str | The metric to be used for reduction 12 | `minimize_loss` | bool | `reduction_metric` is a loss 13 | 14 | The reduction arguments are always invoked through `Scan()`. A typical case involves something on the lines of the below example. 15 | 16 | ```python 17 | talos.Scan(... 18 | reduction_method='correlation', 19 | reduction_interval=50, 20 | reduction_window=25, 21 | reduction_threshold=0.2, 22 | reduction_metric='mae', 23 | minimize_loss=True, 24 | ...) 25 | ``` 26 | Here correlation reducer is used every 50 permutations, looking back 25 permutations, and ignoring any correlation that is below 0.2. The reduction is performed with the goal of minimizing mae. 27 | 28 | In practice this means that Talos will find the hyperparameter with the strongest correlation with high mae (undesirable) and then find which individual value has the highest correlation within the identified hyperparameter. Once found, assuming the correlation is higher than `reduction_threshold`, all permutations in the parameter space with that value of the given hyperparameter are dropped. 29 | 30 | ### Available Reducers 31 | 32 | Choose one of the below in `reduction_method`: 33 | 34 | - `correlation` (same as `spearman`) 35 | - `spearman` 36 | - `kendall` 37 | - `pearson` 38 | - `trees` 39 | - `forrest` 40 | - `local_strategy` # allows dynamically changing local strategy 41 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Quick start 2 | 3 | ```python 4 | pip install talos 5 | ``` 6 | See [here](Install_Options.md) for more options. 7 | 8 | 9 | # Minimal Example 10 | 11 | Your Keras model **WITHOUT** Talos: 12 | 13 | ```python 14 | model = Sequential() 15 | model.add(Dense(12, input_dim=8, activation='relu')) 16 | model.add(Dense(1, activation='sigmoid')) 17 | model.compile(loss='binary_crossentropy', optimizer='adam') 18 | model.fit(x, y) 19 | ``` 20 | Your Keras model **WITH** Talos: 21 | 22 | 23 | ```python 24 | model = Sequential() 25 | model.add(Dense(12, input_dim=8, activation=params['activation'])) 26 | model.add(Dense(1, activation='sigmoid')) 27 | model.compile(loss='binary_crossentropy', params['optimizer']) 28 | model.fit(x, y) 29 | 30 | ``` 31 | See the code complete example [here](http). 32 | 33 | # Typical Use-cases 34 | 35 | The most common use-case of Talos is a hyperparameter scan based on an already created Keras or TensorFlow model. In addition to the [input model](), a hyperparameter scan with Talos involves `talos.Scan()` command and a [parameter dictionary](). 36 | 37 | After completing an experiment, results can be analyzed and visualized. Once a decision have been made if a) the experiment should be reconfigured and continue or b) sufficient level of performance have already been found, model candidates can be automatically evaluated and used for prediction. 38 | 39 | Talos also supports easy deployment of models and experiment assets from the experiment environment to production or other systems. 40 | 41 | # System Requirements 42 | 43 | - Linux, Mac OSX or Windows system 44 | - Python 3.5 or higher (Talos versions 0.5.0 and below support 2.7) 45 | - TensorFlow, Theano, or CNTK 46 | 47 | Talos incorporates grid, random, and probabilistic hyperparameter optimization strategies, with focus on maximizing the flexibility, efficiency, and result of random strategy. Talos users benefit from access to pseudo, quasi, true, and quantum random methods. 48 | -------------------------------------------------------------------------------- /docs/Restore.md: -------------------------------------------------------------------------------- 1 | # Restore() 2 | 3 | The `Deploy()` .zip package can be read back into a copy of the original experiment assets with `Restore()`. 4 | 5 | ```python 6 | from talos import Restore 7 | 8 | restore = Restore('experiment_name.zip') 9 | ``` 10 | NOTE: In the `Deploy()` phase, '.zip' is automatically added to the deploy package file name and must be added here manually. 11 | 12 | ## Restore Arguments 13 | 14 | Parameter | Default | Description 15 | --------- | ------- | ----------- 16 | `path_to_zip` | None | full path to the `Deploy` asset zip file 17 | 18 | 19 | ## Restore Properties 20 | 21 | The `Deploy()` .zip package can be read back into a copy of the original experiment assets with `Restore()`. The object consists of: 22 | 23 | - details of the scan 24 | - model 25 | - results of the experiment 26 | - sample of x data 27 | - sample of y data 28 | 29 | **`details`** returns a pandas DataFrame with various meta-information about the experiment. 30 | 31 | ```python 32 | restore.details 33 | ``` 34 |
35 | 36 | **`model`** returns a Keras model ready to rock. 37 | 38 | ```python 39 | restore.model 40 | ``` 41 |
42 | 43 | **`params`** returns the params dictionary used in the experiment. 44 | 45 | ```python 46 | restore.params 47 | ``` 48 |
49 | 50 | **`results`** returns a pandas DataFrame with the results for the experiment together with the hyperparameter permutation details. 51 | 52 | ```python 53 | restore.results 54 | ``` 55 | 56 |
57 | 58 | **`x`** returns a small sample of the data (features) used for training the model. 59 | 60 | ```python 61 | restore.x 62 | ``` 63 | 64 |
65 | 66 | **`y`** returns a small sample of the data (labels) used for training the model. 67 | 68 | ```python 69 | restore.y 70 | ``` 71 | 72 |
73 | 74 | 75 | `details` | The class object returned by Scan() upon completion of the experiment. 76 | `model` | Input data (features) in the same format as used in Scan(), but should not be the same data (or it will not be much of validation). 77 | `params` | Input data (labels) in the same format as used in Scan(), but should not be the same data (or it will not be much of validation). 78 | `results` | The number of models to be evaluated. If set to 10, then 10 models with the highest metric value are evaluated. See below. 79 | `x` | 80 | `y` | A 81 | -------------------------------------------------------------------------------- /docs/Roadmap.md: -------------------------------------------------------------------------------- 1 | ## What we do? 2 | 3 | Keep pushing the boundaries of automated deep learning workflows. Start where everyone else stops. Do it with tender loving care. 4 | 5 | ## Why we do it? 6 | 7 | So that more people would have access to the undiluted potential and power of advanced deep learning. So that those who already have access, would feel less pain and instead have more joy. 8 | 9 | ## Who we do it for? 10 | 11 | "Researchers first" drives Talos development. Everything is done for researchers, by researchers. 12 | 13 | ## How we use time? 14 | 15 | - 1/3 of time for **PLANNING** 16 | - 1/3 of **PLANNING** for designing the future 17 | - 1/3 of **PLANNING** for writing specification 18 | - 1/3 of **PLANNING** for creating documentation 19 | 20 | - 1/3 of time for **TESTING** 21 | - 1/2 of **TESTING** time to hands-on use 22 | - 1/4 of **TESTING** time for creating new tests 23 | - 1/4 of **TESTING** time for improving current tests 24 | 25 | - 1/3 of time for **CODING** 26 | - 1/3 of **CODING** time for new features 27 | - 1/3 of **CODING** time for improving current features 28 | - 1/3 of **CODING** time for fixing broken features 29 | 30 | ## How to contribute? 31 | 32 | - by using Talos 33 | - by writing about Talos 34 | - by creating examples of using Talos 35 | - by recommending Talos 36 | - by testing Talos 37 | - by contributing code (or tests) 38 | - by making feature requests 39 | - by improving the documentation 40 | -------------------------------------------------------------------------------- /docs/Templates.md: -------------------------------------------------------------------------------- 1 | # Templates Overview 2 | 3 | Talos provides access to sets of templates consisting of the assets required by `Scan()` i.e. datasets, parameter dictionaries, and input models. In addition, `talos.templates.pipelines` consist of ready pipelines that combined the assets into a `Scan()` experiment and run it. These are mainly provided for educational, testing, and development purposes. 4 | 5 | Each category of templates consists at least assets based on four popular machine learning datasets: 6 | 7 | - Wisconsin Breast Cancer | [dataset info](https://archive.ics.uci.edu/ml/datasets/Breast+Cancer+Wisconsin+(Diagnostic)) 8 | - Cervical Cancer Screening | [dataset info](https://arxiv.org/pdf/1812.10383.pdf) 9 | - Iris | [dataset info](https://www.semanticscholar.org/topic/Iris-flower-data-set/620769) 10 | - Titanic Survival | [dataset info](https://www.kaggle.com/c/titanic) 11 | 12 | In addition, some categories (e.g. datasets) include additional templates. These are listed below and can be accessed through the corresponding namespace without previous knowledge. 13 | 14 |
15 | 16 | # Datasets 17 | 18 | Datasets are preprocessed so that they can be used directly as inputs for deep learning models. Datasets are accessed through `talos.templates.datasets`. For example: 19 | 20 | ```python 21 | talos.templates.datasets.breast_cancer() 22 | 23 | ``` 24 | 25 | #### Available Datasets 26 | 27 | - breast_cancer 28 | - cervical_cancer 29 | - icu_mortality 30 | - telco_churn 31 | - titanic 32 | - iris 33 | - mnist 34 | 35 |
36 | 37 | # Params 38 | 39 | Params consist of an indicative and somewhat meaningful parameter space boundaries that can be used as the parameter dictionary for `Scan()` experiments. Parameter dictionaries are accessed through `talos.templates.params`. For example: 40 | 41 | ```python 42 | talos.templates.params.breast_cancer() 43 | ``` 44 | 45 | #### Available Params 46 | 47 | - breast_cancer 48 | - cervical_cancer 49 | - titanic 50 | - iris 51 | 52 |
53 | 54 | # Models 55 | 56 | Models consist of Keras models that can be used as an input model for `Scan()` experiments. Models are accessed through `talos.templates.models`. For example: 57 | 58 | ```python 59 | talos.templates.models.breast_cancer() 60 | ``` 61 | 62 | #### Available Models 63 | 64 | - breast_cancer 65 | - cervical_cancer 66 | - titanic 67 | - iris 68 | 69 |
70 | 71 | # Pipelines 72 | 73 | Pipelines are self-contained `Scan()` experiments where you simply execute the command and an experiment is performed. Pipelines are accessed through `talos.templates.pipelines`. For example: 74 | 75 | ```python 76 | scan_object = talos.templates.pipelines.breast_cancer() 77 | ``` 78 | 79 | #### Available Pipelines 80 | 81 | - breast_cancer 82 | - cervical_cancer 83 | - titanic 84 | - iris 85 | -------------------------------------------------------------------------------- /docs/Workflow.md: -------------------------------------------------------------------------------- 1 | # Workflow 2 | 3 | The goal of a deep learning experiment is to find one or more model candidates that meet a given performance expectation. Talos makes this process as easy and streamlined as possible. Talos provides an intuitive API to manage both semi-automated and fully-automated deep learning workflows. The below example highlights a typical semi-automated workflow from idea to production ready model. 4 | 5 | ![alt text](_media/talos_deep_learning_workflow.png") 6 | -------------------------------------------------------------------------------- /docs/_coverpage.md: -------------------------------------------------------------------------------- 1 | ![logo](_media/talos_logo_bg.png) 2 | 3 | ## v1.4 4 | 5 | > Hyperparameter Experiments with Tensorflow, PyTorch and Keras 6 | 7 | [GitHub](https://github.com/autonomio/talos/) 8 | [Getting Started](https://autonomio.github.io/talos/#/README?id=quick-start) 9 | 10 | ![color](#f0f0f0) 11 | -------------------------------------------------------------------------------- /docs/_media/talos_deep_learning_workflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomio/talos/9cbbc42a69396aaeaf25d68af51d52e6a575e3d9/docs/_media/talos_deep_learning_workflow.png -------------------------------------------------------------------------------- /docs/_media/talos_logo_bg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomio/talos/9cbbc42a69396aaeaf25d68af51d52e6a575e3d9/docs/_media/talos_logo_bg.png -------------------------------------------------------------------------------- /docs/_media/talos_logo_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomio/talos/9cbbc42a69396aaeaf25d68af51d52e6a575e3d9/docs/_media/talos_logo_icon.png -------------------------------------------------------------------------------- /docs/_sidebar.md: -------------------------------------------------------------------------------- 1 | - Getting Started 2 | 3 | - [Quick start](README.md#quick-start) 4 | - [Minimal example](README.md#minimal-example) 5 | - [Typical Use-cases](README.md#typical-use-cases) 6 | - [System Requirements](README.md#system-requirements) 7 | - [Install Options](Install_Options.md) 8 | 9 | - Supported Backends 10 | 11 | - [TensorFlow](Backends.md#Tensorflow) 12 | - [PyTorch](Backends.md#PyTorch) 13 | - [Keras](Backends.md#Keras) 14 | 15 | - Overview 16 | 17 | - [Key Features](Overview.md#key-features) 18 | - [How to Use](Overview.md#how-to-use) 19 | - [Creating an experiment](Overview.md#creating-an-experiment) 20 | - [Workflow](Workflow.md) 21 | 22 | - Examples 23 | 24 | - [Typical](Examples_Typical.md) 25 | - [Multiple outputs](Examples_Multiple_Outputs.md) 26 | - [Multiple inputs](Examples_Multiple_Inputs.md) 27 | - [Using a generator](Examples_Generator.md) 28 | - [AutoML](Examples_AutoML.md) 29 | - [PyTorch](Examples_PyTorch.md) 30 | 31 | - Core Commands 32 | 33 | - [Scan](Scan.md) 34 | - [Analyze](Analyze.md) 35 | - [Evaluate](Evaluate.md) 36 | - [Predict](Predict.md) 37 | - [Deploy](Deploy.md) 38 | - [Restore](Restore.md) 39 | 40 | - Optimization Strategies 41 | 42 | - [Overview](Optimization_Strategies.md#optimization-strategies) 43 | - [Random Search](Optimization_Strategies.md#random-search) 44 | - [Grid Search](Optimization_Strategies.md#grid-search) 45 | - [Early Stopping](Optimization_Strategies.md#early-stopping) 46 | - [Probabilistic Reduction](Probabilistic_Reduction.md) 47 | - [Custom Reducers](Custom_Reducers.md) 48 | - [Local Strategy](Local_Strategy.md) 49 | - [Gamify](Gamify.md) 50 | 51 | - Utilities 52 | 53 | - [Energy Draw](Energy_Draw.md) 54 | - [Generator](Generator.md) 55 | - [Hidden Layers](Hidden_Layers.md) 56 | - [Learning Rate Normalizer](Learning_Rate_Normalizer.md) 57 | - [Metrics](Metrics.md) 58 | - [Monitoring](Monitoring.md) 59 | - [Parallelism](Parallelism.md) 60 | - [Shapes](Hidden_Layers.md#shapes) 61 | 62 | - Templates 63 | 64 | - [Overview](Templates.md) 65 | - [Data](Templates.md#datasets) 66 | - [Params](Templates.md#params) 67 | - [Models](Templates.md#models) 68 | - [Pipelines](Templates.md#pipelines) 69 | 70 | - AutoML 71 | 72 | - [AutoModel](AutoModel.md) 73 | - [AutoParams](AutoParams.md) 74 | - [AutoScan](AutoScan.md) 75 | - [AutoPredict](AutoPredict.md) 76 | 77 | - Troubleshooting 78 | 79 | - [Asking help](Asking_Help.md) 80 | - [Roadmap](Roadmap.md) 81 | -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Talos Docs 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 |
17 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /examples/iris.py: -------------------------------------------------------------------------------- 1 | # first import things as you would usually 2 | from tensorflow.keras.models import Sequential 3 | from tensorflow.keras.layers import Dense, Dropout 4 | from tensorflow.keras.losses import categorical_crossentropy 5 | from tensorflow.keras.activations import relu, softmax 6 | 7 | # import talos 8 | import talos 9 | 10 | # load rthe iris dataset 11 | x, y = talos.templates.datasets.iris() 12 | 13 | # then define the parameter boundaries 14 | 15 | p = {'first_neuron': [8, 16, 32], 16 | 'batch_size': [2, 3, 4]} 17 | 18 | 19 | # then define your Keras model 20 | def iris_model(x_train, y_train, x_val, y_val, params): 21 | 22 | model = Sequential() 23 | model.add(Dense(params['first_neuron'], 24 | input_dim=x_train.shape[1], 25 | activation='relu')) 26 | model.add(Dropout(.5)) 27 | model.add(Dense(y_train.shape[1], activation='softmax')) 28 | 29 | model.compile(optimizer='adam', 30 | loss='categorical_crossentropy', 31 | metrics=['acc']) 32 | 33 | out = model.fit(x_train, y_train, 34 | batch_size=params['batch_size'], 35 | epochs=50, 36 | verbose=0, 37 | validation_data=[x_val, y_val]) 38 | 39 | return out, model 40 | 41 | 42 | # and run the scan 43 | h = talos.Scan(x, y, 44 | params=p, 45 | experiment_name='talos-debug', 46 | model=iris_model, 47 | round_limit=10) 48 | -------------------------------------------------------------------------------- /examples/version-check.py: -------------------------------------------------------------------------------- 1 | import tensorflow 2 | print('TensorFlow %s' % tensorflow.__version__) 3 | 4 | import talos 5 | print('Talos %s' % talos.__version__) 6 | 7 | import sys 8 | print('Python %s' % sys.version.split()[0]) -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomio/talos/9cbbc42a69396aaeaf25d68af51d52e6a575e3d9/logo.png -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling"] 3 | build-backend = "hatchling.build" 4 | 5 | [project] 6 | name = "talos" 7 | dynamic = ["version"] 8 | description = "Talos Hyperparameter Tuning for Keras" 9 | readme = "README.md" 10 | license = "MIT" 11 | authors = [ 12 | { name = "Mikko Kotila", email = "mailme@mikkokotila.com" }, 13 | ] 14 | maintainers = [ 15 | { name = "Mikko Kotila", email = "mailme@mikkokotila.com" }, 16 | ] 17 | classifiers = [ 18 | "Intended Audience :: Science/Research", 19 | "License :: OSI Approved :: MIT License", 20 | "Operating System :: MacOS", 21 | "Operating System :: Microsoft :: Windows :: Windows 10", 22 | "Operating System :: POSIX", 23 | "Operating System :: Unix", 24 | "Programming Language :: Python :: 2.7", 25 | "Programming Language :: Python :: 3.5", 26 | "Programming Language :: Python :: 3.6", 27 | "Topic :: Scientific/Engineering :: Artificial Intelligence", 28 | "Topic :: Scientific/Engineering :: Human Machine Interfaces", 29 | "Topic :: Scientific/Engineering :: Mathematics", 30 | ] 31 | dependencies = [ 32 | "scikit-learn", 33 | "statsmodels >= 0.11.0", 34 | "tensorflow == 2.14.1", 35 | "astetik", 36 | "chances", 37 | "kerasplotlib", 38 | "numpy", 39 | "pandas", 40 | "requests", 41 | "tqdm", 42 | "wrangle", 43 | ] 44 | 45 | [project.urls] 46 | Download = "https://github.com/autonomio/talos/" 47 | Homepage = "http://autonom.io" 48 | 49 | [tool.hatch.version] 50 | path = "talos/__init__.py" 51 | 52 | [tool.hatch.build.targets.sdist] 53 | include = [ 54 | "/talos", 55 | ] 56 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | wrangle 2 | pandas 3 | numpy 4 | keras 5 | astetik 6 | tqdm 7 | scikit-learn 8 | chances 9 | kerasplotlib 10 | requests 11 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # 3 | # Copyright (C) 2022 Mikko Kotila 4 | 5 | DESCRIPTION = "Talos Hyperparameter Tuning for Keras" 6 | LONG_DESCRIPTION = """\ 7 | Talos radically changes the ordinary deep learning workflow by 8 | fully automating hyperparameter tuning and model evaluation. 9 | Talos supports Tensorflow and Pytorch. 10 | 11 | Talos exposes backend functionality entirely and there is 12 | no new syntax or templates to learn. 13 | """ 14 | 15 | DISTNAME = 'talos' 16 | MAINTAINER = 'Mikko Kotila' 17 | MAINTAINER_EMAIL = 'mailme@mikkokotila.com' 18 | URL = 'http://autonom.io' 19 | LICENSE = 'MIT' 20 | DOWNLOAD_URL = 'https://github.com/autonomio/talos/' 21 | VERSION = '1.3.3' 22 | 23 | 24 | try: 25 | from setuptools import setup 26 | _has_setuptools = True 27 | except ImportError: 28 | from distutils.core import setup 29 | 30 | install_requires = ['tensorflow>=2.0.0', 31 | 'statsmodels>=0.11.0', 32 | 'wrangle', 33 | 'numpy', 34 | 'pandas', 35 | 'astetik', 36 | 'scikit-learn', 37 | 'tqdm', 38 | 'chances', 39 | 'kerasplotlib', 40 | 'requests'] 41 | 42 | 43 | if __name__ == "__main__": 44 | 45 | setup(name=DISTNAME, 46 | author=MAINTAINER, 47 | author_email=MAINTAINER_EMAIL, 48 | maintainer=MAINTAINER, 49 | maintainer_email=MAINTAINER_EMAIL, 50 | description=DESCRIPTION, 51 | long_description=LONG_DESCRIPTION, 52 | license=LICENSE, 53 | url=URL, 54 | version=VERSION, 55 | download_url=DOWNLOAD_URL, 56 | install_requires=install_requires, 57 | packages=['talos', 58 | 'talos.scan', 59 | 'talos.templates', 60 | 'talos.utils', 61 | 'talos.model', 62 | 'talos.parameters', 63 | 'talos.reducers', 64 | 'talos.metrics', 65 | 'talos.commands', 66 | 'talos.logging', 67 | 'talos.autom8', 68 | 'talos.callbacks'], 69 | 70 | classifiers=['Intended Audience :: Science/Research', 71 | 'Programming Language :: Python :: 2.7', 72 | 'Programming Language :: Python :: 3.5', 73 | 'Programming Language :: Python :: 3.6', 74 | 'License :: OSI Approved :: MIT License', 75 | 'Topic :: Scientific/Engineering :: Human Machine Interfaces', 76 | 'Topic :: Scientific/Engineering :: Artificial Intelligence', 77 | 'Topic :: Scientific/Engineering :: Mathematics', 78 | 'Operating System :: POSIX', 79 | 'Operating System :: Unix', 80 | 'Operating System :: MacOS', 81 | 'Operating System :: Microsoft :: Windows :: Windows 10']) 82 | -------------------------------------------------------------------------------- /talos/__init__.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | warnings.simplefilter('ignore') 3 | 4 | # import commands 5 | from .scan.Scan import Scan 6 | from .commands.analyze import Analyze 7 | from .commands.analyze import Analyze as Reporting 8 | from .commands.predict import Predict 9 | from .commands.deploy import Deploy 10 | from .commands.evaluate import Evaluate 11 | from .commands.restore import Restore 12 | 13 | # import extras 14 | from . import utils 15 | from . import templates 16 | from . import autom8 17 | from . import callbacks 18 | 19 | # the purpose of everything below is to keep the namespace completely clean 20 | 21 | template_sub = [templates.datasets, 22 | templates.models, 23 | templates.params, 24 | templates.pipelines] 25 | 26 | keep_from_templates = ['iris', 'cervical_cancer', 'titanic', 'breast_cancer', 27 | 'icu_mortality', 'telco_churn', 'mnist'] 28 | 29 | for sub in template_sub: 30 | for key in list(sub.__dict__): 31 | if key.startswith('__') is False: 32 | if key not in keep_from_templates: 33 | delattr(sub, key) 34 | 35 | del commands, scan, model, metrics, key 36 | del sub, keep_from_templates, template_sub, warnings 37 | 38 | __version__ = "1.4" 39 | -------------------------------------------------------------------------------- /talos/autom8/__init__.py: -------------------------------------------------------------------------------- 1 | from .automodel import AutoModel 2 | from .autoparams import AutoParams 3 | from .autopredict import AutoPredict 4 | from .autoscan import AutoScan 5 | 6 | del automodel, autoparams, autopredict, autoscan 7 | -------------------------------------------------------------------------------- /talos/autom8/autopredict.py: -------------------------------------------------------------------------------- 1 | def AutoPredict(scan_object, 2 | x_val, 3 | y_val, 4 | x_pred, 5 | task, 6 | metric='val_acc', 7 | n_models=10, 8 | folds=5, 9 | shuffle=True, 10 | asc=False): 11 | 12 | '''Automatically handles the process of finding the best models from a 13 | completed `Scan()` experiment, evaluates those models, and uses the winner 14 | to make predictions on input data. 15 | 16 | NOTE: the input data must be in same format as 'x' that was 17 | used in `Scan()`. 18 | 19 | Parameters 20 | ---------- 21 | scan_object : Scan() object 22 | A Scan() process needs to be completed first, and then the resulting 23 | object can be used as input here. 24 | x_val : ndarray or list of ndarray 25 | Data to be used for 'x' in evaluation. Note that should be in the same 26 | format as the data which was used in the Scan() but not the same data. 27 | y_val : ndarray or list of ndarray 28 | Data to be used for 'y' in evaluation. Note that should be in the same 29 | format as the data which was used in the Scan() but not the same data. 30 | y_pred : ndarray or list of ndarray 31 | Input data to be used for the actual predictions in evaluation. Note 32 | it should be in the same format as the data which was used in the 33 | Scan() but not the same data. 34 | task : string 35 | 'binary', 'multi_class', 'multi_label', or 'continuous'. 36 | metric : str 37 | The metric to be used for deciding which models are promising. 38 | Basically the 'n' argument and 'metric' argument are combined to pick 39 | 'n' best performing models based on 'metric'. 40 | n_models : str 41 | Number of promising models to be included in the evaluation process. 42 | Time increase linearly with number of models. 43 | folds : int 44 | Number of folds to be used in cross-validation. 45 | shuffle : bool 46 | If the data should be shuffled before cross-validation. 47 | average : str 48 | This parameter is required for multiclass/multilabel targets. If None, 49 | the scores for each class are returned. Otherwise, this determines 50 | the type of averaging performed on the data: 51 | 52 | 'binary': 53 | Only report results for the class specified by pos_label. 54 | This is applicable only if targets (y_{true,pred}) are binary. 55 | 56 | 'micro': 57 | Calculate metrics globally by counting the total true positives, 58 | false negatives and false positives. 59 | 60 | 'macro': 61 | Calculate metrics for each label, and find their unweighted mean. 62 | This does not take label imbalance into account. 63 | 64 | 'weighted': 65 | Calculate metrics for each label, and find their average weighted 66 | by support (the number of true instances for each label). This alters 67 | 'macro' to account for label imbalance; it can result in an F-score 68 | that is not between precision and recall. 69 | 70 | 'samples': 71 | Calculate metrics for each instance, and find their average 72 | (only meaningful for multilabel classification where this differs 73 | from accuracy_score). 74 | asc : bool 75 | This needs to be True for evaluation metrics that need to be minimized, 76 | and False when a metric needs to be maximized. 77 | 78 | ''' 79 | 80 | import numpy as np 81 | 82 | # evaluate and add the evaluation scores 83 | scan_object.evaluate_models(x_val, 84 | y_val, 85 | n_models=n_models, 86 | task=task, 87 | metric=metric, 88 | folds=folds, 89 | shuffle=shuffle, 90 | asc=False) 91 | 92 | # get the best model based on evaluated score 93 | scan_object.preds_model = scan_object.best_model('eval_f1score_mean') 94 | 95 | # make predictions with the model 96 | scan_object.preds_probabilities = scan_object.preds_model.predict(x_pred) 97 | 98 | # make (class) predictiosn with the model 99 | preds = scan_object.preds_model.predict(x_pred) 100 | scan_object.preds_classes = np.argmax(preds, axis=1) 101 | 102 | # get the hyperparameter for the model 103 | scan_object.preds_parameters = scan_object.data.sort_values('eval_f1score_mean', 104 | ascending=False).iloc[0] 105 | 106 | print(">> Added model, probabilities, classes, and parameters to scan_object") 107 | 108 | return scan_object 109 | -------------------------------------------------------------------------------- /talos/autom8/autoscan.py: -------------------------------------------------------------------------------- 1 | class AutoScan: 2 | 3 | def __init__(self, 4 | task, 5 | experiment_name, 6 | max_param_values=None): 7 | 8 | '''Configure the `AutoScan()` experiment and then use 9 | the property `start` in the returned class object to start 10 | the actual experiment. 11 | 12 | `task` | str | 'binary', 'multi_class', 'multi_label', or 'continuous' 13 | `max_param_values` | int | Number of parameter values to be included. 14 | Note, this will only work when `params` is 15 | not passed as kwargs in `AutoScan.start`. 16 | ''' 17 | 18 | self.task = task 19 | self.max_param_values = max_param_values 20 | self.experiment_name = experiment_name 21 | 22 | def start(self, x, y, **kwargs): 23 | 24 | '''Start the scan. Note that you can use `Scan()` arguments as you 25 | would otherwise directly interacting with `Scan()`. 26 | 27 | `x` | array or list of arrays | prediction features 28 | `y` | array or list of arrays | prediction outcome variable 29 | `kwargs` | arguments | any `Scan()` argument can be passed here 30 | 31 | ''' 32 | 33 | import talos 34 | 35 | m = talos.autom8.AutoModel(self.task, self.experiment_name).model 36 | 37 | try: 38 | kwargs['params'] 39 | scan_object = talos.Scan(x, y, 40 | model=m, 41 | experiment_name=self.experiment_name, 42 | **kwargs) 43 | except KeyError: 44 | p = talos.autom8.AutoParams(task=self.task) 45 | 46 | if self.max_param_values is not None: 47 | p.resample_params(self.max_param_values) 48 | params = p.params 49 | scan_object = talos.Scan(x=x, 50 | y=y, 51 | params=params, 52 | model=m, 53 | experiment_name=self.experiment_name, 54 | **kwargs) 55 | 56 | return scan_object 57 | -------------------------------------------------------------------------------- /talos/callbacks/__init__.py: -------------------------------------------------------------------------------- 1 | from kerasplotlib import TrainingLog as TrainingPlot 2 | from .experiment_log import ExperimentLog 3 | from .power_draw import PowerDraw 4 | 5 | del experiment_log, power_draw 6 | -------------------------------------------------------------------------------- /talos/callbacks/experiment_log.py: -------------------------------------------------------------------------------- 1 | from tensorflow.keras.callbacks import Callback 2 | 3 | 4 | class ExperimentLog(Callback): 5 | 6 | def __init__(self, 7 | experiment_name, 8 | params): 9 | 10 | '''Takes as input the name of the experiment which will be 11 | used for creating a .log file with the outputs and the params 12 | dictionary from the input model in `Scan()` 13 | 14 | experiment_name | str | must match the experiment_name in `Scan()` 15 | params | dict | the params dictionary from the input model in `Scan() 16 | 17 | ''' 18 | 19 | super(ExperimentLog, self).__init__() 20 | 21 | import glob 22 | import os 23 | 24 | # get the experiment id first 25 | list_of_files = glob.glob('./' + experiment_name + '/*.csv') 26 | 27 | try: 28 | latest_file = max(list_of_files, key=os.path.getmtime) 29 | except ValueError: 30 | print("\nERROR: `experiment_name` has to match `Scan(experiment_name)`\n") 31 | 32 | self.name = latest_file.replace('.csv', '') + '.log' 33 | 34 | # rest of the config variables 35 | self.params = params 36 | self.counter = 1 37 | self.new_file = True 38 | 39 | def on_train_begin(self, logs={}): 40 | 41 | import random 42 | self.hash = hex(abs(hash(str(random.random())))) 43 | self.final_out = [] 44 | 45 | def on_train_end(self, logs={}): 46 | 47 | f = open(self.name, 'a+') 48 | [f.write(','.join(map(str, i)) + '\n') for i in self.final_out] 49 | f.close() 50 | 51 | def on_epoch_begin(self, epoch, logs={}): 52 | 53 | self.epoch_out = [] 54 | 55 | def on_epoch_end(self, epoch, logs={}): 56 | 57 | if len(self.final_out) == 0: 58 | 59 | try: 60 | open(self.name, 'r') 61 | except FileNotFoundError: 62 | 63 | self.epoch_out.append('id') 64 | self.epoch_out.append('epoch') 65 | 66 | for key in logs.keys(): 67 | 68 | # add to the epoch out list 69 | self.epoch_out.append(key) 70 | 71 | self.final_out.append(self.epoch_out) 72 | self.epoch_out = [] 73 | 74 | self.epoch_out.append(self.hash) 75 | self.epoch_out.append(epoch + 1) 76 | 77 | for key in logs.keys(): 78 | 79 | # round the values 80 | rounded = round(logs[key], 4) 81 | 82 | # add to the epoch out list 83 | self.epoch_out.append(rounded) 84 | 85 | # add to the final out list 86 | self.final_out.append(self.epoch_out) 87 | -------------------------------------------------------------------------------- /talos/callbacks/power_draw.py: -------------------------------------------------------------------------------- 1 | from tensorflow.keras.callbacks import Callback 2 | 3 | 4 | class PowerDraw(Callback): 5 | 6 | '''A callback for recording GPU power draw (watts) on epoch begin and end. 7 | 8 | Example use: 9 | 10 | power_draw = PowerDraw() 11 | 12 | model.fit(...callbacks=[power_draw]...) 13 | 14 | history = talos.utils.power_draw_append(history, power_draw) 15 | 16 | ''' 17 | 18 | def __init__(self): 19 | 20 | super(PowerDraw, self).__init__() 21 | 22 | import os 23 | import time 24 | 25 | self.os = os 26 | self.time = time.time 27 | self.command = "nvidia-smi -i 0 -q | grep -i 'power draw' | tr -s ' ' | cut -d ' ' -f5" 28 | 29 | def on_train_begin(self, logs={}): 30 | self.log = {} 31 | self.log['epoch_begin'] = [] 32 | self.log['epoch_end'] = [] 33 | self.log['seconds'] = [] 34 | 35 | def on_epoch_begin(self, batch, logs=None): 36 | self.epoch_start_time = self.time() 37 | temp = self.os.popen(self.command).read() 38 | temp = float(temp.strip()) 39 | self.log['epoch_begin'].append(temp) 40 | 41 | def on_epoch_end(self, batch, logs=None): 42 | temp = self.os.popen(self.command).read() 43 | temp = float(temp.strip()) 44 | self.log['epoch_end'].append(temp) 45 | seconds = round(self.time() - self.epoch_start_time, 3) 46 | self.log['seconds'].append(seconds) 47 | -------------------------------------------------------------------------------- /talos/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomio/talos/9cbbc42a69396aaeaf25d68af51d52e6a575e3d9/talos/commands/__init__.py -------------------------------------------------------------------------------- /talos/commands/deploy.py: -------------------------------------------------------------------------------- 1 | class Deploy: 2 | 3 | '''Functionality for deploying a model to a filename''' 4 | 5 | def __init__(self, 6 | scan_object, 7 | model_name, 8 | metric, 9 | asc=False, 10 | saved=False, 11 | custom_objects=None): 12 | 13 | '''Deploy a model to be used later or in a different system. 14 | 15 | NOTE: for a metric that is to be minimized, set asc=True or otherwise 16 | you will end up with the model that has the highest loss. 17 | 18 | Deploy() takes in the object from Scan() and creates a package locally 19 | that can be later activated with Restore(). 20 | 21 | scan_object | object | The object that is returned from Scan() upon 22 | completion. 23 | model_name | str | Name for the .zip file to be created. 24 | metric | str | The metric to be used for picking the best model. 25 | asc | bool | Make this True for metrics that are to be minimized 26 | (e.g. loss), and False when the metric is to be 27 | maximized (e.g. acc). 28 | saved | bool | if a model saved on local machine should be used 29 | custom_objects | dict | if the model has a custom object, pass it here 30 | 31 | ''' 32 | 33 | import os 34 | 35 | self.scan_object = scan_object 36 | os.mkdir(model_name) 37 | self.path = model_name + '/' + model_name 38 | self.model_name = model_name 39 | self.metric = metric 40 | self.asc = asc 41 | self.data = scan_object.data 42 | 43 | from ..utils.best_model import best_model, activate_model 44 | self.best_model = best_model(scan_object, metric, asc) 45 | self.model = activate_model(scan_object, 46 | self.best_model, 47 | saved, 48 | custom_objects) 49 | 50 | # runtime 51 | self.save_model_as() 52 | self.save_details() 53 | self.save_data() 54 | self.save_results() 55 | self.save_params() 56 | self.save_readme() 57 | self.package() 58 | 59 | def save_model_as(self): 60 | 61 | '''Model Saver 62 | WHAT: Saves a trained model so it can be loaded later 63 | for predictions by predictor(). 64 | ''' 65 | 66 | model_json = self.model.to_json() 67 | with open(self.path + "_model.json", "w") as json_file: 68 | json_file.write(model_json) 69 | 70 | self.model.save_weights(self.path + "_model.h5") 71 | print("Deploy package" + " " + self.model_name + " " + "have been saved.") 72 | 73 | def save_details(self): 74 | 75 | self.scan_object.details.to_csv(self.path + '_details.txt') 76 | 77 | def save_data(self): 78 | 79 | import pandas as pd 80 | 81 | # input data is <= 2d 82 | try: 83 | x = pd.DataFrame(self.scan_object.x[:100]) 84 | y = pd.DataFrame(self.scan_object.y[:100]) 85 | 86 | # input data is > 2d 87 | except ValueError: 88 | x = pd.DataFrame() 89 | y = pd.DataFrame() 90 | print("data is not 2d, dummy data written instead.") 91 | 92 | x.to_csv(self.path + '_x.csv', header=None, index=None) 93 | y.to_csv(self.path + '_y.csv', header=None, index=None) 94 | 95 | def save_results(self): 96 | 97 | self.scan_object.data.to_csv(self.path + '_results.csv') 98 | 99 | def save_params(self): 100 | 101 | import numpy as np 102 | 103 | np.save(self.path + '_params', self.scan_object.params) 104 | 105 | def save_readme(self): 106 | 107 | txt = 'To activate the assets in the Talos deploy package: \n\n from talos.commands.restore import Restore \n a = Restore(\'path_to_asset\')\n\nNow you will have an object similar to the Scan object, which can be used with other Talos commands as you would be able to with the Scan object' 108 | 109 | text_file = open(self.path.split('/')[0] + '/README.txt', "w") 110 | text_file.write(txt) 111 | text_file.close() 112 | 113 | def package(self): 114 | 115 | import shutil 116 | 117 | shutil.make_archive(self.model_name, 'zip', self.model_name) 118 | shutil.rmtree(self.model_name) 119 | -------------------------------------------------------------------------------- /talos/commands/evaluate.py: -------------------------------------------------------------------------------- 1 | class Evaluate: 2 | 3 | '''Class for evaluating models based on the Scan() object''' 4 | 5 | def __init__(self, scan_object): 6 | 7 | '''Takes in as input a Scan() object. 8 | e = evaluate(scan_object) and see docstring 9 | for e() for more information.''' 10 | 11 | self.scan_object = scan_object 12 | self.data = scan_object.data 13 | 14 | def evaluate(self, 15 | x, 16 | y, 17 | task, 18 | metric, 19 | model_id=None, 20 | folds=5, 21 | shuffle=True, 22 | asc=False, 23 | saved=False, 24 | custom_objects=None, 25 | multi_input=False, 26 | print_out=False): 27 | 28 | '''Evaluate a model based on f1_score (all except regression) 29 | or mae (for regression). Supports 'binary', 'multi_class', 30 | 'multi_label', and 'regression' evaluation. 31 | 32 | x | array | The input data for making predictions 33 | y | array | The ground truth for x 34 | model_id | int | It's possible to evaluate a specific model based 35 | on ID. 36 | folds | int | Number of folds to use for cross-validation 37 | sort_metric | string | A column name referring to the metric that 38 | was used in the scan_object as a performance 39 | metric. This is used for sorting the results 40 | to pick for evaluation. 41 | shuffle | bool | Data is shuffled before evaluation. 42 | task | string | 'binary', 'multi_class', 'multi_label', or 43 | 'continuous'. 44 | asc | bool | False if the metric is to be optimized upwards 45 | (e.g. accuracy or f1_score) 46 | saved | bool | if a model saved on local machine should be used 47 | custom_objects | dict | if the model has a custom object, pass it here 48 | multi_input | bool | if multi-input model is evaluated, set to True 49 | print_out | bool | Print out the results. 50 | 51 | TODO: add possibility to input custom metrics. 52 | 53 | ''' 54 | 55 | import numpy as np 56 | import sklearn as sk 57 | 58 | out = [] 59 | if model_id is None: 60 | from ..utils.best_model import best_model 61 | model_id = best_model(self.scan_object, metric, asc) 62 | 63 | from ..utils.best_model import activate_model 64 | model = activate_model(self.scan_object, 65 | model_id, 66 | saved=saved, 67 | custom_objects=custom_objects) 68 | 69 | from ..utils.validation_split import kfold 70 | kx, ky = kfold(x, y, folds, shuffle, multi_input) 71 | 72 | for i in range(folds): 73 | 74 | y_pred = model.predict(kx[i], verbose=0) 75 | 76 | if task == 'binary': 77 | y_pred = np.array(y_pred) >= .5 78 | scores = sk.metrics.f1_score(y_pred, ky[i], average='binary') 79 | 80 | elif task == 'multi_class': 81 | y_pred = y_pred.argmax(axis=-1) 82 | scores = sk.metrics.f1_score(y_pred, ky[i], average='macro') 83 | 84 | if task == 'multi_label': 85 | y_pred = model.predict(kx[i]).argmax(axis=1) 86 | scores = sk.metrics.f1_score(y_pred, 87 | ky[i].argmax(axis=1), 88 | average='macro') 89 | 90 | elif task == 'continuous': 91 | y_pred = model.predict(kx[i]) 92 | scores = sk.metrics.mean_absolute_error(y_pred, ky[i]) 93 | 94 | out.append(scores) 95 | 96 | if print_out is True: 97 | print("mean : %.2f \n std : %.2f" % (np.mean(out), np.std(out))) 98 | 99 | return out 100 | -------------------------------------------------------------------------------- /talos/commands/predict.py: -------------------------------------------------------------------------------- 1 | class Predict: 2 | 3 | '''Class for making predictions on the models that are stored 4 | in the Scan() object''' 5 | 6 | def __init__(self, scan_object): 7 | 8 | '''Takes in as input a Scan() object and returns and object 9 | with properties for `predict` and `predict_classes`''' 10 | 11 | self.scan_object = scan_object 12 | self.data = scan_object.data 13 | 14 | def predict(self, 15 | x, 16 | metric, 17 | asc, 18 | model_id=None, 19 | saved=False, 20 | custom_objects=None): 21 | 22 | '''Makes a probability prediction from input x. If model_id 23 | is not given, then best_model will be used. 24 | 25 | x | array | data to be used for the predictions 26 | model_id | int | the id of the model from the Scan() object 27 | metric | str | the metric to be used for picking best model 28 | asc | bool | True if `metric` is something to be minimized 29 | saved | bool | if a model saved on local machine should be used 30 | custom_objects | dict | if the model has a custom object, 31 | pass it here 32 | 33 | ''' 34 | 35 | if model_id is None: 36 | from ..utils.best_model import best_model 37 | model_id = best_model(self.scan_object, metric, asc) 38 | 39 | from ..utils.best_model import activate_model 40 | model = activate_model(self.scan_object, 41 | model_id, 42 | saved, 43 | custom_objects) 44 | 45 | return model.predict(x) 46 | 47 | def predict_classes(self, 48 | x, 49 | metric, 50 | asc, 51 | task, 52 | model_id=None, 53 | saved=False, 54 | custom_objects=None): 55 | 56 | '''Makes a class prediction from input x. If model_id 57 | is not given, then best_model will be used. 58 | 59 | x | array | data to be used for the predictions 60 | model_id | int | the id of the model from the Scan() object 61 | metric | str | the metric to be used for picking best model 62 | asc | bool | True if `metric` is something to be minimized 63 | task | string | 'binary' or 'multi_label' 64 | saved | bool | if a model saved on local machine should be used 65 | custom_objects | dict | if the model has a custom object, pass it here 66 | ''' 67 | 68 | import numpy as np 69 | 70 | if model_id is None: 71 | from ..utils.best_model import best_model 72 | model_id = best_model(self.scan_object, metric, asc) 73 | 74 | from ..utils.best_model import activate_model 75 | model = activate_model(self.scan_object, 76 | model_id, 77 | saved, 78 | custom_objects) 79 | 80 | # make (class) predictions with the model 81 | preds = model.predict(x) 82 | 83 | if task == 'binary': 84 | return np.where(preds >= 0.5, 1, 0) 85 | 86 | elif task == 'multi_label': 87 | return np.argmax(preds, 1) 88 | 89 | else: 90 | msg = 'Only `binary` and `multi_label` are supported' 91 | raise AttributeError(msg) 92 | -------------------------------------------------------------------------------- /talos/commands/restore.py: -------------------------------------------------------------------------------- 1 | class Restore: 2 | 3 | '''Restores the scan_object that had been stored locally as a result 4 | of talos.Deploy(scan_object, 'example') 5 | 6 | USE: 7 | 8 | diabetes = ta.Scan(x, y, p, input_model) 9 | ta.Deploy(diabetes, 'diabetes') 10 | ta.Restore('diabetes.zip') 11 | 12 | ''' 13 | 14 | def __init__(self, path_to_zip): 15 | 16 | from zipfile import ZipFile 17 | 18 | import pandas as pd 19 | import numpy as np 20 | 21 | # create paths 22 | self.path_to_zip = path_to_zip 23 | self.extract_to = path_to_zip.replace('.zip', '') 24 | self.package_name = self.extract_to.split('/')[-1] 25 | self.file_prefix = self.extract_to + '/' + self.package_name 26 | 27 | # extract the zip 28 | # unpack_archive(self.path_to_zip, self.extract_to) 29 | z = ZipFile(self.path_to_zip, mode='r') 30 | z.extractall(self.extract_to) 31 | 32 | # add params dictionary 33 | self.params = np.load(self.file_prefix + '_params.npy', 34 | allow_pickle=True).item() 35 | 36 | # add experiment details 37 | self.details = pd.read_csv(self.file_prefix + '_details.txt', 38 | header=None) 39 | 40 | # add x data sample 41 | self.x = pd.read_csv(self.file_prefix + '_x.csv', header=None) 42 | 43 | # add y data sample 44 | self.y = pd.read_csv(self.file_prefix + '_y.csv', header=None) 45 | 46 | # add model 47 | from talos.utils.load_model import load_model 48 | self.model = load_model(self.file_prefix + '_model') 49 | 50 | # add results 51 | self.results = pd.read_csv(self.file_prefix + '_results.csv') 52 | self.results.drop('Unnamed: 0', axis=1, inplace=True) 53 | 54 | # clean up 55 | del self.extract_to, self.file_prefix 56 | del self.package_name, self.path_to_zip 57 | -------------------------------------------------------------------------------- /talos/logging/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomio/talos/9cbbc42a69396aaeaf25d68af51d52e6a575e3d9/talos/logging/__init__.py -------------------------------------------------------------------------------- /talos/logging/logging_finish.py: -------------------------------------------------------------------------------- 1 | def logging_finish(self): 2 | 3 | from .results import result_todf 4 | 5 | # save the results 6 | self = result_todf(self) 7 | 8 | return self 9 | -------------------------------------------------------------------------------- /talos/logging/logging_run.py: -------------------------------------------------------------------------------- 1 | def logging_run(self, round_start, start, model_history): 2 | 3 | import time 4 | 5 | # count the duration of the round 6 | self._round_seconds = time.time() - start 7 | 8 | # set end time and log 9 | round_end = time.strftime('%D-%H%M%S') 10 | self.round_times.append([round_start, round_end, self._round_seconds]) 11 | 12 | # handle first round only things 13 | if self.first_round: 14 | 15 | # capture the history keys for later 16 | self._all_keys = list(model_history.history.keys()) 17 | self._metric_keys = [k for k in self._all_keys if 'val_' not in k] 18 | self._val_keys = [k for k in self._all_keys if 'val_' in k] 19 | 20 | # create a header column for output 21 | _results_header = ['round_epochs'] + self._all_keys + self._param_dict_keys 22 | self.result.append(_results_header) 23 | 24 | # save the results 25 | from .results import save_result 26 | save_result(self) 27 | 28 | # avoid doing this again 29 | self.first_round = False 30 | 31 | # create log and other stats 32 | from ..metrics.entropy import epoch_entropy 33 | self.epoch_entropy.append(epoch_entropy(self, model_history.history)) 34 | 35 | # get round results to the results table and save it 36 | from .results import run_round_results 37 | _round_results = run_round_results(self, model_history) 38 | 39 | self.result.append(_round_results) 40 | 41 | from .results import save_result 42 | save_result(self) 43 | 44 | # return the Scan() self 45 | return self 46 | -------------------------------------------------------------------------------- /talos/logging/results.py: -------------------------------------------------------------------------------- 1 | def run_round_results(self, out): 2 | 3 | '''Called from logging/logging_run.py 4 | 5 | THE MAIN FUNCTION FOR CREATING RESULTS FOR EACH ROUNDself. 6 | Takes in the history object from model.fit() and handles it. 7 | 8 | NOTE: The epoch level data will be dropped here each round. 9 | 10 | ''' 11 | 12 | self._round_epochs = len(list(out.history.values())[0]) 13 | 14 | _round_result_out = [self._round_epochs] 15 | 16 | # record the last epoch result 17 | for key in out.history.keys(): 18 | _round_result_out.append(out.history[key][-1]) 19 | 20 | # record the round hyper-parameters 21 | for key in self.round_params.keys(): 22 | _round_result_out.append(self.round_params[key]) 23 | 24 | return _round_result_out 25 | 26 | 27 | def save_result(self): 28 | 29 | '''SAVES THE RESULTS/PARAMETERS TO A CSV SPECIFIC TO THE EXPERIMENT''' 30 | 31 | import numpy as np 32 | 33 | np.savetxt(self._experiment_log, 34 | self.result, 35 | fmt='%s', 36 | delimiter=',') 37 | 38 | 39 | def result_todf(self): 40 | 41 | '''ADDS A DATAFRAME VERSION OF THE RESULTS TO THE CLASS OBJECT''' 42 | 43 | import pandas as pd 44 | 45 | # create dataframe for results 46 | cols = self.result[0] 47 | self.result = pd.DataFrame(self.result[1:]) 48 | self.result.columns = cols 49 | 50 | return self 51 | 52 | 53 | def peak_epochs_todf(self): 54 | 55 | import pandas as pd 56 | 57 | return pd.DataFrame(self.peak_epochs, columns=self.peak_epochs[0]).drop(0) 58 | -------------------------------------------------------------------------------- /talos/metrics/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomio/talos/9cbbc42a69396aaeaf25d68af51d52e6a575e3d9/talos/metrics/__init__.py -------------------------------------------------------------------------------- /talos/metrics/entropy.py: -------------------------------------------------------------------------------- 1 | def epoch_entropy(self, history): 2 | 3 | '''Called from logging/logging_run.py 4 | 5 | Computes the entropy for epoch metric 6 | variation. If validation is on, 7 | then returns KL divergence instead of 8 | simple Shannon entropy. When Keras 9 | validation_freq is on, Shannon entropy 10 | is returned. Basically, all experiments 11 | should use validation, so Shannon is 12 | provided mearly as a fallback. 13 | 14 | ''' 15 | 16 | import warnings 17 | from scipy.stats import entropy 18 | 19 | warnings.simplefilter('ignore') 20 | 21 | out = [] 22 | 23 | # set the default entropy mode to shannon 24 | mode = 'shannon' 25 | 26 | # try to make sure each metric has validation 27 | if len(self._metric_keys) == len(self._val_keys): 28 | # make sure that the length of the arrays are same 29 | for i in range(len(self._metric_keys)): 30 | if len(history[self._metric_keys[i]]) == len(history[self._val_keys[i]]): 31 | mode = 'kl_divergence' 32 | else: 33 | break 34 | 35 | # handle the case where only shannon entropy can be used 36 | if mode == 'shannon': 37 | for i in range(len(self._metric_keys)): 38 | out.append(entropy(history[self._metric_keys[i]])) 39 | 40 | # handle the case where kl divergence can be used 41 | elif mode == 'kl_divergence': 42 | for i in range(len(self._metric_keys)): 43 | out.append(entropy(history[self._val_keys[i]], 44 | history[self._metric_keys[i]])) 45 | 46 | return out 47 | -------------------------------------------------------------------------------- /talos/metrics/keras_metrics.py: -------------------------------------------------------------------------------- 1 | def mae(y_true, y_pred): 2 | from tensorflow.keras import backend as K 3 | return K.mean(K.abs(y_pred - y_true), axis=-1) 4 | 5 | 6 | def mse(y_true, y_pred): 7 | from tensorflow.keras import backend as K 8 | return K.mean(K.square(y_pred - y_true), axis=-1) 9 | 10 | 11 | def rmae(y_true, y_pred): 12 | from tensorflow.keras import backend as K 13 | return K.sqrt(K.mean(K.abs(y_pred - y_true), axis=-1)) 14 | 15 | 16 | def rmse(y_true, y_pred): 17 | from tensorflow.keras import backend as K 18 | return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1)) 19 | 20 | 21 | def mape(y_true, y_pred): 22 | from tensorflow.keras import backend as K 23 | diff = K.abs((y_true - y_pred) / K.clip(K.abs(y_true), 24 | K.epsilon(), 25 | None)) 26 | return 100. * K.mean(diff, axis=-1) 27 | 28 | 29 | def msle(y_true, y_pred): 30 | from tensorflow.keras import backend as K 31 | first_log = K.log(K.clip(y_pred, K.epsilon(), None) + 1.) 32 | second_log = K.log(K.clip(y_true, K.epsilon(), None) + 1.) 33 | return K.mean(K.square(first_log - second_log), axis=-1) 34 | 35 | 36 | def rmsle(y_true, y_pred): 37 | from tensorflow.keras import backend as K 38 | first_log = K.log(K.clip(y_pred, K.epsilon(), None) + 1.) 39 | second_log = K.log(K.clip(y_true, K.epsilon(), None) + 1.) 40 | return K.sqrt(K.mean(K.square(first_log - second_log), axis=-1)) 41 | 42 | 43 | def matthews(y_true, y_pred): 44 | 45 | from tensorflow.keras import backend as K 46 | y_pred_pos = K.round(K.clip(y_pred, 0, 1)) 47 | y_pred_neg = 1 - y_pred_pos 48 | 49 | y_pos = K.round(K.clip(y_true, 0, 1)) 50 | y_neg = 1 - y_pos 51 | 52 | tp = K.sum(y_pos * y_pred_pos) 53 | tn = K.sum(y_neg * y_pred_neg) 54 | 55 | fp = K.sum(y_neg * y_pred_pos) 56 | fn = K.sum(y_pos * y_pred_neg) 57 | 58 | numerator = (tp * tn - fp * fn) 59 | denominator = K.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn)) 60 | 61 | return numerator / (denominator + K.epsilon()) 62 | 63 | 64 | def precision(y_true, y_pred): 65 | 66 | from tensorflow.keras import backend as K 67 | true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) 68 | predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1))) 69 | precision = true_positives / (predicted_positives + K.epsilon()) 70 | return precision 71 | 72 | 73 | def recall(y_true, y_pred): 74 | 75 | from tensorflow.keras import backend as K 76 | true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) 77 | possible_positives = K.sum(K.round(K.clip(y_true, 0, 1))) 78 | recall = true_positives / (possible_positives + K.epsilon()) 79 | return recall 80 | 81 | 82 | def fbeta(y_true, y_pred, beta=1): 83 | 84 | from tensorflow.keras import backend as K 85 | if beta < 0: 86 | raise ValueError('The lowest choosable beta is zero (only precision).') 87 | 88 | # If there are no true positives, fix the F score at 0 like sklearn. 89 | #if K.sum(K.round(K.clip(y_true, 0, 1))) == 0: 90 | # return 0 91 | 92 | p = precision(y_true, y_pred) 93 | r = recall(y_true, y_pred) 94 | bb = beta ** 2 95 | fbeta_score = (1 + bb) * (p * r) / (bb * p + r + K.epsilon()) 96 | return fbeta_score 97 | 98 | 99 | def f1score(y_true, y_pred): 100 | 101 | return fbeta(y_true, y_pred, beta=1) 102 | -------------------------------------------------------------------------------- /talos/model/__init__.py: -------------------------------------------------------------------------------- 1 | from .early_stopper import early_stopper 2 | from .hidden_layers import hidden_layers 3 | from .normalizers import lr_normalizer 4 | -------------------------------------------------------------------------------- /talos/model/early_stopper.py: -------------------------------------------------------------------------------- 1 | from tensorflow.keras.callbacks import EarlyStopping 2 | 3 | 4 | def early_stopper(epochs=None, 5 | monitor='val_loss', 6 | mode='moderate', 7 | min_delta=None, 8 | patience=None): 9 | 10 | '''EARLY STOP CALLBACK 11 | 12 | Helps prevent wasting time when loss is not becoming 13 | better. Offers two pre-determined settings 'moderate' 14 | and 'strict' and allows input of list with two values: 15 | 16 | `epochs` | int | The number of epochs for the permutation e.g. params['epochs'] 17 | `monitor` | int | The metric to monitor for change 18 | `mode` | str | One of the presets `lazy`, `moderate`, `strict` or `None` 19 | `min_delta` | float | The limit for change at which point flag is raised 20 | `patience` | str | the number of epochs before termination from flag 21 | 22 | ''' 23 | if mode == 'lazy': 24 | _es_out = EarlyStopping(monitor=monitor, 25 | min_delta=0, 26 | patience=int(epochs / 3), 27 | verbose=0, mode='auto') 28 | 29 | if mode == 'moderate': 30 | _es_out = EarlyStopping(monitor=monitor, 31 | min_delta=0, 32 | patience=int(epochs / 10), 33 | verbose=0, mode='auto') 34 | elif mode == 'strict': 35 | _es_out = EarlyStopping(monitor=monitor, 36 | min_delta=0, 37 | patience=2, 38 | verbose=0, mode='auto') 39 | else: 40 | _es_out = EarlyStopping(monitor=monitor, 41 | min_delta=mode[0], 42 | patience=mode[1], 43 | verbose=0, mode='auto') 44 | return _es_out 45 | -------------------------------------------------------------------------------- /talos/model/hidden_layers.py: -------------------------------------------------------------------------------- 1 | def hidden_layers(model, params, last_neuron): 2 | '''HIDDEN LAYER Generator 3 | 4 | NOTE: 'shapes', 'first_neuron', 'dropout', and 'hidden_layers' need 5 | to be present in the params dictionary. 6 | 7 | Hidden layer generation for the cases where number 8 | of layers is used as a variable in the optimization process. 9 | Handles things in a way where any number of layers can be tried 10 | with matching hyperparameters.''' 11 | 12 | # check for the params that are required for hidden_layers 13 | 14 | from tensorflow.keras.layers import Dense, Dropout 15 | from .network_shape import network_shape 16 | from ..utils.exceptions import TalosParamsError 17 | 18 | required = ['shapes', 'first_neuron', 'dropout', 'hidden_layers', 'activation'] 19 | for param in required: 20 | if param not in params: 21 | message = "hidden_layers requires '" + param + "' in params" 22 | raise TalosParamsError(message) 23 | 24 | layer_neurons = network_shape(params, last_neuron) 25 | 26 | for i in range(params['hidden_layers']): 27 | model.add(Dense( 28 | layer_neurons[i], 29 | kernel_initializer=params.get( 30 | 'kernel_initializer', 31 | 'glorot_uniform' 32 | ), 33 | kernel_regularizer=params.get('kernel_regularizer'), 34 | bias_initializer=params.get('bias_initializer', 'zeros'), 35 | bias_regularizer=params.get('bias_regularizer'), 36 | use_bias=params.get('use_bias', True), 37 | activity_regularizer=params.get('activity_regularizer'), 38 | kernel_constraint=params.get('kernel_constraint'), 39 | bias_constraint=params.get('bias_constraint'), 40 | activation=params.get('activation') 41 | )) 42 | model.add(Dropout(params['dropout'])) 43 | -------------------------------------------------------------------------------- /talos/model/ingest_model.py: -------------------------------------------------------------------------------- 1 | def ingest_model(self): 2 | 3 | '''Ingests the model that is input by the user 4 | through Scan() model paramater.''' 5 | 6 | return self.model(self.x_train, 7 | self.y_train, 8 | self.x_val, 9 | self.y_val, 10 | self.round_params) 11 | -------------------------------------------------------------------------------- /talos/model/network_shape.py: -------------------------------------------------------------------------------- 1 | def network_shape(params, last_neuron): 2 | 3 | '''Provides the ability to include network shape in experiments. If params 4 | dictionary for the round contains float value for params['shapes'] then 5 | a linear contraction towards the last_neuron value. The higher the value, 6 | the fewer layers it takes to reach lesser than last_neuron. 7 | 8 | Supports three inbuilt shapes 'brick', 'funnel', and 'triangle'. 9 | 10 | 11 | params : dict 12 | Scan() params for a single roundself. 13 | last_neuron : int 14 | Number of neurons on the output layer in the Keras model. 15 | ''' 16 | import numpy as np 17 | from ..utils.exceptions import TalosParamsError 18 | 19 | layers = params['hidden_layers'] 20 | shape = params['shapes'] 21 | first_neuron = params['first_neuron'] 22 | out = [] 23 | n = first_neuron 24 | 25 | # the case where hidden_layers is zero 26 | if layers == 0: 27 | return [0] 28 | 29 | # the cases where an angle is applied 30 | if isinstance(shape, float): 31 | 32 | for i in range(layers): 33 | 34 | n *= 1 - shape 35 | 36 | if n > last_neuron: 37 | out.append(int(n)) 38 | else: 39 | out.append(last_neuron) 40 | 41 | # the case where a rectantular shape is used 42 | elif shape == 'brick': 43 | out = [first_neuron] * layers 44 | 45 | elif shape == 'funnel': 46 | for i in range(layers + 1): 47 | n -= int((first_neuron - last_neuron) / layers) 48 | out.append(n) 49 | out.pop(-1) 50 | 51 | elif shape == 'triangle': 52 | out = np.linspace(first_neuron, 53 | last_neuron, 54 | layers+2, 55 | dtype=int).tolist() 56 | 57 | out.pop(0) 58 | out.pop(-1) 59 | out.reverse() 60 | 61 | else: 62 | message = "'shapes' must be float or in ['funnel', 'brick', 'triangle']" 63 | raise TalosParamsError(message) 64 | 65 | return out 66 | -------------------------------------------------------------------------------- /talos/model/normalizers.py: -------------------------------------------------------------------------------- 1 | def lr_normalizer(lr, optimizer): 2 | """Assuming a default learning rate 1, rescales the learning rate 3 | such that learning rates amongst different optimizers are more or less 4 | equivalent. 5 | 6 | Parameters 7 | ---------- 8 | lr : float 9 | The learning rate. 10 | optimizer : keras optimizer 11 | The optimizer. For example, Adagrad, Adam, RMSprop. 12 | """ 13 | 14 | from tensorflow.keras.optimizers.legacy import SGD, Adam, Adagrad, Adamax, RMSprop 15 | from tensorflow.keras.optimizers.legacy import Adagrad 16 | from talos.utils.exceptions import TalosModelError 17 | 18 | if optimizer == SGD or optimizer == Adagrad: 19 | lr /= 100.0 20 | elif optimizer == Adam or optimizer == RMSprop: 21 | lr /= 1000.0 22 | elif optimizer == Adamax: 23 | lr /= 500.0 24 | else: 25 | raise TalosModelError(str(optimizer) + " is not supported by lr_normalizer") 26 | 27 | return lr 28 | -------------------------------------------------------------------------------- /talos/model/output_layer.py: -------------------------------------------------------------------------------- 1 | def output_layer(task, last_activation, y_train, y_val): 2 | 3 | import numpy as np 4 | 5 | # output layer 6 | if task == 'binary': 7 | activation = last_activation 8 | last_neuron = 1 9 | 10 | elif task == 'multi_class': 11 | activation = last_activation 12 | last_neuron = len(np.unique(np.hstack((y_train, y_val)))) 13 | 14 | elif task == 'multi_label': 15 | activation = last_activation 16 | last_neuron = y_train.shape[1] 17 | 18 | elif task == 'continuous': 19 | activation = None 20 | last_neuron = 1 21 | 22 | return activation, last_neuron 23 | -------------------------------------------------------------------------------- /talos/parameters/DistributeParamSpace.py: -------------------------------------------------------------------------------- 1 | class DistributeParamSpace: 2 | 3 | def __init__(self, 4 | params, 5 | param_keys, 6 | random_method='uniform_mersenne', 7 | fraction_limit=None, 8 | round_limit=None, 9 | time_limit=None, 10 | boolean_limit=None, 11 | machines=2): 12 | 13 | '''Splits ParamSpace object based on number 14 | of machines. 15 | 16 | params | object | ParamSpace class object 17 | machines | int | number of machines to split for 18 | 19 | NOTE: `Scan()` limits will not be applied if ParamSpace object 20 | is passed directly into `Scan()` as `params` argument so they 21 | should be passed directly into `DistributeParamSpace` instead. 22 | 23 | ''' 24 | 25 | from talos.parameters.ParamSpace import ParamSpace 26 | 27 | self._params = ParamSpace(params=params, 28 | param_keys=param_keys, 29 | random_method='uniform_mersenne', 30 | fraction_limit=fraction_limit, 31 | round_limit=round_limit, 32 | time_limit=time_limit, 33 | boolean_limit=boolean_limit) 34 | 35 | self.machines = machines 36 | 37 | self.param_spaces = self._split_param_space() 38 | 39 | def _split_param_space(self): 40 | 41 | '''Takes in a ParamSpace object and splits it so that 42 | it can be used in DistributeScan experiments.''' 43 | 44 | import numpy as np 45 | import copy 46 | 47 | out = {} 48 | 49 | # randomly shuffle the param_space 50 | rand = np.random.default_rng() 51 | rand.shuffle(self._params.param_space, axis=0) 52 | 53 | # split into n arras 54 | param_spaces = np.array_split(self._params.param_space, self.machines) 55 | 56 | # remove keys to allow copy 57 | param_keys = self._params.param_keys 58 | self._params.param_keys = [] 59 | 60 | # create the individual ParamSpace objects 61 | for i in range(self.machines): 62 | 63 | out[i] = copy.deepcopy(self._params) 64 | out[i].param_space = param_spaces[i] 65 | out[i].dimensions = len(out[i].param_space) 66 | out[i].param_index = list(range(out[i].dimensions)) 67 | out[i].param_keys = param_keys 68 | 69 | return out 70 | -------------------------------------------------------------------------------- /talos/parameters/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomio/talos/9cbbc42a69396aaeaf25d68af51d52e6a575e3d9/talos/parameters/__init__.py -------------------------------------------------------------------------------- /talos/reducers/GamifyMap.py: -------------------------------------------------------------------------------- 1 | class GamifyMap: 2 | 3 | def __init__(self, scan_object): 4 | 5 | '''GamifyMap handles the management of the 6 | dictionary that contains the information about 7 | hyperparameters, which is exchanged in and out 8 | during the `Scan()` experiment. 9 | ''' 10 | 11 | self.params = scan_object.param_object.params 12 | self.scan_object = scan_object 13 | self.generate_gamify_dict() 14 | self.gamify_map = self.generate_gamify_dict_map() 15 | 16 | # parse together the output file name 17 | _folder = './' + scan_object.experiment_name + '/' 18 | _id = scan_object._experiment_id 19 | self._filename = _folder + _id 20 | 21 | def run_updates(self): 22 | 23 | for key in self.gamify_dict.keys(): 24 | for val in self.gamify_dict[key].keys(): 25 | if self.gamify_dict[key][val] != self.updated_dict[key][val]: 26 | 27 | label = list(self.params.keys())[int(key)] 28 | value = self.params[label][int(val)] 29 | 30 | self.gamify_dict[key][val] = self.updated_dict[key][val] 31 | self.scan_object.param_object.remove_is(label, value) 32 | 33 | return self.scan_object 34 | 35 | def back_to_original(self, gamify_from_json): 36 | 37 | gamify_dict = {} 38 | for key in gamify_from_json: 39 | param_vals = {} 40 | for val in gamify_from_json[key]: 41 | param_vals[self.gamify_map[key][val][1]] = gamify_from_json[key][val][:2] 42 | gamify_dict[self.gamify_map[key][val][0]] = param_vals 43 | 44 | return gamify_dict 45 | 46 | def generate_gamify_dict_map(self): 47 | 48 | gamify_dict_map = {} 49 | 50 | for i, key in enumerate(self.params.keys()): 51 | param_vals = {} 52 | for ii, val in enumerate(self.params[key]): 53 | param_vals[str(ii)] = [key, val] 54 | gamify_dict_map[str(i)] = param_vals 55 | 56 | return gamify_dict_map 57 | 58 | def generate_gamify_dict(self): 59 | 60 | '''This is done once at the beginning 61 | of the experiment. 62 | 63 | NOTE: This will be all stringified, so an index 64 | mapping system will be used to convert back to 65 | actual forms later.''' 66 | 67 | gamify_dict = {} 68 | 69 | for i, key in enumerate(self.params.keys()): 70 | param_vals = {} 71 | for ii, val in enumerate(self.params[key]): 72 | param_vals[str(ii)] = ['active', 0, str(key), str(val)] 73 | gamify_dict[str(i)] = param_vals 74 | 75 | self.gamify_dict = gamify_dict 76 | 77 | def export_json(self): 78 | 79 | import json 80 | 81 | with open(self._filename + '.json', 'w') as fp: 82 | json.dump(self.gamify_dict, fp) 83 | 84 | def import_json(self): 85 | 86 | import json 87 | 88 | with open(self._filename + '.json', 'r') as fp: 89 | out = json.load(fp) 90 | 91 | self.updated_dict = out 92 | -------------------------------------------------------------------------------- /talos/reducers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomio/talos/9cbbc42a69396aaeaf25d68af51d52e6a575e3d9/talos/reducers/__init__.py -------------------------------------------------------------------------------- /talos/reducers/correlation.py: -------------------------------------------------------------------------------- 1 | def correlation(self, method): 2 | 3 | '''This is called from reduce_run.py. 4 | 5 | Performs a spearman rank order correlation 6 | based reduction. First looks for a parameter 7 | that correlates with reduction_metric and 8 | correlation meets reduction_threshold and 9 | then converts the match parameter into 10 | a 2d multilabel shape. Then new correlation 11 | against reduction_metric is performed to identify 12 | which particular value is to be dropped. 13 | 14 | ''' 15 | 16 | import numpy as np 17 | 18 | # transform the data properly first 19 | from .reduce_utils import cols_to_multilabel 20 | data = cols_to_multilabel(self) 21 | 22 | # get the correlations 23 | corr_values = data.corr(method)[self.reduction_metric] 24 | 25 | # drop the reduction metric row 26 | corr_values.drop(self.reduction_metric, inplace=True) 27 | 28 | # drop labels where value is NaN 29 | corr_values.dropna(inplace=True) 30 | 31 | # if all nans, then stop 32 | if len(corr_values) <= 1: 33 | return self 34 | 35 | # sort based on the metric type 36 | corr_values.sort_values(ascending=self.minimize_loss, inplace=True) 37 | 38 | # if less than threshold, then stop 39 | if abs(corr_values[-1]) < self.reduction_threshold: 40 | return self 41 | 42 | # get the strongest correlation 43 | corr_values = corr_values.index[-1] 44 | 45 | # get the label, value, and dtype from the column header 46 | label, dtype, value = corr_values.split('~') 47 | 48 | # convert things back to their original dtype 49 | value = np.array([value]).astype(dtype)[0] 50 | 51 | # this is where we modify the parameter space accordingly 52 | self.param_object.remove_is(label, value) 53 | 54 | return self 55 | -------------------------------------------------------------------------------- /talos/reducers/forrest.py: -------------------------------------------------------------------------------- 1 | def forrest(self): 2 | 3 | '''Random Forrest based reduction strategy. Somewhat more 4 | aggressive than for example 'spearman' because there are no 5 | negative values, but instead the highest positive correlation 6 | is minused from all the values so that max value is 0, and then 7 | values are turned into positive. The one with the highest positive 8 | score in the end will be dropped. This means that anything with 9 | 0 originally, is a candidate for dropping. Because there are multiple 10 | zeroes in many cases, there is an element of randomness on which one 11 | is dropped. 12 | 13 | ''' 14 | 15 | import wrangle 16 | import numpy as np 17 | 18 | # handle conversion to multi_labels 19 | from .reduce_utils import cols_to_multilabel 20 | data = cols_to_multilabel(self) 21 | 22 | # get the correlations 23 | corr_values = wrangle.df_corr_randomforest(data, self.reduction_metric) 24 | 25 | # drop labels where value is NaN 26 | corr_values.dropna(inplace=True) 27 | 28 | # handle the turning around of values (see docstring for more info) 29 | corr_values -= corr_values[0] 30 | corr_values = corr_values.abs() 31 | 32 | # get the strongest correlation 33 | corr_values = corr_values.index[-1] 34 | 35 | # get the label, value, and dtype from the column header 36 | label, dtype, value = corr_values.split('~') 37 | 38 | # convert things back to their original dtype 39 | value = np.array([value]).astype(dtype)[0] 40 | 41 | # this is where we modify the parameter space accordingly 42 | self.param_object.remove_is(label, value) 43 | 44 | return self 45 | -------------------------------------------------------------------------------- /talos/reducers/gamify.py: -------------------------------------------------------------------------------- 1 | def gamify(self): 2 | 3 | '''Will apply reduction changes based on edits on the 4 | the produced .json file in the experiment folder''' 5 | 6 | if self.param_object.round_counter == 1: 7 | 8 | # create the gamify object 9 | from .GamifyMap import GamifyMap 10 | g = GamifyMap(self) 11 | 12 | # keep in scan_object 13 | self._gamify_object = g 14 | 15 | # do the first export in the experiment folder 16 | g.export_json() 17 | 18 | return self 19 | 20 | # for every round check if there are changes 21 | self._gamify_object.import_json() 22 | self = self._gamify_object.run_updates() 23 | self._gamify_object.export_json() 24 | 25 | return self 26 | -------------------------------------------------------------------------------- /talos/reducers/limit_by_metric.py: -------------------------------------------------------------------------------- 1 | def limit_by_metric(self): 2 | 3 | '''Takes as input metric, threshold, and loss and 4 | and returs a True if metric threshold have been 5 | met and False if not. 6 | 7 | USE: space.check_metric(model_history) 8 | ''' 9 | 10 | metric = self.performance_target[0] 11 | threshold = self.performance_target[1] 12 | loss = self.performance_target[2] 13 | 14 | if loss is True: 15 | return self.model_history.history[metric][-1] <= threshold 16 | elif loss is False: 17 | return self.model_history.history[metric][-1] >= threshold 18 | -------------------------------------------------------------------------------- /talos/reducers/local_strategy.py: -------------------------------------------------------------------------------- 1 | def local_strategy(self): 2 | 3 | try: 4 | import importlib 5 | importlib.reload(talos_strategy) 6 | self = talos_strategy(self) 7 | except NameError: 8 | try: 9 | from talos_strategy import talos_strategy 10 | self = talos_strategy(self) 11 | except ImportError: 12 | print("No talos_strategy.py found in pwd. Nothing is done.") 13 | 14 | return self 15 | -------------------------------------------------------------------------------- /talos/reducers/reduce_run.py: -------------------------------------------------------------------------------- 1 | def reduce_run(self): 2 | 3 | '''The process run script for reduce 4 | procedures; takes care of everything 5 | related with reduction. When new 6 | reduction methods are added, they need 7 | to be added as options here. 8 | 9 | To add new reducers, create a file in /reducers 10 | which is where this file is located. In that file, 11 | take as input self from Scan() and give as output 12 | either False, which does nothing, or a tuple of 13 | 'value' and 'label' where value is a parameter 14 | value and label is parameter name. For example 15 | batch_size and 128. Then add a reference to 16 | reduce_run.py and make sure that you process 17 | the self.param_object.param_index there before 18 | wrapping up. 19 | 20 | ''' 21 | 22 | from .correlation import correlation 23 | from .forrest import forrest 24 | from .trees import trees 25 | from .gamify import gamify 26 | 27 | from .local_strategy import local_strategy 28 | from .limit_by_metric import limit_by_metric 29 | 30 | # check if performance target is met 31 | if self.performance_target is not None: 32 | status = limit_by_metric(self) 33 | 34 | # handle the case where performance target is met 35 | if status == True: 36 | self.param_object.param_index = [] 37 | print("Target %.3f have been met." % self.performance_target[1]) 38 | 39 | # stop here if no reduction method is set 40 | if self.reduction_method is None: 41 | return self 42 | 43 | # setup what's required for updating progress bar 44 | left = (self.param_object.round_counter + 1) 45 | right = self.reduction_interval 46 | len_before_reduce = len(self.param_object.param_index) 47 | 48 | # check if monte carlo can do something 49 | if self.reduction_method == 'gamify': 50 | self = gamify(self) 51 | 52 | # apply window based reducers 53 | if left % right == 0: 54 | 55 | # check if correlation reducer can do something 56 | if self.reduction_method in ['pearson', 'kendall', 'spearman']: 57 | self = correlation(self, self.reduction_method) 58 | 59 | # check if correlation reducer can do something 60 | if self.reduction_method == 'correlation': 61 | self = correlation(self, 'spearman') 62 | 63 | # check if random forrest can do something 64 | if self.reduction_method == 'forrest': 65 | self = forrest(self) 66 | 67 | # check if random forrest can do something 68 | if self.reduction_method == 'trees': 69 | self = trees(self) 70 | 71 | if self.reduction_method == 'local_strategy': 72 | self = local_strategy(self) 73 | 74 | # finish up by updating progress bar 75 | total_reduced = len_before_reduce - len(self.param_object.param_index) 76 | total_reduced = max(0, total_reduced) 77 | self.pbar.update(total_reduced) 78 | 79 | if total_reduced > 0: 80 | # print out the the status 81 | drop_share = total_reduced / len_before_reduce * 100 82 | print("Total %.1f%% permutations reduced" % drop_share) 83 | 84 | return self 85 | -------------------------------------------------------------------------------- /talos/reducers/reduce_utils.py: -------------------------------------------------------------------------------- 1 | def cols_to_multilabel(self): 2 | 3 | '''Utility function for correlation and other reducers 4 | that require transforming hyperparameter values into 5 | multilabel values before applying the reduction strategy.''' 6 | 7 | import wrangle 8 | import pandas as pd 9 | 10 | # read in the experiment log 11 | data = pd.read_csv(self._experiment_log) 12 | 13 | # apply recuction window 14 | data = data.tail(self.reduction_window) 15 | 16 | # drop all other metric columns except reduction_metric 17 | data = data[[self.reduction_metric] + self._param_dict_keys] 18 | 19 | # convert all hyperparameter columns to multi label columns 20 | for col in data.iloc[:, 1:].columns: 21 | 22 | # get the dtype of the column data 23 | col_dtype = data[col].dtype 24 | 25 | # parse column name to contain label, value and dtype 26 | data = wrangle.col_to_multilabel(data, 27 | col, 28 | extended_colname=True, 29 | extended_separator='~' + str(col_dtype) + '~') 30 | 31 | return data 32 | -------------------------------------------------------------------------------- /talos/reducers/sample_reducer.py: -------------------------------------------------------------------------------- 1 | def sample_reducer(limit, max_value, random_method): 2 | 3 | '''Sample Reducer (Helper) 4 | 5 | NOTE: The Scan() object is in self.main_self because 6 | the object being passed here is ParamGrid() object where 7 | the Scan() object is attached as self.main_self. 8 | 9 | Utilize 'grid_downsample', 'shuffle', and 'random_method' 10 | to reduce the param_grid before starting the experiment. 11 | This is the simplest method in Talos for dealing with curse 12 | of dimensionality. 13 | 14 | Options are uniform random, stratified random, latin hypercube 15 | sampling, and latin hypercube with sudoku style constraint. 16 | 17 | Returns the reduced param_grid as numpy array. 18 | 19 | ''' 20 | 21 | import chances as ch 22 | 23 | # calculate the size of the downsample 24 | if isinstance(limit, float): 25 | n = int(max_value * limit) 26 | if isinstance(limit, int): 27 | n = limit 28 | 29 | max_value = int(max_value) 30 | 31 | # throw an error if 32 | from ..utils.exceptions import TalosDataError 33 | if n < 1: 34 | raise TalosDataError("Limiters lead to < 1 permutations.") 35 | 36 | # Initialize Randomizer() 37 | r = ch.Randomizer(max_value, n) 38 | 39 | # use the user selected method 40 | if random_method == 'sobol': 41 | out = r.sobol() 42 | elif random_method == 'quantum': 43 | out = r.quantum() 44 | elif random_method == 'halton': 45 | out = r.halton() 46 | elif random_method == 'korobov_matrix': 47 | out = r.korobov_matrix() 48 | elif random_method == 'latin_sudoku': 49 | out = r.latin_sudoku() 50 | elif random_method == 'latin_matrix': 51 | out = r.latin_matrix() 52 | elif random_method == 'latin_improved': 53 | out = r.latin_improved() 54 | elif random_method == 'uniform_mersenne': 55 | out = r.uniform_mersenne() 56 | elif random_method == 'uniform_crypto': 57 | out = r.uniform_crypto() 58 | elif random_method == 'ambience': 59 | out = r.ambience() 60 | else: 61 | print('No eligble random_method found. Using uniform_mersenne.') 62 | out = r.uniform_mersenne() 63 | 64 | return out 65 | -------------------------------------------------------------------------------- /talos/reducers/trees.py: -------------------------------------------------------------------------------- 1 | def trees(self, quantile=.8): 2 | 3 | '''Extra Trees based reduction strategy. Like 'forrest', somewhat more 4 | aggressive than for example 'spearman' because there are no 5 | negative values, but instead the highest positive correlation 6 | is minused from all the values so that max value is 0, and then 7 | values are turned into positive. The one with the highest positive 8 | score in the end will be dropped. This means that anything with 9 | 0 originally, is a candidate for dropping. Because there are multiple 10 | zeroes in many cases, there is an element of randomness on which one 11 | is dropped. 12 | 13 | ''' 14 | 15 | import wrangle 16 | import numpy as np 17 | 18 | # handle conversion to multi_labels 19 | from .reduce_utils import cols_to_multilabel 20 | data = cols_to_multilabel(self) 21 | 22 | # because extra trees wants label as 'y' we first transform with quantile 23 | quantile_value = data[self.reduction_metric].quantile(quantile) 24 | data[self.reduction_metric] = data[self.reduction_metric] > quantile_value 25 | 26 | # get the correlations 27 | corr_values = wrangle.df_corr_extratrees(data, self.reduction_metric) 28 | 29 | # drop labels where value is NaN 30 | corr_values.dropna(inplace=True) 31 | 32 | # handle the turning around of values (see docstring for more info) 33 | corr_values -= corr_values[0] 34 | corr_values = corr_values.abs() 35 | 36 | # get the strongest correlation 37 | corr_values = corr_values.index[-1] 38 | 39 | # get the label, value, and dtype from the column header 40 | label, dtype, value = corr_values.split('~') 41 | 42 | # convert things back to their original dtype 43 | value = np.array([value]).astype(dtype)[0] 44 | 45 | # this is where we modify the parameter space accordingly 46 | self.param_object.remove_is(label, value) 47 | 48 | return self 49 | -------------------------------------------------------------------------------- /talos/scan/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomio/talos/9cbbc42a69396aaeaf25d68af51d52e6a575e3d9/talos/scan/__init__.py -------------------------------------------------------------------------------- /talos/scan/scan_finish.py: -------------------------------------------------------------------------------- 1 | def scan_finish(self): 2 | 3 | attrs_final = ['data', 'x', 'y', 'learning_entropy', 'round_times', 4 | 'params', 'saved_models', 'saved_weights', 'round_history'] 5 | 6 | attrs_to_keep = attrs_final + ['random_method', 'grid_downsample', 7 | 'reduction_interval', 'reduce_loss', 8 | 'reduction_method', 'reduction_metric', 9 | 'reduction_threshold', 'reduction_window', 10 | 'experiment_name', 'round_history'] 11 | 12 | import time 13 | import pandas as pd 14 | 15 | # create a dataframe with permutation times 16 | self.round_times = pd.DataFrame(self.round_times) 17 | self.round_times.columns = ['start', 'end', 'duration'] 18 | 19 | # combine epoch entropy tables 20 | self.learning_entropy = pd.DataFrame(self.epoch_entropy) 21 | self.learning_entropy.columns = self._metric_keys 22 | 23 | # clean the results into a dataframe 24 | self.data = self.result 25 | 26 | # keep experiment_id temporarily 27 | _experiment_id = self._experiment_id 28 | 29 | # remove redundant columns 30 | keys = list(self.__dict__.keys()) 31 | for key in keys: 32 | if key not in attrs_to_keep: 33 | delattr(self, key) 34 | 35 | # summarize single inputs in dictionary 36 | out = {} 37 | 38 | for key in list(self.__dict__.keys()): 39 | if key not in attrs_final: 40 | out[key] = self.__dict__[key] 41 | 42 | out['experiment_id'] = _experiment_id 43 | out['complete_time'] = time.strftime('%D/%H:%M') 44 | 45 | try: 46 | out['x_shape'] = self.x.shape 47 | except AttributeError: 48 | out['x_shape'] = 'multi-input' 49 | 50 | try: 51 | out['y_shape'] = self.y.shape 52 | except AttributeError: 53 | out['y_shape'] = 'multi-input' 54 | 55 | # final cleanup 56 | keys = list(self.__dict__.keys()) 57 | for key in keys: 58 | if key not in attrs_final: 59 | delattr(self, key) 60 | 61 | # add details dictionary as series 62 | self.details = pd.Series(out) 63 | 64 | # add best_model 65 | from ..scan.scan_addon import func_best_model, func_evaluate 66 | 67 | self.best_model = func_best_model.__get__(self) 68 | self.evaluate_models = func_evaluate.__get__(self) 69 | 70 | # reset the index 71 | self.data.index = range(len(self.data)) 72 | 73 | # add the timings 74 | self.data = self.round_times.merge(self.data, left_index=True, right_index=True) 75 | 76 | return self 77 | -------------------------------------------------------------------------------- /talos/scan/scan_prepare.py: -------------------------------------------------------------------------------- 1 | def scan_prepare(self): 2 | 3 | '''Includes all preparation procedures up until starting the first scan 4 | through scan_run()''' 5 | 6 | from .scan_utils import initialize_log 7 | 8 | self._experiment_log = initialize_log(self) 9 | 10 | # for the case where x_val or y_val is missing when other is present 11 | self.custom_val_split = False 12 | if (self.x_val is not None and self.y_val is None) or \ 13 | (self.x_val is None and self.y_val is not None): 14 | raise RuntimeError("If x_val/y_val is inputted, other must as well.") 15 | 16 | elif self.x_val is not None and self.y_val is not None: 17 | self.custom_val_split = True 18 | 19 | # handle the case where self.params is dictionary 20 | if isinstance(self.params, dict): 21 | 22 | # create reference for parameter keys 23 | self._param_dict_keys = list(self.params.keys()) 24 | 25 | # create the parameter object and move to self 26 | from ..parameters.ParamSpace import ParamSpace 27 | self.param_object = ParamSpace(params=self.params, 28 | param_keys=self._param_dict_keys, 29 | random_method=self.random_method, 30 | fraction_limit=self.fraction_limit, 31 | round_limit=self.round_limit, 32 | time_limit=self.time_limit, 33 | boolean_limit=self.boolean_limit) 34 | 35 | # handle the case when self.params already is ParamSpace object 36 | elif 'talos.parameters.ParamSpace.ParamSpace' in str(type(self.params)): 37 | 38 | self._param_dict_keys = list(self.params.param_keys) 39 | self.param_object = self.params 40 | 41 | else: 42 | raise TypeError('params has to be either dict or ParamSpace object.') 43 | 44 | # mark that it's a first round 45 | self.first_round = True 46 | 47 | # create various stores 48 | self.round_history = [] 49 | self.peak_epochs = [] 50 | self.epoch_entropy = [] 51 | self.round_times = [] 52 | self.result = [] 53 | self.saved_models = [] 54 | self.saved_weights = [] 55 | 56 | # handle validation split 57 | from ..utils.validation_split import validation_split 58 | self = validation_split(self) 59 | 60 | # set data and len 61 | self._data_len = len(self.x) 62 | 63 | return self 64 | -------------------------------------------------------------------------------- /talos/scan/scan_round.py: -------------------------------------------------------------------------------- 1 | def scan_round(self): 2 | 3 | '''The main operational function that manages the experiment 4 | on the level of execution of each round.''' 5 | 6 | import time 7 | import gc 8 | 9 | # print round params 10 | if self.print_params is True: 11 | print(self.round_params) 12 | 13 | # set start time 14 | round_start = time.strftime('%D-%H%M%S') 15 | start = time.time() 16 | 17 | # fit the model 18 | from ..model.ingest_model import ingest_model 19 | self.model_history, self.round_model = ingest_model(self) 20 | self.round_history.append(self.model_history.history) 21 | 22 | # handle logging of results 23 | from ..logging.logging_run import logging_run 24 | self = logging_run(self, round_start, start, self.model_history) 25 | 26 | # apply reductions 27 | from ..reducers.reduce_run import reduce_run 28 | self = reduce_run(self) 29 | 30 | # handle the case where the actual model is to be saved 31 | if self.save_models: 32 | 33 | dir_name = str(len(self.round_history) - 1) 34 | file_path = self._saved_models_path + '/' + dir_name 35 | self.round_model.save(file_path) 36 | 37 | # handle other cases 38 | else: 39 | try: 40 | # save model and weights 41 | self.saved_models.append(self.round_model.to_json()) 42 | 43 | if self.save_weights: 44 | self.saved_weights.append(self.round_model.get_weights()) 45 | else: 46 | self.saved_weights.append(None) 47 | 48 | except AttributeError as e: 49 | # make sure that the error message is from torch 50 | if str(e) == "'Model' object has no attribute 'to_json'": 51 | if self.save_weights: 52 | self.saved_models.append(self.round_model.state_dict()) 53 | else: 54 | self.saved_weights.append(None) 55 | 56 | # clear tensorflow sessions 57 | if self.clear_session is True: 58 | 59 | del self.round_model 60 | gc.collect() 61 | 62 | # try TF specific and pass for everyone else 63 | try: 64 | from tensorflow.keras import backend as K 65 | K.clear_session() 66 | except ImportError: 67 | pass 68 | 69 | return self 70 | -------------------------------------------------------------------------------- /talos/scan/scan_run.py: -------------------------------------------------------------------------------- 1 | def scan_run(self): 2 | 3 | '''The high-level management of the scan procedures 4 | onwards from preparation. Manages round_run()''' 5 | 6 | from tqdm import tqdm 7 | 8 | from .scan_prepare import scan_prepare 9 | self = scan_prepare(self) 10 | 11 | # initiate the progress bar 12 | self.pbar = tqdm(total=len(self.param_object.param_index), 13 | disable=self.disable_progress_bar) 14 | 15 | # the main cycle of the experiment 16 | while True: 17 | 18 | # get the parameters 19 | self.round_params = self.param_object.round_parameters() 20 | 21 | # break when there is no more permutations left 22 | if self.round_params is False: 23 | break 24 | # otherwise proceed with next permutation 25 | from .scan_round import scan_round 26 | self = scan_round(self) 27 | self.pbar.update(1) 28 | 29 | # close progress bar before finishing 30 | self.pbar.close() 31 | 32 | # finish 33 | from ..logging.logging_finish import logging_finish 34 | self = logging_finish(self) 35 | 36 | from .scan_finish import scan_finish 37 | self = scan_finish(self) 38 | -------------------------------------------------------------------------------- /talos/scan/scan_utils.py: -------------------------------------------------------------------------------- 1 | def initialize_log(self): 2 | 3 | import time 4 | import os 5 | 6 | # create the experiment folder (unless one is already there) 7 | try: 8 | path = os.getcwd() 9 | os.mkdir(path + '/' + self.experiment_name) 10 | except FileExistsError: 11 | pass 12 | 13 | # create unique experiment_id 14 | self._experiment_id = time.strftime('%D%H%M%S').replace('/', '') 15 | 16 | # place saved models on a sub-folder 17 | if self.save_models: 18 | self._saved_models_path = self.experiment_name + '/' + self._experiment_id 19 | file_path = path + '/' + self._saved_models_path 20 | os.mkdir(file_path) 21 | 22 | _file_name = self._experiment_id + '.csv' 23 | _experiment_log = './' + self.experiment_name + '/' + _file_name 24 | 25 | f = open(_experiment_log, 'w') 26 | f.write('') 27 | f.close() 28 | 29 | return _experiment_log 30 | -------------------------------------------------------------------------------- /talos/templates/__init__.py: -------------------------------------------------------------------------------- 1 | from . import datasets 2 | from . import models 3 | from . import params 4 | from . import pipelines 5 | -------------------------------------------------------------------------------- /talos/templates/params.py: -------------------------------------------------------------------------------- 1 | def titanic(debug=False): 2 | 3 | from tensorflow.keras.optimizers.legacy import Adam, Adagrad 4 | 5 | # here use a standard 2d dictionary for inputting the param boundaries 6 | p = {'lr': (0.5, 5, 10), 7 | 'first_neuron': [4, 8, 16], 8 | 'batch_size': [20, 30, 40], 9 | 'dropout': (0, 0.5, 5), 10 | 'optimizer': [Adam(), Adagrad()], 11 | 'epochs': [50, 100, 150], 12 | 'losses': ['LogCosh', 'binary_crossentropy'], 13 | 'shapes': ['brick', 'triangle', 0.2], 14 | 'hidden_layers': [0, 1, 2, 3, 4], 15 | 'activation': ['relu', 'elu'], 16 | 'last_activation': ['sigmoid']} 17 | 18 | if debug: 19 | 20 | p = {'lr': [0.1, 0.2], 21 | 'first_neuron': [4, 8], 22 | 'batch_size': [20, 30], 23 | 'dropout': [0.2, 0.3], 24 | 'optimizer': [Adam(), Adagrad()], 25 | 'epochs': [50, 100], 26 | 'losses': ['LogCosh', 'binary_crossentropy'], 27 | 'shapes': ['brick', 'triangle', 0.2], 28 | 'hidden_layers': [0, 1], 29 | 'activation': ['relu', 'elu'], 30 | 'last_activation': ['sigmoid']} 31 | 32 | return p 33 | 34 | 35 | def iris(): 36 | 37 | from tensorflow.keras.optimizers.legacy import Adam, Adagrad 38 | from tensorflow.keras.activations import relu, elu, softmax 39 | 40 | # here use a standard 2d dictionary for inputting the param boundaries 41 | p = {'lr': (0.5, 5, 10), 42 | 'first_neuron': [4, 8, 16, 32, 64], 43 | 'hidden_layers': [0, 1, 2, 3, 4], 44 | 'batch_size': (2, 30, 10), 45 | 'epochs': [50, 100, 150], 46 | 'dropout': (0, 0.5, 5), 47 | 'weight_regulizer': [None], 48 | 'emb_output_dims': [None], 49 | 'shapes': ['brick', 'triangle', 0.2], 50 | 'optimizer': [Adam, Adagrad], 51 | 'losses': ['LogCosh', 'categorical_crossentropy'], 52 | 'activation': [relu, elu], 53 | 'last_activation': [softmax]} 54 | 55 | return p 56 | 57 | 58 | def breast_cancer(): 59 | 60 | from tensorflow.keras.optimizers.legacy import Adam, Adagrad, RMSprop 61 | from tensorflow.keras.activations import relu, elu, sigmoid 62 | 63 | # then we can go ahead and set the parameter space 64 | p = {'lr': (0.5, 5, 10), 65 | 'first_neuron': [4, 8, 16, 32, 64], 66 | 'hidden_layers': [0, 1, 2], 67 | 'batch_size': (2, 30, 10), 68 | 'epochs': [50, 100, 150], 69 | 'dropout': (0, 0.5, 5), 70 | 'shapes': ['brick', 'triangle', 'funnel'], 71 | 'optimizer': [Adam, Adagrad, RMSprop], 72 | 'losses': ['LogCosh', 'binary_crossentropy'], 73 | 'activation': [relu, elu], 74 | 'last_activation': [sigmoid]} 75 | 76 | return p 77 | 78 | 79 | def cervical_cancer(): 80 | return breast_cancer() 81 | -------------------------------------------------------------------------------- /talos/templates/pipelines.py: -------------------------------------------------------------------------------- 1 | def breast_cancer(round_limit=2, random_method='uniform_mersenne'): 2 | 3 | '''Performs a Scan with Iris dataset and simple dense net''' 4 | import talos as ta 5 | scan_object = ta.Scan(ta.templates.datasets.breast_cancer()[0], 6 | ta.templates.datasets.breast_cancer()[1], 7 | ta.templates.params.breast_cancer(), 8 | ta.templates.models.breast_cancer, 9 | 'test', 10 | round_limit=round_limit) 11 | 12 | return scan_object 13 | 14 | 15 | def cervical_cancer(round_limit=2, random_method='uniform_mersenne'): 16 | 17 | '''Performs a Scan with Iris dataset and simple dense net''' 18 | import talos as ta 19 | scan_object = ta.Scan(ta.templates.datasets.cervical_cancer()[0], 20 | ta.templates.datasets.cervical_cancer()[1], 21 | ta.templates.params.cervical_cancer(), 22 | ta.templates.models.cervical_cancer, 23 | 'test', 24 | round_limit=round_limit) 25 | 26 | return scan_object 27 | 28 | 29 | def iris(round_limit=2, random_method='uniform_mersenne'): 30 | 31 | '''Performs a Scan with Iris dataset and simple dense net''' 32 | import talos as ta 33 | scan_object = ta.Scan(ta.templates.datasets.iris()[0], 34 | ta.templates.datasets.iris()[1], 35 | ta.templates.params.iris(), 36 | ta.templates.models.iris, 37 | 'test', 38 | round_limit=round_limit) 39 | 40 | return scan_object 41 | 42 | 43 | def titanic(round_limit=2, random_method='uniform_mersenne', debug=False): 44 | 45 | '''Performs a Scan with Iris dataset and simple dense net''' 46 | import talos as ta 47 | 48 | scan_object = ta.Scan(ta.templates.datasets.titanic()[0].astype('float32'), 49 | ta.templates.datasets.titanic()[1].astype('float32'), 50 | ta.templates.params.titanic(debug), 51 | ta.templates.models.titanic, 52 | 'test', 53 | random_method=random_method, 54 | round_limit=round_limit) 55 | 56 | return scan_object 57 | -------------------------------------------------------------------------------- /talos/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # In this init we load everything under utils in the Talos namespace 2 | 3 | from ..model.normalizers import lr_normalizer 4 | from ..model.hidden_layers import hidden_layers 5 | from ..model.early_stopper import early_stopper 6 | from .generator import generator 7 | from . import gpu_utils 8 | import talos.metrics.keras_metrics as metrics 9 | from .sequence_generator import SequenceGenerator 10 | from .rescale_meanzero import rescale_meanzero 11 | from .torch_history import TorchHistory 12 | from wrangle import array_split as val_split 13 | from .power_draw_append import power_draw_append 14 | from .recover_best_model import recover_best_model 15 | 16 | del sequence_generator 17 | -------------------------------------------------------------------------------- /talos/utils/best_model.py: -------------------------------------------------------------------------------- 1 | def best_model(self, metric, asc): 2 | 3 | '''Picks the best model based on a given metric and 4 | returns the index number for the model. 5 | 6 | NOTE: for loss 'asc' should be True''' 7 | 8 | best = self.data.sort_values(metric, ascending=asc).iloc[0].name 9 | 10 | return best 11 | 12 | 13 | def activate_model(self, model_id, saved=False, custom_objects=None): 14 | 15 | '''Loads the model from the json that is stored in the Scan object 16 | or from local 17 | 18 | model_id | int | the sequential id of the model 19 | saved | bool | if a model saved on local machine should be used 20 | custom_object | dict | if the model has a custom object, pass it here 21 | 22 | ''' 23 | 24 | import tensorflow as tf 25 | from tensorflow.keras.models import model_from_json 26 | 27 | if saved: 28 | 29 | file_path = self.details['experiment_name'] 30 | file_path += '/' + self.details['experiment_id'] 31 | file_path += '/' + str(model_id) 32 | 33 | model = tf.keras.models.load_model(file_path, 34 | custom_objects=custom_objects) 35 | 36 | else: 37 | model = model_from_json(self.saved_models[model_id]) 38 | model.set_weights(self.saved_weights[model_id]) 39 | 40 | return model 41 | -------------------------------------------------------------------------------- /talos/utils/exceptions.py: -------------------------------------------------------------------------------- 1 | class TalosReturnError(Exception): 2 | pass 3 | 4 | 5 | class TalosParamsError(Exception): 6 | pass 7 | 8 | 9 | class TalosTypeError(Exception): 10 | pass 11 | 12 | 13 | class TalosModelError(Exception): 14 | pass 15 | 16 | 17 | class TalosDataError(Exception): 18 | pass 19 | -------------------------------------------------------------------------------- /talos/utils/generator.py: -------------------------------------------------------------------------------- 1 | def generator(x, y, batch_size): 2 | 3 | '''Creates a data generator for Keras fit_generator(). ''' 4 | 5 | import numpy as np 6 | 7 | samples_per_epoch = x.shape[0] 8 | number_of_batches = samples_per_epoch / batch_size 9 | counter = 0 10 | 11 | while 1: 12 | 13 | x_batch = np.array(x[batch_size*counter:batch_size*(counter+1)]).astype('float32') 14 | y_batch = np.array(y[batch_size*counter:batch_size*(counter+1)]).astype('float32') 15 | counter += 1 16 | 17 | yield x_batch, y_batch 18 | 19 | if counter >= number_of_batches: 20 | counter = 0 21 | -------------------------------------------------------------------------------- /talos/utils/gpu_utils.py: -------------------------------------------------------------------------------- 1 | def parallel_gpu_jobs(allow_growth=True, fraction=.5): 2 | 3 | '''Sets the max used memory as a fraction for tensorflow 4 | backend 5 | 6 | allow_growth :: True of False 7 | 8 | fraction :: a float value (e.g. 0.5 means 4gb out of 8gb) 9 | 10 | ''' 11 | 12 | import keras.backend as K 13 | import tensorflow as tf 14 | 15 | gpu_options = tf.compat.v1.GPUOptions(allow_growth=allow_growth, 16 | per_process_gpu_memory_fraction=fraction) 17 | config = tf.compat.v1.ConfigProto(gpu_options=gpu_options) 18 | session = tf.compat.v1.Session(config=config) 19 | tf.compat.v1.keras.backend.set_session(session) 20 | 21 | 22 | def multi_gpu(model, gpus=None, cpu_merge=True, cpu_relocation=False): 23 | 24 | '''Takes as input the model, and returns a model 25 | based on the number of GPUs available on the machine 26 | or alternatively the 'gpus' user input. 27 | 28 | NOTE: this needs to be used before model.compile() in the 29 | model inputted to Scan in the form: 30 | 31 | from talos.utils.gpu_utils import multi_gpu 32 | model = multi_gpu(model) 33 | 34 | ''' 35 | 36 | from tensorflow.keras.utils import multi_gpu_model 37 | 38 | return multi_gpu_model(model, 39 | gpus=gpus, 40 | cpu_merge=cpu_merge, 41 | cpu_relocation=cpu_relocation) 42 | 43 | 44 | def force_cpu(): 45 | 46 | '''Force CPU on a GPU system 47 | ''' 48 | 49 | import tensorflow.keras.backend as K 50 | import tensorflow as tf 51 | 52 | config = tf.compat.v1.ConfigProto(device_count={'GPU': 0}) 53 | session = tf.compat.v1.Session(config=config) 54 | tf.compat.v1.keras.backend.set_session(session) 55 | -------------------------------------------------------------------------------- /talos/utils/load_model.py: -------------------------------------------------------------------------------- 1 | from tensorflow.keras.models import model_from_json 2 | 3 | 4 | def load_model(saved_model): 5 | 6 | '''Load a Model from local disk 7 | 8 | Takes as input .json and .h5 file with model 9 | and weights and returns a model that can be then 10 | used for predictions. 11 | 12 | saved_model :: name of the saved model without 13 | suffix (e.g. 'iris_model' and not 'iris_model.json') 14 | 15 | ''' 16 | 17 | json_file = open(saved_model + ".json", 'r') 18 | loaded_model_json = json_file.read() 19 | json_file.close() 20 | model = model_from_json(loaded_model_json) 21 | model.load_weights(saved_model + '.h5') 22 | 23 | return model 24 | -------------------------------------------------------------------------------- /talos/utils/power_draw_append.py: -------------------------------------------------------------------------------- 1 | def power_draw_append(history, power_draw): 2 | 3 | '''For appending the data from PowerDrawCallback to the history object 4 | and allowing the data to be captured in the experiment log in Talos. 5 | 6 | history | object | tf.keras model history object 7 | power_draw | object | PowerDrawCallback object 8 | 9 | ''' 10 | 11 | import numpy as np 12 | 13 | joined = power_draw.log['epoch_begin'] + power_draw.log['epoch_end'] 14 | history.history['watts_min'] = [min(joined)] 15 | history.history['watts_max'] = [max(joined)] 16 | history.history['seconds'] = [sum(power_draw.log['seconds'])] 17 | 18 | # get average watts per epoc 19 | epoch_begin = np.array(power_draw.log['epoch_begin']) 20 | epoch_end = np.array(power_draw.log['epoch_end']) 21 | avg_watts = (epoch_begin + epoch_end) / 2 22 | 23 | watt_seconds = round(sum(avg_watts * np.array(power_draw.log['seconds'])), 2) 24 | history.history['Ws'] = [watt_seconds] 25 | 26 | return history 27 | -------------------------------------------------------------------------------- /talos/utils/recover_best_model.py: -------------------------------------------------------------------------------- 1 | def recover_best_model(x_train, 2 | y_train, 3 | x_val, 4 | y_val, 5 | experiment_log, 6 | input_model, 7 | metric, 8 | multi_input=False, 9 | x_cross=None, 10 | y_cross=None, 11 | n_models=5, 12 | task='multi_label'): 13 | 14 | '''Recover best models from Talos experiment log. 15 | 16 | x_train | array | same as was used in the experiment 17 | y_train | array | same as was used in the experiment 18 | x_val | array | same as was used in the experiment 19 | y_val | array | same as was used in the experiment 20 | experiment_log | str | path to the Talos experiment log 21 | input_model | function | model used in the experiment 22 | metric | str | use this metric to pick evaluation candidates 23 | multi_input | bool | set to True if multi-input model 24 | x_cross | array | data for the cross-validation or None for use x_val 25 | y_cross | array | data for the cross-validation or None for use y_val 26 | n_models | int | number of models to cross-validate 27 | task | str | binary, multi_class, multi_label or continuous 28 | 29 | Returns a pandas dataframe with the cross-validation results 30 | and the models. 31 | 32 | ''' 33 | 34 | import pandas as pd 35 | import sklearn as sk 36 | import numpy as np 37 | 38 | from talos.utils.validation_split import kfold 39 | 40 | # read the experiment log into a dataframe 41 | df = pd.read_csv(experiment_log) 42 | 43 | # handle input data scenarios 44 | if x_cross is None or y_cross is None: 45 | x_cross = x_val 46 | y_cross = y_val 47 | 48 | # for final output 49 | results = [] 50 | models = [] 51 | 52 | for i in range(n_models): 53 | 54 | # get the params for the model and train it 55 | params = df.sort_values(metric, ascending=False) 56 | params = params.drop(metric, axis=1).iloc[i].to_dict() 57 | _history, model = input_model(x_train, y_train, x_val, y_val, params) 58 | 59 | # start kfold cross-validation 60 | out = [] 61 | folds = 5 62 | kx, ky = kfold(x_cross, y_cross, folds, True, multi_input) 63 | 64 | for i in range(folds): 65 | 66 | y_pred = model.predict(kx[i]).argmax(axis=1) 67 | 68 | if task == 'binary': 69 | y_pred = y_pred >= .5 70 | scores = sk.metrics.f1_score(y_pred, ky[i], average='binary') 71 | 72 | elif task == 'multi_class': 73 | y_pred = y_pred.argmax(axis=-1) 74 | scores = sk.metrics.f1_score(y_pred, ky[i], average='macro') 75 | 76 | if task == 'multi_label': 77 | y_pred = model.predict(kx[i]).argmax(axis=1) 78 | scores = sk.metrics.f1_score(y_pred, 79 | ky[i].argmax(axis=1), 80 | average='macro') 81 | 82 | elif task == 'continuous': 83 | y_pred = model.predict(kx[i]) 84 | scores = sk.metrics.mean_absolute_error(y_pred, ky[i]) 85 | 86 | out.append(scores) 87 | 88 | results.append(np.mean(out)) 89 | models.append(model) 90 | 91 | out = df.sort_values(metric, ascending=False).head(n_models) 92 | out['crossval_mean_f1score'] = results 93 | 94 | return out, models 95 | -------------------------------------------------------------------------------- /talos/utils/rescale_meanzero.py: -------------------------------------------------------------------------------- 1 | def rescale_meanzero(x): 2 | 3 | '''Rescales an array to mean-zero. 4 | 5 | x | array | the dataset to be rescaled 6 | ''' 7 | 8 | import wrangle 9 | import pandas as pd 10 | 11 | return wrangle.df_rescale_meanzero(pd.DataFrame(x)).values 12 | -------------------------------------------------------------------------------- /talos/utils/sequence_generator.py: -------------------------------------------------------------------------------- 1 | from tensorflow.keras.utils import Sequence 2 | 3 | 4 | class SequenceGenerator(Sequence): 5 | 6 | def __init__(self, x_set, y_set, batch_size): 7 | 8 | self.x, self.y = x_set, y_set 9 | self.batch_size = batch_size 10 | 11 | def __len__(self): 12 | 13 | import numpy as np 14 | 15 | return int(np.ceil(len(self.x) / float(self.batch_size))) 16 | 17 | def __getitem__(self, idx): 18 | batch_x = self.x[idx * self.batch_size:(idx + 1) * self.batch_size] 19 | batch_y = self.y[idx * self.batch_size:(idx + 1) * self.batch_size] 20 | 21 | return batch_x, batch_y 22 | -------------------------------------------------------------------------------- /talos/utils/test_utils.py: -------------------------------------------------------------------------------- 1 | def create_param_space(data, no_of_metrics=2): 2 | 3 | '''Takes as input experiment log dataframe and returns 4 | ParamSpace object 5 | 6 | data | DataFrame | Talos experiment log as pandas dataframe 7 | no_of_metrics | int | number of metrics in the dataframe 8 | 9 | ''' 10 | 11 | from talos.parameters.ParamSpace import ParamSpace 12 | 13 | params = {} 14 | 15 | for col in data.iloc[:, no_of_metrics:].columns: 16 | params[col] = data[col].unique().tolist() 17 | 18 | param_keys = list(params.keys()) 19 | 20 | return ParamSpace(params, param_keys) 21 | -------------------------------------------------------------------------------- /talos/utils/torch_history.py: -------------------------------------------------------------------------------- 1 | class TorchHistory: 2 | 3 | '''This is a helper for replicating the history object 4 | behavior of Keras to make Talos Scan() API consistent between 5 | the two backends.''' 6 | 7 | def __init__(self): 8 | 9 | self.history = {} 10 | 11 | def init_history(self): 12 | self.history = {} 13 | 14 | def append_history(self, history_data, label): 15 | if label not in self.history.keys(): 16 | self.history[label] = [] 17 | self.history[label].append(history_data) 18 | 19 | def append_loss(self, _loss): 20 | self.append_history(_loss, 'loss') 21 | 22 | def append_metric(self, _metric): 23 | self.append_history(_metric, 'metric') 24 | 25 | def append_val_loss(self, _loss): 26 | self.append_history(_loss, 'val_loss') 27 | 28 | def append_val_metric(self, _loss): 29 | self.append_history(_loss, 'val_metric') 30 | -------------------------------------------------------------------------------- /talos/utils/validation_split.py: -------------------------------------------------------------------------------- 1 | def validation_split(self): 2 | 3 | '''Defines the attributes `x_train`, `y_train`, `x_val` and `y_val`. 4 | The validation sets are determined by the attribute val_split, 5 | which is a number in (0, 1) which determines the proportion of 6 | the input data to be allocated for cross-validation.''' 7 | 8 | # data input is list but multi_input is not set to True 9 | if isinstance(self.x, list) and self.multi_input is False: 10 | 11 | raise TypeError("For multi-input x, set multi_input to True") 12 | 13 | # If split is done in `Scan()` do nothing 14 | if self.custom_val_split: 15 | 16 | self.x_train = self.x 17 | self.y_train = self.y 18 | 19 | return self 20 | 21 | # Otherwise start by shuffling 22 | import wrangle 23 | self.x, self.y = wrangle.array_random_shuffle(x=self.x, 24 | y=self.y, 25 | multi_input=self.multi_input) 26 | 27 | # deduce the midway point for input data 28 | limit = int(len(self.y) * (1 - self.val_split)) 29 | 30 | # handle the case where x is multi-input 31 | if self.multi_input: 32 | 33 | self.x_train = [] 34 | self.x_val = [] 35 | 36 | for ar in self.x: 37 | self.x_train.append(ar[:limit]) 38 | self.x_val.append(ar[limit:]) 39 | 40 | # handle the case where x is not multi-input 41 | else: 42 | 43 | self.x_train = self.x[:limit] 44 | self.x_val = self.x[limit:] 45 | 46 | # handle y data same for both cases 47 | self.y_train = self.y[:limit] 48 | self.y_val = self.y[limit:] 49 | 50 | return self 51 | 52 | 53 | def kfold(x, y, folds=10, shuffled=True, multi_input=False): 54 | 55 | import wrangle 56 | 57 | # data input is list but multi_input is not set to True 58 | if isinstance(x, list) and multi_input is False: 59 | raise TypeError("For multi-input x, set multi_input to True") 60 | 61 | if shuffled is True: 62 | x, y = wrangle.array_random_shuffle(x, y, multi_input) 63 | 64 | out_x = [] 65 | out_y = [] 66 | 67 | # establish the fold size 68 | y_len = len(y) 69 | step = int(y_len / folds) 70 | 71 | lo = 0 72 | hi = step 73 | 74 | # create folds one by one 75 | for _i in range(folds): 76 | 77 | # handle the case for multi-input model 78 | if multi_input: 79 | fold_x = [] 80 | for ar in x: 81 | fold_x.append(ar[lo:hi]) 82 | out_x.append(fold_x) 83 | 84 | # handle the case where model is not multi-input 85 | else: 86 | out_x.append(x[lo:hi]) 87 | 88 | out_y.append(y[lo:hi]) 89 | 90 | lo += step 91 | hi += step 92 | 93 | return out_x, out_y 94 | -------------------------------------------------------------------------------- /test-ci.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | if __name__ == '__main__': 4 | 5 | from tests.commands import * 6 | 7 | scan_object = test_scan() 8 | test_latest() 9 | recover_best_model() 10 | test_random_methods() 11 | test_autom8() 12 | test_templates() 13 | test_analyze(scan_object) 14 | test_lr_normalizer() 15 | test_predict() 16 | test_reducers() 17 | test_rest(scan_object) 18 | 19 | print("\n All tests successfully completed :) Good work. \n ") 20 | -------------------------------------------------------------------------------- /test-local.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export MPLBACKEND=agg 4 | 5 | # run memory pressure test 6 | mprof run ./tests/performance/memory_pressure.py 7 | python3 ./tests/performance/memory_pressure_check.py 8 | 9 | # run CI tests 10 | python3 test-ci.py 11 | 12 | # cleanup 13 | rm mprofile_*.dat 14 | rm ./test/*.log 15 | rm ./test/*.csv 16 | rm -rf test_* 17 | rm -rf test -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/autonomio/talos/9cbbc42a69396aaeaf25d68af51d52e6a575e3d9/tests/__init__.py -------------------------------------------------------------------------------- /tests/__main__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | sys.path.append('../talos') 4 | -------------------------------------------------------------------------------- /tests/commands/__init__.py: -------------------------------------------------------------------------------- 1 | from .recover_best_model import recover_best_model 2 | from .test_analyze import test_analyze 3 | from .test_autom8 import test_autom8 4 | from .test_latest import test_latest 5 | from .test_lr_normalizer import test_lr_normalizer 6 | from .test_predict import test_predict 7 | from .test_random_methods import test_random_methods 8 | from .test_reducers import test_reducers 9 | from .test_rest import test_rest 10 | from .test_scan import test_scan 11 | from .test_templates import test_templates 12 | -------------------------------------------------------------------------------- /tests/commands/recover_best_model.py: -------------------------------------------------------------------------------- 1 | def recover_best_model(): 2 | 3 | import os 4 | 5 | from talos.utils import recover_best_model 6 | import talos 7 | import tensorflow 8 | 9 | experiment_log = 'test_q/' + os.listdir('test_q')[0] 10 | 11 | x, y = talos.templates.datasets.iris() 12 | input_model = talos.templates.models.iris 13 | 14 | x = x[:50] 15 | y = y[:50] 16 | 17 | # define the input model 18 | def iris_model(x_train, y_train, x_val, y_val, params): 19 | 20 | from tensorflow.keras.models import Sequential 21 | from tensorflow.keras.layers import Dense 22 | 23 | model = Sequential() 24 | model.add(Dense(params['first_neuron'], input_dim=4, activation=params['activation'])) 25 | 26 | talos.utils.hidden_layers(model, params, 3) 27 | 28 | model.add(Dense(3, activation='softmax')) 29 | model.compile(optimizer=params['optimizer'], loss=params['losses'], metrics=['acc']) 30 | 31 | out = model.fit(x_train, y_train, 32 | batch_size=params['batch_size'], 33 | epochs=params['epochs'], 34 | validation_data=(x_val, y_val), 35 | verbose=0) 36 | 37 | return out, model 38 | 39 | recover_best_model(x_train=x, 40 | y_train=y, 41 | x_val=x, 42 | y_val=y, 43 | experiment_log=experiment_log, 44 | input_model=iris_model, 45 | metric='acc', 46 | x_cross=x, 47 | y_cross=y, 48 | n_models=5, 49 | task='multi_label') 50 | -------------------------------------------------------------------------------- /tests/commands/test_analyze.py: -------------------------------------------------------------------------------- 1 | def test_analyze(scan_object): 2 | 3 | '''Tests all the attributes available in the Reporting() object''' 4 | 5 | print('\n >>> Start Analyze()... \n') 6 | 7 | import talos 8 | 9 | # for now test with old name 10 | r = talos.Reporting(scan_object) 11 | 12 | # read from file 13 | #list_of_files = glob.glob('./test_latest/' + '/*.csv') 14 | 15 | #r = talos.Reporting(list_of_files[-1]) 16 | 17 | # and then from scan object 18 | r = talos.Analyze(scan_object) 19 | 20 | # test the object properties 21 | r.best_params('val_loss', ['val_acc']) 22 | r.correlate('val_loss', ['val_acc', 23 | 'start', 24 | 'end', 25 | 'activation', 26 | 'optimizer', 27 | 'losses', 28 | 'shapes']) 29 | r.data 30 | r.high('val_acc') 31 | r.low('val_acc') 32 | 33 | # r.plot_bars('first_neuron', 'val_acc', 'dropout', 'hidden_layers') 34 | r.plot_box('first_neuron', 'val_acc') 35 | r.plot_corr('val_loss', ['val_acc', 36 | 'start', 37 | 'end', 38 | 'activation', 39 | 'optimizer', 40 | 'losses', 41 | 'shapes']) 42 | r.plot_hist('val_acc') 43 | r.plot_kde('val_acc') 44 | r.plot_line('val_acc') 45 | r.plot_regs('val_acc', 'val_loss') 46 | r.rounds() 47 | r.rounds2high('val_acc') 48 | r.table('val_loss', ['val_acc']) 49 | 50 | print('finish Analyze() \n') 51 | -------------------------------------------------------------------------------- /tests/commands/test_autom8.py: -------------------------------------------------------------------------------- 1 | def test_autom8(): 2 | 3 | import talos 4 | import wrangle 5 | 6 | from tensorflow.keras.optimizers.legacy import Adam 7 | 8 | print('\n >>> start AutoParams()... \n') 9 | 10 | p = talos.autom8.AutoParams() 11 | p.params 12 | p = talos.autom8.AutoParams(p.params) 13 | p.resample_params(5) 14 | 15 | p.activations(['relu']) 16 | p.batch_size(20, 50, 2) 17 | p.dropout(0, 0.22, 0.04) 18 | p.epochs(5, 10, 1) 19 | p.kernel_initializers(['zeros']) 20 | p.last_activations(['softmax']) 21 | p.layers(0, 2, 1) 22 | p.losses([talos.utils.metrics.f1score]) 23 | p.lr([0.01]) 24 | p.networks(['dense']) 25 | p.neurons(1, 5, 1) 26 | p.optimizers([Adam]) 27 | p.shapes(['brick']) 28 | p.shapes_slope(0, .2, .01) 29 | 30 | p.resample_params(1) 31 | 32 | print('finised AutoParams() \n') 33 | 34 | # # # # # # # # # # # # # # # # 35 | 36 | print('\n >>> start AutoModel(), AutoScan() and AutoPredict()... \n') 37 | 38 | x, y = wrangle.utils.create_synth_data('binary', 50, 10, 1) 39 | p.losses(['binary_crossentropy']) 40 | auto = talos.autom8.AutoScan('binary', 'test_a', 1) 41 | scan_object = auto.start(x, y, params=p.params) 42 | talos.autom8.AutoPredict(scan_object, x, y, x, 'binary') 43 | 44 | x, y = wrangle.utils.create_synth_data('multi_label', 50, 10, 4) 45 | p.losses(['categorical_crossentropy']) 46 | auto = talos.autom8.AutoScan('multi_label', 'test_b', 1) 47 | auto.start(x, y) 48 | talos.autom8.AutoPredict(scan_object, x, y, x, 'multi_label') 49 | 50 | x, y = wrangle.utils.create_synth_data('multi_class', 50, 10, 3) 51 | p.losses(['sparse_categorical_crossentropy']) 52 | auto = talos.autom8.AutoScan('multi_class', 'test_c', 1) 53 | auto.start(x, y, params=p.params) 54 | talos.autom8.AutoPredict(scan_object, x, y, x, 'multi_class') 55 | 56 | x, y = wrangle.utils.create_synth_data('continuous', 50, 10, 1) 57 | p.losses(['mae']) 58 | auto = talos.autom8.AutoScan('continuous', 'test_d', 1) 59 | auto.start(x, y, params=p.params) 60 | talos.autom8.AutoPredict(scan_object, x, y, x, 'continuous') 61 | 62 | print('finised AutoModel(), AutoScan() and AutoPredict() \n') 63 | 64 | # # # # # # # # # # # # # # # # 65 | -------------------------------------------------------------------------------- /tests/commands/test_latest.py: -------------------------------------------------------------------------------- 1 | def test_latest(): 2 | 3 | import warnings 4 | 5 | warnings.simplefilter('ignore') 6 | 7 | print('\n >>> start Latest Features... \n') 8 | 9 | import talos 10 | from tensorflow.keras.models import Sequential 11 | from tensorflow.keras.layers import Dense 12 | 13 | x, y = talos.templates.datasets.iris() 14 | 15 | p = {'activation': ['relu', 'elu'], 16 | 'optimizer': ['Adagrad', 'Adam'], 17 | 'losses': ['LogCosh'], 18 | 'shapes': ['brick'], 19 | 'first_neuron': [16, 32, 64, 128], 20 | 'hidden_layers': [0, 1, 2, 3], 21 | 'dropout': [.2, .3, .4], 22 | 'batch_size': [20, 30, 40, 50], 23 | 'epochs': [10]} 24 | 25 | from talos.parameters.DistributeParamSpace import DistributeParamSpace 26 | 27 | param_spaces = DistributeParamSpace(params=p, 28 | param_keys=p.keys(), 29 | machines=100) 30 | 31 | def iris_model(x_train, y_train, x_val, y_val, params): 32 | 33 | model = Sequential() 34 | model.add(Dense(params['first_neuron'], 35 | input_dim=4, 36 | activation=params['activation'])) 37 | 38 | talos.utils.hidden_layers(model, params, 3) 39 | 40 | model.add(Dense(3, activation='softmax')) 41 | model.compile(optimizer=params['optimizer'], 42 | loss=params['losses'], metrics=['acc']) 43 | 44 | out = model.fit(x_train, 45 | y_train, 46 | callbacks=[talos.callbacks.ExperimentLog('test_latest', params)], 47 | batch_size=params['batch_size'], 48 | epochs=params['epochs'], 49 | validation_data=(x_val, y_val), 50 | verbose=0) 51 | 52 | return out, model 53 | 54 | print(type(param_spaces.param_spaces[0])) 55 | 56 | scan_object = talos.Scan(x, 57 | y, 58 | model=iris_model, 59 | params=param_spaces.param_spaces[0], 60 | experiment_name='test_latest', 61 | round_limit=5, 62 | reduction_method='gamify', 63 | save_models=True) 64 | 65 | scan_object.best_model(saved=True) 66 | predict = talos.Predict(scan_object) 67 | predict.predict(x, 'val_acc', False, saved=True) 68 | 69 | scan_object = talos.Scan(x, y, 70 | model=iris_model, 71 | params=p, 72 | experiment_name='test_latest', 73 | round_limit=5, 74 | reduction_method='gamify', 75 | save_weights=False) 76 | 77 | print('finised Latest Features \n') 78 | -------------------------------------------------------------------------------- /tests/commands/test_lr_normalizer.py: -------------------------------------------------------------------------------- 1 | def test_lr_normalizer(): 2 | '''Test learning rate normalizer to confirm an invalid type is 3 | recognized and throws TalosModelError.''' 4 | 5 | from talos.model.normalizers import lr_normalizer 6 | from talos.utils.exceptions import TalosModelError 7 | 8 | print('Testing lr_normalizer() and invalid optimizer type...') 9 | 10 | # Using string as proxy for any invalid class 11 | # (ex., tensorflow-sourced optimizer) 12 | bad_optimizer = 'test' 13 | 14 | try: 15 | lr_normalizer(1, bad_optimizer) 16 | except TalosModelError: 17 | print('Invalid model optimizer caught successfully!') 18 | else: 19 | print('Invalid (string) model optimizer type not caught.') 20 | -------------------------------------------------------------------------------- /tests/commands/test_predict.py: -------------------------------------------------------------------------------- 1 | def test_predict(): 2 | 3 | print("\n >>> start Predict()...") 4 | 5 | import talos 6 | 7 | x, y = talos.templates.datasets.iris() 8 | p = talos.templates.params.iris() 9 | model = talos.templates.models.iris 10 | 11 | x = x[:50] 12 | y = y[:50] 13 | 14 | scan_object = talos.Scan(x=x, 15 | y=y, 16 | params=p, 17 | model=model, 18 | experiment_name='test_iris', round_limit=2) 19 | 20 | predict = talos.Predict(scan_object) 21 | 22 | _preds = predict.predict(x, 'val_acc', False) 23 | _preds = predict.predict_classes(x, 'val_acc', False, task='multi_label') 24 | 25 | print('finised Predict() \n') 26 | 27 | # # # # # # # # # # # # # # # # # # 28 | -------------------------------------------------------------------------------- /tests/commands/test_random_methods.py: -------------------------------------------------------------------------------- 1 | def test_random_methods(): 2 | 3 | '''Tests all the available random methods 4 | in reducers/sample_reducer.py that are invoked 5 | that are invoked through Scan(random_method)''' 6 | 7 | print('\n >>> start Random Methods... \n') 8 | 9 | import talos 10 | 11 | random_methods = ['sobol', 12 | 'quantum', 13 | 'halton', 14 | 'korobov_matrix', 15 | 'latin_sudoku', 16 | 'latin_matrix', 17 | 'latin_improved', 18 | 'uniform_mersenne', 19 | 'uniform_crypto', 20 | 'ambience' 21 | ] 22 | 23 | for method in random_methods: 24 | talos.templates.pipelines.titanic(random_method=method, debug=True) 25 | 26 | print('finish Random Methods \n') 27 | -------------------------------------------------------------------------------- /tests/commands/test_reducers.py: -------------------------------------------------------------------------------- 1 | def test_reducers(): 2 | 3 | print("\n >>> start reducers...") 4 | 5 | import talos 6 | 7 | x, y = talos.templates.datasets.iris() 8 | p = talos.templates.params.iris() 9 | model = talos.templates.models.iris 10 | 11 | x = x[:50] 12 | y = y[:50] 13 | 14 | for strategy in ['trees', 15 | 'forrest', 16 | 'correlation', 17 | 'gamify', 18 | 'local_strategy']: 19 | 20 | talos.Scan(x=x, 21 | y=y, 22 | params=p, 23 | model=model, 24 | experiment_name='test_iris', 25 | round_limit=2, 26 | reduction_method=strategy, 27 | reduction_interval=1) 28 | 29 | print('finised reducers \n') 30 | 31 | # # # # # # # # # # # # # # # # # # 32 | -------------------------------------------------------------------------------- /tests/commands/test_rest.py: -------------------------------------------------------------------------------- 1 | def test_rest(scan_object): 2 | 3 | print('\n >>> start testing the rest... \n') 4 | 5 | import talos 6 | 7 | import random 8 | 9 | deploy_filename = 'test_' + str(random.randint(1, 20000000000)) 10 | 11 | print('\n ...Deploy()... \n') 12 | talos.Deploy(scan_object, deploy_filename, 'val_acc') 13 | 14 | print('\n ...Restore()... \n') 15 | restored = talos.Restore(deploy_filename + '.zip') 16 | 17 | x, y = talos.templates.datasets.breast_cancer() 18 | x = x[:50] 19 | y = y[:50] 20 | 21 | x_train, y_train, x_val, y_val = talos.utils.val_split(x, y, .2) 22 | x = talos.utils.rescale_meanzero(x) 23 | 24 | callbacks = [talos.utils.early_stopper(10), 25 | talos.callbacks.ExperimentLog('test', {})] 26 | 27 | metrics = [talos.utils.metrics.f1score, 28 | talos.utils.metrics.fbeta, 29 | talos.utils.metrics.mae, 30 | talos.utils.metrics.mape, 31 | talos.utils.metrics.matthews, 32 | talos.utils.metrics.mse, 33 | talos.utils.metrics.msle, 34 | talos.utils.metrics.precision, 35 | talos.utils.metrics.recall, 36 | talos.utils.metrics.rmae, 37 | talos.utils.metrics.rmse, 38 | talos.utils.metrics.rmsle] 39 | 40 | from tensorflow.keras.models import Sequential 41 | from tensorflow.keras.layers import Dense 42 | 43 | print('\n ...callbacks and metrics... \n') 44 | 45 | model1 = Sequential() 46 | model1.add(Dense(10, input_dim=x.shape[1])) 47 | model1.add(Dense(1)) 48 | model1.compile('adam', 'LogCosh', metrics=metrics) 49 | model1.fit(x, y, callbacks=callbacks) 50 | 51 | print('\n ...generator... \n') 52 | 53 | model2 = Sequential() 54 | model2.add(Dense(10, input_dim=x.shape[1])) 55 | model2.add(Dense(1)) 56 | model2.compile('adam', 'LogCosh') 57 | model2.fit_generator(talos.utils.generator(x, y, 10), 5) 58 | 59 | print('\n ...SequenceGenerator... \n') 60 | 61 | model3 = Sequential() 62 | model3.add(Dense(10, input_dim=x.shape[1])) 63 | model3.add(Dense(1)) 64 | model3.compile('adam', 'LogCosh') 65 | model3.fit_generator(talos.utils.SequenceGenerator(x, y, 10)) 66 | 67 | print('\n ...gpu_utils... \n') 68 | 69 | talos.utils.gpu_utils.force_cpu() 70 | talos.utils.gpu_utils.parallel_gpu_jobs() 71 | 72 | print('\n ...gpu_utils... \n') 73 | 74 | from talos.utils.test_utils import create_param_space 75 | create_param_space(restored.results, 8) 76 | 77 | print('finished testing the rest \n') 78 | -------------------------------------------------------------------------------- /tests/commands/test_templates.py: -------------------------------------------------------------------------------- 1 | def test_templates(): 2 | 3 | print("\n >>> start templates ...") 4 | 5 | import talos 6 | 7 | x, y = talos.templates.datasets.titanic() 8 | x = x[:50] 9 | y = y[:50] 10 | model = talos.templates.models.titanic 11 | p = talos.templates.params.titanic() 12 | talos.Scan(x, y, p, model, 'test_', round_limit=2) 13 | 14 | x, y = talos.templates.datasets.iris() 15 | 16 | x = x[:50] 17 | y = y[:50] 18 | model = talos.templates.models.iris 19 | p = talos.templates.params.iris() 20 | talos.Scan(x, y, p, model, 'test_', round_limit=2) 21 | 22 | x, y = talos.templates.datasets.cervical_cancer() 23 | x = x[:50] 24 | y = y[:50] 25 | model = talos.templates.models.cervical_cancer 26 | p = talos.templates.params.cervical_cancer() 27 | talos.Scan(x, y, p, model, 'test_', round_limit=2) 28 | 29 | x, y = talos.templates.datasets.breast_cancer() 30 | x = x[:50] 31 | y = y[:50] 32 | model = talos.templates.models.breast_cancer 33 | p = talos.templates.params.breast_cancer() 34 | talos.Scan(x, y, p, model, 'test_', round_limit=2) 35 | 36 | x, y = talos.templates.datasets.icu_mortality(50) 37 | x, y = talos.templates.datasets.telco_churn(.3) 38 | x, y, x1, y1 = talos.templates.datasets.mnist() 39 | x, y = talos.templates.datasets.breast_cancer() 40 | x, y = talos.templates.datasets.cervical_cancer() 41 | x, y = talos.templates.datasets.titanic() 42 | 43 | talos.templates.pipelines.breast_cancer(random_method='uniform_mersenne') 44 | talos.templates.pipelines.cervical_cancer(random_method='sobol') 45 | talos.templates.pipelines.iris(random_method='uniform_crypto') 46 | talos.templates.pipelines.titanic(random_method='korobov_matrix') 47 | 48 | print("finish templates \n") 49 | -------------------------------------------------------------------------------- /tests/performance/memory_pressure.py: -------------------------------------------------------------------------------- 1 | import talos 2 | from talos.utils import SequenceGenerator 3 | 4 | from tensorflow.keras.models import Sequential 5 | from tensorflow.keras.layers import Dense, Dropout, Flatten 6 | from tensorflow.keras.layers import Conv2D 7 | 8 | p = {'activation': ['relu'], 9 | 'optimizer': ['Adam'], 10 | 'losses': ['categorical_crossentropy'], 11 | 'dropout': [.2], 12 | 'batch_size': [256], 13 | 'epochs': [1, 1, 1, 1, 1]} 14 | 15 | x_train, y_train, x_val, y_val = talos.templates.datasets.mnist() 16 | 17 | @profile 18 | def talos_version(): 19 | 20 | def mnist_model(x_train, y_train, x_val, y_val, params): 21 | 22 | model = Sequential() 23 | model.add(Conv2D(32, kernel_size=(3, 3), activation=params['activation'], input_shape=(28, 28, 1))) 24 | model.add(Flatten()) 25 | model.add(Dense(128, activation=params['activation'])) 26 | model.add(Dropout(params['dropout'])) 27 | model.add(Dense(10, activation='softmax')) 28 | 29 | model.compile(optimizer=params['optimizer'], 30 | loss=params['losses'], 31 | metrics=['acc', talos.utils.metrics.f1score]) 32 | 33 | out = model.fit_generator(SequenceGenerator(x_train, 34 | y_train, 35 | batch_size=params['batch_size']), 36 | epochs=params['epochs'], 37 | validation_data=[x_val, y_val], 38 | callbacks=[], 39 | workers=4, 40 | verbose=0) 41 | 42 | return out, model 43 | 44 | scan_object = talos.Scan(x=x_train, 45 | y=y_train, 46 | x_val=x_val, 47 | y_val=y_val, 48 | params=p, 49 | model=mnist_model, 50 | experiment_name='mnist', 51 | save_weights=False) 52 | 53 | 54 | if __name__ == "__main__": 55 | 56 | talos_version() 57 | -------------------------------------------------------------------------------- /tests/performance/memory_pressure_check.py: -------------------------------------------------------------------------------- 1 | if __name__ == '__main__': 2 | 3 | import numpy as np 4 | import pandas as pd 5 | import os 6 | 7 | print('\n Memory Pressure Test Starts...\n') 8 | 9 | for i in os.listdir(): 10 | if 'mprofile_' in i: 11 | df = pd.read_csv(i, sep=' ', error_bad_lines=False) 12 | 13 | df.columns = ['null', 'memory', 'time'] 14 | df.drop('null', 1, inplace=True) 15 | 16 | std_limit = 5 17 | highest_limit = 800 18 | 19 | std = np.std(np.array(df.memory.values[1500:])) 20 | highest = df.memory.max() 21 | 22 | if std > std_limit: 23 | raise Exception('MEMORY TEST FAILED: Standard deviation of memory pressure is %d which is above the %d limit' % (std, std_limit)) 24 | 25 | if highest > highest_limit: 26 | raise Exception('MEMORY TEST FAILED: Max memory is %d which is above the %d limit' % (highest, highest_limit)) 27 | 28 | print("\n Memory Pressure Test Passed \n") 29 | --------------------------------------------------------------------------------