├── setup.py ├── README.md ├── LICENSE ├── .gitignore └── adaptive_tools.py /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.6 2 | # -*-Python-*- 3 | 4 | from setuptools import setup 5 | 6 | 7 | setup(name='adaptive_tools', 8 | version='0.2.1', 9 | description='Tools that extend the use of https://github.com/python-adaptive/adaptive.', 10 | url='https://github.com/basnijholt/adaptive_tools', 11 | author='Bas Nijholt', 12 | license='BSD 3-clause', 13 | py_modules=["adaptive_tools"], 14 | ) 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # adaptive-tools 2 | Tools for [python-adaptive](https://github.com/python-adaptive/adaptive). 3 | 4 | 5 | ## Installation 6 | ``` 7 | pip install -U https://github.com/basnijholt/adaptive-tools/archive/master.zip 8 | ``` 9 | 10 | ## Usage 11 | ``` 12 | from adaptive_tools import Learner2D 13 | learner = Learner2D(...) 14 | learner.save('filename.p') 15 | # do your thing 16 | 17 | 18 | # next time 19 | learner = Learner2D(...) 20 | learner.load('filename.p') 21 | ``` 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2018, Bas Nijholt 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | -------------------------------------------------------------------------------- /adaptive_tools.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.6 2 | # -*-Python-*- 3 | 4 | import asyncio 5 | import math 6 | 7 | import adaptive 8 | import toolz 9 | 10 | 11 | ################################################### 12 | # Running multiple runners, each on its own core. # 13 | ################################################### 14 | 15 | def run_learner_in_ipyparallel_client(learner, goal, interval, save_kwargs, client_kwargs, targets=None): 16 | import ipyparallel 17 | import zmq 18 | import adaptive 19 | import asyncio 20 | 21 | client = ipyparallel.Client(context=zmq.Context(), **client_kwargs) 22 | client[:].use_cloudpickle() 23 | loop = asyncio.new_event_loop() 24 | runner = adaptive.Runner(learner, executor=client.executor(targets), goal=goal, ioloop=loop) 25 | save_task = runner.start_periodic_saving(save_kwargs, interval) 26 | loop.run_until_complete(runner.task) 27 | client.shutdown(targets) 28 | return learner 29 | 30 | 31 | default_client_kwargs = dict(profile='pbs', timeout=300, hostname='hpc05') 32 | default_save_kwargs = dict(folder='tmp-{}') 33 | 34 | 35 | def runners_in_executor(learners, client, nrunners, goal=None, interval=3600, 36 | save_kwargs=default_save_kwargs, 37 | client_kwargs=default_client_kwargs): 38 | if goal is None and interval == 0: 39 | raise Exception('Turn on periodic saving if there is no goal.') 40 | 41 | futs = [] 42 | split_leaners = split(learners, nrunners) 43 | split_targets = split(range(nrunners, len(client)), nrunners) 44 | for i, (_learners, targets) in enumerate(zip(split_leaners, split_targets)): 45 | learner = adaptive.BalancingLearner(_learners) 46 | fut = client[i].executor.submit( 47 | run_learner_in_ipyparallel_client, learner, 48 | goal, interval, save_kwargs, client_kwargs, targets 49 | ) 50 | futs.append(fut) 51 | return futs 52 | 53 | 54 | def split(lst, n_parts): 55 | n = math.ceil(len(lst) / n_parts) 56 | return toolz.partition_all(n, lst) 57 | --------------------------------------------------------------------------------