├── .coveragerc ├── .github └── workflows │ └── tests.yml ├── .gitignore ├── .readthedocs.yaml ├── CHANGES.rst ├── LICENSE.txt ├── MANIFEST.in ├── README.rst ├── automan ├── __init__.py ├── api.py ├── automation.py ├── cluster_manager.py ├── conda_cluster_manager.py ├── edm_cluster_manager.py ├── jobs.py ├── tests │ ├── __init__.py │ ├── example.py │ ├── test_automation.py │ ├── test_cluster_manager.py │ ├── test_jobs.py │ └── test_utils.py └── utils.py ├── docs ├── Makefile ├── make.bat ├── requirements.txt └── source │ ├── conf.py │ ├── index.rst │ ├── overview.rst │ ├── reference │ ├── automan.rst │ └── index.rst │ └── tutorial.rst ├── examples ├── edm_conda_cluster │ ├── README.md │ ├── automate_conda.py │ ├── automate_edm.py │ ├── environments.yml │ ├── powers.py │ ├── requirements.txt │ └── square.py └── tutorial │ ├── README.md │ ├── automate1.py │ ├── automate2.py │ ├── automate3.py │ ├── automate4.py │ ├── powers.py │ └── square.py ├── pyproject.toml └── setup.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | source = automan 4 | omit = 5 | */tests/* 6 | automan/api.py 7 | 8 | [report] 9 | exclude_lines = 10 | # Have to re-enable the standard pragma 11 | pragma: no cover 12 | except ImportError: 13 | raise NotImplementedError() 14 | if __name__ == .__main__.: 15 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: pull_request 4 | 5 | jobs: 6 | tests: 7 | strategy: 8 | matrix: 9 | os: [ubuntu-latest, windows-latest, macos-latest] 10 | python-version: [3.8, 3.12] 11 | 12 | runs-on: ${{ matrix.os }} 13 | 14 | steps: 15 | - uses: actions/checkout@v2 16 | - name: Set up Python ${{ matrix.python-version }} 17 | uses: actions/setup-python@v2 18 | with: 19 | python-version: ${{ matrix.python-version }} 20 | - name: Install dependencies 21 | run: | 22 | python -m pip install --upgrade pip 23 | python -m pip install pytest coverage 24 | python -m pip install -v . 25 | - name: Run tests 26 | run: | 27 | coverage erase 28 | coverage run -m pytest -v 29 | - name: Report 30 | if: ${{ success() }} 31 | run: coverage report 32 | - name: Upload Coverage to Codecov 33 | uses: codecov/codecov-action@v1 34 | with: 35 | env_vars: ${{ matrix.os }}, ${{ matrix.python-version }} 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | *~ 3 | .coverage 4 | .cache/ 5 | build/ 6 | dist/ 7 | *.egg-info/ 8 | *.pytest_cache/ 9 | examples/tutorial/config.json 10 | examples/tutorial/outputs 11 | examples/tutorial/manuscript 12 | examples/tutorial/.automan 13 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | build: 4 | os: "ubuntu-24.04" 5 | tools: 6 | python: "3.12" 7 | 8 | sphinx: 9 | configuration: docs/source/conf.py 10 | 11 | python: 12 | install: 13 | - requirements: docs/requirements.txt 14 | - method: pip 15 | path: . 16 | -------------------------------------------------------------------------------- /CHANGES.rst: -------------------------------------------------------------------------------- 1 | 0.6 2 | ~~~~ 3 | 4 | * Release date: 9th July, 2024 5 | * Fix virtualenv download link. 6 | * Fix tests to run on Python 3.12 and to run from an installation. 7 | * Fix bug with job status not working for killed processes. 8 | * Seven PRs were merged. 9 | 10 | 0.5 11 | ~~~~ 12 | 13 | * Release date: 4th November, 2021 14 | * Provide some handy functions to `generate many simulations from parameters 15 | `_. 16 | * Add ability to `add arbitrary tasks/problems to the automator 17 | `_ 18 | using the ``add_task`` method. 19 | * Allow specification of negative ``n_core`` and ``n_thread`` for the job 20 | information. This is documented in the section on `additional computational 21 | resources 22 | `_. 23 | * Improve ability to customize the styles used with ``compare_runs``. 24 | * Add a ``--rm-remote-output`` argument to the command line arguments. 25 | * Add a convenient ``FileCommandTask``. 26 | * Improve ability to customize command line arguments. 27 | * Fix issue with too many processes and open files. 28 | * Fix an issue with command tasks executing on a remote host and waiting. 29 | * Use github actions for tests. 30 | * 12 PRs were merged. 31 | 32 | 33 | 0.4 34 | ~~~~ 35 | 36 | * Release date: 26th November, 2018. 37 | * Support for inter Problem/Simulation/Task dependencies. 38 | * Print more useful messages when running tasks. 39 | * Fix bug with computing the available cores. 40 | * Improve handling of cases when tasks fail with errors. 41 | * Fix a few subtle bugs in the task runner. 42 | * Minor bug fixes. 43 | 44 | 45 | 0.3 46 | ~~~~ 47 | 48 | * Release date: 5th September, 2018. 49 | * Complete online documentation and examples. 50 | * Much improved and generalized cluster management with support for conda and 51 | edm in addition to virtualenvs. 52 | * Support multiple projects that use different bootstrap scripts. 53 | * Better testing for the cluster management. 54 | * Do not rewrite the path to python executables and run them as requested by 55 | the user. 56 | * Removed any lingering references or use of ``pysph``. 57 | * Change the default root to ``automan`` instead of ``pysph_auto``. 58 | * Support filtering cases with a callable. 59 | * Fix bug where a simulation with an error would always be re-run. 60 | * Fix bug caused due to missing ``--nfs`` option to automator CLI. 61 | 62 | 63 | 0.2 64 | ~~~~ 65 | 66 | * Release date: 28th August, 2017. 67 | * First public release of complete working package with features described in 68 | the paper. 69 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Unless otherwise specified by LICENSE.txt files in individual 2 | directories, all code is 3 | 4 | Copyright (c) 2009-2015, the PySPH developers 5 | All rights reserved. 6 | 7 | Redistribution and use in source and binary forms, with or without 8 | modification, are permitted provided that the following conditions are 9 | met: 10 | 11 | 1. Redistributions of source code must retain the above copyright 12 | notice, this list of conditions and the following disclaimer. 13 | 2. Redistributions in binary form must reproduce the above copyright 14 | notice, this list of conditions and the following disclaimer in 15 | the documentation and/or other materials provided with the 16 | distribution. 17 | 3. Neither the name of the copyright holder nor the names of its contributors 18 | may be used to endorse or promote products derived from this software 19 | without specific prior written permission. 20 | 21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include MANIFEST.in *.py *.rst *.txt *.yml *.toml 2 | recursive-include docs *.* 3 | recursive-include examples *.py *.yml *.txt *.md 4 | recursive-exclude examples/tutorial/.automan *.* 5 | recursive-exclude docs/build *.* 6 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | automan: a simple automation framework 2 | -------------------------------------- 3 | 4 | |CI Status| |Coverage Status| |Documentation Status| 5 | 6 | .. |CI Status| image:: https://github.com/pypr/automan/actions/workflows/tests.yml/badge.svg 7 | :target: https://github.com/pypr/automan/actions 8 | 9 | .. |Coverage Status| image:: https://codecov.io/gh/pypr/automan/branch/main/graph/badge.svg 10 | :target: https://codecov.io/gh/pypr/automan 11 | 12 | .. |Documentation Status| image:: https://readthedocs.org/projects/automan/badge/?version=latest 13 | :target: https://automan.readthedocs.io/en/latest/?badge=latest 14 | 15 | 16 | This framework allows you to automate your computational pipelines. 17 | ``automan`` is open source and distributed under the terms of the 3-clause BSD 18 | license. 19 | 20 | Features 21 | -------- 22 | 23 | It is designed to automate the drudge work of managing many numerical 24 | simulations. As an automation framework it does the following: 25 | 26 | - helps you organize your simulations. 27 | - helps you orchestrate running simulations and then post-processing the 28 | results from these. 29 | - helps you reuse code for the post processing of your simulation data. 30 | - execute all your simulations and post-processing with one command. 31 | - optionally distribute your simulations among other computers on your 32 | network. 33 | 34 | This greatly facilitates reproducibility. Automan is written in pure Python 35 | and is easy to install. 36 | 37 | 38 | Installation 39 | ------------- 40 | 41 | You should be able to install automan using pip_ as:: 42 | 43 | $ pip install automan 44 | 45 | If you want to run on the bleeding edge, you may also clone this repository, 46 | change directory into the created directory and run either:: 47 | 48 | $ python setup.py install 49 | 50 | or:: 51 | 52 | $ python setup.py develop 53 | 54 | 55 | .. _pip: https://pip.pypa.io/en/stable/ 56 | 57 | 58 | Documentation 59 | ------------- 60 | 61 | Documentation for this project is available at https://automan.rtfd.io 62 | 63 | There is a paper on ``automan`` that motivates and describes the software: 64 | 65 | - Prabhu Ramachandran, "automan: A Python-Based Automation Framework for 66 | Numerical Computing," in Computing in Science & Engineering, vol. 20, no. 5, 67 | pp. 81-97, 2018. `doi:10.1109/MCSE.2018.05329818 68 | `_ 69 | 70 | A draft of this paper is available here: https://arxiv.org/abs/1712.04786 71 | 72 | There are more than ten research publications that use automan to automate the 73 | entire paper. To see complete examples of these research publications using 74 | this framework, see the following: 75 | 76 | - The EDAC-SPH paper: https://gitlab.com/prabhu/edac_sph 77 | - All the repositories/papers here: https://gitlab.com/pypr 78 | - ML/AI related research paper using automan: https://github.com/nn4pde/SPINN 79 | 80 | The ``README.rst`` in these repositories will document how to set everything 81 | up. The automation script will typically be called ``automate.py``. 82 | 83 | A simpler example project which uses automan is here: 84 | https://github.com/mesnardo/automan-example 85 | 86 | 87 | 88 | The package name 89 | ---------------- 90 | 91 | The name automan comes from an old serial with the same name. Most 92 | other names were taken on pypi. 93 | -------------------------------------------------------------------------------- /automan/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.7.dev0' 2 | -------------------------------------------------------------------------------- /automan/api.py: -------------------------------------------------------------------------------- 1 | from .jobs import Job, Worker, LocalWorker, RemoteWorker, Scheduler # noqa 2 | 3 | from .automation import ( # noqa 4 | Automator, CommandTask, FileCommandTask, Problem, PySPHProblem, PySPHTask, 5 | RunAll, Simulation, SolveProblem, Task, TaskRunner, WrapperTask 6 | ) 7 | 8 | from .utils import ( # noqa 9 | compare_runs, dprod, filter_by_name, filter_cases, mdict, 10 | opts2path 11 | ) 12 | 13 | from .cluster_manager import ClusterManager # noqa 14 | from .conda_cluster_manager import CondaClusterManager # noqa 15 | from .edm_cluster_manager import EDMClusterManager # noqa 16 | -------------------------------------------------------------------------------- /automan/automation.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | from fnmatch import fnmatch 4 | import glob 5 | import json 6 | import os 7 | import shlex 8 | import shutil 9 | import sys 10 | import time 11 | import traceback 12 | 13 | from .jobs import Job 14 | 15 | 16 | class Task(object): 17 | """Basic task to run. Subclass this to do whatever is needed. 18 | 19 | This class is very similar to luigi's Task class. 20 | """ 21 | def __init__(self, depends=None): 22 | # Depends is a list and available for all tasks. 23 | self.depends = depends if depends is not None else [] 24 | 25 | def complete(self): 26 | """Should return True/False indicating success of task. 27 | 28 | If the task was just executed (in this invocation) but failed, raise 29 | any Exception that is a subclass of Exception as this signals an 30 | error to the task execution engine. 31 | 32 | If the task was executed in an earlier invocation of the automation, 33 | then just return True/False so as to be able to re-run the simulation. 34 | """ 35 | return all([os.path.exists(x) for x in self.output()]) 36 | 37 | def output(self): 38 | """Return list of output paths. 39 | """ 40 | return [] 41 | 42 | def run(self, scheduler): 43 | """Run the task, using the given scheduler. 44 | 45 | Using the scheduler is optional but recommended for any long-running 46 | tasks. It is safe to raise an exception immediately when running the 47 | task but for long running tasks, the exception will not matter and the 48 | `complete` method should do. 49 | """ 50 | pass 51 | 52 | def requires(self): 53 | """Return iterable of tasks this task requires. 54 | 55 | It is important that one either return tasks that are idempotent or 56 | return the same instance as this method is called repeatedly. 57 | """ 58 | return self.depends 59 | 60 | 61 | class WrapperTask(Task): 62 | """A task that wraps other tasks and is done when all its requirements 63 | are done. 64 | """ 65 | def complete(self): 66 | return all(r.complete() for r in self.requires()) 67 | 68 | 69 | class TaskRunner(object): 70 | """Run given tasks using the given scheduler. 71 | """ 72 | def __init__(self, tasks, scheduler): 73 | """Constructor. 74 | 75 | **Parameters** 76 | 77 | tasks: iterable of `Task` instances. 78 | scheduler: `automan.jobs.Scheduler` instance 79 | """ 80 | self.scheduler = scheduler 81 | self.todo = [] 82 | self.task_status = dict() 83 | self.task_outputs = set() 84 | self.repeat_tasks = set() 85 | for task in tasks: 86 | self.add_task(task) 87 | 88 | # #### Private protocol ############################################## 89 | 90 | def _check_error_in_running_tasks(self): 91 | running = self._get_tasks_with_status('running') 92 | for task in running: 93 | if self._check_status_of_task(task) == 'error': 94 | return True 95 | return False 96 | 97 | def _check_status_of_requires(self, task): 98 | status = [self._check_status_of_task(t) for t in task.requires()] 99 | 100 | if 'error' in status: 101 | return 'error' 102 | if all(x is True for x in status): 103 | return 'done' 104 | else: 105 | return 'running' 106 | 107 | def _check_status_of_task(self, task): 108 | status = self.task_status.get(task) 109 | if status == 'not started': 110 | return False 111 | elif status == 'done': 112 | return True 113 | else: 114 | complete = False 115 | try: 116 | complete = task.complete() 117 | self.task_status[task] = 'done' if complete else 'running' 118 | except Exception: 119 | complete = 'error' 120 | self.task_status[task] = 'error' 121 | return complete 122 | 123 | def _get_tasks_with_status(self, status): 124 | return [ 125 | t for t, s in self.task_status.items() 126 | if s == status and t not in self.repeat_tasks 127 | ] 128 | 129 | def _is_output_registered(self, task): 130 | # Note, this has a side-effect of registering the task's output 131 | # when called. 132 | output = task.output() 133 | output_str = str(output) 134 | if output and output_str in self.task_outputs: 135 | self.repeat_tasks.add(task) 136 | return True 137 | else: 138 | if output: 139 | self.task_outputs.add(output_str) 140 | return False 141 | 142 | def _run(self, task): 143 | try: 144 | print("\nRunning task %s..." % task) 145 | self.task_status[task] = 'running' 146 | task.run(self.scheduler) 147 | status = 'running' 148 | except Exception: 149 | traceback.print_exc() 150 | status = 'error' 151 | self.task_status[task] = 'error' 152 | return status 153 | 154 | def _show_remaining_tasks(self, replace_line=False): 155 | start, end = ('\r', '') if replace_line else ('', '\n') 156 | running = self._get_tasks_with_status('running') 157 | print("{start}{pending} tasks pending and {running} tasks running". 158 | format( 159 | start=start, pending=len(self.todo), running=len(running) 160 | ), end=end) 161 | sys.stdout.flush() 162 | 163 | def _wait_for_running_tasks(self, wait): 164 | print("\nWaiting for already running tasks...") 165 | running = self._get_tasks_with_status('running') 166 | while len(running) > 0: 167 | for t in running: 168 | self._check_status_of_task(t) 169 | time.sleep(wait) 170 | running = self._get_tasks_with_status('running') 171 | errors = self._get_tasks_with_status('error') 172 | n_err = len(errors) 173 | print("{n_err} jobs had errors.".format(n_err=n_err)) 174 | return n_err 175 | 176 | # #### Public protocol ############################################## 177 | 178 | def add_task(self, task): 179 | if task in self.task_status or self._is_output_registered(task): 180 | # This task is already added or another task produces exactly 181 | # the same output, so do nothing. 182 | return 183 | 184 | if not task.complete(): 185 | self.todo.append(task) 186 | self.task_status[task] = 'not started' 187 | for req in task.requires(): 188 | self.add_task(req) 189 | else: 190 | self.task_status[task] = 'done' 191 | 192 | def run(self, wait=5): 193 | '''Run the tasks that were given. 194 | 195 | Wait for the given amount of time to poll for completed tasks. 196 | 197 | Returns the number of tasks that had errors. 198 | ''' 199 | self._show_remaining_tasks() 200 | status = 'running' 201 | while len(self.todo) > 0 and status != 'error': 202 | to_remove = [] 203 | for i in range(len(self.todo) - 1, -1, -1): 204 | task = self.todo[i] 205 | status = self._check_status_of_requires(task) 206 | if self._check_error_in_running_tasks(): 207 | status = 'error' 208 | 209 | if status == 'error': 210 | break 211 | elif status == 'done': 212 | to_remove.append(task) 213 | status = self._run(task) 214 | 215 | for task in to_remove: 216 | self.todo.remove(task) 217 | 218 | if len(self.todo) > 0: 219 | self._show_remaining_tasks(replace_line=True) 220 | time.sleep(wait) 221 | 222 | n_errors = self._wait_for_running_tasks(wait) 223 | if n_errors == 0: 224 | print("Finished!") 225 | else: 226 | print("Please fix the issues and re-run.") 227 | return n_errors 228 | 229 | 230 | class CommandTask(Task): 231 | """Convenience class to run a command via the framework. The class provides 232 | a method to run the simulation and also check if the simulation is 233 | completed. The command should ideally produce all of its outputs inside an 234 | output directory that is specified. 235 | 236 | """ 237 | 238 | def __init__(self, command, output_dir, job_info=None, depends=None): 239 | """Constructor 240 | 241 | **Parameters** 242 | 243 | command: str or list: command to run; $output_dir is substituted. 244 | output_dir: str : path of output directory. 245 | job_info: dict: dictionary of job information. 246 | depends: list: list of tasks this depends on. 247 | 248 | """ 249 | super().__init__(depends=depends) 250 | if isinstance(command, str): 251 | self.command = shlex.split(command) 252 | else: 253 | self.command = command 254 | self.command = [x.replace('$output_dir', output_dir) 255 | for x in self.command] 256 | self.output_dir = output_dir 257 | self.job_info = job_info if job_info is not None else {} 258 | self.job_proxy = None 259 | self._copy_proc = None 260 | # This is a sentinel set to true when the job is finished 261 | # the data is copied to a local machine and cleaned on the remote. 262 | self._finished = False 263 | # This file will be created if the job exited with an error. 264 | self._error_status_file = os.path.join( 265 | self.output_dir, 'command_exited_with_error' 266 | ) 267 | self._job = None 268 | 269 | def __str__(self): 270 | return ('%s, output in: %s ' % 271 | (self.__class__.__name__, self.output_dir)) 272 | 273 | # #### Public protocol ########################################### 274 | 275 | def complete(self): 276 | """Should return True/False indicating success of task. 277 | """ 278 | job_proxy = self.job_proxy 279 | if job_proxy is None: 280 | return self._is_done() 281 | elif self._finished: 282 | if os.path.exists(self._error_status_file): 283 | raise RuntimeError( 284 | 'Error in task with output in %s.' % self.output_dir 285 | ) 286 | return True 287 | else: 288 | return self._copy_output_and_check_status() 289 | 290 | def run(self, scheduler): 291 | # Remove the error status file if it exists and we are going to run. 292 | if os.path.exists(self._error_status_file): 293 | os.remove(self._error_status_file) 294 | self.job_proxy = scheduler.submit(self.job) 295 | 296 | def clean(self): 297 | """Clean out any generated results. 298 | 299 | This completely removes the output directory. 300 | 301 | """ 302 | if os.path.exists(self.output_dir): 303 | shutil.rmtree(self.output_dir) 304 | 305 | def output(self): 306 | """Return list of output paths. 307 | """ 308 | return [self.output_dir] 309 | 310 | def requires(self): 311 | return self.depends 312 | 313 | # #### Private protocol ########################################### 314 | 315 | @property 316 | def job(self): 317 | if self._job is None: 318 | self._job = Job( 319 | command=self.command, output_dir=self.output_dir, 320 | **self.job_info 321 | ) 322 | return self._job 323 | 324 | def _is_done(self): 325 | """Returns True if the simulation completed. 326 | """ 327 | if (not os.path.exists(self.output_dir)) \ 328 | or os.path.exists(self._error_status_file): 329 | return False 330 | else: 331 | return self.job.status() == 'done' 332 | 333 | def _check_if_copy_complete(self): 334 | proc = self._copy_proc 335 | if proc is None: 336 | # Local job so no copy needed. 337 | return True 338 | else: 339 | if proc.poll() is None: 340 | return False 341 | else: 342 | if self.job_proxy is not None: 343 | self.job_proxy.clean() 344 | self._finished = True 345 | return True 346 | 347 | def _copy_output_and_check_status(self): 348 | jp = self.job_proxy 349 | status = jp.status() 350 | if status == 'done': 351 | if self._copy_proc is None: 352 | self._copy_proc = jp.copy_output('.') 353 | return self._check_if_copy_complete() 354 | elif status == 'error': 355 | cmd = ' '.join(self.command) 356 | msg = '\n***************** ERROR *********************\n' 357 | msg += 'On host %s Job %s failed!' % (jp.worker.host, cmd) 358 | print(msg) 359 | print(jp.get_stderr()) 360 | proc = jp.copy_output('.') 361 | if proc is not None: 362 | proc.wait() 363 | jp.clean() 364 | print('***************** ERROR **********************') 365 | with open(self._error_status_file, 'w') as fp: 366 | fp.write('') 367 | self._finished = True 368 | raise RuntimeError(msg) 369 | return False 370 | 371 | 372 | class PySPHTask(CommandTask): 373 | """Convenience class to run a PySPH simulation via an automation 374 | framework. 375 | 376 | This task automatically adds the output directory specification for pysph 377 | so users to not need to add it. 378 | 379 | """ 380 | 381 | def __init__(self, command, output_dir, job_info=None, depends=None): 382 | """Constructor 383 | 384 | **Parameters** 385 | 386 | command: str or list: command to run; $output_dir is substituted. 387 | output_dir: str : path of output directory. 388 | job_info: dict: dictionary of job information. 389 | depends: list: list of tasks this depends on. 390 | 391 | """ 392 | super(PySPHTask, self).__init__(command, output_dir, job_info, depends) 393 | self.command += ['-d', output_dir] 394 | 395 | # #### Private protocol ########################################### 396 | 397 | def _is_done(self): 398 | """Returns True if the simulation completed. 399 | """ 400 | if not os.path.exists(self.output_dir): 401 | return False 402 | job_status = self.job.status() 403 | if job_status == 'error': 404 | # If job information exists, it trumps everything else 405 | # as it stores the process exit status which is usually 406 | # a much better indicator of the job status. 407 | return False 408 | else: 409 | info_fname = self._get_info_filename() 410 | if not info_fname or not os.path.exists(info_fname): 411 | return False 412 | with open(info_fname) as fp: 413 | d = json.load(fp) 414 | return d.get('completed') 415 | 416 | def _get_info_filename(self): 417 | files = glob.glob(os.path.join(self.output_dir, '*.info')) 418 | if len(files) > 0: 419 | return files[0] 420 | else: 421 | return None 422 | 423 | 424 | class FileCommandTask(CommandTask): 425 | """Convenience class to run a command which produces as output one or more 426 | files. The difference from the CommandTask is that this does not place its 427 | outputs in a separate directory. 428 | 429 | """ 430 | def __init__(self, command, files, job_info=None, depends=None): 431 | """Constructor 432 | 433 | **Parameters** 434 | 435 | command: str or list: command to run; $output_dir is substituted. 436 | output_dir: str : path of output directory. 437 | files: list(str): relative paths of output files. 438 | job_info: dict: dictionary of job information. 439 | depends: list: list of tasks this depends on. 440 | 441 | """ 442 | self.files = files 443 | output_dir = os.path.join(files[0] + '.job_info') 444 | super().__init__( 445 | command, output_dir, job_info=job_info, depends=depends 446 | ) 447 | 448 | def clean(self): 449 | """Clean out any generated results. 450 | 451 | This completely removes the output directory. 452 | 453 | """ 454 | if os.path.exists(self.output_dir): 455 | shutil.rmtree(self.output_dir) 456 | for f in self.files: 457 | if os.path.exists(f): 458 | os.remove(f) 459 | 460 | def output(self): 461 | """Return list of output paths. 462 | """ 463 | return self.files 464 | 465 | 466 | class Problem(object): 467 | """This class represents a numerical problem or computational 468 | problem of interest that needs to be solved. 469 | 470 | The class helps one run a variety of commands (or simulations), 471 | and then assemble/compare the results from those in the `run` 472 | method. This is perhaps easily understood with an example. Let 473 | us say one wishes to run the elliptical drop example problem with 474 | the standard SPH and TVF and compare the results and their 475 | convergence properties while also keep track of the computational 476 | time. To do this one will have to run several simulations, then 477 | collect and process the results. This is achieved by subclassing 478 | this class and implementing the following methods: 479 | 480 | - `get_name(self)`: returns a string of the name of the problem. All 481 | results and simulations are collected inside a directory with 482 | this name. 483 | - `get_commands(self)`: returns a sequence of (directory_name, 484 | command_string, job_info, depends) tuples. These are to be executed 485 | before the `run` method is called. 486 | - `get_requires(self)`: returns a sequence of (name, task) tuples. These 487 | are to be exeuted before the `run` method is called. 488 | - `run(self)`: Processes the completed simulations to make plots etc. 489 | 490 | See the `EllipticalDrop` example class below to see a full implementation. 491 | 492 | """ 493 | 494 | # The Task class to create for the cases, change to suit your needs. 495 | task_cls = CommandTask 496 | 497 | def __init__(self, simulation_dir, output_dir): 498 | """Constructor. 499 | 500 | **Parameters** 501 | 502 | simulation_dir : str : directory where simulation output goes. 503 | output_dir : str : directory where outputs from `run` go. 504 | """ 505 | self.out_dir = output_dir 506 | self.sim_dir = simulation_dir 507 | 508 | # Setup the simulation instances in the cases. 509 | self.cases = None 510 | self.setup() 511 | 512 | def _make_depends(self, depends): 513 | if not depends: 514 | return [] 515 | deps = [] 516 | for x in depends: 517 | if isinstance(x, Task): 518 | deps.append(x) 519 | elif isinstance(x, Simulation): 520 | if x.depends: 521 | my_depends = self._make_depends(x.depends) 522 | else: 523 | my_depends = None 524 | task = self.task_cls( 525 | x.command, self.input_path(x.name), x.job_info, 526 | depends=my_depends 527 | ) 528 | deps.append(task) 529 | else: 530 | raise RuntimeError( 531 | 'Invalid dependency: {0} for problem {1}'.format( 532 | x, self 533 | ) 534 | ) 535 | 536 | return deps 537 | 538 | # #### Public protocol ########################################### 539 | 540 | def input_path(self, *args): 541 | """Given any arguments, relative to the simulation dir, return 542 | the absolute path. 543 | """ 544 | return os.path.join(self.sim_dir, self.get_name(), *args) 545 | 546 | simulation_path = input_path 547 | 548 | def output_path(self, *args): 549 | """Given any arguments relative to the output_dir return the 550 | absolute path. 551 | """ 552 | return os.path.join(self.out_dir, self.get_name(), *args) 553 | 554 | def setup(self): 555 | """Called by init, so add any initialization here. 556 | """ 557 | pass 558 | 559 | def make_output_dir(self): 560 | """Convenience to make the output directory if needed. 561 | """ 562 | base = self.output_path() 563 | if not os.path.exists(base): 564 | os.makedirs(base) 565 | 566 | def get_name(self): 567 | """Return the name of this problem, this name is used as a 568 | directory for the simulation and the outputs. 569 | """ 570 | # Return a sane default instead of forcing the user to do this. 571 | return self.__class__.__name__ 572 | 573 | def get_commands(self): 574 | """Return a sequence of (name, command_string, job_info_dict) 575 | or (name, command_string, job_info_dict, depends). 576 | 577 | The name represents the command being run and is used as a subdirectory 578 | for generated output. 579 | 580 | The command_string is the command that needs to be run. 581 | 582 | The job_info_dict is a dictionary with any additional info to be used 583 | by the job, these are additional arguments to the 584 | `automan.jobs.Job` class. It may be None if nothing special need 585 | be passed. 586 | 587 | The depends is any dependencies this simulation has in terms of other 588 | simulations/tasks. 589 | 590 | """ 591 | if self.cases is not None: 592 | return [ 593 | (x.name, x.command, x.job_info, x.depends) for x in self.cases 594 | ] 595 | else: 596 | return [] 597 | 598 | def get_requires(self): 599 | """Return a sequence of tuples of form (name, task). 600 | 601 | The name represents the command being run and is used as 602 | a subdirectory for generated output. 603 | 604 | The task is a `automan.automation.Task` instance. 605 | """ 606 | base = self.get_name() 607 | result = [] 608 | for cmd_info in self.get_commands(): 609 | name, cmd, job_info = cmd_info[:3] 610 | deps = cmd_info[3] if len(cmd_info) == 4 else [] 611 | sim_output_dir = self.input_path(name) 612 | depends = self._make_depends(deps) 613 | task = self.task_cls( 614 | cmd, sim_output_dir, job_info, depends=depends 615 | ) 616 | task_name = '%s.%s' % (base, name) 617 | result.append((task_name, task)) 618 | return result 619 | 620 | def get_outputs(self): 621 | """Get a list of outputs generated by this problem. By default it 622 | returns the output directory (as a single element of a list). 623 | """ 624 | return [self.output_path()] 625 | 626 | def run(self): 627 | """Run any analysis code for the simulations completed. This 628 | is usually run after the simulation commands are completed. 629 | """ 630 | pass 631 | 632 | def clean(self): 633 | """Cleanup any generated output from the analysis code. This does not 634 | clean the output of any nested commands. 635 | """ 636 | for path in self.get_outputs(): 637 | if os.path.exists(path): 638 | if os.path.isdir(path): 639 | shutil.rmtree(path) 640 | elif os.path.isfile(path): 641 | os.remove(path) 642 | 643 | 644 | class PySPHProblem(Problem): 645 | task_cls = PySPHTask 646 | 647 | 648 | def key_to_option(key): 649 | """Convert a dictionary key to a valid command line option. This simply 650 | replaces underscores with dashes. 651 | """ 652 | return key.replace('_', '-') 653 | 654 | 655 | def kwargs_to_command_line(kwargs): 656 | """Convert a dictionary of keyword arguments to a list of command-line 657 | options. If the value of the key is None, no value is passed. 658 | 659 | **Examples** 660 | 661 | >>> sorted(kwargs_to_command_line(dict(some_arg=1, something_else=None))) 662 | ['--some-arg=1', '--something-else'] 663 | """ 664 | cmd_line = [] 665 | for key, value in kwargs.items(): 666 | option = key_to_option(key) 667 | if value is None: 668 | arg = "--{option}".format(option=option) 669 | else: 670 | arg = "--{option}={value}".format( 671 | option=option, value=str(value) 672 | ) 673 | 674 | cmd_line.append(arg) 675 | return cmd_line 676 | 677 | 678 | class Simulation(object): 679 | """A convenient class to abstract code for a particular simulation. 680 | Simulation objects are typically created by ``Problem`` instances in order 681 | to abstract and simulate repetitive code for a particular simulation. 682 | 683 | For example if one were comparing the elliptical_drop example, one could 684 | instantiate a Simulation object as follows:: 685 | 686 | >>> s = Simlation('outputs/sph', 'pysph run elliptical_drop') 687 | 688 | One can pass any additional command line arguments as follows:: 689 | 690 | >>> s = Simlation( 691 | ... 'outputs/sph', 'pysph run elliptical_drop', timestep=0.005 692 | ... ) 693 | >>> s.command 694 | 'pysph run elliptical_drop --timestep=0.001' 695 | >>> s.input_path('results.npz') 696 | 'outputs/sph/results.npz' 697 | 698 | The extra parameters can be used to filter and compare different 699 | simulations. One can define additional plot methods for a particular 700 | subclass and use these to easily plot results for different cases. 701 | 702 | One can also pass any additional parameters to the `automan.jobs.Job` 703 | class via the job_info kwarg so as to run the command suitably. For 704 | example:: 705 | 706 | >>> s = Simlation('outputs/sph', 'pysph run elliptical_drop', 707 | ... job_info=dict(n_thread=4)) 708 | 709 | The object has other methods that are convenient when comparing plots. 710 | Along with the ``compare_cases``, ``filter_cases`` and ``filter_by_name`` 711 | this is an extremely powerful way to automate and compare results. 712 | 713 | """ 714 | def __init__(self, root, base_command, job_info=None, depends=None, **kw): 715 | """Constructor 716 | 717 | **Parameters** 718 | 719 | root: str 720 | Path to simulation output directory. 721 | base_command: str 722 | Base command to run. 723 | job_info: dict 724 | Extra arguments to the `automan.jobs.Job` class. 725 | depends: list 726 | List of other simulations/tasks this simulation depends on. 727 | **kw: dict 728 | Additional parameters to pass to command. 729 | """ 730 | self.root = root 731 | self.name = os.path.basename(root) 732 | self.base_command = base_command 733 | self.job_info = job_info 734 | self.depends = depends if depends is not None else [] 735 | self.params = dict(kw) 736 | self._results = None 737 | 738 | def input_path(self, *args): 739 | """Given any arguments, relative to the simulation dir, return 740 | the absolute path. 741 | """ 742 | return os.path.join(self.root, *args) 743 | 744 | @property 745 | def command(self): 746 | return self.base_command + ' ' + self.get_command_line_args() 747 | 748 | @property 749 | def data(self): 750 | if self._results is None: 751 | import numpy 752 | self._results = numpy.load(self.input_path('results.npz')) 753 | return self._results 754 | 755 | def get_labels(self, labels): 756 | render = self.render_parameter 757 | if isinstance(labels, str): 758 | return render(labels) 759 | else: 760 | s = [render(x) for x in labels] 761 | s = [x for x in s if len(x) > 0] 762 | return r', '.join(s) 763 | 764 | def kwargs_to_command_line(self, kwargs): 765 | return kwargs_to_command_line(kwargs) 766 | 767 | def get_command_line_args(self): 768 | return ' '.join(self.kwargs_to_command_line(self.params)) 769 | 770 | def render_parameter(self, param): 771 | """Return string to be used for labels for given parameter. 772 | """ 773 | if param not in self.params: 774 | return '' 775 | value = self.params[param] 776 | if value is None: 777 | return r'%s' % param 778 | else: 779 | return r'%s=%s' % (param, self.params[param]) 780 | 781 | 782 | ############################################################################ 783 | # Convenient classes that can be used to easily automate a collection 784 | # of problems. 785 | 786 | class SolveProblem(Task): 787 | """Solves a particular `Problem`. This runs all the commands that the 788 | problem requires and then runs the problem instance's run method. 789 | 790 | The match argument is a string which when provided helps run only a subset 791 | of the requirements for the problem. 792 | 793 | The force argument specifies that the problem should be cleaned, so as to 794 | re-run any post-processing. 795 | """ 796 | 797 | def __init__(self, problem, match='', force=False, depends=None): 798 | super().__init__(depends=depends) 799 | self.problem = problem 800 | self.match = match 801 | self.force = force 802 | if self.force: 803 | self.problem.clean() 804 | self._requires = [ 805 | self._make_task(task) 806 | for name, task in self.problem.get_requires() 807 | if len(match) == 0 or fnmatch(name, match) 808 | ] 809 | 810 | def _make_task(self, obj): 811 | if isinstance(obj, Task): 812 | return obj 813 | elif isinstance(obj, Problem): 814 | return SolveProblem( 815 | problem=obj, match=self.match, force=self.force 816 | ) 817 | elif isinstance(obj, type) and issubclass(obj, Problem): 818 | problem = obj(self.problem.sim_dir, self.problem.out_dir) 819 | return SolveProblem( 820 | problem=problem, match=self.match, force=self.force 821 | ) 822 | else: 823 | raise RuntimeError( 824 | 'Unknown requirement: {0}, for problem: {1}.'.format( 825 | obj, self.problem 826 | ) 827 | ) 828 | 829 | def __str__(self): 830 | return 'Problem named %s' % self.problem.get_name() 831 | 832 | def complete(self): 833 | if len(self.match) == 0: 834 | return super(SolveProblem, self).complete() 835 | else: 836 | return all(r.complete() for r in self.requires()) 837 | 838 | def output(self): 839 | return self.problem.get_outputs() 840 | 841 | def run(self, scheduler): 842 | if len(self.match) == 0: 843 | self.problem.run() 844 | 845 | def requires(self): 846 | return self._requires + self.depends 847 | 848 | 849 | class RunAll(WrapperTask): 850 | """Solves a given collection of problems. 851 | """ 852 | 853 | def __init__(self, simulation_dir, output_dir, problem_classes, 854 | force=False, match='', depends=None): 855 | super().__init__(depends=depends) 856 | self.simulation_dir = simulation_dir 857 | self.output_dir = output_dir 858 | self.force = force 859 | self.match = match 860 | self.problems = self._make_problems(problem_classes) 861 | self._requires = self._get_requires() 862 | 863 | # #### Private protocol ############################################### 864 | 865 | def _get_requires(self): 866 | return [ 867 | SolveProblem(problem=x, match=self.match, force=self.force) 868 | for x in self.problems 869 | ] 870 | 871 | def _make_problems(self, problem_classes): 872 | problems = [] 873 | for klass in problem_classes: 874 | problem = klass(self.simulation_dir, self.output_dir) 875 | problems.append(problem) 876 | return problems 877 | 878 | # #### Public protocol ################################################ 879 | 880 | def requires(self): 881 | return self._requires + self.depends 882 | 883 | 884 | class Automator(object): 885 | """Main class to automate a collection of problems. 886 | 887 | This processess command line options and runs all tasks with a scheduler 888 | that is configured using the ``config.json`` file if it is present. Here is 889 | typical usage:: 890 | 891 | >>> all_problems = [EllipticalDrop] 892 | >>> automator = Automator('outputs', 'figures', all_problems) 893 | >>> automator.run() 894 | 895 | The class also creates a `automan.cluster_manager.ClusterManager` 896 | instance and integrates the cluster management features as well. This 897 | allows a user to automate their results across a collection of remote 898 | machines accessible only by ssh. 899 | 900 | """ 901 | def __init__(self, simulation_dir, output_dir, all_problems, 902 | cluster_manager_factory=None): 903 | """Constructor. 904 | 905 | **Parameters** 906 | 907 | simulation_dir : str 908 | Root directory to generate simulation results in. 909 | output_dir: str 910 | Root directory where outputs will be generated by Problem 911 | instances. 912 | all_problems: sequence of `Problem` classes. 913 | Sequence of problem classes to automate. 914 | cluster_manager_factory: callable 915 | Callable should return `cluster_manager.ClusterManager` instance. 916 | None will use the default one. 917 | """ 918 | self.simulation_dir = simulation_dir 919 | self.output_dir = output_dir 920 | self.all_problems = all_problems 921 | self.named_tasks = {} 922 | self.tasks = [] 923 | self.post_proc_tasks = [] 924 | self.runner = None 925 | self.cluster_manager = None 926 | self.runall_task = None 927 | self._args = None 928 | if cluster_manager_factory is None: 929 | from automan.cluster_manager import ClusterManager 930 | self.cluster_manager_factory = ClusterManager 931 | else: 932 | self.cluster_manager_factory = cluster_manager_factory 933 | 934 | # #### Public Protocol ######################################## 935 | 936 | def add_task(self, task, name=None, post_proc=False): 937 | """Add a task or a problem instance to also execute. 938 | 939 | If the `name` is specified then it is a treated as a named task wherein 940 | it must be only invoked explicitly via the command line when asked. 941 | 942 | If `post_proc` is True then the task is given an additional dependency 943 | if possible such that the task is run after the `RunAll` task is 944 | completed. 945 | 946 | **Parameters** 947 | 948 | task: Task or Problem instance: Task or Problem to add. 949 | name: str: name of the task (optional). 950 | post_proc: bool: Add a dependency to the task with the RunAll task. 951 | 952 | """ 953 | if isinstance(task, type) and issubclass(task, Problem): 954 | p = task( 955 | simulation_dir=self.simulation_dir, output_dir=self.output_dir 956 | ) 957 | _task = SolveProblem(p) 958 | elif isinstance(task, Problem): 959 | _task = SolveProblem(task) 960 | elif isinstance(task, Task): 961 | _task = task 962 | else: 963 | raise ValueError( 964 | 'Invalid task: must be Problem class/instance or Task.' 965 | ) 966 | if name is not None: 967 | self.named_tasks[name] = _task 968 | else: 969 | self.tasks.append(_task) 970 | 971 | if post_proc: 972 | self.post_proc_tasks.append(_task) 973 | 974 | def run(self, argv=None): 975 | """Start the automation. 976 | """ 977 | self._setup(argv) 978 | self._setup_tasks() 979 | self.runner.run() 980 | 981 | # #### Private Protocol ######################################## 982 | 983 | def _check_positional_arguments(self, problems): 984 | names = [c.__name__ for c in self.all_problems] + ['all'] 985 | lower_names = [x.lower() for x in names] 986 | lower_names.extend(list(self.named_tasks.keys())) 987 | for p in problems: 988 | if p.lower() not in lower_names: 989 | print("ERROR: %s not a valid problem/task!" % p) 990 | print("Valid names are %s" % ', '.join(names)) 991 | self.parser.exit(1) 992 | 993 | def _get_exclude_paths(self): 994 | """Returns a list of exclude paths suitable for passing on to rsync to 995 | exclude syncing some directories on remote machines. 996 | """ 997 | paths = [] 998 | for path in [self.simulation_dir, self.output_dir]: 999 | if not path.endswith('/'): 1000 | paths.append(path + '/') 1001 | return paths 1002 | 1003 | def _parse_args(self, argv): 1004 | '''Parse command line arguments. 1005 | 1006 | Override this when the CLI arguments are customized. 1007 | ''' 1008 | self._args = self.parser.parse_args(argv) 1009 | return self._args 1010 | 1011 | def _select_problem_classes(self, problems): 1012 | if 'all' in problems: 1013 | return self.all_problems 1014 | else: 1015 | lower_names = [x.lower() for x in problems] 1016 | return [cls for cls in self.all_problems 1017 | if cls.__name__.lower() in lower_names] 1018 | 1019 | def _setup(self, argv): 1020 | if self.runner is None: 1021 | self._setup_argparse() 1022 | args = self._parse_args(argv) 1023 | 1024 | self._check_positional_arguments(args.problem) 1025 | 1026 | self.cluster_manager = self.cluster_manager_factory( 1027 | config_fname=args.config, 1028 | exclude_paths=self._get_exclude_paths() 1029 | ) 1030 | from .cluster_manager import BootstrapError 1031 | 1032 | if len(args.host) > 0: 1033 | try: 1034 | self.cluster_manager.add_worker( 1035 | args.host, args.home, args.nfs 1036 | ) 1037 | except BootstrapError: 1038 | pass 1039 | return 1040 | elif len(args.host) == 0 and args.update_remote: 1041 | self.cluster_manager.update(not args.no_rebuild) 1042 | elif len(args.rm_remote_output) > 0: 1043 | self.cluster_manager.delete( 1044 | self.simulation_dir, args.rm_remote_output) 1045 | 1046 | problem_classes = self._select_problem_classes(args.problem) 1047 | task = RunAll( 1048 | simulation_dir=self.simulation_dir, 1049 | output_dir=self.output_dir, 1050 | problem_classes=problem_classes, 1051 | force=args.force, match=args.match 1052 | ) 1053 | self.runall_task = task 1054 | 1055 | self.scheduler = self.cluster_manager.create_scheduler() 1056 | self.runner = TaskRunner([task], self.scheduler) 1057 | 1058 | def _setup_argparse(self): 1059 | import argparse 1060 | desc = "Automation script to run simulations." 1061 | parser = argparse.ArgumentParser( 1062 | description=desc 1063 | ) 1064 | all_problem_names = [c.__name__ for c in self.all_problems] 1065 | all_problem_names += list(self.named_tasks.keys()) + ['all'] 1066 | parser.add_argument( 1067 | 'problem', nargs='*', default=["all"], 1068 | help="Specifies problem/task to run as a string " 1069 | "(case-insensitive), valid names are %s. " 1070 | "Defaults to running all of the problems." 1071 | % all_problem_names 1072 | ) 1073 | 1074 | parser.add_argument( 1075 | '-a', '--add-node', action="store", dest="host", type=str, 1076 | default='', help="Add a new remote worker." 1077 | ) 1078 | parser.add_argument( 1079 | '-c', '--config', action="store", dest="config", 1080 | default="config.json", help="Configuration file to use." 1081 | ) 1082 | parser.add_argument( 1083 | '--home', action="store", dest="home", type=str, 1084 | default='', 1085 | help='Home directory of the remote worker (to be used with -a)' 1086 | ) 1087 | parser.add_argument( 1088 | '--nfs', action="store_true", dest="nfs", 1089 | default=False, 1090 | help=('Does the remote remote worker share the filesystem ' 1091 | '(to be used with -a)') 1092 | ) 1093 | parser.add_argument( 1094 | '-f', '--force', action="store_true", default=False, dest='force', 1095 | help='Redo the plots even if they were already made.' 1096 | ) 1097 | parser.add_argument( 1098 | '-m', '--match', action="store", type=str, default='', 1099 | dest='match', help="Name of the problem to run (uses fnmatch)" 1100 | ) 1101 | parser.add_argument( 1102 | '--no-rebuild', action="store_true", 1103 | dest="no_rebuild", default=False, 1104 | help="Do not rebuild the sources on update, just update the files." 1105 | ) 1106 | parser.add_argument( 1107 | '-u', '--update-remote', action='store_true', 1108 | dest='update_remote', default=False, 1109 | help='Update remote worker machines.' 1110 | ) 1111 | parser.add_argument( 1112 | '--rm-remote-output', nargs='*', action="store", 1113 | dest="rm_remote_output", type=str, default='', 1114 | help="remove output folder from the mentioned machines use" 1115 | "'all' to remove from all host (except localhost and host where " 1116 | "nfs is true)" 1117 | ) 1118 | 1119 | self.parser = parser 1120 | 1121 | def _setup_tasks(self): 1122 | for task in self.post_proc_tasks: 1123 | task.depends.append(self.runall_task) 1124 | 1125 | # Add generic tasks. 1126 | for task in self.tasks: 1127 | self.runner.add_task(task) 1128 | 1129 | # Add named tasks only if specifically requested on CLI. 1130 | for name, task in self.named_tasks.items(): 1131 | if name in self._args.problem: 1132 | self.runner.add_task(task) 1133 | 1134 | # Reset the tasks so we can use the automator interactively. 1135 | self.post_proc_tasks = [] 1136 | self.tasks = [] 1137 | self.named_tasks = {} 1138 | -------------------------------------------------------------------------------- /automan/cluster_manager.py: -------------------------------------------------------------------------------- 1 | """Code to bootstrap and update the project so a remote host can be used as a 2 | worker to help with the automation of tasks. 3 | 4 | This requires ssh/scp and rsync to work on all machines. 5 | 6 | This is currently only tested on Linux machines. 7 | 8 | """ 9 | 10 | import json 11 | import os 12 | import shlex 13 | import stat 14 | import subprocess 15 | import sys 16 | from textwrap import dedent 17 | 18 | try: 19 | from urllib import urlopen 20 | except ImportError: 21 | from urllib.request import urlopen 22 | 23 | 24 | class BootstrapError(Exception): 25 | pass 26 | 27 | 28 | class ClusterManager(object): 29 | """The cluster manager class. 30 | 31 | This class primarily helps setup software on a remote worker machine such 32 | that it can run any computational jobs from the automation framework. 33 | 34 | The general directory structure of a remote worker machine is as follows:: 35 | 36 | remote_home/ # Could be ~ 37 | automan/ # Root of automation directory (configurable) 38 | envs/ # python virtual environments for use. 39 | my_project/ # Current directory for specific projects. 40 | 41 | The project directories are synced from this machine to the remote worker. 42 | 43 | The "my_project" is the root of the directory with the automation script 44 | and this should contain the required sources that need to be executed. One 45 | can use a list of source directories which will be copied over but it is 46 | probably most convenient to put it all in the root of the project directory 47 | to keep everything self-contained. 48 | 49 | The `ClusterManager` class manages these remote workers by helping setup 50 | the directories, bootstrapping the Python virtualenv and also keeping these 51 | up-to-date as project directory is changed on the local machine. 52 | 53 | The class therefore has two primary public methods, 54 | 55 | 1. `add_worker(self, host, home, nfs)` which adds a new worker machine by 56 | bootstrapping the machine with the software and the appropriate source 57 | directories. 58 | 59 | 2. `update()`, which keeps the directory and software up-to-date. 60 | 61 | The class variables BOOTSTRAP and UPDATE are the content of scripts 62 | uploaded to these machines and should be extended by users to do what they 63 | wish. 64 | 65 | The class creates a ``config.json`` in the current working directory that 66 | may be edited by a user. It also creates a directory called 67 | ``.{self.root}`` which defaults to ``.automan``. The bootstrap and 68 | update scripts are put here and may be edited by the user for any new 69 | hosts. 70 | 71 | One may override the `_get_python, _get_helper_scripts`, and 72 | `_get_bootstrap_code, _get_update_code` methods to change this to use other 73 | package managers like edm or conda. See the conda_cluster_manager for an 74 | example. 75 | 76 | """ 77 | 78 | ####################################################### 79 | # These scripts are used to bootstrap the installation 80 | # and update them. 81 | BOOTSTRAP = dedent("""\ 82 | #!/bin/bash 83 | 84 | set -e 85 | if hash virtualenv 2>/dev/null; then 86 | virtualenv -p python3 --system-site-packages envs/{project_name} 87 | else 88 | python3 virtualenv.pyz --system-site-packages envs/{project_name} 89 | fi 90 | source envs/{project_name}/bin/activate 91 | 92 | pip install automan 93 | 94 | # Run any requirements.txt from the user 95 | cd {project_name} 96 | if [ -f "requirements.txt" ] ; then 97 | pip install -r requirements.txt 98 | fi 99 | """) 100 | 101 | UPDATE = dedent("""\ 102 | #!/bin/bash 103 | 104 | set -e 105 | source envs/{project_name}/bin/activate 106 | # Run any requirements.txt from the user 107 | cd {project_name} 108 | if [ -f "requirements.txt" ] ; then 109 | pip install -r requirements.txt 110 | fi 111 | """) 112 | ####################################################### 113 | 114 | def __init__(self, root='automan', sources=None, 115 | config_fname='config.json', exclude_paths=None, 116 | testing=False): 117 | """Create a cluster manager instance. 118 | 119 | **Parameters** 120 | 121 | root: str 122 | The name of the root directory where all the files on the remote 123 | will be created. 124 | sources: list 125 | A list of source directories to sync. 126 | config_fname: str 127 | The name of the config file to create. 128 | exclude_paths: list 129 | A list of paths to exclude while syncing. This is in a form suitable 130 | to pass to rsync. 131 | testing: bool 132 | Use this while testing. This allows us to run unit tests for remotes 133 | on the local machine. 134 | 135 | """ 136 | self.root = root 137 | self.workers = [] 138 | self.sources = sources 139 | self.scripts_dir = os.path.abspath('.' + self.root) 140 | self.exclude_paths = exclude_paths if exclude_paths else [] 141 | self.testing = testing 142 | 143 | # This is setup by the config and is the name of 144 | # the project directory. 145 | self.project_name = None 146 | 147 | # The config file will always trump any direct settings 148 | # unless there is no config file. 149 | self.config_fname = config_fname 150 | self._read_config() 151 | if not os.path.exists(self.scripts_dir): 152 | os.makedirs(self.scripts_dir) 153 | 154 | # ### Private Protocol ######################################## 155 | def _bootstrap(self, host, home): 156 | helper_scripts = self._get_helper_scripts() 157 | base_cmd = ("cd {home}; mkdir -p {root}/envs; " 158 | "mkdir -p {root}/{project_name}/.{root}").format( 159 | home=home, root=self.root, 160 | project_name=self.project_name 161 | ) 162 | self._ssh_run_command(host, base_cmd) 163 | 164 | abs_root = os.path.join(home, self.root) 165 | if helper_scripts: 166 | real_host = '' if self.testing else '{host}:'.format(host=host) 167 | cmd = "scp {helper_scripts} {host}{root}".format( 168 | host=real_host, root=abs_root, helper_scripts=helper_scripts 169 | ) 170 | self._run_command(cmd) 171 | 172 | self._update_sources(host, home) 173 | 174 | cmd = "cd {abs_root}; ./{project_name}/.{root}/bootstrap.sh".format( 175 | abs_root=abs_root, root=self.root, project_name=self.project_name 176 | ) 177 | try: 178 | self._ssh_run_command(host, cmd) 179 | except subprocess.CalledProcessError: 180 | msg = dedent(""" 181 | ****************************************************************** 182 | Bootstrapping of remote host {host} failed. 183 | All files have been copied to the host. 184 | 185 | Please take a look at 186 | {abs_root}/{project_name}/.{root}/bootstrap.sh 187 | and try to fix it. 188 | 189 | You should run it from within the {root} directory as: 190 | 191 | ./{project_name}/.{root}/bootstrap.sh 192 | 193 | Once the bootstrap.sh script runs successfully, the worker can be 194 | used without any further steps. 195 | 196 | The default bootstrap script is in 197 | {scripts_dir} 198 | and can be edited by you. These will be used for any new hosts 199 | you add. 200 | ****************************************************************** 201 | """.format(abs_root=abs_root, root=self.root, host=host, 202 | scripts_dir=self.scripts_dir, 203 | project_name=self.project_name) 204 | ) 205 | print(msg) 206 | raise BootstrapError(msg) 207 | else: 208 | print("Bootstrapping {host} succeeded!".format(host=host)) 209 | 210 | def _get_bootstrap_code(self): 211 | return self.BOOTSTRAP.format(project_name=self.project_name) 212 | 213 | def _get_python(self, host, home): 214 | return os.path.join( 215 | home, self.root, 216 | 'envs/{project_name}/bin/python'.format( 217 | project_name=self.project_name 218 | ) 219 | ) 220 | 221 | def _get_update_code(self): 222 | return self.UPDATE.format(project_name=self.project_name) 223 | 224 | def _get_helper_scripts(self): 225 | """Return a space separated string of script files that you need copied over to 226 | the remote host. 227 | 228 | When overriding this, you can return None or '' if you do not need any. 229 | 230 | """ 231 | script = os.path.join(self.scripts_dir, 'virtualenv.pyz') 232 | if not os.path.exists(script): 233 | print("Downloading latest virtualenv.pyz") 234 | url = 'https://bootstrap.pypa.io/virtualenv.pyz' 235 | opener = urlopen(url) 236 | with open(script, 'wb') as f: 237 | f.write(opener.read()) 238 | return script 239 | 240 | def _read_config(self): 241 | if os.path.exists(self.config_fname): 242 | with open(self.config_fname) as f: 243 | data = json.load(f) 244 | self.root = data['root'] 245 | self.project_name = data['project_name'] 246 | self.sources = data['sources'] 247 | self.workers = data['workers'] 248 | else: 249 | if self.sources is None or len(self.sources) == 0: 250 | project_dir = os.path.abspath(os.getcwd()) 251 | self.project_name = os.path.basename(project_dir) 252 | self.sources = [project_dir] 253 | self.workers = [dict(host='localhost', home='', nfs=False)] 254 | self._write_config() 255 | self.scripts_dir = os.path.abspath('.' + self.root) 256 | 257 | def _rebuild(self, host, home): 258 | abs_root = os.path.join(home, self.root) 259 | base_cmd = "cd {abs_root}; ./{project_name}/.{root}/update.sh".format( 260 | abs_root=abs_root, root=self.root, project_name=self.project_name 261 | ) 262 | self._ssh_run_command(host, base_cmd) 263 | 264 | def _run_command(self, cmd, **kw): 265 | print(cmd) 266 | subprocess.check_call(shlex.split(cmd), **kw) 267 | 268 | def _ssh_run_command(self, host, base_cmd): 269 | if self.testing: 270 | command = base_cmd 271 | print(command) 272 | subprocess.check_call(command, shell=True) 273 | else: 274 | command = "ssh {host} '{cmd}'".format(host=host, cmd=base_cmd) 275 | self._run_command(command) 276 | 277 | def _sync_dir(self, host, src, dest): 278 | options = "" 279 | kwargs = dict() 280 | if os.path.isdir(os.path.join(src, '.git')): 281 | exclude = 'git -C {src} ls-files --exclude-standard -oi '\ 282 | '--directory '.format(src=src) 283 | options = '--exclude-from=-' 284 | proc = subprocess.Popen( 285 | shlex.split(exclude), 286 | stdout=subprocess.PIPE 287 | ) 288 | kwargs['stdin'] = proc.stdout 289 | if self.exclude_paths: 290 | for path in self.exclude_paths: 291 | options += ' --exclude="%s"' % path 292 | 293 | real_host = '' if self.testing else '{host}:'.format(host=host) 294 | command = "rsync -a {options} {src} {host}{dest} ".format( 295 | options=options, src=src, host=real_host, dest=dest 296 | ) 297 | self._run_command(command, **kwargs) 298 | 299 | def _update_sources(self, host, home): 300 | for local_dir in self.sources: 301 | remote_dir = os.path.join(home, self.root + '/') 302 | self._sync_dir(host, local_dir, remote_dir) 303 | 304 | scripts_dir = self.scripts_dir 305 | bootstrap_code = self._get_bootstrap_code() 306 | update_code = self._get_update_code() 307 | scripts = {'bootstrap.sh': bootstrap_code, 308 | 'update.sh': update_code} 309 | for script, code in scripts.items(): 310 | fname = os.path.join(scripts_dir, script) 311 | if not os.path.exists(fname): 312 | # Create the scripts if they don't exist. 313 | with open(fname, 'w') as f: 314 | f.write(code) 315 | 316 | script_files = [os.path.join(scripts_dir, x) for x in scripts] 317 | for fname in script_files: 318 | mode = os.stat(fname).st_mode 319 | os.chmod(fname, mode | stat.S_IXUSR | stat.S_IXGRP) 320 | 321 | path = os.path.join(home, self.root, self.project_name, 322 | '.' + self.root) 323 | real_host = '' if self.testing else '{host}:'.format(host=host) 324 | cmd = "scp {script_files} {host}{path}".format( 325 | host=real_host, path=path, script_files=' '.join(script_files) 326 | ) 327 | self._run_command(cmd) 328 | 329 | def _delete_outputs(self, host, home, sim_dir): 330 | path = os.path.join(home, self.root, self.project_name, 331 | sim_dir) 332 | real_host = '' if self.testing else '{host}'.format(host=host) 333 | cmd = "rm -rf {path}".format( 334 | path=path 335 | ) 336 | self._ssh_run_command(real_host, cmd) 337 | 338 | def _write_config(self): 339 | print("Writing %s" % self.config_fname) 340 | data = dict( 341 | project_name=self.project_name, 342 | root=self.root, 343 | sources=self.sources, 344 | workers=self.workers 345 | ) 346 | with open(self.config_fname, 'w') as f: 347 | json.dump(data, f, indent=2) 348 | 349 | # ### Public Protocol ######################################## 350 | 351 | def add_worker(self, host, home, nfs): 352 | if host == 'localhost': 353 | self.workers.append(dict(host=host, home=home, nfs=nfs)) 354 | else: 355 | curdir = os.path.basename(os.getcwd()) 356 | if nfs: 357 | python = sys.executable 358 | chdir = curdir 359 | else: 360 | python = self._get_python(host, home) 361 | chdir = os.path.join(home, self.root, curdir) 362 | self.workers.append( 363 | dict(host=host, home=home, nfs=nfs, python=python, chdir=chdir) 364 | ) 365 | 366 | self._write_config() 367 | if host != 'localhost' and not nfs: 368 | self._bootstrap(host, home) 369 | 370 | def update(self, rebuild=True): 371 | for worker in self.workers: 372 | host = worker.get('host') 373 | home = worker.get('home') 374 | nfs = worker.get('nfs', False) 375 | if host != 'localhost' and not nfs: 376 | self._update_sources(host, home) 377 | if rebuild: 378 | self._rebuild(host, home) 379 | 380 | def delete(self, sim_dir, remotes): 381 | hosts = [h.get('host') for h in self.workers] 382 | if remotes[-1] == 'all': 383 | remotes = hosts.copy() 384 | else: 385 | for remote in remotes: 386 | if remote not in hosts: 387 | print('%s remote is not a worker' % (remote)) 388 | sys.exit(1) 389 | 390 | for worker in self.workers: 391 | host = worker.get('host') 392 | home = worker.get('home') 393 | nfs = worker.get('nfs', False) 394 | if host in remotes: 395 | if host != 'localhost' and not nfs: 396 | self._delete_outputs(host, home, sim_dir) 397 | 398 | 399 | def create_scheduler(self): 400 | """Return a `automan.jobs.Scheduler` from the configuration. 401 | """ 402 | from .jobs import Scheduler 403 | 404 | scheduler = Scheduler(root='.') 405 | for worker in self.workers: 406 | host = worker.get('host') 407 | nfs = worker.get('nfs', False) 408 | if host == 'localhost': 409 | scheduler.add_worker(dict(host='localhost')) 410 | else: 411 | python = worker.get('python') 412 | chdir = worker.get('chdir') 413 | config = dict(host=host, python=python, chdir=chdir, nfs=nfs) 414 | if self.testing: 415 | config['testing'] = True 416 | scheduler.add_worker(config) 417 | return scheduler 418 | 419 | def cli(self, argv=None): 420 | """This is just a demonstration of how this class could be used. 421 | """ 422 | import argparse 423 | parser = argparse.ArgumentParser(description='Setup remote workers.') 424 | 425 | parser.add_argument( 426 | '-a', '--add-node', action="store", dest="host", type=str, 427 | default='', help="Add a new remote worker." 428 | ) 429 | parser.add_argument( 430 | '--home', action="store", dest="home", type=str, 431 | default='', 432 | help='Home directory of the remote worker (to be used with -a)' 433 | ) 434 | parser.add_argument( 435 | '--nfs', action="store_true", dest="nfs", 436 | default=False, 437 | help=('Does the remote remote worker share the filesystem ' 438 | '(to be used with -a)') 439 | ) 440 | parser.add_argument( 441 | '--no-rebuild', action="store_true", dest="no_rebuild", 442 | default=False, help="Do not rebuild the sources on sync." 443 | ) 444 | 445 | args = parser.parse_args(argv) 446 | 447 | if len(args.host) == 0: 448 | self.update(not args.no_rebuild) 449 | else: 450 | self.add_worker(args.host, args.home, args.nfs) 451 | -------------------------------------------------------------------------------- /automan/conda_cluster_manager.py: -------------------------------------------------------------------------------- 1 | import os 2 | from textwrap import dedent 3 | 4 | from .cluster_manager import ClusterManager 5 | 6 | 7 | class CondaClusterManager(ClusterManager): 8 | 9 | # The path to conda root on the remote, this is a relative path 10 | # and is relative to the home directory. 11 | CONDA_ROOT = 'miniconda3' 12 | 13 | BOOTSTRAP = dedent("""\ 14 | #!/bin/bash 15 | 16 | set -e 17 | CONDA_ROOT={conda_root} 18 | ENV_FILE="{project_name}/environments.yml" 19 | if [ -f $ENV_FILE ] ; then 20 | ~/$CONDA_ROOT/bin/conda env create -q -f $ENV_FILE -n {project_name} 21 | else 22 | ~/$CONDA_ROOT/bin/conda create -y -q -n {project_name} psutil execnet 23 | fi 24 | 25 | source ~/$CONDA_ROOT/bin/activate {project_name} 26 | pip install automan 27 | 28 | cd {project_name} 29 | if [ -f "requirements.txt" ] ; then 30 | pip install -r requirements.txt 31 | fi 32 | """) 33 | 34 | UPDATE = dedent("""\ 35 | #!/bin/bash 36 | 37 | set -e 38 | CONDA_ROOT={conda_root} 39 | ENV_FILE="{project_name}/environments.yml" 40 | if [ -f $ENV_FILE ] ; then 41 | ~/$CONDA_ROOT/bin/conda env update -q -f $ENV_FILE -n {project_name} 42 | fi 43 | 44 | source ~/$CONDA_ROOT/bin/activate {project_name} 45 | 46 | cd {project_name} 47 | if [ -f "requirements.txt" ] ; then 48 | pip install -r requirements.txt 49 | fi 50 | """) 51 | 52 | def _get_bootstrap_code(self): 53 | return self.BOOTSTRAP.format( 54 | project_name=self.project_name, conda_root=self.CONDA_ROOT 55 | ) 56 | 57 | def _get_python(self, host, home): 58 | return os.path.join( 59 | home, self.CONDA_ROOT, 60 | 'envs/{project_name}/bin/python'.format( 61 | project_name=self.project_name 62 | ) 63 | ) 64 | 65 | def _get_update_code(self): 66 | return self.UPDATE.format( 67 | project_name=self.project_name, conda_root=self.CONDA_ROOT 68 | ) 69 | 70 | def _get_helper_scripts(self): 71 | """Return a space separated string of script files that you need copied over to 72 | the remote host. 73 | 74 | When overriding this, you can return None or '' if you do not need any. 75 | 76 | """ 77 | return '' 78 | -------------------------------------------------------------------------------- /automan/edm_cluster_manager.py: -------------------------------------------------------------------------------- 1 | import os 2 | from textwrap import dedent 3 | 4 | from .cluster_manager import ClusterManager 5 | 6 | 7 | class EDMClusterManager(ClusterManager): 8 | 9 | # The path to edm root on the remote, this is a relative path 10 | # and is relative to the home directory. 11 | EDM_ROOT = '.edm' 12 | 13 | ENV_FILE = "bundled_env.json" 14 | 15 | BOOTSTRAP = dedent("""\ 16 | #!/bin/bash 17 | 18 | set -e 19 | ENV_FILE="{project_name}/{env_file}" 20 | 21 | if hash edm 2>/dev/null; then 22 | EDM_EXE=edm 23 | else 24 | EDM_EXE=~/{edm_root}/bin/edm 25 | fi 26 | 27 | if [ -f $ENV_FILE ] ; then 28 | $EDM_EXE -q envs import --force {project_name} -f $ENV_FILE 29 | else 30 | $EDM_EXE -q envs create --force {project_name} --version 3.6 31 | $EDM_EXE -q install psutil execnet -y -e {project_name} 32 | fi 33 | 34 | $EDM_EXE run -e {project_name} -- pip install automan 35 | 36 | cd {project_name} 37 | if [ -f "requirements.txt" ] ; then 38 | $EDM_EXE run -e {project_name} -- pip install -r requirements.txt 39 | fi 40 | """) 41 | 42 | UPDATE = dedent("""\ 43 | #!/bin/bash 44 | 45 | set -e 46 | ENV_FILE="{project_name}/{env_file}" 47 | 48 | if hash edm 2>/dev/null; then 49 | EDM_EXE=edm 50 | else 51 | EDM_EXE=~/{edm_root}/bin/edm 52 | fi 53 | 54 | if [ -f $ENV_FILE ] ; then 55 | $EDM_EXE -q envs import --force {project_name} -f $ENV_FILE 56 | fi 57 | 58 | cd {project_name} 59 | if [ -f "requirements.txt" ] ; then 60 | $EDM_EXE run -e {project_name} -- pip install -r requirements.txt 61 | fi 62 | """) 63 | 64 | def _get_bootstrap_code(self): 65 | return self.BOOTSTRAP.format( 66 | project_name=self.project_name, edm_root=self.EDM_ROOT, 67 | env_file=self.ENV_FILE 68 | ) 69 | 70 | def _get_python(self, host, home): 71 | return os.path.join( 72 | home, self.EDM_ROOT, 73 | 'envs/{project_name}/bin/python'.format( 74 | project_name=self.project_name 75 | ) 76 | ) 77 | 78 | def _get_update_code(self): 79 | return self.UPDATE.format( 80 | project_name=self.project_name, edm_root=self.EDM_ROOT, 81 | env_file=self.ENV_FILE 82 | ) 83 | 84 | def _get_helper_scripts(self): 85 | """Return a space separated string of script files that you need copied over to 86 | the remote host. 87 | 88 | When overriding this, you can return None or '' if you do not need any. 89 | 90 | """ 91 | return '' 92 | -------------------------------------------------------------------------------- /automan/jobs.py: -------------------------------------------------------------------------------- 1 | # Standard libraray imports 2 | from __future__ import print_function 3 | 4 | from collections import deque 5 | import json 6 | import multiprocessing 7 | import os 8 | import shlex 9 | import shutil 10 | import subprocess 11 | import sys 12 | import time 13 | 14 | # External module imports. 15 | import psutil 16 | 17 | 18 | def _make_command_list(command): 19 | if not isinstance(command, (list, tuple)): 20 | return shlex.split(command) 21 | else: 22 | return command 23 | 24 | 25 | def free_cores(): 26 | free = (1.0 - psutil.cpu_percent(interval=0.5)/100.) 27 | ncore = free*psutil.cpu_count(logical=False) 28 | return round(ncore, 0) 29 | 30 | 31 | def total_cores(): 32 | return psutil.cpu_count(logical=False) 33 | 34 | 35 | def cores_required(n_core): 36 | if n_core < 0: 37 | return int(total_cores()/(-n_core)) 38 | else: 39 | return n_core 40 | 41 | 42 | def threads_required(n_thread, n_core): 43 | if n_thread < 0: 44 | return int(cores_required(n_core)*(-n_thread)) 45 | else: 46 | return n_thread 47 | 48 | 49 | class Job(object): 50 | def __init__(self, command, output_dir, n_core=1, n_thread=1, env=None): 51 | """Constructor 52 | 53 | Note that `n_core` is used to schedule a task on a machine which has 54 | that many free cores. This is not used to run the job but only used by 55 | the scheduler. The number can be any integer. When the number is 56 | negative, it will use the value of `total_cores()/(-n_core)`. This 57 | value may be used to set the number of threads as discussed below. 58 | 59 | `n_thread` is used to set the `OMP_NUM_THREADS`. Note that if 60 | `n_thread` is set to `None`, the environment variable is not set. If a 61 | positive integer is given that specific number is used. If the number 62 | is negative, then the number of threads is set to `n_core*(-n_thread)`, 63 | i.e. the product of the number of cores and the negative of the number 64 | given. 65 | 66 | """ 67 | self.command = _make_command_list(command) 68 | self._given_env = env 69 | self.env = dict(os.environ) 70 | if env is not None: 71 | self.env.update(env) 72 | if n_thread is not None: 73 | nt = threads_required(n_thread, n_core) 74 | self.env['OMP_NUM_THREADS'] = str(nt) 75 | self.n_core = n_core 76 | self.n_thread = n_thread 77 | self.output_dir = output_dir 78 | self.output_already_exists = os.path.exists(self.output_dir) 79 | self.stderr = os.path.join(self.output_dir, 'stderr.txt') 80 | self.stdout = os.path.join(self.output_dir, 'stdout.txt') 81 | self._info_file = os.path.join(self.output_dir, 'job_info.json') 82 | self.proc = None 83 | 84 | def substitute_in_command(self, basename, substitute): 85 | """Replace occurrence of given basename with the substitute. 86 | 87 | This is useful where the user asks to run ['python', 'script.py'] and 88 | we wish to change the 'python' to a specific Python. Normally this is 89 | not needed as the PATH is set to pick up the right Python. However, in 90 | the rare cases where this rewriting is needed, this method is 91 | available. 92 | 93 | """ 94 | args = [] 95 | for arg in self.command: 96 | if os.path.basename(arg) == basename: 97 | args.append(substitute) 98 | else: 99 | args.append(arg) 100 | self.command = args 101 | 102 | def to_dict(self): 103 | state = dict() 104 | for key in ('command', 'output_dir', 'n_core', 'n_thread'): 105 | state[key] = getattr(self, key) 106 | state['env'] = self._given_env 107 | return state 108 | 109 | def pretty_command(self): 110 | return ' '.join(self.command) 111 | 112 | def get_stderr(self): 113 | with open(self.stderr) as fp: 114 | return fp.read() 115 | 116 | def get_stdout(self): 117 | with open(self.stdout) as fp: 118 | return fp.read() 119 | 120 | def get_info(self): 121 | return self._read_info() 122 | 123 | def _write_info(self, info): 124 | with open(self._info_file, 'w') as fp: 125 | json.dump(info, fp) 126 | 127 | def _read_info(self): 128 | if not os.path.exists(self._info_file): 129 | return {'status': 'not started'} 130 | with open(self._info_file, 'r') as fp: 131 | try: 132 | return json.load(fp) 133 | except ValueError: 134 | return {'status': 'running'} 135 | 136 | def _run(self): # pragma: no cover 137 | # This is run in a multiprocessing.Process instance so does not 138 | # get covered. 139 | stdout = open(self.stdout, 'wb') 140 | stderr = open(self.stderr, 'wb') 141 | 142 | proc = subprocess.Popen( 143 | self.command, stdout=stdout, stderr=stderr, env=self.env 144 | ) 145 | 146 | info = dict( 147 | start=time.ctime(), end='', status='running', 148 | exitcode=None, pid=proc.pid 149 | ) 150 | self._write_info(info) 151 | 152 | proc.wait() 153 | status = 'error' if proc.returncode != 0 else 'done' 154 | info.update(end=time.ctime(), status=status, exitcode=proc.returncode) 155 | self._write_info(info) 156 | stdout.close() 157 | stderr.close() 158 | 159 | def run(self): 160 | if not os.path.exists(self.output_dir): 161 | os.makedirs(self.output_dir) 162 | self._write_info(dict(status='running', pid=None)) 163 | self.proc = multiprocessing.Process( 164 | target=self._run 165 | ) 166 | self.proc.start() 167 | 168 | def join(self): 169 | self.proc.join() 170 | 171 | def status(self): 172 | info = self._read_info() 173 | if self.proc is None and info.get('status') == 'running': 174 | # Either the process creating the job or the job itself 175 | # was killed. 176 | pid = info.get('pid') 177 | if pid is not None: 178 | try: 179 | proc = psutil.Process(pid) 180 | if not proc.is_running(): 181 | return 'error' 182 | except psutil.NoSuchProcess: 183 | return 'error' 184 | elif self.proc is not None and info.get('status') != 'running': 185 | if not self.proc.is_alive(): 186 | self.join() 187 | self.proc = None 188 | return info.get('status') 189 | 190 | def clean(self, force=False): 191 | if self.output_already_exists and not force: 192 | if os.path.exists(self.stdout): 193 | os.remove(self.stdout) 194 | os.remove(self.stderr) 195 | elif os.path.exists(self.output_dir): 196 | shutil.rmtree(self.output_dir) 197 | 198 | 199 | ############################################ 200 | # This class is meant to be used by execnet alone. 201 | class _RemoteManager(object): # pragma: no cover 202 | # This is run via execnet so coverage does not catch these. 203 | # This is used by the RemoteWorker and that is tested, so we should 204 | # be safe not explicitly covering this. 205 | def __init__(self): 206 | self.jobs = dict() 207 | self.job_count = 0 208 | self._setup_path() 209 | 210 | def _setup_path(self): 211 | py_dir = os.path.dirname(sys.executable) 212 | env_path = os.environ.get('PATH').split(os.pathsep) 213 | if py_dir not in env_path: 214 | env_path.insert(0, py_dir) 215 | os.environ['PATH'] = os.pathsep.join(env_path) 216 | 217 | def run(self, job_data): 218 | job = Job(**job_data) 219 | job.run() 220 | ret_val = self.job_count 221 | self.jobs[ret_val] = job 222 | self.job_count += 1 223 | return ret_val 224 | 225 | def status(self, job_id): 226 | if job_id in self.jobs: 227 | return self.jobs[job_id].status() 228 | else: 229 | return 'invalid job id %d' % job_id 230 | 231 | def clean(self, job_id, force=False): 232 | if job_id in self.jobs: 233 | return self.jobs[job_id].clean(force) 234 | else: 235 | return 'invalid job id %d' % job_id 236 | 237 | def get_stdout(self, job_id): 238 | return self.jobs[job_id].get_stdout() 239 | 240 | def get_stderr(self, job_id): 241 | return self.jobs[job_id].get_stderr() 242 | 243 | def get_info(self, job_id): 244 | return self.jobs[job_id].get_info() 245 | 246 | 247 | def serve(channel): # pragma: no cover 248 | """Serve the remote manager via execnet. 249 | """ 250 | manager = _RemoteManager() 251 | while True: 252 | msg, data = channel.receive() 253 | if msg == 'free_cores': 254 | channel.send(free_cores()) 255 | elif msg == 'total_cores': 256 | channel.send(total_cores()) 257 | else: 258 | channel.send(getattr(manager, msg)(*data)) 259 | ############################################ 260 | 261 | 262 | class Worker(object): 263 | def __init__(self): 264 | self.jobs = dict() 265 | self.running_jobs = set() 266 | self._total_cores = None 267 | 268 | def _check_running_jobs(self): 269 | for i in self.running_jobs.copy(): 270 | self.status(i) 271 | 272 | def free_cores(self): 273 | return free_cores() 274 | 275 | def cores_required(self, n_core): 276 | if n_core < 0: 277 | return int(self.total_cores()/(-n_core)) 278 | else: 279 | return n_core 280 | 281 | def total_cores(self): 282 | if self._total_cores is None: 283 | self._total_cores = total_cores() 284 | return self._total_cores 285 | 286 | def can_run(self, req_core): 287 | """Returns True if the worker can run a job with the required cores. 288 | """ 289 | n_core = self.cores_required(req_core) 290 | if n_core == 0: 291 | return True 292 | free = self.free_cores() 293 | result = False 294 | if free >= n_core: 295 | self._check_running_jobs() 296 | jobs = self.jobs 297 | n_cores_used = sum( 298 | [self.cores_required(jobs[i].n_core) 299 | for i in self.running_jobs] 300 | ) 301 | if (self.total_cores() - n_cores_used) >= n_core: 302 | result = True 303 | return result 304 | 305 | def run(self, job): 306 | """Runs the job and returns a JobProxy for the job.""" 307 | raise NotImplementedError() 308 | 309 | def status(self, job_id): 310 | """Returns status of the job.""" 311 | raise NotImplementedError() 312 | 313 | def copy_output(self, job_id, dest): 314 | raise NotImplementedError() 315 | 316 | def clean(self, job_id, force=False): 317 | raise NotImplementedError() 318 | 319 | def get_stdout(self, job_id): 320 | raise NotImplementedError() 321 | 322 | def get_stderr(self, job_id): 323 | raise NotImplementedError() 324 | 325 | def get_info(self, job_id): 326 | raise NotImplementedError() 327 | 328 | 329 | class JobProxy(object): 330 | def __init__(self, worker, job_id, job): 331 | self.worker = worker 332 | self.job_id = job_id 333 | self.job = job 334 | 335 | def free_cores(self): 336 | return self.worker.free_cores() 337 | 338 | def total_cores(self): 339 | return self.worker.total_cores() 340 | 341 | def run(self): 342 | print("JobProxy cannot be run") 343 | 344 | def status(self): 345 | return self.worker.status(self.job_id) 346 | 347 | def copy_output(self, dest): 348 | return self.worker.copy_output(self.job_id, dest) 349 | 350 | def clean(self, force=False): 351 | return self.worker.clean(self.job_id, force) 352 | 353 | def get_stdout(self): 354 | return self.worker.get_stdout(self.job_id) 355 | 356 | def get_stderr(self): 357 | return self.worker.get_stderr(self.job_id) 358 | 359 | def get_info(self): 360 | return self.worker.get_info(self.job_id) 361 | 362 | 363 | class LocalWorker(Worker): 364 | def __init__(self): 365 | super(LocalWorker, self).__init__() 366 | self.host = 'localhost' 367 | self.job_count = 0 368 | 369 | def get_config(self): 370 | return dict(host='localhost') 371 | 372 | def run(self, job): 373 | count = self.job_count 374 | print("Running %s" % job.pretty_command()) 375 | self.jobs[count] = job 376 | self.running_jobs.add(count) 377 | job.run() 378 | self.job_count += 1 379 | return JobProxy(self, count, job) 380 | 381 | def status(self, job_id): 382 | s = self.jobs[job_id].status() 383 | rj = self.running_jobs 384 | if s != 'running': 385 | rj.discard(job_id) 386 | return s 387 | 388 | def copy_output(self, job_id, dest): 389 | return 390 | 391 | def clean(self, job_id, force=False): 392 | if force: 393 | self.jobs[job_id].clean(force) 394 | 395 | def get_stdout(self, job_id): 396 | return self.jobs[job_id].get_stdout() 397 | 398 | def get_stderr(self, job_id): 399 | return self.jobs[job_id].get_stderr() 400 | 401 | def get_info(self, job_id): 402 | return self.jobs[job_id].get_info() 403 | 404 | 405 | class RemoteWorker(Worker): 406 | def __init__(self, host, python, chdir=None, testing=False, 407 | nfs=False): 408 | super(RemoteWorker, self).__init__() 409 | self.host = host 410 | self.python = python 411 | self.chdir = chdir 412 | self.testing = testing 413 | self.nfs = nfs 414 | if testing: 415 | spec = 'popen//python={python}'.format(python=python) 416 | else: 417 | spec = 'ssh={host}//python={python}'.format( 418 | host=host, python=python 419 | ) 420 | if chdir is not None: 421 | spec += '//chdir={chdir}'.format(chdir=chdir) 422 | 423 | import execnet 424 | self.gw = execnet.makegateway(spec) 425 | self.channel = self.gw.remote_exec( 426 | "from automan import jobs; jobs.serve(channel)" 427 | ) 428 | 429 | def get_config(self): 430 | return dict(host=self.host, python=self.python, chdir=self.chdir) 431 | 432 | def _call_remote(self, method, *data): 433 | ch = self.channel 434 | ch.send((method, data)) 435 | return ch.receive() 436 | 437 | def free_cores(self): 438 | return self._call_remote('free_cores', None) 439 | 440 | def total_cores(self): 441 | if self._total_cores is None: 442 | self._total_cores = self._call_remote('total_cores', None) 443 | return self._total_cores 444 | 445 | def run(self, job): 446 | print("Running %s" % job.pretty_command()) 447 | job_id = self._call_remote('run', job.to_dict()) 448 | self.jobs[job_id] = job 449 | self.running_jobs.add(job_id) 450 | return JobProxy(self, job_id, job) 451 | 452 | def status(self, job_id): 453 | s = self._call_remote('status', job_id) 454 | rj = self.running_jobs 455 | if s != 'running': 456 | rj.discard(job_id) 457 | return s 458 | 459 | def copy_output(self, job_id, dest): 460 | job = self.jobs[job_id] 461 | if self.testing: 462 | src = os.path.join(self.chdir, job.output_dir) 463 | real_dest = os.path.join(dest, job.output_dir) 464 | args = [ 465 | sys.executable, '-c', 466 | 'import sys,shutil; shutil.copytree(sys.argv[1], sys.argv[2])', 467 | src, real_dest 468 | ] 469 | elif not self.nfs: 470 | src = '{host}:{path}'.format( 471 | host=self.host, path=os.path.join(self.chdir, job.output_dir) 472 | ) 473 | real_dest = os.path.join(dest, os.path.dirname(job.output_dir)) 474 | args = ['scp', '-qr', src, real_dest] 475 | else: 476 | args = [] 477 | 478 | if args: 479 | print("\n" + " ".join(args)) 480 | proc = subprocess.Popen(args) 481 | return proc 482 | else: 483 | return 484 | 485 | def clean(self, job_id, force=False): 486 | return self._call_remote('clean', job_id, force) 487 | 488 | def get_stdout(self, job_id): 489 | return self._call_remote('get_stdout', job_id) 490 | 491 | def get_stderr(self, job_id): 492 | return self._call_remote('get_stderr', job_id) 493 | 494 | def get_info(self, job_id): 495 | return self._call_remote('get_info', job_id) 496 | 497 | 498 | class Scheduler(object): 499 | def __init__(self, root='.', worker_config=(), wait=5): 500 | self.workers = deque() 501 | self.worker_config = list(worker_config) 502 | self.root = os.path.abspath(os.path.expanduser(root)) 503 | self.wait = wait 504 | self._completed_jobs = [] 505 | self.jobs = [] 506 | 507 | def _create_worker(self): 508 | conf = self.worker_config[len(self.workers)] 509 | host = conf.get('host') 510 | print("Starting worker on %s." % host) 511 | if host == 'localhost': 512 | w = LocalWorker() 513 | else: 514 | w = RemoteWorker(**conf) 515 | self.workers.append(w) 516 | return w 517 | 518 | def _get_active_workers(self): 519 | completed = [] 520 | workers = set() 521 | for job in self.jobs: 522 | if job.status() in ['error', 'done']: 523 | completed.append(job) 524 | else: 525 | workers.add(job.worker.host) 526 | 527 | for job in completed: 528 | self.jobs.remove(job) 529 | self._completed_jobs.append(job) 530 | 531 | return workers 532 | 533 | def _rotate_existing_workers(self): 534 | worker = self.workers[0] 535 | self.workers.rotate(-1) 536 | return worker 537 | 538 | def _get_worker(self, n_core): 539 | n_configs = len(self.worker_config) 540 | n_running = len(self.workers) 541 | if n_running == n_configs: 542 | worker = self._rotate_existing_workers() 543 | else: 544 | active_workers = self._get_active_workers() 545 | if n_running > len(active_workers): 546 | for w in self.workers: 547 | if (w.host not in active_workers) and \ 548 | w.can_run(n_core): 549 | worker = w 550 | break 551 | else: 552 | worker = self._create_worker() 553 | else: 554 | worker = self._create_worker() 555 | return worker 556 | 557 | def save(self, fname): 558 | config = dict(root=self.root) 559 | config['workers'] = self.worker_config 560 | with open(fname, 'w') as fp: 561 | json.dump(config, fp, indent=2) 562 | 563 | def load(self, fname): 564 | with open(fname) as fp: 565 | config = json.load(fp) 566 | self.root = config.get('root') 567 | self.worker_config = config.get('workers') 568 | 569 | def add_worker(self, conf): 570 | self.worker_config.append(conf) 571 | 572 | def submit(self, job): 573 | proxy = None 574 | slept = False 575 | while proxy is None: 576 | for i in range(len(self.worker_config)): 577 | worker = self._get_worker(job.n_core) 578 | if worker.can_run(job.n_core): 579 | if slept: 580 | print() 581 | slept = False 582 | print("Job run by %s" % worker.host) 583 | proxy = worker.run(job) 584 | self.jobs.append(proxy) 585 | break 586 | else: 587 | time.sleep(self.wait) 588 | slept = True 589 | print("\rWaiting for free worker ...", end='') 590 | sys.stdout.flush() 591 | return proxy 592 | -------------------------------------------------------------------------------- /automan/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pypr/automan/e0e1abbda4323b1ef306f4aa2b0d54702ded3a93/automan/tests/__init__.py -------------------------------------------------------------------------------- /automan/tests/example.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import sys 4 | 5 | 6 | def write_pysph_data(directory): 7 | info_fname = os.path.join(directory, 'example.info') 8 | data = { 9 | 'completed': True, 10 | 'output_dir': directory, 11 | 'fname': 'example' 12 | } 13 | with open(info_fname, 'w') as fp: 14 | json.dump(data, fp) 15 | 16 | with open(os.path.join(directory, 'results.dat'), 'w') as fp: 17 | fp.write(str(sys.argv[1:])) 18 | 19 | 20 | def main(): 21 | from argparse import ArgumentParser 22 | p = ArgumentParser() 23 | p.add_argument( 24 | '-d', '--directory', default='example_output', 25 | help='Output directory' 26 | ) 27 | p.add_argument( 28 | '--update-h', action="store_true", dest='update_h', 29 | help='Update h' 30 | ) 31 | p.add_argument( 32 | '--no-update-h', action="store_false", dest='update_h', 33 | help='Do not update h' 34 | ) 35 | o = p.parse_args() 36 | 37 | if not os.path.exists(o.directory): 38 | os.mkdir(o.directory) 39 | 40 | write_pysph_data(o.directory) 41 | 42 | 43 | if __name__ == '__main__': 44 | main() 45 | -------------------------------------------------------------------------------- /automan/tests/test_automation.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import os 4 | import sys 5 | import tempfile 6 | import unittest 7 | 8 | try: 9 | from unittest import mock 10 | except ImportError: 11 | import mock 12 | 13 | from automan.automation import ( 14 | Automator, CommandTask, FileCommandTask, Problem, PySPHProblem, RunAll, 15 | Simulation, SolveProblem, TaskRunner 16 | ) 17 | try: 18 | from automan.jobs import Scheduler, RemoteWorker 19 | except ImportError: 20 | raise unittest.SkipTest('test_jobs requires psutil') 21 | 22 | from automan.cluster_manager import ClusterManager 23 | 24 | from automan.tests.test_jobs import wait_until, safe_rmtree 25 | 26 | 27 | class MySimulation(Simulation): 28 | @property 29 | def data(self): 30 | if self._results is None: 31 | with open(self.input_path('results.dat')) as fp: 32 | self._results = fp.read() 33 | return self._results 34 | 35 | 36 | class EllipticalDrop(PySPHProblem): 37 | """We define a simple example problem which we will run using the 38 | automation framework. 39 | 40 | In this case we run two variants of the elliptical drop problem. 41 | 42 | The setup method defines the cases to run which are simply Simulation 43 | instances. 44 | 45 | The get_commands returns the actual commands to run. 46 | 47 | The run method does the post-processing, after the simulations are done. 48 | 49 | """ 50 | 51 | def get_name(self): 52 | return 'elliptical_drop' 53 | 54 | def setup(self): 55 | # Two cases, one with update_h and one without. 56 | cmd = 'python -m automan.tests.example' 57 | 58 | # If self.cases is set, the get_requires method will do the right 59 | # thing. 60 | self.cases = [ 61 | MySimulation( 62 | root=self.input_path('update_h'), 63 | base_command=cmd, 64 | job_info=dict(n_core=1, n_thread=1), 65 | update_h=None 66 | ), 67 | MySimulation( 68 | root=self.input_path('no_update_h'), 69 | base_command=cmd, 70 | job_info=dict(n_core=1, n_thread=1), 71 | no_update_h=None 72 | ), 73 | ] 74 | 75 | def run(self): 76 | self.make_output_dir() 77 | no_update = self.cases[0].data 78 | update = self.cases[1].data 79 | output = open(self.output_path('result.txt'), 'w') 80 | output.write('no_update_h: %s\n' % no_update) 81 | output.write('update_h: %s\n' % update) 82 | output.close() 83 | 84 | 85 | class TestAutomationBase(unittest.TestCase): 86 | def setUp(self): 87 | self.cwd = os.getcwd() 88 | self.root = tempfile.mkdtemp() 89 | os.chdir(self.root) 90 | self.sim_dir = 'sim' 91 | self.output_dir = 'output' 92 | patch = mock.patch( 93 | 'automan.jobs.free_cores', return_value=2 94 | ) 95 | patch.start() 96 | self.addCleanup(patch.stop) 97 | 98 | def tearDown(self): 99 | os.chdir(self.cwd) 100 | if os.path.exists(self.root): 101 | safe_rmtree(self.root) 102 | 103 | 104 | class TestTaskRunner(TestAutomationBase): 105 | def _make_scheduler(self): 106 | worker = dict(host='localhost') 107 | s = Scheduler(root='.', worker_config=[worker], wait=0.1) 108 | return s 109 | 110 | def _get_time(self, path): 111 | with open(os.path.join(path, 'stdout.txt')) as f: 112 | t = float(f.read()) 113 | return t 114 | 115 | @mock.patch('automan.jobs.total_cores', return_value=2) 116 | def test_task_runner_waits_for_tasks_in_the_end(self, m_t_cores): 117 | # Given 118 | s = self._make_scheduler() 119 | cmd = 'python -c "import sys, time; time.sleep(0.1); sys.exit(1)"' 120 | ct1_dir = os.path.join(self.sim_dir, '1') 121 | ct2_dir = os.path.join(self.sim_dir, '2') 122 | ct3_dir = os.path.join(self.sim_dir, '3') 123 | ct1 = CommandTask(cmd, output_dir=ct1_dir) 124 | ct2 = CommandTask(cmd, output_dir=ct2_dir) 125 | ct3 = CommandTask(cmd, output_dir=ct3_dir) 126 | 127 | # When 128 | t = TaskRunner(tasks=[ct1, ct2, ct3], scheduler=s) 129 | n_errors = t.run(wait=0.1) 130 | 131 | # Then 132 | # All the tasks may have been run but those that ran will fail. 133 | self.assertEqual(n_errors + len(t.todo), 3) 134 | self.assertTrue(n_errors > 0) 135 | 136 | @mock.patch('automan.jobs.total_cores', return_value=2) 137 | def test_task_runner_checks_for_error_in_running_tasks(self, m_t_cores): 138 | # Given 139 | s = self._make_scheduler() 140 | cmd = 'python -c "import sys, time; time.sleep(0.1); sys.exit(1)"' 141 | ct1_dir = os.path.join(self.sim_dir, '1') 142 | ct2_dir = os.path.join(self.sim_dir, '2') 143 | ct3_dir = os.path.join(self.sim_dir, '3') 144 | job_info = dict(n_core=2, n_thread=2) 145 | ct1 = CommandTask(cmd, output_dir=ct1_dir, job_info=job_info) 146 | ct2 = CommandTask(cmd, output_dir=ct2_dir, job_info=job_info) 147 | ct3 = CommandTask(cmd, output_dir=ct3_dir, job_info=job_info) 148 | 149 | # When 150 | t = TaskRunner(tasks=[ct1, ct2, ct3], scheduler=s) 151 | 152 | # Then 153 | self.assertEqual(len(t.todo), 3) 154 | 155 | # When 156 | n_errors = t.run(wait=0.1) 157 | 158 | # Then 159 | # In this case, two tasks should have run and one should not have run 160 | # as the other two had errors. 161 | self.assertEqual(n_errors, 2) 162 | self.assertEqual(len(t.todo), 1) 163 | self.assertTrue(os.path.exists(ct3_dir)) 164 | self.assertTrue(os.path.exists(ct2_dir)) 165 | self.assertFalse(os.path.exists(ct1_dir)) 166 | 167 | @mock.patch('automan.jobs.total_cores', return_value=2) 168 | def test_task_runner_doesnt_block_on_problem_with_error(self, m_t_cores): 169 | # Given 170 | class A(Problem): 171 | def get_requires(self): 172 | cmd = ('python -c "import sys, time; time.sleep(0.1); ' 173 | 'sys.exit(1)"') 174 | ct = CommandTask(cmd, output_dir=self.input_path()) 175 | return [('task1', ct)] 176 | 177 | def run(self): 178 | self.make_output_dir() 179 | 180 | s = self._make_scheduler() 181 | 182 | # When 183 | task = RunAll( 184 | simulation_dir=self.sim_dir, output_dir=self.output_dir, 185 | problem_classes=[A] 186 | ) 187 | t = TaskRunner(tasks=[task], scheduler=s) 188 | n_error = t.run(wait=0.1) 189 | 190 | # Then 191 | self.assertTrue(n_error > 0) 192 | 193 | # For the case where there is a match expression 194 | # When 195 | problem = A(self.sim_dir, self.output_dir) 196 | problem.clean() 197 | 198 | task = RunAll( 199 | simulation_dir=self.sim_dir, output_dir=self.output_dir, 200 | problem_classes=[A], match='*task1' 201 | ) 202 | t = TaskRunner(tasks=[task], scheduler=s) 203 | n_error = t.run(wait=0.1) 204 | 205 | # Then 206 | self.assertTrue(n_error > 0) 207 | 208 | def test_task_runner_does_not_add_repeated_tasks(self): 209 | # Given 210 | s = self._make_scheduler() 211 | cmd = 'python -c "print(1)"' 212 | ct1 = CommandTask(cmd, output_dir=self.sim_dir) 213 | ct2 = CommandTask(cmd, output_dir=self.sim_dir) 214 | 215 | # When 216 | t = TaskRunner(tasks=[ct1, ct2, ct1], scheduler=s) 217 | 218 | # Then 219 | self.assertEqual(len(t.todo), 1) 220 | 221 | def test_problem_depending_on_other_problems(self): 222 | # Given 223 | class A(Problem): 224 | def get_requires(self): 225 | cmd = 'python -c "print(1)"' 226 | # Can return tasks ... 227 | ct = CommandTask(cmd, output_dir=self.sim_dir) 228 | return [('task1', ct)] 229 | 230 | def run(self): 231 | self.make_output_dir() 232 | 233 | class B(Problem): 234 | def get_requires(self): 235 | # or return Problem instances ... 236 | return [('a', A(self.sim_dir, self.out_dir))] 237 | 238 | def run(self): 239 | self.make_output_dir() 240 | 241 | class C(Problem): 242 | def get_requires(self): 243 | # ... or Problem subclasses 244 | return [('a', A), ('b', B)] 245 | 246 | def run(self): 247 | self.make_output_dir() 248 | 249 | s = self._make_scheduler() 250 | 251 | # When 252 | task = RunAll( 253 | simulation_dir=self.sim_dir, output_dir=self.output_dir, 254 | problem_classes=[A, B, C] 255 | ) 256 | t = TaskRunner(tasks=[task], scheduler=s) 257 | 258 | # Then 259 | self.assertEqual(len(t.todo), 5) 260 | # Basically only one instance of CommandTask should be created. 261 | names = [x.__class__.__name__ for x in t.todo] 262 | problems = [x.problem for x in t.todo if isinstance(x, SolveProblem)] 263 | self.assertEqual(names.count('RunAll'), 1) 264 | self.assertEqual(names.count('CommandTask'), 1) 265 | self.assertEqual(names.count('SolveProblem'), 3) 266 | self.assertEqual(len(problems), 3) 267 | self.assertEqual( 268 | sorted(x.__class__.__name__ for x in problems), 269 | ['A', 'B', 'C'] 270 | ) 271 | 272 | # When 273 | t.run(wait=0.1) 274 | 275 | # Then. 276 | self.assertEqual(t.todo, []) 277 | 278 | # The output dirs should exist now. 279 | for name in ('A', 'B', 'C'): 280 | self.assertTrue( 281 | os.path.exists(os.path.join(self.output_dir, name)) 282 | ) 283 | 284 | # When 285 | # We set force to True. 286 | task = RunAll( 287 | simulation_dir=self.sim_dir, output_dir=self.output_dir, 288 | problem_classes=[A, B, C], force=True 289 | ) 290 | 291 | # Then 292 | # Make sure that all the output directories are deleted as this will 293 | for name in ('A', 'B', 'C'): 294 | self.assertFalse( 295 | os.path.exists(os.path.join(self.output_dir, name)) 296 | ) 297 | 298 | def test_problem_with_bad_requires_raises_error(self): 299 | # Given 300 | class D(Problem): 301 | def get_requires(self): 302 | return [('a', 'A')] 303 | 304 | # When 305 | self.assertRaises( 306 | RuntimeError, 307 | SolveProblem, D(self.sim_dir, self.output_dir) 308 | ) 309 | 310 | def test_tasks_with_dependencies(self): 311 | # Given 312 | s = self._make_scheduler() 313 | cmd = 'python -c "import time; print(time.time())"' 314 | ct1_dir = os.path.join(self.sim_dir, '1') 315 | ct2_dir = os.path.join(self.sim_dir, '2') 316 | ct3_dir = os.path.join(self.sim_dir, '3') 317 | ct1 = CommandTask(cmd, output_dir=ct1_dir) 318 | ct2 = CommandTask(cmd, output_dir=ct2_dir, depends=[ct1]) 319 | ct3 = CommandTask(cmd, output_dir=ct3_dir, depends=[ct1, ct2]) 320 | 321 | # When 322 | t = TaskRunner(tasks=[ct1, ct2, ct3], scheduler=s) 323 | 324 | # Then 325 | self.assertEqual(len(t.todo), 3) 326 | 327 | # When 328 | t.run(wait=0.1) 329 | 330 | wait_until(lambda: not ct3.complete()) 331 | 332 | # Then. 333 | # Ensure that the tasks are run in the right order. 334 | ct1_t, ct2_t, ct3_t = [ 335 | self._get_time(x) for x in (ct1_dir, ct2_dir, ct3_dir) 336 | ] 337 | self.assertTrue(ct2_t > ct1_t) 338 | self.assertTrue(ct3_t > ct2_t) 339 | 340 | def test_simulation_with_dependencies(self): 341 | # Given 342 | class A(Problem): 343 | def setup(self): 344 | cmd = 'python -c "import time; print(time.time())"' 345 | s1 = Simulation(self.input_path('1'), cmd) 346 | s2 = Simulation(self.input_path('2'), cmd, depends=[s1]) 347 | s3 = Simulation(self.input_path('3'), cmd, depends=[s1, s2]) 348 | self.cases = [s1, s2, s3] 349 | 350 | def run(self): 351 | self.make_output_dir() 352 | 353 | s = self._make_scheduler() 354 | 355 | # When 356 | problem = A(self.sim_dir, self.output_dir) 357 | task = SolveProblem(problem) 358 | t = TaskRunner(tasks=[task], scheduler=s) 359 | 360 | # Then 361 | self.assertEqual(len(t.todo), 4) 362 | # Basically only one instance of CommandTask should be created. 363 | names = [x.__class__.__name__ for x in t.todo] 364 | self.assertEqual(names.count('CommandTask'), 3) 365 | self.assertEqual(names.count('SolveProblem'), 1) 366 | 367 | # When 368 | t.run(wait=0.1) 369 | wait_until(lambda: not task.complete()) 370 | 371 | # Then 372 | ct1_t, ct2_t, ct3_t = [ 373 | self._get_time(problem.input_path(x)) for x in ('1', '2', '3') 374 | ] 375 | self.assertTrue(ct2_t > ct1_t) 376 | self.assertTrue(ct3_t > ct2_t) 377 | 378 | 379 | class TestLocalAutomation(TestAutomationBase): 380 | def _make_scheduler(self): 381 | worker = dict(host='localhost') 382 | s = Scheduler(root='.', worker_config=[worker]) 383 | return s 384 | 385 | def test_automation(self): 386 | # Given. 387 | problem = EllipticalDrop(self.sim_dir, self.output_dir) 388 | s = self._make_scheduler() 389 | t = TaskRunner(tasks=[SolveProblem(problem=problem)], scheduler=s) 390 | 391 | # When. 392 | t.run(wait=1) 393 | 394 | # Then. 395 | sim1 = os.path.join(self.root, self.sim_dir, 396 | 'elliptical_drop', 'no_update_h') 397 | self.assertTrue(os.path.exists(sim1)) 398 | sim2 = os.path.join(self.root, self.sim_dir, 399 | 'elliptical_drop', 'update_h') 400 | self.assertTrue(os.path.exists(sim2)) 401 | 402 | results = os.path.join(self.root, self.output_dir, 403 | 'elliptical_drop', 'result.txt') 404 | self.assertTrue(os.path.exists(results)) 405 | data = open(results).read() 406 | self.assertTrue('no_update_h' in data) 407 | self.assertTrue('update_h' in data) 408 | 409 | # When. 410 | problem = EllipticalDrop(self.sim_dir, self.output_dir) 411 | t = TaskRunner(tasks=[SolveProblem(problem=problem)], scheduler=s) 412 | 413 | # Then. 414 | self.assertEqual(len(t.todo), 0) 415 | 416 | # When 417 | problem.clean() 418 | 419 | # Then. 420 | out1 = os.path.join(self.root, self.output_dir, 421 | 'elliptical_drop', 'update_h') 422 | out2 = os.path.join(self.root, self.output_dir, 423 | 'elliptical_drop', 'no_update_h') 424 | self.assertFalse(os.path.exists(out1)) 425 | self.assertFalse(os.path.exists(out2)) 426 | self.assertTrue(os.path.exists(sim1)) 427 | self.assertTrue(os.path.exists(sim2)) 428 | 429 | def test_nothing_is_run_when_output_exists(self): 430 | # Given. 431 | s = self._make_scheduler() 432 | output = os.path.join(self.output_dir, 'elliptical_drop') 433 | os.makedirs(output) 434 | 435 | # When 436 | problem = EllipticalDrop(self.sim_dir, self.output_dir) 437 | t = TaskRunner(tasks=[SolveProblem(problem=problem)], scheduler=s) 438 | 439 | # Then. 440 | self.assertEqual(len(t.todo), 0) 441 | 442 | 443 | class TestRemoteAutomation(TestLocalAutomation): 444 | def setUp(self): 445 | super(TestRemoteAutomation, self).setUp() 446 | self.other_dir = tempfile.mkdtemp() 447 | p = mock.patch.object( 448 | RemoteWorker, 'free_cores', return_value=2.0 449 | ) 450 | p.start() 451 | self.addCleanup(p.stop) 452 | 453 | def tearDown(self): 454 | super(TestRemoteAutomation, self).tearDown() 455 | if os.path.exists(self.other_dir): 456 | safe_rmtree(self.other_dir) 457 | 458 | def _make_scheduler(self): 459 | workers = [ 460 | dict(host='localhost'), 461 | dict(host='test_remote', 462 | python=sys.executable, chdir=self.other_dir, testing=True) 463 | ] 464 | try: 465 | import execnet 466 | except ImportError: 467 | raise unittest.SkipTest('This test requires execnet') 468 | return Scheduler(root=self.sim_dir, worker_config=workers) 469 | 470 | def test_job_with_error_is_handled_correctly(self): 471 | # Given. 472 | problem = EllipticalDrop(self.sim_dir, self.output_dir) 473 | problem.cases[0].base_command += ' --xxx' 474 | s = self._make_scheduler() 475 | t = TaskRunner(tasks=[SolveProblem(problem=problem)], scheduler=s) 476 | 477 | # When. 478 | try: 479 | t.run(wait=1) 480 | except RuntimeError: 481 | pass 482 | 483 | # Then. 484 | 485 | # Ensure that the directories are copied over when they have errors. 486 | sim1 = os.path.join(self.root, self.sim_dir, 487 | 'elliptical_drop', 'no_update_h') 488 | self.assertTrue(os.path.exists(sim1)) 489 | sim2 = os.path.join(self.root, self.sim_dir, 490 | 'elliptical_drop', 'update_h') 491 | self.assertTrue(os.path.exists(sim2)) 492 | 493 | # Ensure that all the correct but already scheduled jobs are completed. 494 | task_status = t.task_status 495 | status_values = list(task_status.values()) 496 | self.assertEqual(status_values.count('error'), 1) 497 | self.assertEqual(status_values.count('done'), 1) 498 | self.assertEqual(status_values.count('not started'), 1) 499 | for t, s in task_status.items(): 500 | if s == 'done': 501 | self.assertTrue(t.complete()) 502 | if s == 'error': 503 | self.assertRaises(RuntimeError, t.complete) 504 | 505 | 506 | class TestCommandTask(TestAutomationBase): 507 | def _make_scheduler(self): 508 | worker = dict(host='localhost') 509 | s = Scheduler(root='.', worker_config=[worker]) 510 | return s 511 | 512 | def test_command_tasks_executes_simple_command(self): 513 | # Given 514 | s = self._make_scheduler() 515 | cmd = 'python -c "print(1)"' 516 | t = CommandTask(cmd, output_dir=self.sim_dir) 517 | 518 | self.assertFalse(t.complete()) 519 | 520 | # When 521 | t.run(s) 522 | wait_until(lambda: not t.complete()) 523 | 524 | # Then 525 | self.assertTrue(t.complete()) 526 | self.assertEqual(t.job_proxy.status(), 'done') 527 | self.assertEqual(t.job_proxy.get_stdout().strip(), '1') 528 | 529 | def test_command_tasks_converts_dollar_output_dir(self): 530 | # Given 531 | s = self._make_scheduler() 532 | cmd = '''python -c "print('$output_dir')"''' 533 | t = CommandTask(cmd, output_dir=self.sim_dir) 534 | 535 | self.assertFalse(t.complete()) 536 | 537 | # When 538 | t.run(s) 539 | wait_until(lambda: not t.complete()) 540 | 541 | # Then 542 | self.assertTrue(t.complete()) 543 | self.assertEqual(t.job_proxy.status(), 'done') 544 | self.assertEqual(t.job_proxy.get_stdout().strip(), self.sim_dir) 545 | 546 | def test_command_tasks_handles_errors_correctly(self): 547 | # Given 548 | s = self._make_scheduler() 549 | cmd = 'python --junk' 550 | t = CommandTask(cmd, output_dir=self.sim_dir) 551 | 552 | self.assertFalse(t.complete()) 553 | 554 | # When 555 | t.run(s) 556 | try: 557 | wait_until(lambda: not t.complete()) 558 | except RuntimeError: 559 | pass 560 | 561 | # Then 562 | self.assertRaises(RuntimeError, t.complete) 563 | self.assertEqual(t.job_proxy.status(), 'error') 564 | 565 | # A new command task should still detect that the run failed, even 566 | # though the output directory exists. In this case, it should 567 | # return False and not raise an error. 568 | 569 | # Given 570 | t = CommandTask(cmd, output_dir=self.sim_dir) 571 | 572 | # When/Then 573 | self.assertFalse(t.complete()) 574 | 575 | 576 | class TestFileCommandTask(TestAutomationBase): 577 | def _make_scheduler(self): 578 | worker = dict(host='localhost') 579 | s = Scheduler(root='.', worker_config=[worker]) 580 | return s 581 | 582 | def test_file_command_tasks_works(self): 583 | # Given 584 | s = self._make_scheduler() 585 | pth = os.path.join(self.sim_dir, 'output.txt') 586 | cmd = 'python -c "from pathlib import Path; Path(%r).touch()"' % pth 587 | t = FileCommandTask(cmd, files=[pth]) 588 | 589 | self.assertFalse(t.complete()) 590 | 591 | # When 592 | t.run(s) 593 | wait_until(lambda: not t.complete()) 594 | 595 | # Then 596 | self.assertTrue(t.complete()) 597 | output_dir = pth + '.job_info' 598 | self.assertEqual(t.output_dir, output_dir) 599 | self.assertTrue(os.path.exists(t.output_dir)) 600 | self.assertEqual(t.job_proxy.status(), 'done') 601 | self.assertTrue(os.path.exists(pth)) 602 | 603 | 604 | class TestRemoteCommandTask(TestAutomationBase): 605 | def _make_scheduler(self): 606 | if not os.path.exists(self.output_dir): 607 | os.makedirs(self.output_dir) 608 | worker = dict( 609 | host='test_remote', python=sys.executable, 610 | chdir=self.output_dir, testing=True 611 | ) 612 | s = Scheduler(root='.', worker_config=[worker]) 613 | return s 614 | 615 | def test_remote_command_tasks_complete_method_works(self): 616 | # Given 617 | s = self._make_scheduler() 618 | cmd = 'python -c "print(1)"' 619 | t = CommandTask(cmd, output_dir=self.sim_dir) 620 | 621 | self.assertFalse(t.complete()) 622 | self.assertFalse(os.path.exists( 623 | os.path.join(self.sim_dir, 'stdout.txt') 624 | )) 625 | 626 | # When 627 | t.run(s) 628 | wait_until(lambda: not t.complete()) 629 | 630 | # Then 631 | self.assertTrue(t.complete()) 632 | self.assertTrue(os.path.exists( 633 | os.path.join(self.sim_dir, 'stdout.txt') 634 | )) 635 | # Test that if we call it repeatedly that it does indeed return True 636 | self.assertTrue(t.complete()) 637 | 638 | 639 | def test_simulation_get_labels(): 640 | # Given 641 | s = Simulation( 642 | 'junk', 'pysph run taylor_green', 643 | nx=25, perturb=0.1, correction=None 644 | ) 645 | 646 | # When 647 | l = s.get_labels('nx') 648 | 649 | # Then 650 | assert l == r'nx=25' 651 | 652 | # When 653 | l = s.get_labels(['nx', 'perturb', 'correction']) 654 | 655 | # Then 656 | assert l == r'nx=25, perturb=0.1, correction' 657 | 658 | 659 | class TestAutomator(TestAutomationBase): 660 | def setUp(self): 661 | super(TestAutomator, self).setUp() 662 | 663 | def test_automator_accepts_cluster_manager(self): 664 | # Given/When 665 | a = Automator('sim', 'output', [EllipticalDrop], 666 | cluster_manager_factory=ClusterManager) 667 | 668 | # Then 669 | self.assertEqual(a.cluster_manager_factory, ClusterManager) 670 | 671 | @mock.patch.object(TaskRunner, 'run') 672 | def test_automator(self, mock_run): 673 | # Given 674 | a = Automator('sim', 'output', [EllipticalDrop]) 675 | 676 | # When 677 | a.run([]) 678 | 679 | # Then 680 | mock_run.assert_called_with() 681 | self.assertEqual(len(a.runner.todo), 4) 682 | 683 | expect = ['RunAll', 'SolveProblem', 'PySPHTask', 'PySPHTask'] 684 | names = [x.__class__.__name__ for x in a.runner.todo] 685 | self.assertEqual(names, expect) 686 | 687 | # When 688 | # Given 689 | a = Automator('sim', 'output', [EllipticalDrop]) 690 | a.run(['-m', '*no_up*']) 691 | 692 | # Then 693 | mock_run.assert_called_with() 694 | self.assertEqual(len(a.runner.todo), 3) 695 | 696 | expect = ['RunAll', 'SolveProblem', 'PySPHTask'] 697 | names = [x.__class__.__name__ for x in a.runner.todo] 698 | self.assertEqual(names, expect) 699 | out_dir = os.path.basename(a.runner.todo[-1].output_dir) 700 | self.assertEqual(out_dir, 'no_update_h') 701 | 702 | @mock.patch.object(TaskRunner, 'run') 703 | def test_automates_only_tasks(self, mock_run): 704 | # Given 705 | a = Automator('sim', 'output', []) 706 | cmd = 'python -c "print(1)"' 707 | task = CommandTask(cmd, output_dir=self.sim_dir) 708 | a.add_task(task) 709 | 710 | # When 711 | a.run([]) 712 | 713 | # Then 714 | mock_run.assert_called_with() 715 | self.assertEqual(len(a.runner.todo), 1) 716 | self.assertEqual(a.runner.todo[-1], task) 717 | 718 | # Given 719 | a = Automator('sim', 'output', []) 720 | cmd = 'python -c "print(1)"' 721 | task = CommandTask(cmd, output_dir=self.sim_dir) 722 | a.add_task(task, name='task') 723 | 724 | # When 725 | a.run([]) 726 | 727 | # Then 728 | mock_run.assert_called_with() 729 | self.assertEqual(len(a.runner.todo), 0) 730 | 731 | # Given 732 | a = Automator('sim', 'output', []) 733 | cmd = 'python -c "print(1)"' 734 | task = CommandTask(cmd, output_dir=self.sim_dir) 735 | a.add_task(task, name='task') 736 | 737 | # When 738 | a.run(['task']) 739 | 740 | # Then 741 | mock_run.assert_called_with() 742 | self.assertEqual(len(a.runner.todo), 1) 743 | self.assertEqual(a.runner.todo[-1], task) 744 | 745 | @mock.patch.object(TaskRunner, 'run') 746 | def test_adding_problem_as_task(self, mock_run): 747 | # Given 748 | a = Automator('sim', 'output', []) 749 | a.add_task(EllipticalDrop) 750 | 751 | # When 752 | a.run([]) 753 | 754 | # Then 755 | self.assertEqual(len(a.runner.todo), 3) 756 | 757 | expect = ['SolveProblem', 'PySPHTask', 'PySPHTask'] 758 | names = [x.__class__.__name__ for x in a.runner.todo] 759 | self.assertEqual(names, expect) 760 | 761 | # Given 762 | a = Automator('sim', 'output', []) 763 | problem = EllipticalDrop('sim', 'output') 764 | a.add_task(problem) 765 | 766 | # When 767 | a.run([]) 768 | 769 | # Then 770 | self.assertEqual(len(a.runner.todo), 3) 771 | 772 | expect = ['SolveProblem', 'PySPHTask', 'PySPHTask'] 773 | names = [x.__class__.__name__ for x in a.runner.todo] 774 | self.assertEqual(names, expect) 775 | self.assertEqual(a.runner.todo[0].problem, problem) 776 | 777 | @mock.patch.object(TaskRunner, 'run') 778 | def test_automates_tasks_and_problems(self, mock_run): 779 | # Given 780 | a = Automator('sim', 'output', [EllipticalDrop]) 781 | cmd = 'python -c "print(1)"' 782 | task = CommandTask(cmd, output_dir=self.sim_dir) 783 | a.add_task(task) 784 | dir2 = os.path.join(self.sim_dir, '2') 785 | task2 = CommandTask(cmd, output_dir=dir2) 786 | a.add_task(task2, post_proc=True) 787 | 788 | # When 789 | a.run([]) 790 | 791 | # Then 792 | self.assertEqual(len(a.runner.todo), 6) 793 | 794 | expect = ['RunAll', 'SolveProblem', 'PySPHTask', 'PySPHTask', 795 | 'CommandTask', 'CommandTask'] 796 | names = [x.__class__.__name__ for x in a.runner.todo] 797 | self.assertEqual(names, expect) 798 | self.assertEqual(a.runner.todo[-2], task) 799 | self.assertEqual(a.runner.todo[-1], task2) 800 | self.assertEqual(task2.depends, [a.runner.todo[0]]) 801 | 802 | @mock.patch.object(TaskRunner, 'run') 803 | def test_automates_named_tasks(self, mock_run): 804 | # Given 805 | a = Automator('sim', 'output', [EllipticalDrop]) 806 | cmd = 'python -c "print(1)"' 807 | task = CommandTask(cmd, output_dir=self.sim_dir) 808 | a.add_task(task, name='task') 809 | dir2 = os.path.join(self.sim_dir, '2') 810 | task2 = CommandTask(cmd, output_dir=dir2) 811 | a.add_task(task2, name='task2', post_proc=True) 812 | 813 | # When 814 | a.run([]) 815 | 816 | # Then 817 | self.assertEqual(len(a.runner.todo), 4) 818 | 819 | expect = ['RunAll', 'SolveProblem', 'PySPHTask', 'PySPHTask'] 820 | names = [x.__class__.__name__ for x in a.runner.todo] 821 | self.assertEqual(names, expect) 822 | self.assertEqual(task2.depends, [a.runner.todo[0]]) 823 | 824 | # Given 825 | a = Automator('sim', 'output', [EllipticalDrop]) 826 | cmd = 'python -c "print(1)"' 827 | task = CommandTask(cmd, output_dir=self.sim_dir) 828 | a.add_task(task, name='task') 829 | 830 | # When 831 | a.run(['task']) 832 | 833 | # Then 834 | self.assertEqual(len(a.runner.todo), 1) 835 | 836 | expect = ['CommandTask'] 837 | names = [x.__class__.__name__ for x in a.runner.todo] 838 | self.assertEqual(names, expect) 839 | self.assertEqual(a.runner.todo[-1], task) 840 | 841 | # Given 842 | a = Automator('sim', 'output', [EllipticalDrop]) 843 | cmd = 'python -c "print(1)"' 844 | task = CommandTask(cmd, output_dir=self.sim_dir) 845 | a.add_task(task, name='task') 846 | 847 | # When 848 | a.run(['all', 'task']) 849 | 850 | # Then 851 | self.assertEqual(len(a.runner.todo), 5) 852 | 853 | expect = ['RunAll', 'SolveProblem', 'PySPHTask', 'PySPHTask', 854 | 'CommandTask'] 855 | names = [x.__class__.__name__ for x in a.runner.todo] 856 | self.assertEqual(names, expect) 857 | self.assertEqual(a.runner.todo[-1], task) 858 | -------------------------------------------------------------------------------- /automan/tests/test_cluster_manager.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import json 4 | import os 5 | from os.path import dirname 6 | import shutil 7 | import sys 8 | import tempfile 9 | from textwrap import dedent 10 | import unittest 11 | 12 | try: 13 | from unittest import mock 14 | except ImportError: 15 | import mock 16 | 17 | from automan.jobs import Job 18 | from automan.cluster_manager import ClusterManager 19 | from automan.conda_cluster_manager import CondaClusterManager 20 | from automan.edm_cluster_manager import EDMClusterManager 21 | from .test_jobs import wait_until 22 | 23 | 24 | ROOT_DIR = dirname(dirname(dirname(__file__))) 25 | 26 | 27 | class MyClusterManager(ClusterManager): 28 | 29 | BOOTSTRAP = dedent("""\ 30 | #!/bin/bash 31 | 32 | set -e 33 | python3 -m venv --system-site-packages envs/{project_name} 34 | source envs/{project_name}/bin/activate 35 | cd ROOT 36 | python setup.py install 37 | """) 38 | 39 | UPDATE = dedent("""\ 40 | #!/bin/bash 41 | echo "update" 42 | """) 43 | 44 | def _get_helper_scripts(self): 45 | return None 46 | 47 | 48 | class TestClusterManager(unittest.TestCase): 49 | 50 | def setUp(self): 51 | self.cwd = os.getcwd() 52 | self.root = tempfile.mkdtemp() 53 | self.proj_root = os.path.join(self.root, 'project') 54 | os.makedirs(self.proj_root) 55 | os.chdir(self.proj_root) 56 | 57 | def tearDown(self): 58 | os.chdir(self.cwd) 59 | if os.path.exists(self.root): 60 | shutil.rmtree(self.root) 61 | 62 | def _get_config(self): 63 | with open(os.path.join(self.proj_root, 'config.json'), 'r') as fp: 64 | data = json.load(fp) 65 | return data 66 | 67 | def test_new_config_created_on_creation(self): 68 | # Given/When 69 | ClusterManager() 70 | 71 | # Then 72 | config = self._get_config() 73 | 74 | self.assertEqual(config.get('root'), 'automan') 75 | self.assertEqual(config.get('project_name'), 76 | os.path.basename(self.proj_root)) 77 | self.assertEqual(os.path.realpath(config.get('sources')[0]), 78 | os.path.realpath(self.proj_root)) 79 | workers = config.get('workers') 80 | self.assertEqual(len(workers), 1) 81 | self.assertEqual(workers[0]['host'], 'localhost') 82 | 83 | @mock.patch.object(ClusterManager, '_bootstrap') 84 | def test_add_worker(self, mock_bootstrap): 85 | # Given 86 | cm = ClusterManager() 87 | 88 | # When 89 | cm.add_worker('host', home='/home/foo', nfs=False) 90 | 91 | # Then 92 | mock_bootstrap.assert_called_with('host', '/home/foo') 93 | config = self._get_config() 94 | workers = config.get('workers') 95 | self.assertEqual(len(workers), 2) 96 | hosts = sorted(x['host'] for x in workers) 97 | self.assertEqual(hosts, ['host', 'localhost']) 98 | 99 | @mock.patch.object(ClusterManager, '_bootstrap') 100 | def test_create_scheduler(self, mock_bootstrap): 101 | # Given 102 | cm = ClusterManager() 103 | cm.add_worker('host', home='/home/foo', nfs=False) 104 | 105 | # When 106 | s = cm.create_scheduler() 107 | 108 | # Then 109 | confs = s.worker_config 110 | hosts = sorted([x['host'] for x in confs]) 111 | self.assertEqual(hosts, ['host', 'localhost']) 112 | 113 | @mock.patch.object(ClusterManager, '_bootstrap') 114 | @mock.patch.object(ClusterManager, '_update_sources') 115 | @mock.patch.object(ClusterManager, '_rebuild') 116 | def test_update(self, mock_rebuild, mock_update_sources, mock_bootstrap): 117 | # Given 118 | cm = ClusterManager() 119 | cm.add_worker('host', home='/home/foo', nfs=False) 120 | 121 | # When 122 | cm.update() 123 | 124 | # Then 125 | mock_update_sources.assert_called_with('host', '/home/foo') 126 | mock_rebuild.assert_called_with('host', '/home/foo') 127 | 128 | @mock.patch.object(ClusterManager, 'add_worker') 129 | @mock.patch.object(ClusterManager, 'update') 130 | def test_cli(self, mock_update, mock_add_worker): 131 | # Given 132 | cm = ClusterManager() 133 | 134 | # When 135 | cm.cli(['-a', 'host', '--home', 'home', '--nfs']) 136 | 137 | # Then 138 | mock_add_worker.assert_called_with('host', 'home', True) 139 | 140 | @unittest.skipIf((sys.version_info < (3, 3)) or 141 | sys.platform.startswith('win'), 142 | 'Test requires Python 3.x and a non-Windows system.') 143 | def test_remote_bootstrap_and_sync(self): 144 | if not os.path.exists('setup.py'): 145 | raise unittest.SkipTest( 146 | 'This test requires to be run from the automan source directory.' 147 | ) 148 | 149 | # Given 150 | cm = MyClusterManager(exclude_paths=['outputs/'], testing=True) 151 | cm.BOOTSTRAP = cm.BOOTSTRAP.replace('ROOT', self.cwd) 152 | output_dir = os.path.join(self.proj_root, 'outputs') 153 | os.makedirs(output_dir) 154 | 155 | # Remove the default localhost worker. 156 | cm.workers = [] 157 | 158 | # When 159 | cm.add_worker('host', home=self.root, nfs=False) 160 | 161 | # Then 162 | self.assertEqual(len(cm.workers), 1) 163 | worker = cm.workers[0] 164 | self.assertEqual(worker['host'], 'host') 165 | project_name = cm.project_name 166 | self.assertEqual(project_name, os.path.basename(self.proj_root)) 167 | py = os.path.join(self.root, 'automan', 'envs', project_name, 168 | 'bin', 'python') 169 | self.assertEqual(worker['python'], py) 170 | chdir = os.path.join(self.root, 'automan', project_name) 171 | self.assertEqual(worker['chdir'], chdir) 172 | 173 | # Given 174 | cmd = ['python', '-c', 'import sys; print(sys.executable)'] 175 | job = Job(command=cmd, output_dir=output_dir) 176 | 177 | s = cm.create_scheduler() 178 | 179 | # When 180 | proxy = s.submit(job) 181 | 182 | # Then 183 | wait_until(lambda: proxy.status() != 'done') 184 | 185 | self.assertEqual(proxy.status(), 'done') 186 | output = proxy.get_stdout().strip() 187 | self.assertEqual(os.path.realpath(output), os.path.realpath(py)) 188 | 189 | # Test to see if updating works. 190 | 191 | # When 192 | with open(os.path.join(self.proj_root, 'script.py'), 'w') as f: 193 | f.write('print("hello")\n') 194 | 195 | cm.update() 196 | 197 | # Then 198 | dest = os.path.join(self.root, 'automan', project_name, 'script.py') 199 | self.assertTrue(os.path.exists(dest)) 200 | 201 | @mock.patch.object(ClusterManager, '_bootstrap') 202 | @mock.patch.object(ClusterManager, '_delete_outputs') 203 | def test_delete_outputs_from_hosts(self, mock_delete_outputs, 204 | mock_bootstrap): 205 | # Given 206 | cm = ClusterManager() 207 | cm.add_worker('host', home='/home/foo', nfs=False) 208 | cm.add_worker('host1', home='/home/foo1', nfs=False) 209 | 210 | # When 211 | cm.delete('outputs', ['host', 'host1']) 212 | 213 | # Then 214 | mock_delete_outputs.assert_any_call('host1', '/home/foo1', 'outputs') 215 | mock_delete_outputs.assert_any_call('host', '/home/foo', 'outputs') 216 | 217 | @mock.patch.object(ClusterManager, '_bootstrap') 218 | @mock.patch.object(ClusterManager, '_delete_outputs') 219 | def test_do_not_delete_outputs_from_hosts_when_nfs_true( 220 | self, mock_delete_outputs, mock_bootstrap 221 | ): 222 | # Given 223 | cm = ClusterManager() 224 | cm.add_worker('host', home='/home/foo', nfs=True) 225 | cm.add_worker('host1', home='/home/foo1', nfs=True) 226 | 227 | # When 228 | cm.delete('outputs', ['host', 'host1']) 229 | 230 | # Then 231 | mock_delete_outputs.assert_not_called() 232 | 233 | @mock.patch.object(ClusterManager, '_bootstrap') 234 | @mock.patch.object(ClusterManager, '_delete_outputs') 235 | def test_delete_outputs_from_all_hosts(self, mock_delete_outputs, 236 | mock_bootstrap): 237 | # Given 238 | cm = ClusterManager() 239 | cm.add_worker('host', home='/home/foo', nfs=False) 240 | cm.add_worker('host1', home='/home/foo1', nfs=False) 241 | 242 | # When 243 | cm.delete('outputs', ['all']) 244 | 245 | # Then 246 | mock_delete_outputs.assert_any_call('host1', '/home/foo1', 'outputs') 247 | mock_delete_outputs.assert_any_call('host', '/home/foo', 'outputs') 248 | 249 | 250 | class TestCondaClusterManager(unittest.TestCase): 251 | def setUp(self): 252 | self.cwd = os.getcwd() 253 | self.root = tempfile.mkdtemp() 254 | os.chdir(self.root) 255 | 256 | def tearDown(self): 257 | os.chdir(self.cwd) 258 | if os.path.exists(self.root): 259 | shutil.rmtree(self.root) 260 | 261 | @mock.patch("automan.conda_cluster_manager.CondaClusterManager.CONDA_ROOT", 262 | 'TEST_ROOT') 263 | def test_overloaded_methods(self): 264 | # Given 265 | cm = CondaClusterManager() 266 | 267 | # When/Then 268 | python = cm._get_python('foo', 'blah') 269 | name = os.path.basename(self.root) 270 | self.assertEqual(python, os.path.join( 271 | 'blah', 'TEST_ROOT', 'envs/{name}/bin/python'.format(name=name) 272 | )) 273 | 274 | code = cm._get_bootstrap_code() 275 | self.assertTrue('TEST_ROOT' in code) 276 | 277 | code = cm._get_update_code() 278 | self.assertTrue('TEST_ROOT' in code) 279 | 280 | self.assertEqual(cm._get_helper_scripts(), '') 281 | 282 | 283 | class TestEDMClusterManager(unittest.TestCase): 284 | def setUp(self): 285 | self.cwd = os.getcwd() 286 | self.root = tempfile.mkdtemp() 287 | os.chdir(self.root) 288 | 289 | def tearDown(self): 290 | os.chdir(self.cwd) 291 | if os.path.exists(self.root): 292 | shutil.rmtree(self.root) 293 | 294 | @mock.patch("automan.edm_cluster_manager.EDMClusterManager.EDM_ROOT", 295 | 'TEST_ROOT') 296 | def test_overloaded_methods(self): 297 | # Given 298 | cm = EDMClusterManager() 299 | 300 | # When/Then 301 | python = cm._get_python('foo', 'bar') 302 | name = os.path.basename(self.root) 303 | self.assertEqual(python, os.path.join( 304 | 'bar', 'TEST_ROOT', 'envs/{name}/bin/python'.format(name=name) 305 | )) 306 | 307 | code = cm._get_bootstrap_code() 308 | self.assertTrue('TEST_ROOT' in code) 309 | 310 | code = cm._get_update_code() 311 | self.assertTrue('TEST_ROOT' in code) 312 | 313 | self.assertEqual(cm._get_helper_scripts(), '') 314 | -------------------------------------------------------------------------------- /automan/tests/test_jobs.py: -------------------------------------------------------------------------------- 1 | import psutil 2 | import multiprocessing 3 | import shutil 4 | import sys 5 | import os 6 | import tempfile 7 | import time 8 | import unittest 9 | try: 10 | from unittest import mock 11 | except ImportError: 12 | import mock 13 | 14 | try: 15 | from automan import jobs 16 | except ImportError: 17 | raise unittest.SkipTest('test_jobs requires psutil') 18 | 19 | 20 | def safe_rmtree(*args, **kw): 21 | if sys.platform.startswith('win'): 22 | try: 23 | shutil.rmtree(*args, **kw) 24 | except WindowsError: 25 | pass 26 | else: 27 | try: 28 | shutil.rmtree(*args, **kw) 29 | except OSError: 30 | pass 31 | 32 | 33 | def test_cores_required(): 34 | with mock.patch('automan.jobs.total_cores', return_value=4.0): 35 | assert jobs.cores_required(0) == 0 36 | assert jobs.cores_required(1) == 1 37 | assert jobs.cores_required(3) == 3 38 | assert jobs.cores_required(-1) == 4 39 | assert jobs.cores_required(-2) == 2 40 | assert jobs.cores_required(-4) == 1 41 | 42 | 43 | def test_threads_required(): 44 | with mock.patch('automan.jobs.total_cores', return_value=4.0): 45 | # n_thread, n_core 46 | assert jobs.threads_required(1, 1) == 1 47 | assert jobs.threads_required(2, 2) == 2 48 | assert jobs.threads_required(2, -1) == 2 49 | assert jobs.threads_required(-1, -1) == 4 50 | assert jobs.threads_required(-2, -1) == 8 51 | assert jobs.threads_required(-2, -2) == 4 52 | assert jobs.threads_required(-4, -1) == 16 53 | 54 | 55 | class TestJob(unittest.TestCase): 56 | def setUp(self): 57 | self.root = tempfile.mkdtemp() 58 | 59 | def tearDown(self): 60 | if os.path.exists(self.root): 61 | safe_rmtree(self.root) 62 | 63 | def test_job_can_handle_string_command(self): 64 | # Given 65 | command = '''\ 66 | python -c 'import sys;sys.stdout.write("1");sys.stderr.write("2")' ''' 67 | j = jobs.Job(command=command, output_dir=self.root) 68 | 69 | # When 70 | j.run() 71 | j.join() 72 | 73 | # Then. 74 | self.assertTrue(isinstance(j.command, list)) 75 | self.assertTrue(j.status(), 'done') 76 | self.assertEqual(j.get_stdout(), '1') 77 | self.assertEqual(j.get_stderr(), '2') 78 | 79 | def test_simple_job(self): 80 | # Given 81 | command = ['python', '-c', 82 | 'import sys;sys.stdout.write("1");sys.stderr.write("2")'] 83 | j = jobs.Job(command=command, output_dir=self.root) 84 | 85 | # When 86 | j.run() 87 | j.join() 88 | 89 | # Then 90 | self.assertEqual(j.status(), 'done') 91 | self.assertEqual(j.output_dir, self.root) 92 | self.assertEqual(j.n_core, 1) 93 | self.assertEqual(j.n_thread, 1) 94 | self.assertEqual(j.get_stdout(), '1') 95 | self.assertEqual(j.get_stderr(), '2') 96 | state = j.to_dict() 97 | expect = dict( 98 | command=command, output_dir=self.root, n_core=1, 99 | n_thread=1, env=None 100 | ) 101 | expect['command'][0] = sys.executable 102 | self.assertDictEqual(state, expect) 103 | 104 | def test_job_substitute_in_command(self): 105 | # Given 106 | j = jobs.Job(command=['python', '-c', 'print(123)'], 107 | output_dir=self.root) 108 | 109 | # When 110 | sub = '/usr/bin/python' 111 | j.substitute_in_command('python', sub) 112 | 113 | # Then 114 | self.assertEqual(j.command[0], sub) 115 | 116 | def test_job_status(self): 117 | # Given/When 118 | j = jobs.Job( 119 | [sys.executable, '-c', 'import time; time.sleep(0.25)'], 120 | output_dir=self.root 121 | ) 122 | 123 | # Then 124 | self.assertEqual(j.status(), 'not started') 125 | 126 | # When 127 | j.run() 128 | 129 | # Then 130 | self.assertEqual(j.status(), 'running') 131 | 132 | # When 133 | j.join() 134 | self.assertEqual(j.status(), 'done') 135 | 136 | # Given 137 | j = jobs.Job( 138 | [sys.executable, '-c', 'asdf'], 139 | output_dir=self.root 140 | ) 141 | # When 142 | j.run() 143 | j.join() 144 | 145 | # Then 146 | self.assertEqual(j.status(), 'error') 147 | self.assertTrue('NameError' in j.get_stderr()) 148 | 149 | def test_proc_reset_when_job_done(self): 150 | # Given/When 151 | j = jobs.Job( 152 | [sys.executable, '-c', 'import time; time.sleep(0.25)'], 153 | output_dir=self.root 154 | ) 155 | 156 | # Then 157 | self.assertEqual(j.status(), 'not started') 158 | self.assertIsNone(j.proc) 159 | 160 | # When 161 | j.run() 162 | 163 | # Then 164 | self.assertEqual(j.status(), 'running') 165 | self.assertIsNotNone(j.proc) 166 | 167 | # When 168 | j.join() 169 | self.assertEqual(j.status(), 'done') 170 | self.assertIsNone(j.proc) 171 | 172 | def test_reset_proc_when_job_status_error(self): 173 | j = jobs.Job( 174 | [sys.executable, '--junk'], 175 | output_dir=self.root, 176 | n_thread=4, 177 | ) 178 | 179 | # Then 180 | self.assertEqual(j.status(), 'not started') 181 | self.assertIsNone(j.proc) 182 | 183 | # When 184 | j.run() 185 | 186 | self.assertEqual(j.status(), 'running') 187 | self.assertIsNotNone(j.proc) 188 | 189 | # When 190 | j.join() 191 | 192 | # Then 193 | self.assertEqual(j.status(), 'error') 194 | self.assertIsNone(j.proc) 195 | 196 | def test_that_job_sets_env_var(self): 197 | # Given/When 198 | j = jobs.Job( 199 | [sys.executable, '-c', 'import os;print(os.environ.get("FOO"))'], 200 | output_dir=self.root, 201 | env=dict(FOO='hello') 202 | ) 203 | j.run() 204 | j.join() 205 | 206 | # Then 207 | print(j.get_stdout(), j.get_stderr()) 208 | self.assertEqual(j.status(), 'done') 209 | self.assertEqual(j.get_stdout().strip(), 'hello') 210 | 211 | @mock.patch('automan.jobs.total_cores', return_value=2.0) 212 | def test_that_job_sets_omp_var(self, mock_total_cores): 213 | j = jobs.Job( 214 | [sys.executable, '-c', 215 | 'import os;print(os.environ.get("OMP_NUM_THREADS"))'], 216 | output_dir=self.root, 217 | n_thread=4, 218 | ) 219 | j.run() 220 | j.join() 221 | 222 | # Then 223 | self.assertEqual(j.status(), 'done') 224 | self.assertEqual(j.get_stdout().strip(), '4') 225 | self.assertEqual(j.env.get('OMP_NUM_THREADS'), '4') 226 | 227 | # When 228 | j = jobs.Job( 229 | [sys.executable, '-c', 'print(1)'], 230 | output_dir=self.root, 231 | n_thread=None, 232 | ) 233 | 234 | # Then 235 | self.assertFalse('OMP_NUM_THREADS' in j.env) 236 | 237 | # When 238 | j = jobs.Job( 239 | [sys.executable, '-c', 'print(1)'], 240 | output_dir=self.root, 241 | n_thread=-2, n_core=1 242 | ) 243 | 244 | # Then 245 | self.assertEqual(j.env.get('OMP_NUM_THREADS'), '2') 246 | 247 | # When 248 | j = jobs.Job( 249 | [sys.executable, '-c', 'print(1)'], 250 | output_dir=self.root, 251 | n_thread=-2, n_core=-1 252 | ) 253 | 254 | # Then 255 | self.assertEqual(j.env.get('OMP_NUM_THREADS'), '4') 256 | 257 | def test_free_cores(self): 258 | n = jobs.free_cores() 259 | self.assertTrue(n >= 0) 260 | self.assertTrue(n <= multiprocessing.cpu_count()) 261 | 262 | def test_total_cores(self): 263 | n = jobs.total_cores() 264 | self.assertTrue(n >= 0) 265 | self.assertTrue(n <= multiprocessing.cpu_count()) 266 | 267 | def test_status_when_job_is_incorrect(self): 268 | j = jobs.Job( 269 | [sys.executable, '--junk'], 270 | output_dir=self.root, 271 | n_thread=4, 272 | ) 273 | j.run() 274 | j.join() 275 | 276 | # Then 277 | self.assertEqual(j.status(), 'error') 278 | info = j.get_info() 279 | self.assertEqual(info['status'], 'error') 280 | self.assertTrue(info['exitcode'] != 0) 281 | 282 | # Now retry and the status should be the same. 283 | j1 = jobs.Job( 284 | [sys.executable, '--junk'], 285 | output_dir=self.root, 286 | n_thread=4, 287 | ) 288 | 289 | self.assertEqual(j1.status(), 'error') 290 | 291 | def test_status_when_job_is_rerun(self): 292 | # Given 293 | command = ['python', '-c', 'print(123)'] 294 | j = jobs.Job(command=command, output_dir=self.root) 295 | j.run() 296 | j.join() 297 | 298 | # When 299 | j1 = jobs.Job(command=command, output_dir=self.root) 300 | 301 | # Then 302 | self.assertEqual(j1.status(), 'done') 303 | 304 | def test_clean_removes_new_output_directory(self): 305 | # Given 306 | out_dir = os.path.join(self.root, 'junk') 307 | command = ['python', '-c', 'print(123)'] 308 | j = jobs.Job(command=command, output_dir=out_dir) 309 | j.run() 310 | j.join() 311 | 312 | # When 313 | j.clean() 314 | 315 | # Then 316 | self.assertFalse(os.path.exists(out_dir)) 317 | 318 | def test_clean_does_not_remove_existing_output_directory(self): 319 | # Given 320 | command = ['python', '-c', 'print(123)'] 321 | j = jobs.Job(command=command, output_dir=self.root) 322 | j.run() 323 | j.join() 324 | 325 | # When 326 | j.clean() 327 | 328 | # Then 329 | self.assertFalse(os.path.exists(j.stdout)) 330 | self.assertFalse(os.path.exists(j.stderr)) 331 | self.assertTrue(os.path.exists(self.root)) 332 | 333 | def test_job_status_when_process_killed(self): 334 | # Given 335 | cmd = ['python', '-c', 'import time; time.sleep(100)'] 336 | j = jobs.Job(command=cmd, output_dir=self.root) 337 | j.run() 338 | wait_until(lambda: j.get_info()['pid'] is None) 339 | 340 | # When 341 | 342 | # Kill the Python process the job is running, as well as its subprocess. 343 | pid = j.get_info()['pid'] 344 | proc = psutil.Process(pid) 345 | j.proc.kill() 346 | proc.kill() 347 | 348 | def _check_proc(): 349 | try: 350 | proc.status() 351 | return True 352 | except psutil.NoSuchProcess: 353 | return False 354 | 355 | wait_until(_check_proc) 356 | 357 | j = jobs.Job(command=cmd, output_dir=self.root) 358 | 359 | # Then 360 | self.assertEqual(j.status(), 'error') 361 | 362 | 363 | def wait_until(cond, timeout=1, wait=0.1): 364 | t = 0.0 365 | while cond(): 366 | time.sleep(wait) 367 | t += wait 368 | if t > timeout: 369 | break 370 | 371 | 372 | class TestLocalWorker(unittest.TestCase): 373 | def setUp(self): 374 | self.root = tempfile.mkdtemp() 375 | 376 | def tearDown(self): 377 | safe_rmtree(self.root) 378 | 379 | @mock.patch('automan.jobs.total_cores', return_value=4.0) 380 | @mock.patch('automan.jobs.free_cores', return_value=2.0) 381 | def test_worker_computes_correct_cores_required( 382 | self, mock_free_cores, mock_total_cores 383 | ): 384 | # Given 385 | w = jobs.Worker() 386 | self.assertEqual(w.total_cores(), 4.0) 387 | self.assertEqual(w.free_cores(), 2.0) 388 | 389 | # When/Then 390 | self.assertEqual(w.cores_required(1), 1) 391 | self.assertEqual(w.cores_required(2), 2) 392 | self.assertEqual(w.cores_required(0), 0) 393 | self.assertEqual(w.cores_required(-1), 4) 394 | self.assertEqual(w.cores_required(-2), 2) 395 | 396 | self.assertEqual(w.can_run(0), True) 397 | self.assertEqual(w.can_run(1), True) 398 | self.assertEqual(w.can_run(2), True) 399 | self.assertEqual(w.can_run(-2), True) 400 | self.assertEqual(w.can_run(-1), False) 401 | 402 | @mock.patch('automan.jobs.free_cores', return_value=2.0) 403 | def test_scheduler_works_with_local_worker(self, mock_free_cores): 404 | # Given 405 | s = jobs.Scheduler(worker_config=[dict(host='localhost')]) 406 | 407 | # When 408 | j = jobs.Job( 409 | [sys.executable, '-c', 'import time; time.sleep(0.05); print(1)'], 410 | output_dir=self.root 411 | ) 412 | proxy = s.submit(j) 413 | 414 | # Then 415 | wait_until(lambda: proxy.status() != 'done', timeout=2) 416 | self.assertEqual(proxy.status(), 'done') 417 | self.assertEqual(proxy.get_stderr(), '') 418 | self.assertEqual(proxy.get_stdout().strip(), '1') 419 | info = proxy.get_info() 420 | self.assertEqual(info['status'], 'done') 421 | self.assertEqual(info['exitcode'], 0) 422 | 423 | 424 | class TestRemoteWorker(unittest.TestCase): 425 | def setUp(self): 426 | self.root = tempfile.mkdtemp() 427 | try: 428 | import execnet 429 | except ImportError: 430 | raise unittest.SkipTest('This test requires execnet') 431 | 432 | def tearDown(self): 433 | safe_rmtree(self.root) 434 | 435 | def test_free_cores(self): 436 | # Given 437 | r = jobs.RemoteWorker( 438 | host='localhost', python=sys.executable, testing=True 439 | ) 440 | # Then. 441 | n = r.free_cores() 442 | self.assertTrue(n >= 0) 443 | self.assertTrue(n <= multiprocessing.cpu_count()) 444 | 445 | def test_simple(self): 446 | # Given 447 | r = jobs.RemoteWorker( 448 | host='localhost', python=sys.executable, testing=True 449 | ) 450 | 451 | # When 452 | j = jobs.Job( 453 | [sys.executable, '-c', 'import time; time.sleep(0.05); print(1)'], 454 | output_dir=self.root 455 | ) 456 | proxy = r.run(j) 457 | 458 | # Then 459 | wait_until(lambda: proxy.status() != 'done') 460 | self.assertEqual(proxy.status(), 'done') 461 | self.assertEqual(proxy.get_stderr(), '') 462 | self.assertEqual(proxy.get_stdout().strip(), '1') 463 | info = proxy.get_info() 464 | self.assertEqual(info['status'], 'done') 465 | self.assertEqual(info['exitcode'], 0) 466 | 467 | def test_remote_worker_does_not_copy_when_nfs_is_set(self): 468 | # Given 469 | r = jobs.RemoteWorker( 470 | host='localhost', python=sys.executable, testing=True, nfs=True 471 | ) 472 | 473 | # When 474 | j = jobs.Job( 475 | [sys.executable, '-c', 'import time; time.sleep(0.05); print(1)'], 476 | output_dir=self.root 477 | ) 478 | proxy = r.run(j) 479 | wait_until(lambda: proxy.status() != 'done') 480 | 481 | # Now set testing to False and check the copying 482 | r.testing = False 483 | ret = r.copy_output(proxy.job_id, '.') 484 | 485 | # Then 486 | self.assertEqual(ret, None) 487 | 488 | 489 | class TestScheduler(unittest.TestCase): 490 | def setUp(self): 491 | self.root = tempfile.mkdtemp() 492 | self.count = 0 493 | try: 494 | import execnet 495 | except ImportError: 496 | raise unittest.SkipTest('This test requires execnet') 497 | 498 | def tearDown(self): 499 | safe_rmtree(self.root) 500 | 501 | def _make_dummy_job(self, n_core=1, sleep=0.05): 502 | output = os.path.join(self.root, 'job%d' % self.count) 503 | job = jobs.Job( 504 | [sys.executable, '-c', 505 | 'import time; time.sleep(%f); print(1)' % sleep], 506 | output_dir=output, 507 | n_core=n_core 508 | ) 509 | self.count += 1 510 | return job 511 | 512 | @mock.patch('automan.jobs.LocalWorker') 513 | def test_scheduler_does_not_start_worker_when_created(self, mock_lw): 514 | # Given 515 | config = [dict(host='localhost')] 516 | 517 | # When 518 | s = jobs.Scheduler(worker_config=config) 519 | 520 | # Then 521 | self.assertEqual(mock_lw.call_count, 0) 522 | self.assertEqual(len(s.workers), 0) 523 | 524 | @mock.patch('automan.jobs.LocalWorker') 525 | def test_scheduler_starts_worker_on_submit(self, mock_lw): 526 | attrs = {'host': 'localhost', 'free_cores.return_value': 2} 527 | mock_lw.return_value = mock.MagicMock(**attrs) 528 | 529 | # Given 530 | config = [dict(host='localhost')] 531 | s = jobs.Scheduler(worker_config=config) 532 | j = jobs.Job( 533 | [sys.executable, '-c', 'print(1)'], 534 | output_dir=self.root 535 | ) 536 | 537 | # When 538 | s.submit(j) 539 | 540 | # Then 541 | self.assertEqual(mock_lw.call_count, 1) 542 | self.assertEqual(len(s.workers), 1) 543 | 544 | @mock.patch.object(jobs.RemoteWorker, 'free_cores', return_value=2.0) 545 | def test_scheduler_only_creates_required_workers(self, mock_remote_worker): 546 | # Given 547 | config = [ 548 | dict(host='host1', python=sys.executable, testing=True), 549 | dict(host='host2', python=sys.executable, testing=True), 550 | ] 551 | s = jobs.Scheduler(worker_config=config) 552 | j = self._make_dummy_job() 553 | 554 | # When 555 | proxy = s.submit(j) 556 | 557 | # Then 558 | self.assertEqual(len(s.workers), 1) 559 | self.assertEqual(proxy.worker.host, 'host1') 560 | 561 | # Wait for this job to end and then see what happens 562 | # When a new job is submitted. 563 | self._wait_while_not_done(proxy, 25) 564 | 565 | # When 566 | j = self._make_dummy_job() 567 | s.submit(j) 568 | 569 | # Then 570 | self.assertEqual(len(s.workers), 1) 571 | self.assertEqual(proxy.worker.host, 'host1') 572 | 573 | # Running two jobs in a row should produce two workers. 574 | # When 575 | j = self._make_dummy_job() 576 | proxy = s.submit(j) 577 | 578 | # Then 579 | self.assertEqual(len(s.workers), 2) 580 | self.assertEqual(proxy.worker.host, 'host2') 581 | 582 | # Adding more should work. 583 | 584 | # When 585 | j = self._make_dummy_job() 586 | proxy = s.submit(j) 587 | j = self._make_dummy_job() 588 | proxy1 = s.submit(j) 589 | 590 | # Then 591 | self.assertEqual(len(s.workers), 2) 592 | self._wait_while_not_done(proxy, 15) 593 | self._wait_while_not_done(proxy1, 15) 594 | 595 | self.assertEqual(proxy.status(), 'done') 596 | self.assertEqual(proxy.worker.host, 'host1') 597 | self.assertEqual(proxy1.worker.host, 'host2') 598 | 599 | @mock.patch('automan.jobs.total_cores', return_value=2.0) 600 | @mock.patch('automan.jobs.free_cores', return_value=2.0) 601 | def test_scheduler_should_not_overload_worker(self, m_total_cores, 602 | m_free_cores): 603 | # Given 604 | n_core = jobs.total_cores() 605 | config = [dict(host='localhost')] 606 | s = jobs.Scheduler(worker_config=config, wait=0.5) 607 | 608 | j1 = self._make_dummy_job(n_core, sleep=0.5) 609 | j2 = self._make_dummy_job(n_core, sleep=0.5) 610 | j3 = self._make_dummy_job(n_core, sleep=0.5) 611 | j4 = self._make_dummy_job(0, sleep=0.5) 612 | 613 | # When 614 | proxy1 = s.submit(j1) 615 | proxy2 = s.submit(j2) 616 | proxy3 = s.submit(j3) 617 | proxy4 = s.submit(j4) 618 | 619 | # Then 620 | self.assertEqual(len(s.workers), 1) 621 | # Basically, submit will wait for the existing jobs to complete. 622 | # Therefore when s.submit(j3) is called it should wait until a 623 | # free worker is available. 624 | self.assertEqual(proxy1.status(), 'done') 625 | self.assertEqual(proxy2.status(), 'done') 626 | self.assertEqual(proxy3.status(), 'running') 627 | 628 | # Proxy4 will be running since it needs no cores. 629 | self.assertEqual(proxy4.status(), 'running') 630 | self._wait_while_not_done(proxy3, 20) 631 | self.assertEqual(proxy3.status(), 'done') 632 | self._wait_while_not_done(proxy4, 15) 633 | self.assertEqual(proxy4.status(), 'done') 634 | 635 | def _wait_while_not_done(self, proxy, n_count, sleep=0.1): 636 | count = 0 637 | while proxy.status() != 'done' and count < n_count: 638 | time.sleep(sleep) 639 | count += 1 640 | -------------------------------------------------------------------------------- /automan/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | from automan.automation import Simulation 3 | from automan.utils import (compare_runs, dprod, filter_cases, mdict, 4 | opts2path) 5 | 6 | 7 | def test_compare_runs_calls_methods_when_given_names(): 8 | # Given 9 | sims = [mock.MagicMock(), mock.MagicMock()] 10 | s0, s1 = sims 11 | s0.get_labels.return_value = s1.get_labels.return_value = 'label' 12 | 13 | # When 14 | compare_runs(sims, 'fig', labels=['x'], exact='exact') 15 | 16 | # Then 17 | s0.exact.assert_called_once_with(color='k', linestyle='-') 18 | s0.fig.assert_called_once_with(color='k', label='label', linestyle='--') 19 | s0.get_labels.assert_called_once_with(['x']) 20 | assert s1.exact.called is False 21 | s1.fig.assert_called_once_with(color='k', label='label', linestyle='-.') 22 | s1.get_labels.assert_called_once_with(['x']) 23 | 24 | 25 | def test_compare_runs_works_when_given_callables(): 26 | # Given 27 | sims = [mock.MagicMock()] 28 | s0 = sims[0] 29 | s0.get_labels.return_value = 'label' 30 | 31 | func = mock.MagicMock() 32 | exact = mock.MagicMock() 33 | 34 | # When 35 | compare_runs(sims, func, labels=['x'], exact=exact) 36 | 37 | # Then 38 | exact.assert_called_once_with(s0, color='k', linestyle='-') 39 | func.assert_called_once_with(s0, color='k', label='label', linestyle='--') 40 | s0.get_labels.assert_called_once_with(['x']) 41 | 42 | 43 | def test_compare_runs_uses_given_styles(): 44 | # Given 45 | sims = [mock.MagicMock()] 46 | s0 = sims[0] 47 | s0.get_labels.return_value = 'label' 48 | 49 | func = mock.MagicMock() 50 | styles = mock.MagicMock() 51 | styles.return_value = iter([dict(linestyle=':', color='b')]) 52 | 53 | # When 54 | compare_runs(sims, func, labels=['x'], styles=styles) 55 | 56 | # Then 57 | func.assert_called_once_with(s0, color='b', label='label', linestyle=':') 58 | s0.get_labels.assert_called_once_with(['x']) 59 | styles.assert_called_once_with(sims) 60 | 61 | 62 | def test_compare_runs_uses_given_styles_returning_iterable(): 63 | # Given 64 | sims = [mock.MagicMock()] 65 | s0 = sims[0] 66 | s0.get_labels.return_value = 'label' 67 | 68 | func = mock.MagicMock() 69 | styles = mock.MagicMock() 70 | # Return an iterable and not an iterator 71 | styles.return_value = [dict(linestyle=':', color='b')] 72 | 73 | # When 74 | compare_runs(sims, func, labels=['x'], styles=styles) 75 | 76 | # Then 77 | func.assert_called_once_with(s0, color='b', label='label', linestyle=':') 78 | s0.get_labels.assert_called_once_with(['x']) 79 | styles.assert_called_once_with(sims) 80 | 81 | 82 | def test_dprod(): 83 | # Given/When 84 | res = dprod(mdict(a=[1, 2], b=['xy']), mdict(c='ab')) 85 | # Then 86 | exp = [{'a': 1, 'b': 'xy', 'c': 'a'}, 87 | {'a': 1, 'b': 'xy', 'c': 'b'}, 88 | {'a': 2, 'b': 'xy', 'c': 'a'}, 89 | {'a': 2, 'b': 'xy', 'c': 'b'}] 90 | assert res == exp 91 | 92 | 93 | def test_filter_cases_works_with_params(): 94 | # Given 95 | sims = [Simulation(root='', base_command='python', param1=i, param2=i+1) 96 | for i in range(5)] 97 | # When 98 | result = filter_cases(sims, param1=2) 99 | 100 | # Then 101 | assert len(result) == 1 102 | assert result[0].params['param1'] == 2 103 | 104 | # When 105 | result = filter_cases(sims, param1=2, param2=2) 106 | 107 | # Then 108 | assert len(result) == 0 109 | 110 | # When 111 | result = filter_cases(sims, param1=3, param2=4) 112 | 113 | # Then 114 | assert len(result) == 1 115 | assert result[0].params['param1'] == 3 116 | assert result[0].params['param2'] == 4 117 | 118 | 119 | def test_filter_cases_works_with_predicate(): 120 | # Given 121 | sims = [Simulation(root='', base_command='python', param1=i, param2=i+1) 122 | for i in range(5)] 123 | 124 | # When 125 | result = filter_cases( 126 | sims, predicate=lambda x: x.params.get('param1', 0) % 2 127 | ) 128 | 129 | # Then 130 | assert len(result) == 2 131 | assert result[0].params['param1'] == 1 132 | assert result[1].params['param1'] == 3 133 | 134 | # When 135 | result = filter_cases( 136 | sims, predicate=2 137 | ) 138 | 139 | # Then 140 | assert len(result) == 0 141 | 142 | # Given 143 | sims = [Simulation(root='', base_command='python', predicate=i) 144 | for i in range(5)] 145 | 146 | # When 147 | result = filter_cases( 148 | sims, predicate=2 149 | ) 150 | 151 | # Then 152 | assert len(result) == 1 153 | assert result[0].params['predicate'] == 2 154 | 155 | 156 | def test_mdict(): 157 | exp = [{'a': 1, 'b': 'x'}, 158 | {'a': 1, 'b': 'y'}, 159 | {'a': 2, 'b': 'x'}, 160 | {'a': 2, 'b': 'y'}] 161 | assert mdict(a=[1, 2], b='xy') == exp 162 | 163 | 164 | def test_opts2path(): 165 | assert opts2path(dict(x=1, y='hello', z=0.1)) == 'x_1_hello_z_0.1' 166 | assert opts2path(dict(x=1, y='hello', z=0.1), keys=['x']) == 'x_1' 167 | res = opts2path(dict(x=1, y='hello', z=0.1), ignore=['x']) 168 | assert res == 'hello_z_0.1' 169 | res = opts2path(dict(x=1, y='hello', z=0.1), kmap=dict(x='XX')) 170 | assert res == 'XX_1_hello_z_0.1' 171 | -------------------------------------------------------------------------------- /automan/utils.py: -------------------------------------------------------------------------------- 1 | """Utility functions for automation scripts. 2 | """ 3 | import collections 4 | import itertools as IT 5 | 6 | 7 | def dprod(a, b): 8 | '''Multiplies the given list of dictionaries `a` and `b`. 9 | 10 | This makes a list of new dictionaries which is the product of the given 11 | two dictionaries. 12 | 13 | **Example** 14 | 15 | >>> dprod(mdict(a=[1, 2], b=['xy']), mdict(c='ab')) 16 | [{'a': 1, 'b': 'xy', 'c': 'a'}, 17 | {'a': 1, 'b': 'xy', 'c': 'b'}, 18 | {'a': 2, 'b': 'xy', 'c': 'a'}, 19 | {'a': 2, 'b': 'xy', 'c': 'b'}] 20 | 21 | ''' 22 | return [ 23 | dict(IT.chain(x.items(), y.items())) for x, y in IT.product(a, b) 24 | ] 25 | 26 | 27 | def styles(sims): 28 | """Cycles over a set of possible styles to use for plotting. 29 | 30 | The method is passed a sequence of the Simulation instances. This should 31 | return an iterator which produces a dictionary each time containing a set 32 | of keyword arguments to be used for a particular plot. 33 | 34 | **Parameters** 35 | 36 | sims: sequence 37 | Sequence of `Simulation` objects. 38 | 39 | **Returns** 40 | 41 | An iterator which produces a dictionary containing a set of kwargs to be 42 | used for the plotting. Can also return an iterable containing 43 | dictionaries. 44 | 45 | """ 46 | ls = [dict(color=x[0], linestyle=x[1]) for x in 47 | IT.product("kbgr", ["-", "--", "-.", ":"])] 48 | return IT.cycle(ls) 49 | 50 | 51 | def compare_runs(sims, method, labels, exact=None, styles=styles): 52 | """Given a sequence of Simulation instances, a method name, the labels to 53 | compare and an optional method name for an exact solution, this calls the 54 | methods with the appropriate parameters for each simulation. 55 | 56 | **Parameters** 57 | 58 | sims: sequence 59 | Sequence of `Simulation` objects. 60 | method: str or callable 61 | Name of a method on each simulation method to call for plotting. 62 | Or a callable which is passed the simulation instance and any kwargs. 63 | labels: sequence 64 | Sequence of parameters to use as labels for the plot. 65 | exact: str or callable 66 | Name of a method that produces an exact solution plot 67 | or a callable that will be called. 68 | styles: callable: returns an iterator/iterable of style keyword arguments. 69 | Defaults to the ``styles`` function defined in this module. 70 | """ 71 | ls = styles(sims) 72 | if isinstance(ls, collections.abc.Iterable): 73 | ls = iter(ls) 74 | if exact is not None: 75 | if isinstance(exact, str): 76 | getattr(sims[0], exact)(**next(ls)) 77 | else: 78 | exact(sims[0], **next(ls)) 79 | for s in sims: 80 | if isinstance(method, str): 81 | m = getattr(s, method) 82 | m(label=s.get_labels(labels), **next(ls)) 83 | else: 84 | method(s, label=s.get_labels(labels), **next(ls)) 85 | 86 | 87 | def filter_cases(runs, predicate=None, **params): 88 | """Given a sequence of simulations and any additional parameters, filter 89 | out all the cases having exactly those parameters and return a list of 90 | them. 91 | 92 | One may also pass a callable to filter the cases using the `predicate` 93 | keyword argument. If this is not a callable, it is treated as a parameter. 94 | If `predicate` is passed though, the other keyword arguments are ignored. 95 | 96 | """ 97 | if predicate is not None: 98 | if callable(predicate): 99 | return list(filter(predicate, runs)) 100 | else: 101 | params['predicate'] = predicate 102 | 103 | def _check_match(run): 104 | for param, expected in params.items(): 105 | if param not in run.params or run.params[param] != expected: 106 | return False 107 | return True 108 | 109 | return list(filter(_check_match, runs)) 110 | 111 | 112 | def filter_by_name(cases, names): 113 | """Filter a sequence of Simulations by their names. That is, if the case 114 | has a name contained in the given `names`, it will be selected. 115 | """ 116 | if isinstance(names, str): 117 | names = [names] 118 | return sorted( 119 | [x for x in cases if x.name in names], 120 | key=lambda x: names.index(x.name) 121 | ) 122 | 123 | 124 | def mdict(**kw): 125 | '''Expands out the passed kwargs into a list of dictionaries. 126 | 127 | Each kwarg value is expected to be a sequence. The resulting list of 128 | dictionaries is the product of the different values and the same keys. 129 | 130 | **Example** 131 | 132 | >>> mdict(a=[1, 2], b='xy') 133 | [{'a': 1, 'b': 'x'}, 134 | {'a': 1, 'b': 'y'}, 135 | {'a': 2, 'b': 'x'}, 136 | {'a': 2, 'b': 'y'}] 137 | 138 | ''' 139 | keys = list(kw.keys()) 140 | return [dict(zip(keys, opts)) for opts in IT.product(*kw.values())] 141 | 142 | 143 | def opts2path(opts, keys=None, ignore=None, kmap=None): 144 | '''Renders the given options as a path name. 145 | 146 | **Parameters** 147 | 148 | opts: dict 149 | dictionary of options 150 | keys: list 151 | Keys of the options use. 152 | ignore: list 153 | Ignore these keys in the options. 154 | kmap: dict 155 | map the key names through this dict. 156 | 157 | **Examples** 158 | 159 | >>> opts2path(dict(x=1, y='hello', z=0.1)) 160 | 'x_1_hello_z_0.1' 161 | >>> opts2path(dict(x=1, y='hello', z=0.1), keys=['x']) 162 | 'x_1' 163 | >>> opts2path(dict(x=1, y='hello', z=0.1), ignore=['x']) 164 | 'hello_z_0.1' 165 | >>> opts2path(dict(x=1, y='hello', z=0.1), kmap=dict(x='XX')) 166 | 'XX_1_hello_z_0.1' 167 | ''' 168 | keys = set(opts.keys()) if keys is None else set(keys) 169 | ignore = [] if ignore is None else ignore 170 | keymap = {} if kmap is None else kmap 171 | for x in ignore: 172 | keys.discard(x) 173 | keys = sorted(x for x in keys if x in opts) 174 | 175 | def _key2name(k): 176 | v = opts[k] 177 | r = keymap.get(k, '') 178 | if r: 179 | return f'{r}_{v}' 180 | elif isinstance(v, str): 181 | return v 182 | else: 183 | return f'{k}_{v}' 184 | 185 | return '_'.join([_key2name(k) for k in keys]) 186 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = automan 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | set SPHINXPROJ=automan 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 20 | echo.installed, then set the SPHINXBUILD environment variable to point 21 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 22 | echo.may add the Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx 2 | sphinx-rtd-theme 3 | 4 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # automan documentation build configuration file, created by 5 | # sphinx-quickstart on Thu Aug 23 22:12:56 2018. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | # 20 | # import os 21 | # import sys 22 | # sys.path.insert(0, os.path.abspath('.')) 23 | import os 24 | 25 | # -- General configuration ------------------------------------------------ 26 | 27 | # If your documentation needs a minimal Sphinx version, state it here. 28 | # 29 | # needs_sphinx = '1.0' 30 | 31 | # Add any Sphinx extension module names here, as strings. They can be 32 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 33 | # ones. 34 | extensions = [ 35 | 'sphinx.ext.autodoc', 36 | 'sphinx.ext.mathjax', 37 | 'sphinx.ext.viewcode', 38 | 'sphinx_rtd_theme' 39 | ] 40 | 41 | # Add any paths that contain templates here, relative to this directory. 42 | templates_path = ['_templates'] 43 | 44 | # The suffix(es) of source filenames. 45 | # You can specify multiple suffix as a list of string: 46 | # 47 | # source_suffix = ['.rst', '.md'] 48 | source_suffix = '.rst' 49 | 50 | # The master toctree document. 51 | master_doc = 'index' 52 | 53 | # General information about the project. 54 | project = 'automan' 55 | copyright = '2018-2025, Prabhu Ramachandran' 56 | author = 'Prabhu Ramachandran' 57 | 58 | # The version info for the project you're documenting, acts as replacement for 59 | # |version| and |release|, also used in various other places throughout the 60 | # built documents. 61 | # 62 | _d = {} 63 | fname = os.path.join(os.pardir, os.pardir, 'automan', '__init__.py') 64 | exec(compile(open(fname).read(), fname, 'exec'), _d) 65 | 66 | # The full version, including alpha/beta/rc tags. 67 | release = _d['__version__'] 68 | 69 | # The short X.Y version. 70 | version = release[:3] 71 | 72 | 73 | # The language for content autogenerated by Sphinx. Refer to documentation 74 | # for a list of supported languages. 75 | # 76 | # This is also used if you do content translation via gettext catalogs. 77 | # Usually you set "language" from the command line for these cases. 78 | language = 'en' 79 | 80 | # List of patterns, relative to source directory, that match files and 81 | # directories to ignore when looking for source files. 82 | # This patterns also effect to html_static_path and html_extra_path 83 | exclude_patterns = [] 84 | 85 | # The name of the Pygments (syntax highlighting) style to use. 86 | pygments_style = 'sphinx' 87 | 88 | # If true, `todo` and `todoList` produce output, else they produce nothing. 89 | todo_include_todos = False 90 | 91 | 92 | # -- Options for HTML output ---------------------------------------------- 93 | 94 | # The theme to use for HTML and HTML Help pages. See the documentation for 95 | # a list of builtin themes. 96 | # 97 | html_theme = 'sphinx_rtd_theme' 98 | 99 | # Theme options are theme-specific and customize the look and feel of a theme 100 | # further. For a list of options available for each theme, see the 101 | # documentation. 102 | # 103 | # html_theme_options = {} 104 | 105 | # Add any paths that contain custom static files (such as style sheets) here, 106 | # relative to this directory. They are copied after the builtin static files, 107 | # so a file named "default.css" will overwrite the builtin "default.css". 108 | html_static_path = ['_static'] 109 | 110 | 111 | # -- Options for HTMLHelp output ------------------------------------------ 112 | 113 | # Output file base name for HTML help builder. 114 | htmlhelp_basename = 'automandoc' 115 | 116 | 117 | # -- Options for LaTeX output --------------------------------------------- 118 | 119 | latex_elements = { 120 | # The paper size ('letterpaper' or 'a4paper'). 121 | # 122 | # 'papersize': 'letterpaper', 123 | 124 | # The font size ('10pt', '11pt' or '12pt'). 125 | # 126 | # 'pointsize': '10pt', 127 | 128 | # Additional stuff for the LaTeX preamble. 129 | # 130 | # 'preamble': '', 131 | 132 | # Latex figure (float) alignment 133 | # 134 | # 'figure_align': 'htbp', 135 | } 136 | 137 | # Grouping the document tree into LaTeX files. List of tuples 138 | # (source start file, target name, title, 139 | # author, documentclass [howto, manual, or own class]). 140 | latex_documents = [ 141 | (master_doc, 'automan.tex', 'automan Documentation', 142 | 'Prabhu Ramachandran', 'manual'), 143 | ] 144 | 145 | 146 | # -- Options for manual page output --------------------------------------- 147 | 148 | # One entry per manual page. List of tuples 149 | # (source start file, name, description, authors, manual section). 150 | man_pages = [ 151 | (master_doc, 'automan', 'automan Documentation', 152 | [author], 1) 153 | ] 154 | 155 | 156 | # -- Options for Texinfo output ------------------------------------------- 157 | 158 | # Grouping the document tree into Texinfo files. List of tuples 159 | # (source start file, target name, title, author, 160 | # dir menu entry, description, category) 161 | texinfo_documents = [ 162 | (master_doc, 'automan', 'automan Documentation', 163 | author, 'automan', 'One line description of project.', 164 | 'Miscellaneous'), 165 | ] 166 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. automan documentation master file, created by 2 | sphinx-quickstart on Thu Aug 23 22:12:56 2018. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to automan's documentation! 7 | =================================== 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | :caption: Contents: 12 | 13 | overview.rst 14 | tutorial.rst 15 | 16 | ************************ 17 | Reference documentation 18 | ************************ 19 | 20 | 21 | .. toctree:: 22 | :maxdepth: 2 23 | 24 | reference/index.rst 25 | 26 | Indices and tables 27 | ================== 28 | 29 | * :ref:`genindex` 30 | * :ref:`modindex` 31 | * :ref:`search` 32 | -------------------------------------------------------------------------------- /docs/source/overview.rst: -------------------------------------------------------------------------------- 1 | Overview 2 | ========== 3 | 4 | ``automan`` is an open source, Python-based automation framework for numerical 5 | computing. 6 | 7 | It is designed to automate the drudge work of managing many numerical 8 | simulations. As an automation framework it does the following: 9 | 10 | - helps you organize your simulations. 11 | - helps you orchestrate running simulations and then post-processing the 12 | results from these. 13 | - helps you reuse code for the post processing of your simulation data. 14 | - execute all your simulations and post-processing with one command. 15 | - optionally distribute your simulations among other computers on your 16 | network. 17 | 18 | This greatly facilitates reproducibility. Automan is written in pure Python 19 | and is easy to install. 20 | 21 | This document should help you use automan to improve your productivity. If you 22 | are interested in a more detailed article about automan see the `automan paper 23 | `_ a draft of 24 | which is available here: https://arxiv.org/abs/1712.04786 25 | 26 | 27 | Installation 28 | ------------- 29 | 30 | The easiest way to install ``automan`` is with pip_:: 31 | 32 | $ pip install automan 33 | 34 | If you wish to run the latest version that has not been relesed you may clone 35 | the git repository:: 36 | 37 | $ git clone https://github.com/pypr/automan 38 | 39 | $ cd automan 40 | 41 | And then run:: 42 | 43 | $ python setup.py develop 44 | 45 | .. _pip: https://pip.pypa.io/en/stable/ 46 | 47 | If you just want to run the latest version and do not have ``git`` you can do this:: 48 | 49 | $ pip install https://github.com/pypr/automan/zipball/main 50 | 51 | Once this is done, move on to the next section that provides a gentle tutorial 52 | introduction to using automan. 53 | 54 | 55 | Citing ``automan`` 56 | ------------------- 57 | 58 | If you find automan useful and wish to cite it you may use the following 59 | article: 60 | 61 | - Prabhu Ramachandran, "automan: A Python-Based Automation Framework for 62 | Numerical Computing," in *Computing in Science & Engineering*, vol. 20, no. 63 | 5, pp. 81-97, 2018. `doi:10.1109/MCSE.2018.05329818 64 | `_ 65 | 66 | You can find a draft of the article here: https://arxiv.org/abs/1712.04786 67 | 68 | 69 | Changelog 70 | ---------- 71 | 72 | .. include:: ../../CHANGES.rst 73 | -------------------------------------------------------------------------------- /docs/source/reference/automan.rst: -------------------------------------------------------------------------------- 1 | Main automation module 2 | ======================= 3 | 4 | .. automodule:: automan.automation 5 | :members: 6 | :undoc-members: 7 | 8 | 9 | Utility functions for automation 10 | ================================= 11 | 12 | .. automodule:: automan.utils 13 | :members: 14 | :undoc-members: 15 | 16 | 17 | Low-level job management module 18 | ================================= 19 | 20 | .. automodule:: automan.jobs 21 | :members: 22 | :undoc-members: 23 | 24 | Cluster management module 25 | ========================= 26 | 27 | .. automodule:: automan.cluster_manager 28 | :members: 29 | :undoc-members: 30 | 31 | .. automodule:: automan.conda_cluster_manager 32 | :members: 33 | :undoc-members: 34 | 35 | .. automodule:: automan.edm_cluster_manager 36 | :members: 37 | :undoc-members: 38 | -------------------------------------------------------------------------------- /docs/source/reference/index.rst: -------------------------------------------------------------------------------- 1 | Reference Documentation 2 | ========================= 3 | 4 | Autogenerated from doc strings using sphinx’s autodoc feature. 5 | 6 | .. toctree:: 7 | :maxdepth: 3 8 | 9 | automan.rst 10 | -------------------------------------------------------------------------------- /examples/edm_conda_cluster/README.md: -------------------------------------------------------------------------------- 1 | # EDM and Conda cluster manager example 2 | 3 | This uses the example from the tutorial but shows how it can be used with the 4 | remote computers running conda or edm. 5 | 6 | ## Conda example 7 | 8 | The `environments.yml` is used to create the environment and there is also an 9 | optional `requirements.txt` which will install packages using `pip`. 10 | 11 | The example assumes that the remote computer has its conda root in `~/miniconda3`. 12 | 13 | To run the example, let us say you have a remote computer running Linux/Mac OS 14 | with password-less SSH setup, let us call this computer `remote_host`, then 15 | you can do the following: 16 | 17 | ``` 18 | $ python automate_conda.py -a remote_host 19 | [...] 20 | Bootstrapping remote_host succeeded! 21 | 22 | $ python automate_conda.py 23 | [...] 24 | ``` 25 | 26 | ## EDM example 27 | 28 | The ``requirements.txt`` is used to setup the environment. We assumne that the 29 | edm root is at ~``~/.edm``. 30 | 31 | Note that if you have already run the ``automate_conda.py`` example the 32 | simulations will be complete, so remove the ``outputs`` and ``manuscript`` 33 | directories first before the following. To run the example you may do: 34 | 35 | ``` 36 | $ python automate_edm.py -a remote_host 37 | [...] 38 | Bootstrapping remote_host succeeded! 39 | 40 | $ python automate_edm.py 41 | [...] 42 | ``` 43 | -------------------------------------------------------------------------------- /examples/edm_conda_cluster/automate_conda.py: -------------------------------------------------------------------------------- 1 | from automan.api import Problem, Automator, Simulation 2 | from automan.api import CondaClusterManager 3 | from matplotlib import pyplot as plt 4 | import numpy as np 5 | 6 | 7 | class Squares(Problem): 8 | def get_name(self): 9 | return 'squares' 10 | 11 | def get_commands(self): 12 | commands = [(str(i), 'python square.py %d' % i, None) 13 | for i in range(1, 8)] 14 | return commands 15 | 16 | def run(self): 17 | self.make_output_dir() 18 | data = [] 19 | for i in range(1, 8): 20 | stdout = self.input_path(str(i), 'stdout.txt') 21 | with open(stdout) as f: 22 | values = [float(x) for x in f.read().split()] 23 | data.append(values) 24 | 25 | data = np.asarray(data) 26 | plt.plot(data[:, 0], data[:, 1], 'o-') 27 | plt.xlabel('x') 28 | plt.ylabel('y') 29 | plt.savefig(self.output_path('squares.pdf')) 30 | 31 | 32 | class Powers(Problem): 33 | def get_name(self): 34 | return 'powers' 35 | 36 | def setup(self): 37 | base_cmd = 'python powers.py --output-dir $output_dir' 38 | self.cases = [ 39 | Simulation( 40 | root=self.input_path(str(i)), 41 | base_command=base_cmd, 42 | power=float(i) 43 | ) 44 | for i in range(1, 5) 45 | ] 46 | 47 | def run(self): 48 | self.make_output_dir() 49 | for case in self.cases: 50 | data = np.load(case.input_path('results.npz')) 51 | plt.plot( 52 | data['x'], data['y'], 53 | label=r'$x^{{%.2f}}$' % case.params['power'] 54 | ) 55 | plt.grid() 56 | plt.xlabel('x') 57 | plt.ylabel('y') 58 | plt.legend() 59 | plt.savefig(self.output_path('powers.pdf')) 60 | 61 | 62 | if __name__ == '__main__': 63 | automator = Automator( 64 | simulation_dir='outputs', 65 | output_dir='manuscript/figures', 66 | all_problems=[Squares, Powers], 67 | cluster_manager_factory=CondaClusterManager 68 | ) 69 | automator.run() 70 | -------------------------------------------------------------------------------- /examples/edm_conda_cluster/automate_edm.py: -------------------------------------------------------------------------------- 1 | from automan.api import Problem, Automator, Simulation 2 | from automan.api import EDMClusterManager 3 | from matplotlib import pyplot as plt 4 | import numpy as np 5 | 6 | 7 | class Squares(Problem): 8 | def get_name(self): 9 | return 'squares' 10 | 11 | def get_commands(self): 12 | commands = [(str(i), 'python square.py %d' % i, None) 13 | for i in range(1, 8)] 14 | return commands 15 | 16 | def run(self): 17 | self.make_output_dir() 18 | data = [] 19 | for i in range(1, 8): 20 | stdout = self.input_path(str(i), 'stdout.txt') 21 | with open(stdout) as f: 22 | values = [float(x) for x in f.read().split()] 23 | data.append(values) 24 | 25 | data = np.asarray(data) 26 | plt.plot(data[:, 0], data[:, 1], 'o-') 27 | plt.xlabel('x') 28 | plt.ylabel('y') 29 | plt.savefig(self.output_path('squares.pdf')) 30 | 31 | 32 | class Powers(Problem): 33 | def get_name(self): 34 | return 'powers' 35 | 36 | def setup(self): 37 | base_cmd = 'python powers.py --output-dir $output_dir' 38 | self.cases = [ 39 | Simulation( 40 | root=self.input_path(str(i)), 41 | base_command=base_cmd, 42 | power=float(i) 43 | ) 44 | for i in range(1, 5) 45 | ] 46 | 47 | def run(self): 48 | self.make_output_dir() 49 | for case in self.cases: 50 | data = np.load(case.input_path('results.npz')) 51 | plt.plot( 52 | data['x'], data['y'], 53 | label=r'$x^{{%.2f}}$' % case.params['power'] 54 | ) 55 | plt.grid() 56 | plt.xlabel('x') 57 | plt.ylabel('y') 58 | plt.legend() 59 | plt.savefig(self.output_path('powers.pdf')) 60 | 61 | 62 | if __name__ == '__main__': 63 | automator = Automator( 64 | simulation_dir='outputs', 65 | output_dir='manuscript/figures', 66 | all_problems=[Squares, Powers], 67 | cluster_manager_factory=EDMClusterManager 68 | ) 69 | automator.run() 70 | -------------------------------------------------------------------------------- /examples/edm_conda_cluster/environments.yml: -------------------------------------------------------------------------------- 1 | name: conda_cluster 2 | dependencies: 3 | - numpy 4 | -------------------------------------------------------------------------------- /examples/edm_conda_cluster/powers.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | import numpy as np 5 | 6 | 7 | def compute_powers(r_max, power): 8 | """Compute the powers of the integers upto r_max and return the result. 9 | """ 10 | result = [] 11 | for i in range(0, r_max + 1): 12 | result.append((i, i**power)) 13 | x = np.arange(0, r_max + 1) 14 | y = np.power(x, power) 15 | return x, y 16 | 17 | 18 | def main(): 19 | p = argparse.ArgumentParser() 20 | p.add_argument( 21 | '--power', type=float, default=2.0, 22 | help='Power to calculate' 23 | ) 24 | p.add_argument( 25 | '--max', type=int, default=10, 26 | help='Maximum integer that we must raise to the given power' 27 | ) 28 | p.add_argument( 29 | '--output-dir', type=str, default='.', 30 | help='Output directory to generate file.' 31 | ) 32 | opts = p.parse_args() 33 | 34 | x, y = compute_powers(opts.max, opts.power) 35 | 36 | fname = os.path.join(opts.output_dir, 'results.npz') 37 | np.savez(fname, x=x, y=y) 38 | 39 | 40 | if __name__ == '__main__': 41 | main() 42 | -------------------------------------------------------------------------------- /examples/edm_conda_cluster/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | -------------------------------------------------------------------------------- /examples/edm_conda_cluster/square.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import sys 3 | x = float(sys.argv[1]) 4 | print(x, x*x) 5 | 6 | -------------------------------------------------------------------------------- /examples/tutorial/README.md: -------------------------------------------------------------------------------- 1 | # Tutorial examples 2 | 3 | This directory contains the scripts and automation scripts that are 4 | discussed in the tutorials available here: 5 | https://automan.readthedocs.io/en/latest/tutorial.html 6 | -------------------------------------------------------------------------------- /examples/tutorial/automate1.py: -------------------------------------------------------------------------------- 1 | from automan.api import Problem, Automator 2 | 3 | 4 | class Squares(Problem): 5 | def get_name(self): 6 | return 'squares' 7 | 8 | def get_commands(self): 9 | return [ 10 | ('1', 'python square.py 1', None), 11 | ('2', 'python square.py 2', None), 12 | ] 13 | 14 | def run(self): 15 | self.make_output_dir() 16 | 17 | 18 | if __name__ == '__main__': 19 | automator = Automator( 20 | simulation_dir='outputs', 21 | output_dir='manuscript/figures', 22 | all_problems=[Squares] 23 | ) 24 | automator.run() 25 | -------------------------------------------------------------------------------- /examples/tutorial/automate2.py: -------------------------------------------------------------------------------- 1 | from automan.api import Problem, Automator 2 | 3 | 4 | class Squares(Problem): 5 | def get_name(self): 6 | return 'squares' 7 | 8 | def get_commands(self): 9 | return [ 10 | ('1', 'python square.py 1', None), 11 | ('2', 'python square.py 2', None), 12 | ('3', 'python square.py 3', None), 13 | ('4', 'python square.py 4', None), 14 | ] 15 | 16 | def run(self): 17 | self.make_output_dir() 18 | data = [] 19 | for i in ('1', '2', '3', '4'): 20 | stdout = self.input_path(i, 'stdout.txt') 21 | with open(stdout) as f: 22 | data.append(f.read().split()) 23 | 24 | output = self.output_path('output.txt') 25 | with open(output, 'w') as o: 26 | o.write(str(data)) 27 | 28 | 29 | if __name__ == '__main__': 30 | automator = Automator( 31 | simulation_dir='outputs', 32 | output_dir='manuscript/figures', 33 | all_problems=[Squares] 34 | ) 35 | 36 | automator.run() 37 | -------------------------------------------------------------------------------- /examples/tutorial/automate3.py: -------------------------------------------------------------------------------- 1 | from automan.api import Problem, Automator 2 | from matplotlib import pyplot as plt 3 | import numpy as np 4 | 5 | 6 | class Squares(Problem): 7 | def get_name(self): 8 | return 'squares' 9 | 10 | def get_commands(self): 11 | commands = [(str(i), 'python square.py %d' % i, None) 12 | for i in range(1, 8)] 13 | return commands 14 | 15 | def run(self): 16 | self.make_output_dir() 17 | data = [] 18 | for i in range(1, 8): 19 | stdout = self.input_path(str(i), 'stdout.txt') 20 | with open(stdout) as f: 21 | values = [float(x) for x in f.read().split()] 22 | data.append(values) 23 | 24 | data = np.asarray(data) 25 | plt.plot(data[:, 0], data[:, 1], 'o-') 26 | plt.xlabel('x') 27 | plt.ylabel('y') 28 | plt.savefig(self.output_path('squares.pdf')) 29 | plt.close() 30 | 31 | 32 | if __name__ == '_main__': 33 | automator = Automator( 34 | simulation_dir='outputs', 35 | output_dir='manuscript/figures', 36 | all_problems=[Squares] 37 | ) 38 | automator.run() 39 | -------------------------------------------------------------------------------- /examples/tutorial/automate4.py: -------------------------------------------------------------------------------- 1 | from automan.api import Problem, Automator 2 | from matplotlib import pyplot as plt 3 | import numpy as np 4 | 5 | 6 | class Squares(Problem): 7 | def get_name(self): 8 | return 'squares' 9 | 10 | def get_commands(self): 11 | commands = [(str(i), 'python square.py %d' % i, None) 12 | for i in range(1, 8)] 13 | return commands 14 | 15 | def run(self): 16 | self.make_output_dir() 17 | data = [] 18 | for i in range(1, 8): 19 | stdout = self.input_path(str(i), 'stdout.txt') 20 | with open(stdout) as f: 21 | values = [float(x) for x in f.read().split()] 22 | data.append(values) 23 | 24 | data = np.asarray(data) 25 | plt.plot(data[:, 0], data[:, 1], 'o-') 26 | plt.xlabel('x') 27 | plt.ylabel('y') 28 | plt.savefig(self.output_path('squares.pdf')) 29 | plt.close() 30 | 31 | 32 | from automan.api import Simulation 33 | 34 | 35 | class Powers(Problem): 36 | def get_name(self): 37 | return 'powers' 38 | 39 | def setup(self): 40 | base_cmd = 'python powers.py --output-dir $output_dir' 41 | self.cases = [ 42 | Simulation( 43 | root=self.input_path(str(i)), 44 | base_command=base_cmd, 45 | power=float(i) 46 | ) 47 | for i in range(1, 5) 48 | ] 49 | 50 | def run(self): 51 | self.make_output_dir() 52 | plt.figure() 53 | for case in self.cases: 54 | data = np.load(case.input_path('results.npz')) 55 | plt.plot( 56 | data['x'], data['y'], 57 | label=r'$x^{{%.2f}}$' % case.params['power'] 58 | ) 59 | plt.grid() 60 | plt.xlabel('x') 61 | plt.ylabel('y') 62 | plt.legend() 63 | plt.savefig(self.output_path('powers.pdf')) 64 | plt.close() 65 | 66 | 67 | if __name__ == '__main__': 68 | automator = Automator( 69 | simulation_dir='outputs', 70 | output_dir='manuscript/figures', 71 | all_problems=[Squares, Powers] 72 | ) 73 | automator.run() 74 | -------------------------------------------------------------------------------- /examples/tutorial/powers.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | import numpy as np 5 | 6 | 7 | def compute_powers(r_max, power): 8 | """Compute the powers of the integers upto r_max and return the result. 9 | """ 10 | x = np.arange(0, r_max + 1) 11 | y = np.power(x, power) 12 | return x, y 13 | 14 | 15 | def main(): 16 | p = argparse.ArgumentParser() 17 | p.add_argument( 18 | '--power', type=float, default=2.0, 19 | help='Power to calculate' 20 | ) 21 | p.add_argument( 22 | '--max', type=int, default=10, 23 | help='Maximum integer that we must raise to the given power' 24 | ) 25 | p.add_argument( 26 | '--output-dir', type=str, default='.', 27 | help='Output directory to generate file.' 28 | ) 29 | opts = p.parse_args() 30 | 31 | x, y = compute_powers(opts.max, opts.power) 32 | 33 | fname = os.path.join(opts.output_dir, 'results.npz') 34 | np.savez(fname, x=x, y=y) 35 | 36 | 37 | if __name__ == '__main__': 38 | main() 39 | -------------------------------------------------------------------------------- /examples/tutorial/square.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import sys 3 | 4 | x = float(sys.argv[1]) 5 | print(x, x*x) 6 | 7 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "wheel>=0.29.0", 4 | "setuptools>=42.0.0" 5 | ] -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | import sys 3 | 4 | 5 | def get_version(): 6 | import os 7 | data = {} 8 | fname = os.path.join('automan', '__init__.py') 9 | exec(compile(open(fname).read(), fname, 'exec'), data) 10 | return data.get('__version__') 11 | 12 | 13 | install_requires = ['psutil', 'execnet', 'setuptools'] 14 | tests_require = ['pytest'] 15 | if sys.version_info.major < 3: 16 | tests_require.append('mock') 17 | install_requires.append('mock') 18 | 19 | classes = """ 20 | Development Status :: 3 - Alpha 21 | Environment :: Console 22 | Intended Audience :: Developers 23 | Intended Audience :: Education 24 | Intended Audience :: End Users/Desktop 25 | Intended Audience :: Science/Research 26 | License :: OSI Approved :: BSD License 27 | Natural Language :: English 28 | Operating System :: OS Independent 29 | Programming Language :: Python 30 | Programming Language :: Python :: 2.7 31 | Programming Language :: Python :: 3 32 | Programming Language :: Python :: 3.3 33 | Programming Language :: Python :: 3.4 34 | Programming Language :: Python :: 3.5 35 | Programming Language :: Python :: 3.6 36 | Topic :: Scientific/Engineering 37 | Topic :: Software Development :: Libraries 38 | Topic :: Utilities 39 | """ 40 | 41 | classifiers = [x.strip() for x in classes.splitlines() if x] 42 | 43 | setup( 44 | name='automan', 45 | version=get_version(), 46 | author='Prabhu Ramachandran', 47 | author_email='prabhu@aero.iitb.ac.in', 48 | description='A simple Python-based automation framework.', 49 | long_description=open('README.rst').read(), 50 | license="BSD", 51 | url='https://github.com/pypr/automan', 52 | classifiers=classifiers, 53 | packages=find_packages(), 54 | install_requires=install_requires, 55 | tests_require=tests_require, 56 | package_dir={'automan': 'automan'}, 57 | ) 58 | --------------------------------------------------------------------------------