├── bot
    ├── .gitignore
    ├── systems
    │   ├── .gitignore
    │   ├── README.md
    │   └── generate.sh
    ├── workloads
    │   ├── .gitignore
    │   ├── README.md
    │   └── generate.sh
    ├── bot_exp.sh
    ├── process_results.py
    └── gen
    │   ├── bot_gen.py
    │   └── sys_gen.py
├── .gitignore
├── doc
    ├── source
    │   ├── reference
    │   │   ├── csimdag.rst
    │   │   ├── cplatform.rst
    │   │   ├── cscheduling.rst
    │   │   ├── tools.bot_gen.rst
    │   │   ├── tools.dag_gen.rst
    │   │   ├── tools.plat_gen.rst
    │   │   ├── tools.experiment.rst
    │   │   ├── index.rst
    │   │   ├── simdag.algorithms.rst
    │   │   └── simdag.rst
    │   ├── _themes
    │   │   └── sphinx_rtd_theme
    │   │   │   ├── static
    │   │   │       ├── fonts
    │   │   │       │   ├── FontAwesome.otf
    │   │   │       │   ├── Lato-Bold.ttf
    │   │   │       │   ├── Lato-Regular.ttf
    │   │   │       │   ├── Inconsolata-Bold.ttf
    │   │   │       │   ├── RobotoSlab-Bold.ttf
    │   │   │       │   ├── RobotoSlab-Regular.ttf
    │   │   │       │   ├── Inconsolata-Regular.ttf
    │   │   │       │   ├── fontawesome-webfont.eot
    │   │   │       │   ├── fontawesome-webfont.ttf
    │   │   │       │   ├── fontawesome-webfont.woff
    │   │   │       │   └── fontawesome-webfont.woff2
    │   │   │       ├── css
    │   │   │       │   ├── badge_only.css.map
    │   │   │       │   └── badge_only.css
    │   │   │       └── js
    │   │   │       │   └── theme.js
    │   │   │   ├── theme.conf
    │   │   │   ├── __init__.py
    │   │   │   ├── searchbox.html
    │   │   │   ├── versions.html
    │   │   │   ├── search.html
    │   │   │   ├── footer.html
    │   │   │   ├── breadcrumbs.html
    │   │   │   └── layout.html
    │   ├── index.rst
    │   └── conf.py
    └── Makefile
├── test
    ├── __init__.py
    ├── data
    │   ├── basic_graph.dot
    │   ├── pl_4hosts.xml
    │   └── pl_4hosts_master.xml
    └── test_capi.py
├── TODO.md
├── get_simgrid.sh
├── pysimgrid
    ├── tools
    │   ├── estimator.py
    │   ├── __init__.py
    │   ├── scale_ccr.py
    │   ├── trace_chart.py
    │   ├── bot_gen.py
    │   └── dax_to_dot.py
    ├── _version.py
    ├── simdag
    │   ├── __init__.py
    │   └── algorithms
    │   │   ├── __init__.py
    │   │   ├── round_robin.py
    │   │   ├── random.py
    │   │   ├── olb.py
    │   │   ├── heft.py
    │   │   ├── mct.py
    │   │   ├── mct_dyn.py
    │   │   ├── lookahead.py
    │   │   ├── peft.py
    │   │   ├── hcpt.py
    │   │   ├── simheft.py
    │   │   ├── batch_dyn.py
    │   │   ├── batch.py
    │   │   └── dls.py
    ├── __init__.py
    ├── xbt.pxd
    ├── common.pxd
    ├── cplatform.pxd
    ├── cplatform.pyx
    └── csimdag.pxd
├── dag
    ├── algorithms.json
    ├── run.sh
    ├── prepare.sh
    └── analyse.py
├── Dockerfile
├── setup.py
├── README.md
├── basic_test.py
├── run_tests.py
└── LICENSE.LGPL


/bot/.gitignore:
--------------------------------------------------------------------------------
1 | *.log


--------------------------------------------------------------------------------
/bot/systems/.gitignore:
--------------------------------------------------------------------------------
1 | cluster*


--------------------------------------------------------------------------------
/bot/workloads/.gitignore:
--------------------------------------------------------------------------------
1 | fixed*
2 | varied*


--------------------------------------------------------------------------------
/bot/bot_exp.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | 
3 | PYTHONPATH=$PYTHONPATH:../ python3 bot_exp.py $@
4 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # build dirs
2 | /**/build*
3 | # dependencies
4 | /opt
5 | # misc
6 | /pysimgrid/*.c
7 | /pysimgrid/*.so
8 | __pycache__


--------------------------------------------------------------------------------
/bot/systems/README.md:
--------------------------------------------------------------------------------
1 | # Synthetic Systems
2 | 
3 | Use `generate.sh` to generate all systems.
4 | See comments inside the script for details.


--------------------------------------------------------------------------------
/bot/workloads/README.md:
--------------------------------------------------------------------------------
1 | # Synthetic Bag-of-tasks Workloads
2 | 
3 | Use `generate.sh` to generate all workloads.
4 | See comments inside the script for details.


--------------------------------------------------------------------------------
/doc/source/reference/csimdag.rst:
--------------------------------------------------------------------------------
1 | *****************
2 | pysimgrid.csimdag
3 | *****************
4 | 
5 | 
6 | .. automodule:: pysimgrid.csimdag
7 |    :members:
8 | 


--------------------------------------------------------------------------------
/doc/source/reference/cplatform.rst:
--------------------------------------------------------------------------------
1 | *******************
2 | pysimgrid.cplatform
3 | *******************
4 | 
5 | .. automodule:: pysimgrid.cplatform
6 |    :members:
7 | 


--------------------------------------------------------------------------------
/doc/source/reference/cscheduling.rst:
--------------------------------------------------------------------------------
1 | *********************
2 | pysimgrid.cscheduling
3 | *********************
4 | 
5 | .. automodule:: pysimgrid.cscheduling
6 |    :members:
7 | 


--------------------------------------------------------------------------------
/doc/source/_themes/sphinx_rtd_theme/static/fonts/FontAwesome.otf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/alexmnazarenko/pysimgrid/HEAD/doc/source/_themes/sphinx_rtd_theme/static/fonts/FontAwesome.otf


--------------------------------------------------------------------------------
/doc/source/_themes/sphinx_rtd_theme/static/fonts/Lato-Bold.ttf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/alexmnazarenko/pysimgrid/HEAD/doc/source/_themes/sphinx_rtd_theme/static/fonts/Lato-Bold.ttf


--------------------------------------------------------------------------------
/doc/source/_themes/sphinx_rtd_theme/static/fonts/Lato-Regular.ttf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/alexmnazarenko/pysimgrid/HEAD/doc/source/_themes/sphinx_rtd_theme/static/fonts/Lato-Regular.ttf


--------------------------------------------------------------------------------
/doc/source/_themes/sphinx_rtd_theme/static/fonts/Inconsolata-Bold.ttf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/alexmnazarenko/pysimgrid/HEAD/doc/source/_themes/sphinx_rtd_theme/static/fonts/Inconsolata-Bold.ttf


--------------------------------------------------------------------------------
/doc/source/_themes/sphinx_rtd_theme/static/fonts/RobotoSlab-Bold.ttf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/alexmnazarenko/pysimgrid/HEAD/doc/source/_themes/sphinx_rtd_theme/static/fonts/RobotoSlab-Bold.ttf


--------------------------------------------------------------------------------
/doc/source/_themes/sphinx_rtd_theme/static/fonts/RobotoSlab-Regular.ttf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/alexmnazarenko/pysimgrid/HEAD/doc/source/_themes/sphinx_rtd_theme/static/fonts/RobotoSlab-Regular.ttf


--------------------------------------------------------------------------------
/doc/source/_themes/sphinx_rtd_theme/static/fonts/Inconsolata-Regular.ttf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/alexmnazarenko/pysimgrid/HEAD/doc/source/_themes/sphinx_rtd_theme/static/fonts/Inconsolata-Regular.ttf


--------------------------------------------------------------------------------
/doc/source/_themes/sphinx_rtd_theme/static/fonts/fontawesome-webfont.eot:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/alexmnazarenko/pysimgrid/HEAD/doc/source/_themes/sphinx_rtd_theme/static/fonts/fontawesome-webfont.eot


--------------------------------------------------------------------------------
/doc/source/_themes/sphinx_rtd_theme/static/fonts/fontawesome-webfont.ttf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/alexmnazarenko/pysimgrid/HEAD/doc/source/_themes/sphinx_rtd_theme/static/fonts/fontawesome-webfont.ttf


--------------------------------------------------------------------------------
/doc/source/_themes/sphinx_rtd_theme/static/fonts/fontawesome-webfont.woff:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/alexmnazarenko/pysimgrid/HEAD/doc/source/_themes/sphinx_rtd_theme/static/fonts/fontawesome-webfont.woff


--------------------------------------------------------------------------------
/doc/source/_themes/sphinx_rtd_theme/static/fonts/fontawesome-webfont.woff2:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/alexmnazarenko/pysimgrid/HEAD/doc/source/_themes/sphinx_rtd_theme/static/fonts/fontawesome-webfont.woff2


--------------------------------------------------------------------------------
/doc/source/reference/tools.bot_gen.rst:
--------------------------------------------------------------------------------
1 | .. _pysimgrid.tools.bot_gen:
2 | 
3 | ***********************
4 | pysimgrid.tools.bot_gen
5 | ***********************
6 | 
7 | 
8 | .. automodule:: pysimgrid.tools.bot_gen
9 | 


--------------------------------------------------------------------------------
/doc/source/reference/tools.dag_gen.rst:
--------------------------------------------------------------------------------
1 | .. _pysimgrid.tools.dag_gen:
2 | 
3 | ***********************
4 | pysimgrid.tools.dag_gen
5 | ***********************
6 | 
7 | 
8 | .. automodule:: pysimgrid.tools.dag_gen
9 | 


--------------------------------------------------------------------------------
/doc/source/reference/tools.plat_gen.rst:
--------------------------------------------------------------------------------
1 | .. _pysimgrid.tools.plat_gen:
2 | 
3 | ************************
4 | pysimgrid.tools.plat_gen
5 | ************************
6 | 
7 | 
8 | .. automodule:: pysimgrid.tools.plat_gen
9 | 


--------------------------------------------------------------------------------
/doc/source/reference/tools.experiment.rst:
--------------------------------------------------------------------------------
1 | .. _pysimgrid.tools.experiment:
2 | 
3 | **************************
4 | pysimgrid.tools.experiment
5 | **************************
6 | 
7 | 
8 | .. automodule:: pysimgrid.tools.experiment
9 | 


--------------------------------------------------------------------------------
/test/__init__.py:
--------------------------------------------------------------------------------
1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
2 | #
3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
4 | #
5 | # License:  Standard 3-clause BSD; see "license.txt" for full license terms
6 | #           and contributor agreement.
7 | 


--------------------------------------------------------------------------------
/TODO.md:
--------------------------------------------------------------------------------
 1 | Functional
 2 | ==========
 3 | 
 4 | * More algorithms
 5 | * Batch run (N platforms x M algorithms + configurations)
 6 | 
 7 | Build
 8 | =====
 9 | 
10 | * Get into SimGrid build to
11 |     * check if there is anything useful that can be enabled
12 |     * disable fortran smpi
13 | * Add more tests
14 | 


--------------------------------------------------------------------------------
/doc/source/_themes/sphinx_rtd_theme/theme.conf:
--------------------------------------------------------------------------------
 1 | [theme]
 2 | inherit = basic
 3 | stylesheet = css/theme.css
 4 | 
 5 | [options]
 6 | typekit_id = hiw1hhg
 7 | analytics_id = 
 8 | sticky_navigation = False
 9 | logo_only =
10 | collapse_navigation = False
11 | display_version = True
12 | navigation_depth = 4
13 | prev_next_buttons_location = bottom
14 | 


--------------------------------------------------------------------------------
/doc/source/_themes/sphinx_rtd_theme/__init__.py:
--------------------------------------------------------------------------------
 1 | """Sphinx ReadTheDocs theme.
 2 | 
 3 | From https://github.com/ryan-roemer/sphinx-bootstrap-theme.
 4 | 
 5 | """
 6 | import os
 7 | 
 8 | __version__ = '0.1.10-alpha'
 9 | __version_full__ = __version__
10 | 
11 | 
12 | def get_html_theme_path():
13 |     """Return list of HTML theme paths."""
14 |     cur_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
15 |     return cur_dir
16 | 


--------------------------------------------------------------------------------
/doc/source/_themes/sphinx_rtd_theme/searchbox.html:
--------------------------------------------------------------------------------
 1 | {%- if builder != 'singlehtml' %}
 2 | <div role="search">
 3 |   <form id="rtd-search-form" class="wy-form" action="{{ pathto('search') }}" method="get">
 4 |     <input type="text" name="q" placeholder="{{ _('Search docs') }}" />
 5 |     <input type="hidden" name="check_keywords" value="yes" />
 6 |     <input type="hidden" name="area" value="default" />
 7 |   </form>
 8 | </div>
 9 | {%- endif %}
10 | 


--------------------------------------------------------------------------------
/doc/source/reference/index.rst:
--------------------------------------------------------------------------------
 1 | *************
 2 | API Reference
 3 | *************
 4 | 
 5 | 
 6 | High-level API
 7 | ==============
 8 | 
 9 | .. toctree::
10 |    :maxdepth: 1
11 | 
12 |    simdag
13 |    simdag.algorithms
14 | 
15 | C API
16 | =====
17 | 
18 | .. toctree::
19 |    :maxdepth: 1
20 | 
21 |    csimdag
22 |    cplatform
23 | 
24 | 
25 | Tools
26 | =====
27 | 
28 | .. toctree::
29 |    :maxdepth: 1
30 | 
31 |    tools.experiment
32 |    tools.dag_gen
33 |    tools.plat_gen
34 |    tools.bot_gen
35 | 
36 | Utils
37 | =====
38 | 
39 | .. toctree::
40 |    :maxdepth: 1
41 | 
42 |    cscheduling
43 | 


--------------------------------------------------------------------------------
/test/data/basic_graph.dot:
--------------------------------------------------------------------------------
 1 | digraph G {
 2 |   c1 [size="2e9"];
 3 |   c2 [size="5e9"];
 4 |   c3 [size="2e9"];
 5 |   c4 [size="5e8"];
 6 |   c5 [size="1e10"];
 7 |   c6 [size="6e9"];
 8 |   c7 [size="3e9"];
 9 |   c8 [size="7e9"];
10 | 
11 |   root -> c1 [size="2e8"];
12 |   root -> c2 [size="1e8"];
13 |   c1 -> c3   [size="5e8"];
14 |   c2 -> c3   [size="5e7"];
15 |   c3 -> c4   [size="5e7"];
16 |   c3 -> c5   [size="5e7"];
17 |   c3 -> c6   [size="5e7"];
18 |   c4 -> c7   [size="3e7"];
19 |   c5 -> c7   [size="3e7"];
20 |   c6 -> c7   [size="3e7"];
21 |   c7 -> c8   [size="3e8"];
22 |   c8 -> end   [size="1e7"];
23 |   c3 -> end  [size="1e8"];
24 | }
25 | 


--------------------------------------------------------------------------------
/doc/Makefile:
--------------------------------------------------------------------------------
 1 | # Minimal makefile for Sphinx documentation
 2 | #
 3 | 
 4 | # You can set these variables from the command line.
 5 | SPHINXOPTS    =
 6 | SPHINXBUILD   = sphinx-build
 7 | SPHINXPROJ    = pysimgrid
 8 | SOURCEDIR     = source
 9 | BUILDDIR      = build
10 | 
11 | # Put it first so that "make" without argument is like "make help".
12 | help:
13 | 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
14 | 
15 | .PHONY: help Makefile
16 | 
17 | # Catch-all target: route all unknown targets to Sphinx using the new
18 | # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
19 | %: Makefile
20 | 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)


--------------------------------------------------------------------------------
/get_simgrid.sh:
--------------------------------------------------------------------------------
 1 | SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 2 | OPT_ROOT="$SCRIPT_DIR/opt"
 3 | PKG_ROOT="$OPT_ROOT/SimGrid"
 4 | VERSION="v3_13"
 5 | 
 6 | echo "Create dirs..."
 7 | mkdir -p "$PKG_ROOT/src"
 8 | 
 9 | echo "Download..."
10 | if [[ ! -f  "$PKG_ROOT/src/simgrid-${VERSION}.tar.gz" ]]; then
11 |   wget -P "$PKG_ROOT/src" https://framagit.org/simgrid/simgrid/-/archive/${VERSION}/simgrid-${VERSION}.tar.gz
12 | fi
13 | 
14 | echo "Unpack..."
15 | (cd "$PKG_ROOT/src" && tar xzf "$PKG_ROOT/src/simgrid-${VERSION}.tar.gz")
16 | 
17 | mkdir "$PKG_ROOT/build"
18 | echo "Configure..."
19 | (cd "$PKG_ROOT/build" && cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX="$PKG_ROOT" ../src/SimGrid-3.13)
20 | echo "Build..."
21 | (cd "$PKG_ROOT/build" && make -j4 install)
22 | 


--------------------------------------------------------------------------------
/pysimgrid/tools/estimator.py:
--------------------------------------------------------------------------------
 1 | from abc import ABCMeta, abstractmethod
 2 | import numpy as np
 3 | 
 4 | 
 5 | class Estimator(metaclass=ABCMeta):
 6 | 
 7 |     @abstractmethod
 8 |     def __init__(self):
 9 |         pass
10 | 
11 |     @abstractmethod
12 |     def generate(self, value):
13 |         pass
14 | 
15 | 
16 | class AccurateEstimator(Estimator):
17 | 
18 |     def __init__(self):
19 |         super().__init__()
20 | 
21 |     def generate(self, value):
22 |         return value
23 | 
24 | 
25 | class SimpleDispersionEstimator(Estimator):
26 | 
27 |     def __init__(self, percentage, seed=1234):
28 |         self.percentage = percentage
29 |         np.random.seed(seed)
30 |         super().__init__()
31 | 
32 |     def generate(self, value):
33 |         low = max(float(value * (1. - self.percentage)), 1.0)
34 |         high = max(float(value * (1. + self.percentage)), 1.0)
35 |         return np.random.uniform(low, high)
36 | 


--------------------------------------------------------------------------------
/pysimgrid/tools/__init__.py:
--------------------------------------------------------------------------------
 1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
 2 | #
 3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
 4 | #
 5 | # This library is free software: you can redistribute it and/or modify
 6 | # it under the terms of the GNU Lesser General Public License as published by
 7 | # the Free Software Foundation, either version 3 of the License, or
 8 | # (at your option) any later version.
 9 | #
10 | # This library is distributed in the hope that it will be useful,
11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | # GNU Lesser General Public License for more details.
14 | #
15 | # You should have received a copy of the GNU Lesser General Public
16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
17 | #
18 | 
19 | from .estimator import AccurateEstimator, SimpleDispersionEstimator


--------------------------------------------------------------------------------
/doc/source/index.rst:
--------------------------------------------------------------------------------
 1 | 
 2 | pysimgrid
 3 | =========
 4 | 
 5 | .. only:: html
 6 | 
 7 |     :Release: |version|
 8 |     :Date: |today|
 9 | 
10 | 
11 | pysimgrid is a framework aimed mainly to simplify DAG scheduling algorithm development, testing and benchmarking.
12 | It is based on an awesome `SimGrid library <http://simgrid.gforge.inria.fr/>`_ that provides a reliable simulation core complemented by python's
13 | ease of development and wide array of libraries (import antigravity, anyone?).
14 | 
15 | Current features:
16 | 
17 | * Simulation of DAG execution on a distributed network
18 | * Array of popular scheduling algorithms (see :ref:`simdag_algorithms`)
19 | * Convenience tools for experiments
20 | 
21 |   * Batch/parallel simulations
22 |   * Platform generator
23 |   * DAG generator
24 |   * BoT generator
25 | 
26 | 
27 | Documentation
28 | =============
29 | 
30 | 
31 | .. toctree::
32 |    :maxdepth: 2
33 |    :caption: Contents:
34 | 
35 |    quick_start
36 |    reference/index
37 | 


--------------------------------------------------------------------------------
/dag/algorithms.json:
--------------------------------------------------------------------------------
 1 | [
 2 |   {
 3 |     "class": "pysimgrid.simdag.algorithms.OLB",
 4 |     "name": "OLB"
 5 |   },
 6 |   {
 7 |     "class": "pysimgrid.simdag.algorithms.MCT",
 8 |     "name": "MCT"
 9 |   },
10 |   {
11 |     "class": "pysimgrid.simdag.algorithms.BatchMin",
12 |     "name": "MinMin"
13 |   },
14 |   {
15 |     "class": "pysimgrid.simdag.algorithms.BatchMax",
16 |     "name": "MaxMin"
17 |   },
18 |   {
19 |     "class": "pysimgrid.simdag.algorithms.BatchSufferage",
20 |     "name": "Sufferage"
21 |   },
22 |   {
23 |     "class": "pysimgrid.simdag.algorithms.DLS",
24 |     "name": "DLS"
25 |   },
26 |   {
27 |     "class": "pysimgrid.simdag.algorithms.HEFT",
28 |     "name": "HEFT"
29 |   },
30 |   {
31 |     "class": "pysimgrid.simdag.algorithms.HCPT",
32 |     "name": "HCPT"
33 |   },
34 |   {
35 |     "class": "pysimgrid.simdag.algorithms.Lookahead",
36 |     "name": "Lookahead"
37 |   },
38 |   {
39 |     "class": "pysimgrid.simdag.algorithms.PEFT",
40 |     "name": "PEFT"
41 |   }
42 | ]
43 | 


--------------------------------------------------------------------------------
/dag/run.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | 
 3 | SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 4 | 
 5 | pushd "$SCRIPT_DIR/.."
 6 | 
 7 | if [ ! -f "$SCRIPT_DIR/exp1.json" ]; then
 8 |   python3 -m pysimgrid.tools.experiment "$SCRIPT_DIR/exp1_systems" "$SCRIPT_DIR/exp1_workflows" "$SCRIPT_DIR/algorithms.json" dag/exp1.json -j8  --simgrid-log-level=error
 9 | fi
10 | 
11 | if [ ! -f "$SCRIPT_DIR/exp1_inf.json" ]; then
12 |   python3 -m pysimgrid.tools.experiment "$SCRIPT_DIR/exp1_systems_inf" "$SCRIPT_DIR/exp1_workflows" "$SCRIPT_DIR/algorithms.json" dag/exp1_inf.json -j8  --simgrid-log-level=error
13 | fi
14 | 
15 | if [ ! -f "$SCRIPT_DIR/exp2.json" ]; then
16 |   python3 -m pysimgrid.tools.experiment "$SCRIPT_DIR/exp2_systems" "$SCRIPT_DIR/exp2_workflows" "$SCRIPT_DIR/algorithms.json" dag/exp2.json -j8
17 | fi
18 | 
19 | if [ ! -f "$SCRIPT_DIR/exp2_inf.json" ]; then
20 |   python3 -m pysimgrid.tools.experiment "$SCRIPT_DIR/exp2_systems_inf" "$SCRIPT_DIR/exp2_workflows" "$SCRIPT_DIR/algorithms.json" dag/exp2_inf.json -j8
21 | fi
22 | 
23 | popd
24 | 


--------------------------------------------------------------------------------
/pysimgrid/_version.py:
--------------------------------------------------------------------------------
 1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
 2 | #
 3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
 4 | #
 5 | # This library is free software: you can redistribute it and/or modify
 6 | # it under the terms of the GNU Lesser General Public License as published by
 7 | # the Free Software Foundation, either version 3 of the License, or
 8 | # (at your option) any later version.
 9 | #
10 | # This library is distributed in the hope that it will be useful,
11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | # GNU Lesser General Public License for more details.
14 | #
15 | # You should have received a copy of the GNU Lesser General Public
16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
17 | #
18 | 
19 | from . import csimdag
20 | 
21 | version = "1.0.0"
22 | simgrid_version = ".".join(map(str, [csimdag.LIBRARY_VERSION_MAJOR, csimdag.LIBRARY_VERSION_MINOR, csimdag.LIBRARY_VERSION_PATCH]))
23 | 


--------------------------------------------------------------------------------
/pysimgrid/simdag/__init__.py:
--------------------------------------------------------------------------------
 1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
 2 | #
 3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
 4 | #
 5 | # This library is free software: you can redistribute it and/or modify
 6 | # it under the terms of the GNU Lesser General Public License as published by
 7 | # the Free Software Foundation, either version 3 of the License, or
 8 | # (at your option) any later version.
 9 | #
10 | # This library is distributed in the hope that it will be useful,
11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | # GNU Lesser General Public License for more details.
14 | #
15 | # You should have received a copy of the GNU Lesser General Public
16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
17 | #
18 | 
19 | from ..csimdag import TaskKind
20 | from ..csimdag import TaskState
21 | from .simulation import Simulation
22 | from .scheduler import Scheduler, StaticScheduler, DynamicScheduler
23 | 


--------------------------------------------------------------------------------
/pysimgrid/__init__.py:
--------------------------------------------------------------------------------
 1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
 2 | #
 3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
 4 | #
 5 | # This library is free software: you can redistribute it and/or modify
 6 | # it under the terms of the GNU Lesser General Public License as published by
 7 | # the Free Software Foundation, either version 3 of the License, or
 8 | # (at your option) any later version.
 9 | #
10 | # This library is distributed in the hope that it will be useful,
11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | # GNU Lesser General Public License for more details.
14 | #
15 | # You should have received a copy of the GNU Lesser General Public
16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
17 | #
18 | 
19 | from . import csimdag
20 | from . import cplatform
21 | from . import simdag
22 | 
23 | from . import _version
24 | __version__ = _version.version
25 | __simgrid_version__ = _version.simgrid_version
26 | 


--------------------------------------------------------------------------------
/test/data/pl_4hosts.xml:
--------------------------------------------------------------------------------
 1 | <?xml version='1.0'?>
 2 | <!DOCTYPE platform SYSTEM "http://simgrid.gforge.inria.fr/simgrid/simgrid.dtd">
 3 | <platform version="4">
 4 |   <AS  id="AS0"  routing="Floyd">
 5 |     <host id="host0" core="1" speed="2e9flops"/>
 6 |     <host id="host1" core="1" speed="2e9flops"/>
 7 |     <host id="host2" core="1" speed="2e9flops"/>
 8 |     <host id="host3" core="1" speed="2e9flops"/>
 9 | 
10 |     <link id="link0" bandwidth="100MBps" latency="0.0001s"/>
11 |     <link id="link1" bandwidth="100MBps" latency="0.0001s"/>
12 |     <link id="link2" bandwidth="100MBps" latency="0.0001s"/>
13 |     <!-- <link id="link3" bandwidth="100000000" latency="0.000100"/>
14 |     <link id="link4" bandwidth="100000000" latency="0.000100"/>
15 |     <link id="link5" bandwidth="100000000" latency="0.000100"/> -->
16 | 
17 |     <route src="host0" dst="host1"><link_ctn id="link0"/></route>
18 |     <route src="host0" dst="host2"><link_ctn id="link1"/></route>
19 |     <route src="host0" dst="host3"><link_ctn id="link2"/></route>
20 |     <!-- <route src="host1" dst="host2"><link_ctn id="link3"/></route>
21 |     <route src="host1" dst="host3"><link_ctn id="link4"/></route>
22 |     <route src="host2" dst="host3"><link_ctn id="link5"/></route> -->
23 |   </AS>
24 | </platform>
25 | 


--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM python:3.6.15-slim as base
 2 | 
 3 | ENV LATEST_VERSION v3_13
 4 | ARG VERSION=${LATEST_VERSION}
 5 | ARG URL=https://framagit.org/simgrid/simgrid/-/archive/${VERSION}/simgrid-${VERSION}.tar.gz
 6 | 
 7 | ARG SCRIPT_DIR=/home/pysimgrid
 8 | ARG OPT_ROOT=$SCRIPT_DIR/opt
 9 | ARG PKG_ROOT=$OPT_ROOT/SimGrid
10 | 
11 | RUN apt update -q && apt install -q wget curl build-essential cmake libboost-context-dev libboost-program-options-dev libboost-filesystem-dev doxygen graphviz-dev libgraphviz-dev -y
12 | 
13 | RUN useradd -ms /bin/bash pysimgrid
14 | WORKDIR /home/pysimgrid/
15 | 
16 | 
17 | RUN mkdir -p $PKG_ROOT/src
18 | RUN wget -P  $PKG_ROOT/src $URL --no-check-certificate
19 | RUN cd $PKG_ROOT/src && tar xzf $PKG_ROOT/src/simgrid-${VERSION}.tar.gz
20 | 
21 | 
22 | RUN cd $PKG_ROOT/src/simgrid-${VERSION} && cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=$PKG_ROOT .
23 | RUN cd $PKG_ROOT/src/simgrid-${VERSION} && make
24 | RUN cd $PKG_ROOT/src/simgrid-${VERSION} && make install
25 | 
26 | 
27 | RUN pip3 install Cython==0.29.36 numpy==1.19.0 setuptools==44.1.0 networkx==2.4
28 | 
29 | 
30 | COPY . $SCRIPT_DIR
31 | 
32 | RUN cd $SCRIPT_DIR && python3 setup.py build_ext --inplace
33 | 
34 | RUN cd $SCRIPT_DIR && python3 run_tests.py
35 | 
36 | RUN chown -R pysimgrid /home/pysimgrid
37 |  
38 | USER pysimgrid
39 | 
40 | CMD ["bash"]
41 | 


--------------------------------------------------------------------------------
/pysimgrid/simdag/algorithms/__init__.py:
--------------------------------------------------------------------------------
 1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
 2 | #
 3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
 4 | #
 5 | # This library is free software: you can redistribute it and/or modify
 6 | # it under the terms of the GNU Lesser General Public License as published by
 7 | # the Free Software Foundation, either version 3 of the License, or
 8 | # (at your option) any later version.
 9 | #
10 | # This library is distributed in the hope that it will be useful,
11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | # GNU Lesser General Public License for more details.
14 | #
15 | # You should have received a copy of the GNU Lesser General Public
16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
17 | #
18 | 
19 | from .batch import BatchMin, BatchMax, BatchSufferage
20 | from .batch_dyn import DynamicBatchMin, DynamicBatchMax, DynamicBatchSufferage
21 | from .dls import DLS
22 | from .hcpt import HCPT
23 | from .heft import HEFT
24 | from .lookahead import Lookahead
25 | from .mct import MCT
26 | from .mct_dyn import DynamicMCT
27 | from .olb import OLB
28 | from .peft import PEFT
29 | from .random import RandomStatic
30 | from .round_robin import RoundRobinStatic
31 | from .simheft import SimHEFT
32 | 


--------------------------------------------------------------------------------
/test/data/pl_4hosts_master.xml:
--------------------------------------------------------------------------------
 1 | <?xml version='1.0'?>
 2 | <!DOCTYPE platform SYSTEM "http://simgrid.gforge.inria.fr/simgrid/simgrid.dtd">
 3 | <platform version="4">
 4 |   <AS  id="AS0"  routing="Floyd">
 5 |     <host id="master" core="1" speed="1e9flops"/>
 6 |     <host id="host0" core="1" speed="2e9flops"/>
 7 |     <host id="host1" core="1" speed="2e9flops"/>
 8 |     <host id="host2" core="1" speed="2e9flops"/>
 9 |     <host id="host3" core="1" speed="2e9flops"/>
10 | 
11 |     <link id="link0" bandwidth="100MBps" latency="0.0001s"/>
12 |     <link id="link1" bandwidth="100MBps" latency="0.0001s"/>
13 |     <link id="link2" bandwidth="100MBps" latency="0.0001s"/>
14 |     <link id="link3" bandwidth="100MBps" latency="0.0001s"/>
15 |     <!-- <link id="link3" bandwidth="100000000" latency="0.000100"/>
16 |     <link id="link4" bandwidth="100000000" latency="0.000100"/>
17 |     <link id="link5" bandwidth="100000000" latency="0.000100"/> -->
18 | 
19 |     <route src="master" dst="host0"><link_ctn id="link0"/></route>
20 |     <route src="master" dst="host1"><link_ctn id="link1"/></route>
21 |     <route src="master" dst="host2"><link_ctn id="link2"/></route>
22 |     <route src="master" dst="host3"><link_ctn id="link3"/></route>
23 |     <!-- <route src="host1" dst="host2"><link_ctn id="link3"/></route>
24 |     <route src="host1" dst="host3"><link_ctn id="link4"/></route>
25 |     <route src="host2" dst="host3"><link_ctn id="link5"/></route> -->
26 |   </AS>
27 | </platform>
28 | 


--------------------------------------------------------------------------------
/doc/source/_themes/sphinx_rtd_theme/versions.html:
--------------------------------------------------------------------------------
 1 | {% if READTHEDOCS %}
 2 | {# Add rst-badge after rst-versions for small badge style. #}
 3 |   <div class="rst-versions" data-toggle="rst-versions" role="note" aria-label="versions">
 4 |     <span class="rst-current-version" data-toggle="rst-current-version">
 5 |       <span class="fa fa-book"> Read the Docs</span>
 6 |       v: {{ current_version }}
 7 |       <span class="fa fa-caret-down"></span>
 8 |     </span>
 9 |     <div class="rst-other-versions">
10 |       <dl>
11 |         <dt>{{ _('Versions') }}</dt>
12 |         {% for slug, url in versions %}
13 |           <dd><a href="{{ url }}">{{ slug }}</a></dd>
14 |         {% endfor %}
15 |       </dl>
16 |       <dl>
17 |         <dt>{{ _('Downloads') }}</dt>
18 |         {% for type, url in downloads %}
19 |           <dd><a href="{{ url }}">{{ type }}</a></dd>
20 |         {% endfor %}
21 |       </dl>
22 |       <dl>
23 |         <dt>{{ _('On Read the Docs') }}</dt>
24 |           <dd>
25 |             <a href="//{{ PRODUCTION_DOMAIN }}/projects/{{ slug }}/?fromdocs={{ slug }}">{{ _('Project Home') }}</a>
26 |           </dd>
27 |           <dd>
28 |             <a href="//{{ PRODUCTION_DOMAIN }}/builds/{{ slug }}/?fromdocs={{ slug }}">{{ _('Builds') }}</a>
29 |           </dd>
30 |       </dl>
31 |       <hr/>
32 |       {% trans %}Free document hosting provided by <a href="http://www.readthedocs.org">Read the Docs</a>.{% endtrans %}
33 | 
34 |     </div>
35 |   </div>
36 | {% endif %}
37 | 
38 | 


--------------------------------------------------------------------------------
/pysimgrid/xbt.pxd:
--------------------------------------------------------------------------------
 1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
 2 | #
 3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
 4 | #
 5 | # This library is free software: you can redistribute it and/or modify
 6 | # it under the terms of the GNU Lesser General Public License as published by
 7 | # the Free Software Foundation, either version 3 of the License, or
 8 | # (at your option) any later version.
 9 | #
10 | # This library is distributed in the hope that it will be useful,
11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | # GNU Lesser General Public License for more details.
14 | #
15 | # You should have received a copy of the GNU Lesser General Public
16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
17 | #
18 | 
19 | """
20 | Very basic wrapper over SimGrid/XBT library types, only the absolutely required functions.
21 | 
22 | Intent is to avoid using them as much as possible. However, SimGrid API sometimes just returns them, so we need at least RO access.
23 | """
24 | 
25 | cdef extern from "xbt/dynar.h":
26 |   ctypedef void* xbt_dynar_t
27 | 
28 |   unsigned long xbt_dynar_length(const xbt_dynar_t dynar);
29 |   int xbt_dynar_is_empty(const xbt_dynar_t dynar);
30 |   void xbt_dynar_get_cpy(const xbt_dynar_t dynar, const unsigned long idx, void* const dst);
31 | 
32 |   void* xbt_dynar_to_array(xbt_dynar_t dynar);
33 | 
34 |   void xbt_dynar_free(xbt_dynar_t* dynar);
35 |   void xbt_dynar_free_container(xbt_dynar_t* dynar);
36 | 
37 | 
38 | cdef extern from "xbt/dict.h":
39 |   ctypedef void* xbt_dict_t
40 | 
41 | 
42 | cdef extern from "xbt/log.h":
43 |   void xbt_log_control_set(const char* cs);
44 | 


--------------------------------------------------------------------------------
/doc/source/_themes/sphinx_rtd_theme/search.html:
--------------------------------------------------------------------------------
 1 | {#
 2 |     basic/search.html
 3 |     ~~~~~~~~~~~~~~~~~
 4 | 
 5 |     Template for the search page.
 6 | 
 7 |     :copyright: Copyright 2007-2013 by the Sphinx team, see AUTHORS.
 8 |     :license: BSD, see LICENSE for details.
 9 | #}
10 | {%- extends "layout.html" %}
11 | {% set title = _('Search') %}
12 | {% set script_files = script_files + ['_static/searchtools.js'] %}
13 | {% block footer %}
14 |   <script type="text/javascript">
15 |     jQuery(function() { Search.loadIndex("{{ pathto('searchindex.js', 1) }}"); });
16 |   </script>
17 |   {# this is used when loading the search index using $.ajax fails,
18 |      such as on Chrome for documents on localhost #}
19 |   <script type="text/javascript" id="searchindexloader"></script>
20 |   {{ super() }}
21 | {% endblock %}
22 | {% block body %}
23 |   <noscript>
24 |   <div id="fallback" class="admonition warning">
25 |     <p class="last">
26 |       {% trans %}Please activate JavaScript to enable the search
27 |       functionality.{% endtrans %}
28 |     </p>
29 |   </div>
30 |   </noscript>
31 | 
32 |   {% if search_performed %}
33 |     <h2>{{ _('Search Results') }}</h2>
34 |     {% if not search_results %}
35 |       <p>{{ _('Your search did not match any documents. Please make sure that all words are spelled correctly and that you\'ve selected enough categories.') }}</p>
36 |     {% endif %}
37 |   {% endif %}
38 |   <div id="search-results">
39 |   {% if search_results %}
40 |     <ul>
41 |     {% for href, caption, context in search_results %}
42 |       <li>
43 |         <a href="{{ pathto(item.href) }}">{{ caption }}</a>
44 |         <p class="context">{{ context|e }}</p>
45 |       </li>
46 |     {% endfor %}
47 |     </ul>
48 |   {% endif %}
49 |   </div>
50 | {% endblock %}
51 | 


--------------------------------------------------------------------------------
/doc/source/reference/simdag.algorithms.rst:
--------------------------------------------------------------------------------
 1 | .. _simdag_algorithms:
 2 | 
 3 | ***************************
 4 | pysimgrid.simdag.algorithms
 5 | ***************************
 6 | 
 7 | Package containing some pre-made scheduling algorithms for benchmarking.
 8 | 
 9 | .. contents::
10 | 
11 | Dynamic scheduling
12 | ==================
13 | 
14 | Dynamic algorithms work along the simulation and schedule only a subset of tasks on each execution (most commonly the tasks that are
15 | ready for execution right now).
16 | 
17 | .. autosummary::
18 |   :nosignatures:
19 | 
20 |   pysimgrid.simdag.algorithms.OLB
21 |   pysimgrid.simdag.algorithms.MCT
22 |   pysimgrid.simdag.algorithms.BatchMin
23 |   pysimgrid.simdag.algorithms.BatchMax
24 |   pysimgrid.simdag.algorithms.BatchSufferage
25 | 
26 | 
27 | .. autoclass:: pysimgrid.simdag.algorithms.OLB
28 | 
29 | .. autoclass:: pysimgrid.simdag.algorithms.MCT
30 | 
31 | .. autoclass:: pysimgrid.simdag.algorithms.BatchMin
32 | 
33 | .. autoclass:: pysimgrid.simdag.algorithms.BatchMax
34 | 
35 | .. autoclass:: pysimgrid.simdag.algorithms.BatchSufferage
36 | 
37 | Static scheduling
38 | =================
39 | 
40 | Static algorithms are executed before the simulation and schedule all tasks at once.
41 | 
42 | .. autosummary::
43 |   :nosignatures:
44 | 
45 |   pysimgrid.simdag.algorithms.HCPT
46 |   pysimgrid.simdag.algorithms.HEFT
47 |   pysimgrid.simdag.algorithms.Lookahead
48 |   pysimgrid.simdag.algorithms.PEFT
49 |   pysimgrid.simdag.algorithms.RandomStatic
50 |   pysimgrid.simdag.algorithms.RoundRobinStatic
51 | 
52 | 
53 | .. autoclass:: pysimgrid.simdag.algorithms.HCPT
54 | 
55 | .. autoclass:: pysimgrid.simdag.algorithms.HEFT
56 | 
57 | .. autoclass:: pysimgrid.simdag.algorithms.Lookahead
58 | 
59 | .. autoclass:: pysimgrid.simdag.algorithms.PEFT
60 | 
61 | .. autoclass:: pysimgrid.simdag.algorithms.RandomStatic
62 | 
63 | .. autoclass:: pysimgrid.simdag.algorithms.RoundRobinStatic
64 | 


--------------------------------------------------------------------------------
/pysimgrid/common.pxd:
--------------------------------------------------------------------------------
 1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
 2 | #
 3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
 4 | #
 5 | # This library is free software: you can redistribute it and/or modify
 6 | # it under the terms of the GNU Lesser General Public License as published by
 7 | # the Free Software Foundation, either version 3 of the License, or
 8 | # (at your option) any later version.
 9 | #
10 | # This library is distributed in the hope that it will be useful,
11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | # GNU Lesser General Public License for more details.
14 | #
15 | # You should have received a copy of the GNU Lesser General Public
16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
17 | #
18 | 
19 | """
20 | General utility definitions.
21 | """
22 | 
23 | # intptr type to expose native pointers to python
24 | from libc.stdint cimport uintptr_t as intptr
25 | # dynamic memory management
26 | from libc.stdlib cimport malloc, free
27 | 
28 | # Import Python reference counting API for some darkmagic (storing pyobjects in C)
29 | cdef extern from "Python.h":
30 |   void Py_INCREF(object o)
31 |   void Py_DECREF(object o)
32 | 
33 | cdef inline unicode unicode_string(s):
34 |   """
35 |   Unicode strings adapter adapted from Cython docs.
36 | 
37 |   Recieves an 'object' argument and ensures it to be a unicode string.
38 |   """
39 |   if type(s) is unicode:
40 |     return <unicode>s
41 |   elif isinstance(s, bytes):
42 |     return (<bytes>s).decode("utf-8")
43 |   elif isinstance(s, unicode):
44 |     return unicode(s)
45 |   else:
46 |     raise TypeError("wrong argument type, unicode string is expected")
47 | 
48 | cdef inline bytes utf8_string(s):
49 |   """
50 |   Ensure a UTF-8 byte string.
51 |   """
52 |   return unicode_string(s).encode("utf-8")
53 | 


--------------------------------------------------------------------------------
/pysimgrid/simdag/algorithms/round_robin.py:
--------------------------------------------------------------------------------
 1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
 2 | #
 3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
 4 | #
 5 | # This library is free software: you can redistribute it and/or modify
 6 | # it under the terms of the GNU Lesser General Public License as published by
 7 | # the Free Software Foundation, either version 3 of the License, or
 8 | # (at your option) any later version.
 9 | #
10 | # This library is distributed in the hope that it will be useful,
11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | # GNU Lesser General Public License for more details.
14 | #
15 | # You should have received a copy of the GNU Lesser General Public
16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
17 | #
18 | 
19 | import logging
20 | import networkx
21 | 
22 | from .. import scheduler
23 | 
24 | 
25 | class RoundRobinStatic(scheduler.StaticScheduler):
26 |     """
27 |     Round robin static scheduler.
28 | 
29 |     The important part there is static - all tasks are scheduled simultaneously.
30 |     For more details please see RandomScheduler description.
31 | 
32 |     Anyway, not a sensible approach by any means.
33 |     """
34 | 
35 |     def get_schedule(self, simulation):
36 |         schedule = {host: [] for host in simulation.hosts}
37 |         master_host = simulation.hosts.by_prop('name', scheduler.Scheduler.MASTER_HOST_NAME)[0]
38 |         exec_hosts = simulation.hosts.by_prop('name', scheduler.Scheduler.MASTER_HOST_NAME, negate=True)
39 |         hosts_count = len(exec_hosts)
40 |         i = 0
41 |         for task in networkx.topological_sort(simulation.get_task_graph()):
42 |             if task.name in scheduler.Scheduler.BOUNDARY_TASKS:
43 |                 host = master_host
44 |             else:
45 |                 host = exec_hosts[i % hosts_count]
46 |                 i += 1
47 |             schedule[host].append(task)
48 |             # logging.info("%s -> %s" % (task.name, host.name))
49 |         return schedule
50 | 


--------------------------------------------------------------------------------
/doc/source/_themes/sphinx_rtd_theme/footer.html:
--------------------------------------------------------------------------------
 1 | <footer>
 2 |   {% if (theme_prev_next_buttons_location == 'bottom' or theme_prev_next_buttons_location == 'both') and (next or prev) %}
 3 |     <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
 4 |       {% if next %}
 5 |         <a href="{{ next.link|e }}" class="btn btn-neutral float-right" title="{{ next.title|striptags|e }}" accesskey="n">{{ _('Next') }} <span class="fa fa-arrow-circle-right"></span></a>
 6 |       {% endif %}
 7 |       {% if prev %}
 8 |         <a href="{{ prev.link|e }}" class="btn btn-neutral" title="{{ prev.title|striptags|e }}" accesskey="p"><span class="fa fa-arrow-circle-left"></span> {{ _('Previous') }}</a>
 9 |       {% endif %}
10 |     </div>
11 |   {% endif %}
12 | 
13 |   <hr/>
14 | 
15 |   <div role="contentinfo">
16 |     <p>
17 |     {%- if show_copyright %}
18 |       {%- if hasdoc('copyright') %}
19 |         {% trans path=pathto('copyright'), copyright=copyright|e %}&copy; <a href="{{ path }}">Copyright</a> {{ copyright }}.{% endtrans %}
20 |       {%- else %}
21 |         {% trans copyright=copyright|e %}&copy; Copyright {{ copyright }}.{% endtrans %}
22 |       {%- endif %}
23 |     {%- endif %}
24 | 
25 |     {%- if build_id and build_url %}
26 |       {% trans build_url=build_url, build_id=build_id %}
27 |         <span class="build">
28 |           Build
29 |           <a href="{{ build_url }}">{{ build_id }}</a>.
30 |         </span>
31 |       {% endtrans %}
32 |     {%- elif commit %}
33 |       {% trans commit=commit %}
34 |         <span class="commit">
35 |           Revision <code>{{ commit }}</code>.
36 |         </span>
37 |       {% endtrans %}
38 |     {%- elif last_updated %}
39 |       {% trans last_updated=last_updated|e %}Last updated on {{ last_updated }}.{% endtrans %}
40 |     {%- endif %}
41 | 
42 |     </p>
43 |   </div>
44 | 
45 |   {%- if show_sphinx %}
46 |   {% trans %}Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>{% endtrans %}.
47 |   {%- endif %}
48 | 
49 |   {%- block extrafooter %} {% endblock %}
50 | 
51 | </footer>
52 | 
53 | 


--------------------------------------------------------------------------------
/bot/process_results.py:
--------------------------------------------------------------------------------
 1 | import argparse
 2 | import numpy as np
 3 | import scipy.stats as st
 4 | 
 5 | 
 6 | def process_results(results_file):
 7 |     results = {}
 8 |     with open(results_file, 'r') as f:
 9 |         while True:
10 |             line = f.readline()
11 |             if not line: break
12 |             system = line.strip()
13 |             workload = f.readline().strip()
14 |             sched_result = {}
15 |             while True:
16 |                 result = f.readline().strip()
17 |                 if result == '': break
18 |                 parts = result.split('\t')
19 |                 sched = parts[0]
20 |                 makespan = float(parts[1])
21 |                 sched_result[sched] = makespan
22 |             for sched, makespan in sched_result.iteritems():
23 |                 if sched not in results:
24 |                     results[sched] = {
25 |                         'MS': [],
26 |                         'NMS': []
27 |                     }
28 |                 results[sched]['MS'].append(makespan)
29 |                 normalized_makespan = makespan / sched_result['OLB']
30 |                 results[sched]['NMS'].append(normalized_makespan)
31 | 
32 |     for sched, sched_results in results.iteritems():
33 |         ms_results = np.array(sched_results['MS'])
34 |         ms_mean = np.mean(ms_results)
35 |         nms_results = np.array(sched_results['NMS'])
36 |         if sched == 'OLB':
37 |             nms_mean = np.mean(nms_results)
38 |             nms_conf_low = nms_mean
39 |             nms_conf_up = nms_mean
40 |         else:
41 |             nms_mean, nms_conf_low, nms_conf_up = mean_confidence_interval(nms_results)
42 |         print('%20s\t%f\t%f [%f, %f]' % (sched, ms_mean, nms_mean, nms_conf_low, nms_conf_up))
43 | 
44 | 
45 | def mean_confidence_interval(data, confidence=0.95):
46 |     mean = np.mean(data)
47 |     conf_int = st.t.interval(confidence, len(data) - 1, loc=mean, scale=st.sem(data))
48 |     return mean, conf_int[0], conf_int[1]
49 | 
50 | 
51 | if __name__ == '__main__':
52 |     parser = argparse.ArgumentParser(description="Process file with experiment results")
53 |     parser.add_argument("file", type=str, help="input file")
54 |     args = parser.parse_args()
55 |     process_results(args.file)


--------------------------------------------------------------------------------
/bot/systems/generate.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | #
 4 | # Homogeneous clusters with infinitely fast network.
 5 | #
 6 | # Speed:     1 GFLOPS
 7 | # Bandwidth: 1e12 MBps
 8 | # Latency:   0 us
 9 | #
10 | python ../gen/sys_gen.py cluster-0comm-hom-10 1 cluster 10 1 1e12 0
11 | python ../gen/sys_gen.py cluster-0comm-hom-100 1 cluster 100 1 1e12 0
12 | python ../gen/sys_gen.py cluster-0comm-hom-1000 1 cluster 1000 1 1e12 0
13 | python ../gen/sys_gen.py cluster-0comm-hom-2000 1 cluster 2000 1 1e12 0
14 | 
15 | #
16 | # Homogeneous clusters with homogeneous network a-la 1Gb Ethernet.
17 | #
18 | # Speed:     1 GFLOPS
19 | # Bandwidth: 100 MBps
20 | # Latency:   100 us
21 | #
22 | python ../gen/sys_gen.py cluster-lan-hom-100 1 cluster 100 1 100 100
23 | 
24 | #
25 | # Heterogeneous clusters with infinitely fast network.
26 | #
27 | # Speed:     1-2(5,10) GFLOPS
28 | # Bandwidth: 1e12 MBps
29 | # Latency:   0 us
30 | #
31 | python ../gen/sys_gen.py cluster-0comm-het2-10 100 cluster 10 1-2 1e12 0
32 | python ../gen/sys_gen.py cluster-0comm-het2-100 100 cluster 100 1-2 1e12 0
33 | python ../gen/sys_gen.py cluster-0comm-het2-1000 100 cluster 1000 1-2 1e12 0
34 | python ../gen/sys_gen.py cluster-0comm-het2-2000 100 cluster 2000 1-2 1e12 0
35 | python ../gen/sys_gen.py cluster-0comm-het5-10 100 cluster 10 1-5 1e12 0
36 | python ../gen/sys_gen.py cluster-0comm-het5-100 100 cluster 100 1-5 1e12 0
37 | python ../gen/sys_gen.py cluster-0comm-het5-1000 100 cluster 1000 1-5 1e12 0
38 | python ../gen/sys_gen.py cluster-0comm-het5-2000 100 cluster 2000 1-5 1e12 0
39 | python ../gen/sys_gen.py cluster-0comm-het10-10 100 cluster 10 1-10 1e12 0
40 | python ../gen/sys_gen.py cluster-0comm-het10-100 100 cluster 100 1-10 1e12 0
41 | python ../gen/sys_gen.py cluster-0comm-het10-1000 100 cluster 1000 1-10 1e12 0
42 | python ../gen/sys_gen.py cluster-0comm-het10-2000 100 cluster 2000 1-10 1e12 0
43 | 
44 | #
45 | # Heterogeneous clusters with homogeneous network a-la 1Gb Ethernet.
46 | #
47 | # Speed:     1-2(5,10) GFLOPS
48 | # Bandwidth: 100 MBps
49 | # Latency:   100 us
50 | #
51 | python ../gen/sys_gen.py cluster-lan-het2-100 100 cluster 100 1-2 100 100
52 | python ../gen/sys_gen.py cluster-lan-het5-100 100 cluster 100 1-5 100 100
53 | python ../gen/sys_gen.py cluster-lan-het10-100 100 cluster 100 1-10 100 100


--------------------------------------------------------------------------------
/bot/workloads/generate.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # All workloads have 1000 tasks.
 4 | 
 5 | #
 6 | # Fixed size tasks with zero I/O.
 7 | #
 8 | # COMP: 100 GFLOPS
 9 | # I/O:  0
10 | #
11 | python ../gen/bot_gen.py 1000 0 1e11 0 fixed-0comm 1
12 | 
13 | #
14 | # Varied size tasks with zero I/O.
15 | #
16 | # COMP: 10-(20,100,1000) GFLOPS
17 | # I/O:  0
18 | #
19 | python ../gen/bot_gen.py 1000 0 u:1e10:2e10 0 varied2-0comm 100
20 | python ../gen/bot_gen.py 1000 0 u:1e10:1e11 0 varied10-0comm 100
21 | python ../gen/bot_gen.py 1000 0 u:1e10:1e12 0 varied100-0comm 100
22 | 
23 | #
24 | # Fixed size tasks with different granularity.
25 | #
26 | # Granularity   COMP (GFLOPS)   I/O (MB)
27 | # 1             100             100 / 10
28 | # 10            100             10  / 1
29 | # 100           100             1   / 0.1
30 | #
31 | python ../gen/bot_gen.py 1000 1e8 1e11 1e7 fixed-gran1 1
32 | python ../gen/bot_gen.py 1000 1e7 1e11 1e6 fixed-gran10 1
33 | python ../gen/bot_gen.py 1000 1e6 1e11 1e5 fixed-gran100 1
34 | 
35 | #
36 | # Varied size tasks with different granularity.
37 | #
38 | # Granularity   COMP (GFLOPS)   I/O (MB)
39 | # 1             10-20           10-20   / 1-2
40 | # 1             10-100          10-100  / 1-10
41 | # 1             10-1000         10-1000 / 1-100
42 | # 10            10-20           1-2     / 0.1-0.2
43 | # 10            10-100          1-10    / 0.1-1
44 | # 10            10-1000         1-100   / 0.1-10
45 | # 100           10-20           0.1-0.2 / 0.01-0.02
46 | # 100           10-100          0.1-1   / 0.01-0.1
47 | # 100           10-1000         0.1-10  / 0.01-1
48 | #
49 | python ../gen/bot_gen.py 1000 u:1e7:2e7 x:1e3 x:0.1 varied2-gran1 100
50 | python ../gen/bot_gen.py 1000 u:1e7:1e8 x:1e3 x:0.1 varied10-gran1 100
51 | python ../gen/bot_gen.py 1000 u:1e7:1e9 x:1e3 x:0.1 varied100-gran1 100
52 | python ../gen/bot_gen.py 1000 u:1e6:2e6 x:1e4 x:0.1 varied2-gran10 100
53 | python ../gen/bot_gen.py 1000 u:1e6:1e7 x:1e4 x:0.1 varied10-gran10 100
54 | python ../gen/bot_gen.py 1000 u:1e6:1e8 x:1e4 x:0.1 varied100-gran10 100
55 | python ../gen/bot_gen.py 1000 u:1e5:2e5 x:1e5 x:0.1 varied2-gran100 100
56 | python ../gen/bot_gen.py 1000 u:1e5:1e6 x:1e5 x:0.1 varied10-gran100 100
57 | python ../gen/bot_gen.py 1000 u:1e5:1e7 x:1e5 x:0.1 varied100-gran100 100


--------------------------------------------------------------------------------
/pysimgrid/simdag/algorithms/random.py:
--------------------------------------------------------------------------------
 1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
 2 | #
 3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
 4 | #
 5 | # This library is free software: you can redistribute it and/or modify
 6 | # it under the terms of the GNU Lesser General Public License as published by
 7 | # the Free Software Foundation, either version 3 of the License, or
 8 | # (at your option) any later version.
 9 | #
10 | # This library is distributed in the hope that it will be useful,
11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | # GNU Lesser General Public License for more details.
14 | #
15 | # You should have received a copy of the GNU Lesser General Public
16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
17 | #
18 | 
19 | import logging
20 | import networkx
21 | import random
22 | 
23 | from .. import scheduler
24 | 
25 | 
26 | class RandomStatic(scheduler.StaticScheduler):
27 |     """
28 |     Random static scheduler.
29 | 
30 |     The important part there is static - all tasks are scheduled simultaneously.
31 |     Resulting schedule is significantly worse than dynamic random schedule. In static
32 |     case task may wait for their parents execution for a quite some time (blocking some other tasks),
33 |     while dynamic approach guarantees that parents are already done.
34 | 
35 |     The point of this scheduler is to be an 'worst' static scheduler as a reference point
36 |     for sensible algorithms.
37 |     """
38 | 
39 |     def get_schedule(self, simulation):
40 |         schedule = {host: [] for host in simulation.hosts}
41 |         master_host = simulation.hosts.by_prop('name', scheduler.Scheduler.MASTER_HOST_NAME)[0]
42 |         exec_hosts = simulation.hosts.by_prop('name', scheduler.Scheduler.MASTER_HOST_NAME, negate=True)
43 |         graph = simulation.get_task_graph()
44 |         for task in networkx.topological_sort(graph):
45 |             if task.name in scheduler.Scheduler.BOUNDARY_TASKS:
46 |                 host = master_host
47 |             else:
48 |                 host = random.choice(exec_hosts)
49 |             schedule[host].append(task)
50 |             # logging.info("%s -> %s" % (task.name, host.name))
51 |         return schedule
52 | 


--------------------------------------------------------------------------------
/pysimgrid/tools/scale_ccr.py:
--------------------------------------------------------------------------------
 1 | import argparse
 2 | 
 3 | 
 4 | def calculate_ccr(file):
 5 |     total_comm = 0.
 6 |     total_comp = 0.
 7 |     with open(file, "r") as input:
 8 |         for line in input:
 9 |             if '->' in line:
10 |                 size = float(line.split('size="')[1].split('"')[0])
11 |                 total_comm += size
12 |             elif 'size="' in line:
13 |                 size = float(line.split('size="')[1].split('"')[0])
14 |                 total_comp += size
15 | 
16 |     return total_comm / total_comp
17 | 
18 | 
19 | def main():
20 |     parser = argparse.ArgumentParser(description="Scale CCR to desired value")
21 |     parser.add_argument("input_file", type=str, help="path to input DOT file")
22 |     parser.add_argument("-c", "--ccr", type=float, help="target CCR value")
23 |     parser.add_argument("-s", "--speed", type=float, help="host speed in Gflop/s")
24 |     parser.add_argument("-b", "--bandwidth", type=float, help="network bandwidth in MBps")
25 |     parser.add_argument("-o", "--output-file", type=str, help="path to output DOT file")
26 | 
27 |     args = parser.parse_args()
28 | 
29 |     input_ccr = calculate_ccr(args.input_file)
30 |     print("Input CCR: %f" % input_ccr)
31 |     if args.speed is not None and args.bandwidth is not None:
32 |         system_factor = (args.speed * 1e9) / (args.bandwidth * 1e6)
33 |         input_ccr *= system_factor
34 |         print("Input system CCR: %f" % input_ccr)
35 |     else:
36 |         system_factor = None
37 | 
38 |     if args.output_file is not None:
39 |         factor = args.ccr / input_ccr
40 |         with open(args.output_file, "w") as out:
41 |             with open(args.input_file, "r") as input:
42 |                 for line in input:
43 |                     if '->' in line:
44 |                         parts = line.split('size="')
45 |                         size = float(parts[1].split('"')[0])
46 |                         new_size = size * factor
47 |                         new_line = '%ssize="%e"%s' % (parts[0], new_size, parts[1].split('"')[1])
48 |                         out.write(new_line)
49 |                     else:
50 |                         out.write(line)
51 | 
52 |         output_ccr = calculate_ccr(args.output_file)
53 |         print("Output CCR: %f" % output_ccr)
54 |         if system_factor is not None:
55 |             print("Output system CCR: %f" % (output_ccr * system_factor))
56 | 
57 | 
58 | if __name__ == "__main__":
59 |     main()
60 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
 1 | from __future__ import print_function
 2 | 
 3 | import os
 4 | import sys
 5 | import platform
 6 | import textwrap
 7 | from setuptools import setup, Extension
 8 | 
 9 | import numpy
10 | 
11 | from Cython.Build import cythonize
12 | 
13 | 
14 | def local_path(*components):
15 |   project_root = os.path.dirname(os.path.realpath(__file__))
16 |   return os.path.normpath(os.path.join(project_root, *components))
17 | 
18 | 
19 | def source_file(filename):
20 |   return os.path.join("pysimgrid", filename) + ".pyx"
21 | 
22 | 
23 | simgrid_env_path = os.environ.get("SIMGRID_ROOT")
24 | SIMGRID_ROOT = simgrid_env_path if simgrid_env_path else local_path("opt/SimGrid")
25 | 
26 | # Note that SimGrid shared library is required at runtime, so you shouldn't
27 | # move/delete used SimGrid installation while using pysimgrid.
28 | if not os.path.isdir(SIMGRID_ROOT):
29 |   print(textwrap.dedent("""\
30 |   SimGrid root directory is not found.
31 | 
32 |   SimGrid can be built manually or using the provided get_simgrid.sh script.
33 |   pysimgrid isn't intended to be used with system-wide installations of SimGrid.
34 | 
35 |   In case of a custom build, you can select your
36 |   install location using SIMGRID_ROOT environment variable.
37 |   """))
38 |   sys.exit(1)
39 | 
40 | EXT_OPTIONS = {
41 |   "libraries": ["simgrid"],
42 |   "include_dirs": [os.path.join(SIMGRID_ROOT, "include"), numpy.get_include()],
43 |   "library_dirs": [os.path.join(SIMGRID_ROOT, "lib")],
44 | }
45 | if platform.system().lower() == "linux":
46 |   EXT_OPTIONS["runtime_library_dirs"] = [os.path.join(SIMGRID_ROOT, "lib")]
47 | elif platform.system().lower() == "darwin":
48 |   EXT_OPTIONS["extra_link_args"] = ["-Wl,-rpath," + os.path.join(SIMGRID_ROOT, "lib")]
49 | 
50 | sdwrapper = Extension("pysimgrid.csimdag",
51 |                       sources=[source_file("csimdag")],
52 |                       **EXT_OPTIONS)
53 | 
54 | plwrapper = Extension("pysimgrid.cplatform",
55 |                       sources=[source_file("cplatform")],
56 |                       **EXT_OPTIONS)
57 | 
58 | scwrapper = Extension("pysimgrid.cscheduling",
59 |                       sources=[source_file("cscheduling")],
60 |                       **EXT_OPTIONS)
61 | 
62 | extensions = [sdwrapper, plwrapper, scwrapper]
63 | extensions = cythonize(extensions, compiler_directives={'embedsignature': True})
64 | 
65 | setup(name="pysimgrid",
66 |       version="1.0.0",
67 |       author="Alexey Nazarenko",
68 |       packages=["pysimgrid", "pysimgrid.simdag"],
69 |       install_requires=['numpy', 'networkx>=2.0'],
70 |       ext_modules=extensions)
71 | 


--------------------------------------------------------------------------------
/pysimgrid/cplatform.pxd:
--------------------------------------------------------------------------------
 1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
 2 | #
 3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
 4 | #
 5 | # This library is free software: you can redistribute it and/or modify
 6 | # it under the terms of the GNU Lesser General Public License as published by
 7 | # the Free Software Foundation, either version 3 of the License, or
 8 | # (at your option) any later version.
 9 | #
10 | # This library is distributed in the hope that it will be useful,
11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | # GNU Lesser General Public License for more details.
14 | #
15 | # You should have received a copy of the GNU Lesser General Public
16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
17 | #
18 | 
19 | cimport xbt
20 | 
21 | cdef extern from "simgrid/simdag.h":
22 |   ctypedef void* SD_link_t
23 |   ctypedef void* sg_host_t
24 | 
25 |   ##########################################################
26 |   # Platform API
27 |   ##########################################################
28 |   size_t sg_host_count();
29 |   sg_host_t* sg_host_list();
30 | 
31 |   sg_host_t sg_host_by_name(const char* name);
32 |   sg_host_t sg_host_by_name_or_create(const char* name);
33 |   const char* sg_host_get_name(sg_host_t host);
34 | 
35 |   double sg_host_speed(sg_host_t host);
36 |   double sg_host_get_available_speed(sg_host_t host);
37 | 
38 |   xbt.xbt_dict_t sg_host_get_properties(sg_host_t host);
39 |   const char* sg_host_get_property_value(sg_host_t host, const char* name);
40 |   void sg_host_dump(sg_host_t ws);
41 | 
42 |   void* sg_host_user(sg_host_t host);
43 |   void sg_host_user_set(sg_host_t host, void* userdata);
44 |   void sg_host_user_destroy(sg_host_t host);
45 | 
46 |   SD_link_t* SD_route_get_list(sg_host_t src, sg_host_t dst);
47 |   int SD_route_get_size(sg_host_t src, sg_host_t dst);
48 |   const char* sg_link_name(SD_link_t link);
49 | 
50 |   double SD_route_get_latency(sg_host_t src, sg_host_t dst);
51 |   double SD_route_get_bandwidth(sg_host_t src, sg_host_t dst);
52 | 
53 |   # undocumneted advanced functions
54 |   #int sg_host_get_nb_pstates(sg_host_t host);
55 |   #int sg_host_get_pstate(sg_host_t host);
56 |   #void sg_host_set_pstate(sg_host_t host, int pstate);
57 |   #size_t sg_host_extension_create(void(*deleter)(void*));
58 |   #void* sg_host_extension_get(sg_host_t host, size_t rank);
59 |   #xbt.xbt_dynar_t sg_hosts_as_dynar();
60 | 
61 | cdef class Host:
62 |   cdef sg_host_t impl
63 |   cdef object user_data
64 | 
65 |   @staticmethod
66 |   cdef Host wrap(sg_host_t impl)
67 | 
68 |   @staticmethod
69 |   cdef list wrap_batch(sg_host_t* hosts, int count)
70 | 


--------------------------------------------------------------------------------
/pysimgrid/tools/trace_chart.py:
--------------------------------------------------------------------------------
 1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
 2 | #
 3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
 4 | #
 5 | # This library is free software: you can redistribute it and/or modify
 6 | # it under the terms of the GNU Lesser General Public License as published by
 7 | # the Free Software Foundation, either version 3 of the License, or
 8 | # (at your option) any later version.
 9 | #
10 | # This library is distributed in the hope that it will be useful,
11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | # GNU Lesser General Public License for more details.
14 | #
15 | # You should have received a copy of the GNU Lesser General Public
16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
17 | #
18 | 
19 | 
20 | import argparse
21 | import sys
22 | import matplotlib.pyplot as plt
23 | 
24 | 
25 | def show_plot(raw_trace):
26 |   trace = raw_trace.strip().split('\n')
27 | 
28 |   types  = {}
29 |   hosts = {}
30 |   links = {}
31 |   schedule = {}
32 |   end = 0
33 | 
34 |   for line in trace:
35 |     if line.startswith('0 '):
36 |       _, alias, type_, name = line.strip().split(' ')
37 |       types[alias] = name
38 |     elif line.startswith('6 '):
39 |       _, _, alias, type_, container, name = line.strip().split(' ')
40 |       if types[type_] == 'HOST':
41 |           hosts[alias] = name
42 |     elif line.startswith('8 '):
43 |       # PageSetVariable
44 |       # Init the work on the schedule
45 |       _, time, type_, container, value = line.strip().split(' ')
46 |       if container in hosts:
47 |         schedule[hosts[container]] = []
48 |     elif line.startswith('9 '):
49 |       # PajeAddVariable
50 |       # Start to calculate a task
51 |       _, time, type_, container, value = line.strip().split(' ')
52 |       if container in hosts:
53 |         schedule[hosts[container]].append([float(time), None])
54 |     elif line.startswith('10 '):
55 |       # PajeSubVariable
56 |       # Finish to calculate a task
57 |       _, time, type_, container, value = line.strip().split(' ')
58 |       if container in hosts:
59 |         schedule[hosts[container]][-1][1] = float(time) - schedule[hosts[container]][-1][0]
60 |         end = max(end, float(time))
61 | 
62 |   fig, ax = plt.subplots()
63 |   for i, host in enumerate(sorted(schedule)):
64 |       ax.broken_barh(schedule[host], (i*2, 1), facecolors="grey")
65 |   ax.set_ylim(0, len(hosts) * 2 + 1)
66 |   ax.set_xlim(0, end)
67 |   ax.set_yticks(range(len(hosts) * 2))
68 |   ax.grid(True)
69 |   plt.show()
70 | 
71 | if __name__ == "__main__":
72 |   parser = argparse.ArgumentParser(description="Host utilization visualization from SimGrid Paje trace files")
73 |   parser.add_argument("trace_file", type=str, help="path to trace file to visualize")
74 |   args = parser.parse_args()
75 |   with open(args.trace_file, "r") as f:
76 |     raw_trace = f.read()
77 |     show_plot(raw_trace)
78 | 


--------------------------------------------------------------------------------
/pysimgrid/simdag/algorithms/olb.py:
--------------------------------------------------------------------------------
 1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
 2 | #
 3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
 4 | #
 5 | # This library is free software: you can redistribute it and/or modify
 6 | # it under the terms of the GNU Lesser General Public License as published by
 7 | # the Free Software Foundation, either version 3 of the License, or
 8 | # (at your option) any later version.
 9 | #
10 | # This library is distributed in the hope that it will be useful,
11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | # GNU Lesser General Public License for more details.
14 | #
15 | # You should have received a copy of the GNU Lesser General Public
16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
17 | #
18 | 
19 | import logging
20 | 
21 | from .. import scheduler
22 | from ... import csimdag
23 | 
24 | 
25 | class OLB(scheduler.DynamicScheduler):
26 |     """
27 |     Opportunistic load balancing scheduler.
28 | 
29 |     Dynamically schedule a free task on a fastest host, disregarding any communication costs.
30 |     Main practical advantage is avoiding any attempts to estimate task performance on given machine.
31 | 
32 |     Can be considered baseline algorithm for benchmarking purposes as one of the simplest approaches
33 |     that actually make sense for DAG scheduling.
34 |     """
35 | 
36 |     def prepare(self, simulation):
37 |         for h in simulation.hosts:
38 |             h.data = {}
39 |         master_hosts = simulation.hosts.by_prop("name", self.MASTER_HOST_NAME)
40 |         self._master_host = master_hosts[0] if master_hosts else None
41 |         if self._master_host:
42 |             for task in simulation.tasks.by_func(lambda t: t.name in self.BOUNDARY_TASKS):
43 |                 task.schedule(self._master_host)
44 |         self._exec_hosts = simulation.hosts.by_prop("name", self.MASTER_HOST_NAME, True)
45 |         self._queue = []
46 | 
47 |     def schedule(self, simulation, changed):
48 |         for h in simulation.hosts:
49 |             h.data["free"] = True
50 |         for task in simulation.tasks[csimdag.TaskState.TASK_STATE_RUNNING, csimdag.TaskState.TASK_STATE_SCHEDULED]:
51 |             task.hosts[0].data["free"] = False
52 |         queue_set = set(self._queue)
53 |         for t in simulation.tasks[csimdag.TaskState.TASK_STATE_SCHEDULABLE]:
54 |             if t not in queue_set:
55 |                 self._queue.append(t)
56 |         while self._queue:
57 |             task = self._queue[0]
58 |             free_hosts = self._exec_hosts.by_data("free", True)
59 |             if free_hosts:
60 |                 self._queue.pop(0)
61 |                 free_hosts = free_hosts.sorted(lambda h: h.speed, reverse=True)
62 |                 target_host = free_hosts[0]
63 |                 task.schedule(target_host)
64 |                 # logging.info("%s -> %s" % (task.name, target_host.name))
65 |                 target_host.data["free"] = False
66 |             else:
67 |                 break
68 | 


--------------------------------------------------------------------------------
/doc/source/_themes/sphinx_rtd_theme/static/css/badge_only.css.map:
--------------------------------------------------------------------------------
1 | {
2 | "version": 3,
3 | "mappings": "CAyDA,SAAY,EACV,qBAAsB,EAAE,UAAW,EAqDrC,QAAS,EARP,IAAK,EAAE,AAAC,EACR,+BAAS,EAEP,MAAO,EAAE,IAAK,EACd,MAAO,EAAE,CAAE,EACb,cAAO,EACL,IAAK,EAAE,GAAI,EC1Gb,SAkBC,EAjBC,UAAW,ECFJ,UAAW,EDGlB,UAAW,EAHqC,KAAM,EAItD,SAAU,EAJsD,KAAM,EAapE,EAAG,EAAE,qCAAwB,EAC7B,EAAG,EAAE,0PAG2D,ECftE,SAAU,EACR,MAAO,EAAE,WAAY,EACrB,UAAW,EAAE,UAAW,EACxB,SAAU,EAAE,KAAM,EAClB,UAAW,EAAE,KAAM,EACnB,UAAW,EAAE,AAAC,EACd,cAAe,EAAE,MAAO,EAG1B,IAAK,EACH,MAAO,EAAE,WAAY,EACrB,cAAe,EAAE,MAAO,EAIxB,KAAG,EACD,MAAO,EAAE,WAAY,EACvB,sCAAiB,EAGf,IAAK,EAAE,MAAY,EAEvB,KAAM,EACJ,cAAe,EAAE,GAAI,EACrB,UAAW,EAAE,EAAG,EAChB,UAAW,EAAE,KAAM,EAEjB,YAAG,EACD,IAAK,EAAE,IAAI,EACb,oDAAiB,EAGf,aAAc,EAAE,OAAQ,EAG9B,cAAe,EACb,MAAO,EAAE,EAAO,EAElB,gBAAiB,EACf,MAAO,EAAE,EAAO,EAElB,oBAAqB,EACnB,MAAO,EAAE,EAAO,EAElB,sBAAuB,EACrB,MAAO,EAAE,EAAO,EAElB,kBAAmB,EACjB,MAAO,EAAE,EAAO,EAElB,oBAAqB,EACnB,MAAO,EAAE,EAAO,EAElB,oBAAqB,EACnB,MAAO,EAAE,EAAO,EAElB,sBAAuB,EACrB,MAAO,EAAE,EAAO,EAElB,qBAAsB,EACpB,MAAO,EAAE,EAAO,EAElB,uBAAwB,EACtB,MAAO,EAAE,EAAO,ECnElB,YAAa,EACX,OAAQ,EAAE,IAAK,EACf,KAAM,EAAE,AAAC,EACT,GAAI,EAAE,AAAC,EACP,IAAK,EC6E+B,IAAK,ED5EzC,IAAK,EEoC+B,MAAyB,EFnC7D,SAAU,EAAE,MAAkC,EAC9C,SAAU,EAAE,iBAAiC,EAC7C,UAAW,EE+CyB,sDAA2D,EF9C/F,MAAO,EC+E6B,EAAG,ED9EvC,cAAC,EACC,IAAK,EE+B6B,MAAK,EF9BvC,cAAe,EAAE,GAAI,EACvB,6BAAgB,EACd,MAAO,EAAE,GAAI,EACf,iCAAoB,EAClB,MAAO,EAAE,GAAqB,EAC9B,eAAgB,EAAE,MAAkC,EACpD,MAAO,EAAE,IAAK,EACd,SAAU,EAAE,IAAK,EACjB,QAAS,EAAE,EAAG,EACd,KAAM,EAAE,MAAO,EACf,IAAK,EEX6B,MAAM,EL4F1C,IAAK,EAAE,AAAC,EACR,iFAAS,EAEP,MAAO,EAAE,IAAK,EACd,MAAO,EAAE,CAAE,EACb,uCAAO,EACL,IAAK,EAAE,GAAI,EGrFX,qCAAG,EACD,IAAK,EEgB2B,MAAyB,EFf3D,0CAAQ,EACN,IAAK,EAAE,GAAI,EACb,4CAAU,EACR,IAAK,EAAE,GAAI,EACb,iDAAiB,EACf,eAAgB,ECQgB,MAAI,EDPpC,IAAK,EEI2B,GAAM,EFHxC,wDAAwB,EACtB,eAAgB,EEmBgB,MAAO,EFlBvC,IAAK,ECzB2B,GAAI,ED0BxC,yCAA8B,EAC5B,MAAO,EAAE,IAAK,EAChB,gCAAmB,EACjB,QAAS,EAAE,EAAG,EACd,MAAO,EAAE,GAAqB,EAC9B,IAAK,EEP6B,GAAY,EFQ9C,MAAO,EAAE,GAAI,EACb,mCAAE,EACA,MAAO,EAAE,IAAK,EACd,KAAM,EAAE,EAAG,EACX,KAAM,EAAE,AAAC,EACT,KAAM,EAAE,KAAM,EACd,MAAO,EAAE,AAAC,EACV,SAAU,EAAE,gBAA6C,EAC3D,mCAAE,EACA,MAAO,EAAE,WAAY,EACrB,KAAM,EAAE,AAAC,EACT,qCAAC,EACC,MAAO,EAAE,WAAY,EACrB,MAAO,EAAE,EAAqB,EAC9B,IAAK,EEfyB,MAAyB,EFgB7D,sBAAW,EACT,IAAK,EAAE,GAAI,EACX,KAAM,EAAE,GAAI,EACZ,IAAK,EAAE,GAAI,EACX,GAAI,EAAE,GAAI,EACV,KAAM,EAAE,GAAI,EACZ,QAAS,ECkByB,IAAK,EDjBvC,iCAAU,EACR,IAAK,EAAE,GAAI,EACb,+BAAQ,EACN,IAAK,EAAE,GAAI,EACb,oDAA+B,EAC7B,SAAU,EAAE,IAAK,EACjB,6DAAQ,EACN,IAAK,EAAE,GAAI,EACb,+DAAU,EACR,IAAK,EAAE,GAAI,EACf,2CAAoB,EAClB,IAAK,EAAE,GAAI,EACX,KAAM,EAAE,GAAI,EACZ,UAAW,EAAE,GAAI,EACjB,MAAO,EAAE,IAAuB,EAChC,MAAO,EAAE,IAAK,EACd,SAAU,EAAE,KAAM,EGhDpB,mCAAsB,EHmDxB,YAAa,EACX,IAAK,EAAE,EAAG,EACV,MAAO,EAAE,GAAI,EACb,kBAAO,EACL,MAAO,EAAE,IAAK,EAClB,EAAG,EACD,IAAK,EAAE,GAAI,EACX,KAAM,EAAE,GAAI",
4 | "sources": ["../../../bower_components/wyrm/sass/wyrm_core/_mixin.sass","../../../bower_components/bourbon/dist/css3/_font-face.scss","../../../sass/_theme_badge_fa.sass","../../../sass/_theme_badge.sass","../../../bower_components/wyrm/sass/wyrm_core/_wy_variables.sass","../../../sass/_theme_variables.sass","../../../bower_components/neat/app/assets/stylesheets/grid/_media.scss"],
5 | "names": [],
6 | "file": "badge_only.css"
7 | }
8 | 


--------------------------------------------------------------------------------
/pysimgrid/simdag/algorithms/heft.py:
--------------------------------------------------------------------------------
 1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
 2 | #
 3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
 4 | #
 5 | # This library is free software: you can redistribute it and/or modify
 6 | # it under the terms of the GNU Lesser General Public License as published by
 7 | # the Free Software Foundation, either version 3 of the License, or
 8 | # (at your option) any later version.
 9 | #
10 | # This library is distributed in the hope that it will be useful,
11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | # GNU Lesser General Public License for more details.
14 | #
15 | # You should have received a copy of the GNU Lesser General Public
16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
17 | #
18 | 
19 | import networkx
20 | 
21 | from .. import scheduler
22 | from ... import cscheduling
23 | 
24 | 
25 | class HEFT(scheduler.StaticScheduler):
26 |   """
27 |   Heterogeneous Earliest Finish Time (HEFT) scheduler.
28 | 
29 |   Many advantages of this method include:
30 | 
31 |   * pretty high performance
32 |   * low time complexity (N**2M)
33 |   * quite simple implementation
34 | 
35 |   The general idea is very simple:
36 | 
37 |   1. Sort tasks according in decreasing ranku order
38 | 
39 |      ranku(task) = ECT(task) + max_among_children(ranku(child) + ECOMT(task, child))
40 | 
41 |      where ECT and ECOMT are evaluated using platform mean speed, bandwidt and latency values
42 | 
43 |      Important property of this ordering is that it is also an topological order (see below).
44 | 
45 |   2. Go over ordered tasks list and schedule each one to the minimize task completion time.
46 | 
47 |      To estimate task completion time, ofc, you need to all task parents to be scheduled already.
48 |      This is achieved automatically as HEFT order is also a topological sort for a tasks.
49 | 
50 |      HEFT scheduling allows insertions to happen, so if some host has an empty and wide enough time slot,
51 |      next task may be added in between already scheduled tasks.
52 | 
53 |   For more details please refer to the original HEFT publication:
54 | 
55 |     H. Topcuoglu, S. Hariri and Min-You Wu, "Performance-effective and low-complexity task
56 |     scheduling for heterogeneous computing", IEEE Transactions on Parallel and Distributed
57 |     Systems, Vol 13, No 3, 2002, pp. 260-274
58 |   """
59 | 
60 |   def get_schedule(self, simulation):
61 |     """
62 |     Overriden.
63 |     """
64 |     nxgraph = simulation.get_task_graph()
65 |     platform_model = cscheduling.PlatformModel(simulation)
66 |     state = cscheduling.SchedulerState(simulation)
67 | 
68 |     ordered_tasks = cscheduling.heft_order(nxgraph, platform_model)
69 | 
70 |     cscheduling.heft_schedule(nxgraph, platform_model, state, ordered_tasks, self._data_transfer_mode.name)
71 |     # store ECT in tasks for QUEUE_ECT data transfer mode
72 |     for task, task_state in state.task_states.items():
73 |       task.data = {"ect": task_state["ect"]}
74 |     expected_makespan = max([state["ect"] for state in state.task_states.values()])
75 |     return state.schedule, expected_makespan
76 | 


--------------------------------------------------------------------------------
/doc/source/reference/simdag.rst:
--------------------------------------------------------------------------------
 1 | ****************
 2 | pysimgrid.simdag
 3 | ****************
 4 | 
 5 | .. contents::
 6 | 
 7 | Simulation
 8 | ==========
 9 | 
10 | Simulation class provides an entry point for the, ugh, simulations.
11 | 
12 | It is intended to be used as context manager, as it is important to ensure proper setup and cleanup
13 | when working with a native library.
14 | 
15 | .. autoclass:: pysimgrid.simdag.Simulation
16 |    :members:
17 | 
18 | Schedulers
19 | ==========
20 | 
21 | Schedulers are the main workhorses during the simulation.
22 | 
23 | While nothing stops you from doing everything manually, scheduler interfaces
24 | provides a useful framework to build your algorithm upon.
25 | 
26 | 
27 | .. autoclass:: pysimgrid.simdag.Scheduler
28 |   :members:
29 | 
30 | .. autoclass:: pysimgrid.simdag.StaticScheduler
31 |   :show-inheritance:
32 | 
33 |   .. automethod:: pysimgrid.simdag.StaticScheduler.get_schedule
34 | 
35 | .. autoclass:: pysimgrid.simdag.DynamicScheduler
36 |   :show-inheritance:
37 | 
38 |   .. automethod:: pysimgrid.simdag.DynamicScheduler.prepare
39 | 
40 |   .. automethod:: pysimgrid.simdag.DynamicScheduler.schedule
41 | 
42 | 
43 | Task kinds
44 | ==========
45 | 
46 | SimGrid tasks can have different types.
47 | 
48 | Parallel tasks API is unfinished for now, there a only 2 important ones:
49 |   *TASK_KIND_COMP_SEQ* - computational task
50 | 
51 |   *TASK_KIND_COMM_E2E* - data transfer task (dependency)
52 | 
53 | :class:`pysimgrid.simdag.Simulation` class provides shortcut methods to access tasks by kind::
54 | 
55 |     import pysimgrid.simdag
56 | 
57 |     with pysimgrid.simdag.Simulation(platform, "task_file.dot") as simulation:
58 |       # get all computational tasks
59 |       tasks = simulation.tasks
60 |       # get all computational tasks "manually"
61 |       tasks = simulation.all_tasks[pysimgrid.simdag.TaskKind.TASK_KIND_COMP_SEQ]
62 |       # get all connections
63 |       connections = simulation.connections
64 |       # get all connections "manually"
65 |       connections = simulation.connections[pysimgrid.simdag.TaskKind.TASK_KIND_COMM_E2E]
66 | 
67 | .. autoclass:: pysimgrid.simdag.TaskKind
68 |   :members:
69 |   :undoc-members:
70 | 
71 | Task states
72 | ===========
73 | 
74 | Task state is provided by the underlying library and can be used to conveniently filter tasks.
75 | 
76 | Let's explain by example::
77 | 
78 |   import pysimgrid.simdag
79 | 
80 |   with pysimgrid.simdag.Simulation(platform, "task_file.dot") as simulation:
81 |     # select all schedulable tasks
82 |     schedulable = simulation.tasks[pysimgrid.simdag.TaskState.TASK_STATE_SCHEDULABLE]
83 |     # select all running OR ready to run tasks
84 |     states = (pysimgrid.simdag.TaskState.TASK_STATE_RUNNING,
85 |               pysimgrid.simdag.TaskState.TASK_STATE_RUNNABLE,
86 |               pysimgrid.simdag.TaskState.TASK_STATE_SCHEDULED)
87 |     ready_or_running = simulation.tasks[states]
88 |     # select all completed tasks
89 |     done = simulation.tasks[pysimgrid.simdag.TaskState.TASK_STATE_DONE]
90 |     # select all fully transfered connections
91 |     conn_done = simulation.connections[pysimgrid.simdag.TaskState.TASK_STATE_DONE]
92 | 
93 | 
94 | .. autoclass:: pysimgrid.simdag.TaskState
95 |    :members:
96 |    :undoc-members:
97 | 


--------------------------------------------------------------------------------
/doc/source/_themes/sphinx_rtd_theme/static/css/badge_only.css:
--------------------------------------------------------------------------------
1 | .fa:before{-webkit-font-smoothing:antialiased}.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;content:""}.clearfix:after{clear:both}@font-face{font-family:FontAwesome;font-weight:normal;font-style:normal;src:url("../font/fontawesome_webfont.eot");src:url("../font/fontawesome_webfont.eot?#iefix") format("embedded-opentype"),url("../font/fontawesome_webfont.woff") format("woff"),url("../font/fontawesome_webfont.ttf") format("truetype"),url("../font/fontawesome_webfont.svg#FontAwesome") format("svg")}.fa:before{display:inline-block;font-family:FontAwesome;font-style:normal;font-weight:normal;line-height:1;text-decoration:inherit}a .fa{display:inline-block;text-decoration:inherit}li .fa{display:inline-block}li .fa-large:before,li .fa-large:before{width:1.875em}ul.fas{list-style-type:none;margin-left:2em;text-indent:-0.8em}ul.fas li .fa{width:0.8em}ul.fas li .fa-large:before,ul.fas li .fa-large:before{vertical-align:baseline}.fa-book:before{content:""}.icon-book:before{content:""}.fa-caret-down:before{content:""}.icon-caret-down:before{content:""}.fa-caret-up:before{content:""}.icon-caret-up:before{content:""}.fa-caret-left:before{content:""}.icon-caret-left:before{content:""}.fa-caret-right:before{content:""}.icon-caret-right:before{content:""}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;border-top:solid 10px #343131;font-family:"Lato","proxima-nova","Helvetica Neue",Arial,sans-serif;z-index:400}.rst-versions a{color:#2980B9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27AE60;*zoom:1}.rst-versions .rst-current-version:before,.rst-versions .rst-current-version:after{display:table;content:""}.rst-versions .rst-current-version:after{clear:both}.rst-versions .rst-current-version .fa{color:#fcfcfc}.rst-versions .rst-current-version .fa-book{float:left}.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#E74C3C;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#F1C40F;color:#000}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:gray;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:solid 1px #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px}.rst-versions.rst-badge .icon-book{float:none}.rst-versions.rst-badge .fa-book{float:none}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book{float:left}.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge .rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width: 768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}img{width:100%;height:auto}}
2 | /*# sourceMappingURL=badge_only.css.map */
3 | 


--------------------------------------------------------------------------------
/pysimgrid/simdag/algorithms/mct.py:
--------------------------------------------------------------------------------
 1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
 2 | #
 3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
 4 | #
 5 | # This library is free software: you can redistribute it and/or modify
 6 | # it under the terms of the GNU Lesser General Public License as published by
 7 | # the Free Software Foundation, either version 3 of the License, or
 8 | # (at your option) any later version.
 9 | #
10 | # This library is distributed in the hope that it will be useful,
11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | # GNU Lesser General Public License for more details.
14 | #
15 | # You should have received a copy of the GNU Lesser General Public
16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
17 | #
18 | 
19 | import logging
20 | 
21 | from .. import scheduler
22 | from ... import csimdag
23 | 
24 | 
25 | class MCT(scheduler.DynamicScheduler):
26 |     """
27 |     Minimum completion time scheduler.
28 | 
29 |     Ready tasks are scheduled on host (probably busy one) that promises the earliest task completion.
30 | 
31 |     Task completion time is estimated as:
32 | 
33 |     ECT(task, host) = max(current_clock, estimated_host_ready_time) + max(comm_time_from_parents) + EET(task, host)
34 |     """
35 | 
36 |     def prepare(self, simulation):
37 |         for h in simulation.hosts:
38 |             h.data = {
39 |                 "est": 0.
40 |             }
41 |         master_hosts = simulation.hosts.by_prop("name", self.MASTER_HOST_NAME)
42 |         self._master_host = master_hosts[0] if master_hosts else None
43 |         if self._master_host:
44 |             for task in simulation.tasks.by_func(lambda t: t.name in self.BOUNDARY_TASKS):
45 |                 task.schedule(self._master_host)
46 |         self._exec_hosts = simulation.hosts.by_prop("name", self.MASTER_HOST_NAME, True)
47 |         self._target_hosts = {}
48 |         self._is_free = {}
49 | 
50 |     def schedule(self, simulation, changed):
51 |         for h in simulation.hosts:
52 |             self._is_free[h] = True
53 |         for task in simulation.tasks[csimdag.TaskState.TASK_STATE_RUNNING, csimdag.TaskState.TASK_STATE_SCHEDULED]:
54 |             self._is_free[task.hosts[0]] = False
55 |         clock = simulation.clock
56 |         for task in simulation.tasks[csimdag.TaskState.TASK_STATE_SCHEDULABLE]:
57 |             target_host = self._target_hosts.get(task)
58 |             if not target_host:
59 |                 sorted_hosts = self._exec_hosts.sorted(lambda h: self.get_ect(clock, task, h))
60 |                 target_host = sorted_hosts[0]
61 |                 self._target_hosts[task] = target_host
62 |                 target_host.data["est"] = self.get_ect(clock, task, target_host)
63 |             if self._is_free[target_host]:
64 |                 task.schedule(target_host)
65 |                 # logging.info("%s -> %s" % (task.name, target_host.name))
66 |                 self._is_free[target_host] = False
67 | 
68 |     @staticmethod
69 |     def get_ect(clock, task, host):
70 |         parent_connections = [p for p in task.parents if p.kind == csimdag.TaskKind.TASK_KIND_COMM_E2E]
71 |         comm_times = [conn.get_ecomt(conn.parents[0].hosts[0], host) for conn in parent_connections]
72 |         return max(host.data["est"], clock) + task.get_eet(host) + (max(comm_times) if comm_times else 0.)
73 | 


--------------------------------------------------------------------------------
/dag/prepare.sh:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env bash
  2 | 
  3 | SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
  4 | pushd "$SCRIPT_DIR/.."
  5 | 
  6 | if [ -z "$DAGGEN_PATH" ]; then
  7 |   echo "Specify path to daggen executable in DAGGEN_PATH environment variable"
  8 |   exit 1
  9 | fi
 10 | if [ ! -f "$DAGGEN_PATH" ]; then
 11 |   echo "daggen executable $DAGGEN_PATH not found"
 12 |   exit 1
 13 | fi
 14 | 
 15 | SEED=1234
 16 | 
 17 | # EXPERIMENT 1 (REAL WORKFLOWS) ****************************************************************************************
 18 | 
 19 | #
 20 | # Real workflows (CyberShake, Epigenomics, LIGO Inspiral, Montage)
 21 | #
 22 | # Tasks: 100
 23 | # See tasks_exp1 directory
 24 | #
 25 | 
 26 | #
 27 | # Heterogeneous clusters with homogeneous network a-la 1Gbit Ethernet.
 28 | #
 29 | # Nodes:     5, 10, 20
 30 | # Speed:     1-4 GFLOPS
 31 | # Bandwidth: 100 MBps
 32 | # Latency:   100 us
 33 | #
 34 | if [ ! -d "$SCRIPT_DIR/exp1_systems" ]; then
 35 |   python3 -m pysimgrid.tools.plat_gen "$SCRIPT_DIR/exp1_systems" 100 $SEED cluster 5 1-4 100 100 --include_master
 36 |   python3 -m pysimgrid.tools.plat_gen "$SCRIPT_DIR/exp1_systems" 100 $SEED cluster 10 1-4 100 100 --include_master
 37 |   python3 -m pysimgrid.tools.plat_gen "$SCRIPT_DIR/exp1_systems" 100 $SEED cluster 20 1-4 100 100 --include_master
 38 | fi
 39 | 
 40 | #
 41 | # Heterogeneous clusters with infinitely fast network (including loopback).
 42 | #
 43 | # Nodes:     5, 10, 20
 44 | # Speed:     1-4 GFLOPS
 45 | # Bandwidth: 1e12 MBps
 46 | # Latency:   0 us
 47 | #
 48 | if [ ! -d "$SCRIPT_DIR/exp1_systems_inf" ]; then
 49 |   python3 -m pysimgrid.tools.plat_gen "$SCRIPT_DIR/exp1_systems_inf" 100 $SEED cluster 5 1-4 1e12 0 \
 50 |         --loopback_bandwidth=1e12 --loopback_latency=0 --include_master
 51 |   python3 -m pysimgrid.tools.plat_gen "$SCRIPT_DIR/exp1_systems_inf" 100 $SEED cluster 10 1-4 1e12 0 \
 52 |         --loopback_bandwidth=1e12 --loopback_latency=0 --include_master
 53 |   python3 -m pysimgrid.tools.plat_gen "$SCRIPT_DIR/exp1_systems_inf" 100 $SEED cluster 20 1-4 1e12 0 \
 54 |         --loopback_bandwidth=1e12 --loopback_latency=0 --include_master
 55 | fi
 56 | 
 57 | # EXPERIMENT 2 (SYNTHETIC WORKFLOWS) ***********************************************************************************
 58 | 
 59 | #
 60 | # Synthetic workflows
 61 | #
 62 | # Tasks:     100
 63 | # Fat:       0.5, 0.65, 0.8 (10, 20, 40 tasks per layer)
 64 | # Density:   0.1
 65 | # Jump:      2
 66 | # Data:      1-1000 MB
 67 | # CCR:       1, 10, 100 MB/Gflop
 68 | #
 69 | if [ ! -d "$SCRIPT_DIR/exp2_workflows" ]; then
 70 |   python3 -m pysimgrid.tools.dag_gen \
 71 |         -n 100 --fat 0.5 0.65 0.8 --density 0.1 --jump 2 \
 72 |         --mindata 1e6 --maxdata 1e9 --ccr 1 10 100 \
 73 |         --repeat 100 --seed $SEED \
 74 |         "$SCRIPT_DIR/exp2_workflows"
 75 | fi
 76 | 
 77 | #
 78 | # Heterogeneous cluster with homogeneous network a-la 1Gbit Ethernet.
 79 | #
 80 | # Nodes:     10
 81 | # Speed:     1-4 GFLOPS
 82 | # Bandwidth: 100 MBps
 83 | # Latency:   100 us
 84 | #
 85 | if [ ! -d "$SCRIPT_DIR/exp2_systems" ]; then
 86 |   python3 -m pysimgrid.tools.plat_gen "$SCRIPT_DIR/exp2_systems" 1 $SEED cluster 10 1-4 100 100 --include_master
 87 | fi
 88 | 
 89 | #
 90 | # Heterogeneous cluster with infinitely fast network (including loopback).
 91 | #
 92 | # Nodes:     10
 93 | # Speed:     1-4 GFLOPS
 94 | # Bandwidth: 1e12 MBps
 95 | # Latency:   0 us
 96 | #
 97 | if [ ! -d "$SCRIPT_DIR/exp2_systems_inf" ]; then
 98 |   python3 -m pysimgrid.tools.plat_gen "$SCRIPT_DIR/exp2_systems_inf" 1 $SEED cluster 10 1-4 1e12 0 \
 99 |         --loopback_bandwidth=1e12 --loopback_latency=0 --include_master
100 | fi
101 | 
102 | popd
103 | 


--------------------------------------------------------------------------------
/bot/gen/bot_gen.py:
--------------------------------------------------------------------------------
  1 | import argparse
  2 | import os
  3 | import random
  4 | import sys
  5 | 
  6 | 
  7 | def generate_tasks(num_tasks, input_size, comp_size, output_size):
  8 |     tasks = []
  9 |     for i in xrange(0, num_tasks):
 10 |         task = {
 11 |             'name': "task%d" % i
 12 |         }
 13 |         tasks.append(task)
 14 | 
 15 |     # populate task input sizes
 16 |     input_sizes = generate_values(input_size, num_tasks)
 17 |     for i in xrange(0, num_tasks):
 18 |         tasks[i]['input_size'] = input_sizes[i]
 19 | 
 20 |     # populate task computational sizes
 21 |     comp_sizes = generate_values(comp_size, num_tasks, input_sizes)
 22 |     for i in xrange(0, num_tasks):
 23 |         tasks[i]['comp_size'] = comp_sizes[i]
 24 | 
 25 |     # populate task output sizes
 26 |     output_sizes = generate_values(output_size, num_tasks, input_sizes)
 27 |     for i in xrange(0, num_tasks):
 28 |         tasks[i]['output_size'] = output_sizes[i]
 29 | 
 30 |     return tasks
 31 | 
 32 | 
 33 | def generate_values(spec, num, inputs=None):
 34 |     try:
 35 |         # fixed value
 36 |         fixed = float(spec)
 37 |         values = [fixed] * num
 38 | 
 39 |     except ValueError:
 40 |         parts = spec.split(':')
 41 |         type = parts[0]
 42 |         params = parts[1:]
 43 | 
 44 |         # uniform distribution: u:min:max
 45 |         if type == "u":
 46 |             min = float(params[0])
 47 |             max = float(params[1])
 48 |             values = [random.uniform(min, max) for _ in xrange(0, num)]
 49 | 
 50 |         # normal distribution: n:mean:std_dev
 51 |         elif type == "n":
 52 |             mean = float(params[0])
 53 |             std_dev = float(params[1])
 54 |             values = [random.normalvariate(mean, std_dev) for _ in xrange(0, num)]
 55 | 
 56 |         # scaled values: x:factor
 57 |         elif type == "x":
 58 |             factor = float(params[0])
 59 |             if inputs is not None:
 60 |                 values = [inputs[i] * factor for i in xrange(0, num)]
 61 |             else:
 62 |                 print "Inputs are not specified"
 63 |                 sys.exit(-1)
 64 | 
 65 |         else:
 66 |             print "Unknown distribution"
 67 |             sys.exit(-1)
 68 | 
 69 |     return values
 70 | 
 71 | 
 72 | def save_as_dot_file(tasks, output_path):
 73 |     with open(output_path, "w") as f:
 74 |         f.write("digraph G {\n")
 75 |         for task in tasks:
 76 |             f.write('  root -> %s [size="%e"]\n' % (task['name'], task['input_size']))
 77 |             f.write('  %s [size="%e"]\n' % (task['name'], task['comp_size']))
 78 |             f.write('  %s -> end [size="%e"]\n' % (task['name'], task['output_size']))
 79 |         f.write("}\n")
 80 | 
 81 | 
 82 | def main():
 83 |     parser = argparse.ArgumentParser(description="Generator of synthetic graphs for bag-of-tasks applications")
 84 |     parser.add_argument("num_tasks", type=int, help="number of tasks")
 85 |     parser.add_argument("input_size", type=str, help="task input size in bytes")
 86 |     parser.add_argument("comp_size", type=str, help="task computational size in flops")
 87 |     parser.add_argument("output_size", type=str, help="task output size in bytes")
 88 |     parser.add_argument("output_dir", type=str, help="output directory")
 89 |     parser.add_argument("num_graphs", type=int, help="number of generated graphs")
 90 |     args = parser.parse_args()
 91 | 
 92 |     os.mkdir(args.output_dir)
 93 | 
 94 |     for i in xrange(0, args.num_graphs):
 95 |         tasks = generate_tasks(args.num_tasks, args.input_size, args.comp_size, args.output_size)
 96 |         file_name = 'bot_%d_%s_%s_%s_%d.dot' % (args.num_tasks, args.input_size, args.comp_size, args.output_size, i)
 97 |         file_path = args.output_dir + '/' + file_name
 98 |         save_as_dot_file(tasks, file_path)
 99 |         print('Generated file: %s' % file_path)
100 | 
101 |     return 0
102 | 
103 | 
104 | if __name__ == '__main__':
105 |     main()


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | pysimgrid
  2 | =========
  3 | 
  4 | The main goal of the project is to create an easy way to implement and benchmark
  5 | scheduling algorithms while leveraging the powerful [SimGrid](http://simgrid.gforge.inria.fr) simulation framework.
  6 | 
  7 | #### Features:
  8 | 
  9 | * Simulate different scheduling distributed platforms
 10 | * Implement your own scheduling algorithms
 11 | * Compare them with well-known approaches (e.g. HEFT). Batteries included )
 12 | * Run your simulations in parallel
 13 | 
 14 | #### Documentation:
 15 | 
 16 | https://alexmnazarenko.github.io/pysimgrid/index.html
 17 | 
 18 | #### Acknowledgements:
 19 | 
 20 | This work is supported by the Russian Foundation for Basic Research
 21 | (projects 15-29-07043, 15-29-07068).
 22 | 
 23 | Examples
 24 | ========
 25 | 
 26 | Running a simulation:
 27 | 
 28 | ```python
 29 | from pysimgrid import simdag
 30 | import pysimgrid.simdag.algorithms as algorithms
 31 | 
 32 | with simdag.Simulation("test/data/pl_4hosts.xml", "test/data/basic_graph.dot") as simulation:
 33 |   scheduler = algorithms.Lookahead(simulation)
 34 |   scheduler.run()
 35 |   print(simulation.clock, scheduler.scheduler_time, scheduler.total_time)
 36 | ```
 37 | 
 38 | 
 39 | Implementing your very own scheduling algorithm:
 40 | 
 41 | ```python
 42 | from pysimgrid import simdag
 43 | import networkx
 44 | 
 45 | class RandomSchedule(simdag.StaticScheduler):
 46 |   def get_schedule(self, simulation):
 47 |     schedule = {host: [] for host in simulation.hosts}
 48 |     graph = simulation.get_task_graph()
 49 |     for task in networkx.topological_sort(graph):
 50 |       schedule[random.choice(simulation.hosts)].append(task)
 51 |     return schedule
 52 | ```
 53 | 
 54 | 
 55 | Dependencies
 56 | ============
 57 | 
 58 | C++
 59 | 
 60 | * C++11 capable compiler (tested with GCC/G++ 4.8.4)
 61 | * [SimGrid](http://simgrid.gforge.inria.fr/download.php) built with graphviz support (build script provided, tested v3.13)
 62 | * [CMake](https://cmake.org/) (for building SimGrid)
 63 | 
 64 | Python
 65 | 
 66 | * python 2.7 or 3.4+
 67 | * [setuptools](https://pypi.python.org/pypi/setuptools)
 68 | * [Cython](http://cython.org/)
 69 | * [numpy](http://www.numpy.org/)
 70 | * [networkx](https://networkx.github.io/)
 71 | 
 72 | 
 73 | 
 74 | Build instructions
 75 | ==================
 76 | 
 77 | Ubuntu 14.04+
 78 | -------------
 79 | 
 80 | Install system dependencies (list is not full):
 81 | 
 82 | ```bash
 83 | sudo apt-get install libboost-context-dev libboost-program-options-dev libboost-filesystem-dev doxygen graphviz-dev
 84 | ```
 85 | 
 86 | Use provided scripts to get dependencies:
 87 | 
 88 | ```bash
 89 | ./get_simgrid.sh
 90 | ```
 91 | 
 92 | Installation:
 93 | 
 94 | ```bash
 95 | python3 setup.py install --user
 96 | ```
 97 | 
 98 | Development build
 99 | -----------------
100 | 
101 | Inplace build
102 | 
103 | ```bash
104 | python3 setup.py build_ext --inplace
105 | ```
106 | 
107 | Test the build:
108 | 
109 | ```bash
110 | python3 run_tests.py
111 | ```
112 | 
113 | FAQ
114 | ===
115 | 
116 | #### 1. Where to get platform definition files? They look scary.
117 | 
118 | SimGrid source distribution contains quite a few of platform examples of different complexity and scale.
119 | 
120 | 
121 | #### 2. Where to get some ready DAGs?
122 | 
123 | * Pegasus workflows, DAX format  
124 |     * See [Pegasus Workfow Generator](https://confluence.pegasus.isi.edu/display/pegasus/WorkflowGenerator)
125 |     * Direct download (warning: ~400Mb) [[link](https://download.pegasus.isi.edu/misc/SyntheticWorkflows.tar.gz)]
126 | 
127 | 
128 | #### 3. What about multi-core tasks?
129 | 
130 | They are not supported for now. Two main reasons:
131 | 
132 | * Existing SimGrid task parsers do not support them, so you need custom task format to even setup this.
133 | * There are some [discouraging words](http://simgrid.gforge.inria.fr/simgrid/latest/doc/platform.html#pf_Cr)
134 | in SimGrid documentation about validity of such simulation.
135 | * API seems to be "in progress"
136 | 


--------------------------------------------------------------------------------
/pysimgrid/simdag/algorithms/mct_dyn.py:
--------------------------------------------------------------------------------
 1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
 2 | #
 3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
 4 | #
 5 | # This library is free software: you can redistribute it and/or modify
 6 | # it under the terms of the GNU Lesser General Public License as published by
 7 | # the Free Software Foundation, either version 3 of the License, or
 8 | # (at your option) any later version.
 9 | #
10 | # This library is distributed in the hope that it will be useful,
11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | # GNU Lesser General Public License for more details.
14 | #
15 | # You should have received a copy of the GNU Lesser General Public
16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
17 | #
18 | 
19 | import logging
20 | 
21 | from .. import scheduler
22 | from ... import csimdag
23 | 
24 | 
25 | class DynamicMCT(scheduler.DynamicScheduler):
26 |     """
27 |     Minimum completion time scheduler.
28 | 
29 |     Ready tasks are scheduled on host (probably busy one) that promises the earliest task completion.
30 | 
31 |     Task completion time is estimated as:
32 | 
33 |     ECT(task, host) = max(current_clock, estimated_host_ready_time) + max(comm_time_from_parents) + EET(task, host)
34 |     """
35 | 
36 |     def prepare(self, simulation):
37 |         for h in simulation.hosts:
38 |             h.data = {
39 |                 "est": 0.
40 |             }
41 |         master_hosts = simulation.hosts.by_prop("name", self.MASTER_HOST_NAME)
42 |         self._master_host = master_hosts[0] if master_hosts else None
43 |         if self._master_host:
44 |             for task in simulation.tasks.by_func(lambda t: t.name in self.BOUNDARY_TASKS):
45 |                 task.schedule(self._master_host)
46 |         self._exec_hosts = simulation.hosts.by_prop("name", self.MASTER_HOST_NAME, True)
47 |         self._started_tasks = set()
48 |         self._estimate_cache = {}
49 | 
50 |     def schedule(self, simulation, changed):
51 |         clock = simulation.clock
52 |         free_hosts = set(self._exec_hosts)
53 |         for task in simulation.tasks[csimdag.TaskState.TASK_STATE_RUNNING, csimdag.TaskState.TASK_STATE_SCHEDULED]:
54 |             host = task.hosts[0]
55 |             free_hosts.discard(host)
56 |             if task.start_time > 0 and task not in self._started_tasks:
57 |                 self._started_tasks.add(task)
58 |                 host.data["est"] = task.start_time + task.get_eet(host)
59 |         host_est = {}
60 |         for h in self._exec_hosts:
61 |             host_est[h] = h.data["est"]
62 |         for task in simulation.tasks[csimdag.TaskState.TASK_STATE_SCHEDULABLE]:
63 |             sorted_hosts = self._exec_hosts.sorted(lambda h: self.get_ect(host_est[h], clock, task, h))
64 |             target_host = sorted_hosts[0]
65 |             host_est[target_host] = self.get_ect(host_est[target_host], clock, task, target_host)
66 |             if target_host in free_hosts:
67 |                 task.schedule(target_host)
68 |                 # logging.info("%s -> %s" % (task.name, target_host.name))
69 |                 target_host.data["est"] = self.get_ect(target_host.data["est"], clock, task, target_host)
70 |                 free_hosts.remove(target_host)
71 |                 if len(free_hosts) == 0:
72 |                     break
73 | 
74 |     def get_ect(self, est, clock, task, host):
75 |         if (task, host) in self._estimate_cache:
76 |             task_time = self._estimate_cache[(task, host)]
77 |         else:
78 |             parent_connections = [p for p in task.parents if p.kind == csimdag.TaskKind.TASK_KIND_COMM_E2E]
79 |             comm_times = [conn.get_ecomt(conn.parents[0].hosts[0], host) for conn in parent_connections]
80 |             task_time = (max(comm_times) if comm_times else 0.) + task.get_eet(host)
81 |             self._estimate_cache[(task, host)] = task_time
82 |         return max(est, clock) + task_time
83 | 


--------------------------------------------------------------------------------
/doc/source/_themes/sphinx_rtd_theme/breadcrumbs.html:
--------------------------------------------------------------------------------
 1 | {# Support for Sphinx 1.3+ page_source_suffix, but don't break old builds. #}
 2 | 
 3 | {% if page_source_suffix %}
 4 | {% set suffix = page_source_suffix %}
 5 | {% else %}
 6 | {% set suffix = source_suffix %}
 7 | {% endif %}
 8 | 
 9 | {% if meta is defined and meta is not none %}
10 | {% set check_meta = True %}
11 | {% else %}
12 | {% set check_meta = False %}
13 | {% endif %}
14 | 
15 | {% if check_meta and 'github_url' in meta %}
16 | {% set display_github = True %}
17 | {% endif %}
18 | 
19 | {% if check_meta and 'bitbucket_url' in meta %}
20 | {% set display_bitbucket = True %}
21 | {% endif %}
22 | 
23 | {% if check_meta and 'gitlab_url' in meta %}
24 | {% set display_gitlab = True %}
25 | {% endif %}
26 | 
27 | <div role="navigation" aria-label="breadcrumbs navigation">
28 | 
29 |   <ul class="wy-breadcrumbs">
30 |     {% block breadcrumbs %}
31 |       <li><a href="{{ pathto(master_doc) }}">{{ _('Docs') }}</a> &raquo;</li>
32 |         {% for doc in parents %}
33 |           <li><a href="{{ doc.link|e }}">{{ doc.title }}</a> &raquo;</li>
34 |         {% endfor %}
35 |       <li>{{ title }}</li>
36 |     {% endblock %}
37 |     {% block breadcrumbs_aside %}
38 |       <li class="wy-breadcrumbs-aside">
39 |         {% if pagename != "search" %}
40 |             {% if display_github %}
41 |             {% if check_meta and 'github_url' in meta %}
42 |               <!-- User defined GitHub URL -->
43 |               <a href="{{ meta['github_url'] }}" class="fa fa-github"> {{ _('Edit on GitHub') }}</a>
44 |             {% else %}
45 |               <a href="https://{{ github_host|default("github.com") }}/{{ github_user }}/{{ github_repo }}/blob/{{ github_version }}{{ conf_py_path }}{{ pagename }}{{ suffix }}" class="fa fa-github"> {{ _('Edit on GitHub') }}</a>
46 |             {% endif %}
47 |           {% elif display_bitbucket %}
48 |             {% if check_meta and 'bitbucket_url' in meta %}
49 |               <!-- User defined Bitbucket URL -->
50 |               <a href="{{ meta['bitbucket_url'] }}" class="fa fa-bitbucket"> {{ _('Edit on Bitbucket') }}</a>
51 |             {% else %}
52 |               <a href="https://bitbucket.org/{{ bitbucket_user }}/{{ bitbucket_repo }}/src/{{ bitbucket_version}}{{ conf_py_path }}{{ pagename }}{{ suffix }}" class="fa fa-bitbucket"> {{ _('Edit on Bitbucket') }}</a>
53 |             {% endif %}
54 |           {% elif display_gitlab %}
55 |             {% if check_meta and 'gitlab_url' in meta %}
56 |               <!-- User defined GitLab URL -->
57 |               <a href="{{ meta['gitlab_url'] }}" class="fa fa-gitlab"> {{ _('Edit on GitLab') }}</a>
58 |             {% else %}
59 |               <a href="https://{{ gitlab_host|default("gitlab.com") }}/{{ gitlab_user }}/{{ gitlab_repo }}/blob/{{ gitlab_version }}{{ conf_py_path }}{{ pagename }}{{ suffix }}" class="fa fa-gitlab"> {{ _('Edit on GitLab') }}</a>
60 |             {% endif %}
61 |           {% elif show_source and source_url_prefix %}
62 |             <a href="{{ source_url_prefix }}{{ pagename }}{{ suffix }}">{{ _('View page source') }}</a>
63 |           {% elif show_source and has_source and sourcename %}
64 |             <a href="{{ pathto('_sources/' + sourcename, true)|e }}" rel="nofollow"> {{ _('View page source') }}</a>
65 |           {% endif %}
66 |         {% endif %}
67 |       </li>
68 |     {% endblock %}
69 |   </ul>
70 | 
71 |   {% if (theme_prev_next_buttons_location == 'top' or theme_prev_next_buttons_location == 'both') and (next or prev) %}
72 |   <div class="rst-breadcrumbs-buttons" role="navigation" aria-label="breadcrumb navigation">
73 |       {% if next %}
74 |         <a href="{{ next.link|e }}" class="btn btn-neutral float-right" title="{{ next.title|striptags|e }}" accesskey="n">Next <span class="fa fa-arrow-circle-right"></span></a>
75 |       {% endif %}
76 |       {% if prev %}
77 |         <a href="{{ prev.link|e }}" class="btn btn-neutral" title="{{ prev.title|striptags|e }}" accesskey="p"><span class="fa fa-arrow-circle-left"></span> Previous</a>
78 |       {% endif %}
79 |   </div>
80 |   {% endif %}
81 |   <hr/>
82 | </div>
83 | 


--------------------------------------------------------------------------------
/pysimgrid/simdag/algorithms/lookahead.py:
--------------------------------------------------------------------------------
 1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
 2 | #
 3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
 4 | #
 5 | # This library is free software: you can redistribute it and/or modify
 6 | # it under the terms of the GNU Lesser General Public License as published by
 7 | # the Free Software Foundation, either version 3 of the License, or
 8 | # (at your option) any later version.
 9 | #
10 | # This library is distributed in the hope that it will be useful,
11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 | # GNU Lesser General Public License for more details.
14 | #
15 | # You should have received a copy of the GNU Lesser General Public
16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
17 | #
18 | 
19 | from ... import cscheduling
20 | from .. import scheduler
21 | from . import heft
22 | 
23 | 
24 | class Lookahead(scheduler.StaticScheduler):
25 |   """
26 |   Lookahead scheduler.
27 | 
28 |   Lookahead aims to be an improved version of HEFT, avoiding some of its greedy scheduling decisions.
29 | 
30 |   Some quick experiments show that it can indeed improve on HEFT schedules, but it comes with a price:
31 |   lookahead scheduler is much slower (as it runs HEFT to an end for each task/host pair).
32 | 
33 |   Main idea is simple:
34 | 
35 |     1. Consider tasks in a HEFT (ranku) order
36 |     2. For each next task and for all hosts:
37 | 
38 |       * schedule task on a host
39 |       * complete schedule using HEFT
40 |       * evaluate makespan
41 | 
42 |     3. Select a host achieving a best makespan
43 | 
44 |   Note: authors actually propose 4 variants of the algorithm and as for now only the first one is implemented.
45 | 
46 |   For more details please refer to the original publication:
47 | 
48 |     L. F. Bittencourt, R. Sakellariou and E. R. M. Madeira, "DAG Scheduling Using a Lookahead
49 |     Variant of the Heterogeneous Earliest Finish Time Algorithm", 18th Euromicro
50 |     Conference on Parallel, Distributed and Network-based Processing, 2010, pp. 27-34
51 |   """
52 |   def get_schedule(self, simulation):
53 |     """
54 |     Overriden.
55 |     """
56 |     nxgraph = simulation.get_task_graph()
57 |     platform_model = cscheduling.PlatformModel(simulation)
58 |     state = cscheduling.SchedulerState(simulation)
59 | 
60 |     ordered_tasks = cscheduling.heft_order(nxgraph, platform_model)
61 |     for idx, task in enumerate(ordered_tasks):
62 |       if cscheduling.try_schedule_boundary_task(task, nxgraph, platform_model, state):
63 |         continue
64 |       current_min = cscheduling.MinSelector()
65 |       for host, timesheet in state.timetable.items():
66 |         if cscheduling.is_master_host(host):
67 |           continue
68 |         temp_state = state.copy()
69 |         est = platform_model.est(host, dict(nxgraph.pred[task]), state)
70 |         eet = platform_model.eet(task, host)
71 |         pos, start, finish = cscheduling.timesheet_insertion(timesheet, est, eet)
72 |         temp_state.update(task, host, pos, start, finish)
73 |         cscheduling.heft_schedule(nxgraph, platform_model, temp_state, ordered_tasks[(idx + 1):],
74 |                                   self._data_transfer_mode.name)
75 |         total_time = max([state["ect"] for state in temp_state.task_states.values()])
76 |         # key order to ensure stable sorting:
77 |         #  first sort by HEFT makespan (as Lookahead requires)
78 |         #  if equal - sort by host speed
79 |         #  if equal - sort by host name (guaranteed to be unique)
80 |         current_min.update((total_time, host.speed, host.name), (host, pos, start, finish))
81 |       host, pos, start, finish = current_min.value
82 |       state.update(task, host, pos, start, finish)
83 | 
84 |     # store ECT in tasks for QUEUE_ECT data transfer mode
85 |     for task, task_state in state.task_states.items():
86 |       task.data = {"ect": task_state["ect"]}
87 |     expected_makespan = max([state["ect"] for state in state.task_states.values()])
88 |     return state.schedule, expected_makespan


--------------------------------------------------------------------------------
/basic_test.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Temporary development test for quick-hacking.
  3 | 
  4 | Probably should be deleted on 'final release'.
  5 | It's pretty illustrative, however.
  6 | 
  7 | For bettter examples on C API wrappers look at test/test_capi.py.
  8 | """
  9 | 
 10 | from __future__ import print_function
 11 | 
 12 | import random
 13 | import logging
 14 | import multiprocessing
 15 | 
 16 | import networkx
 17 | 
 18 | from pysimgrid import simdag
 19 | import pysimgrid.simdag.algorithms as algorithms
 20 | 
 21 | _LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
 22 | _LOG_FORMAT = "[%(name)s] [%(levelname)5s] [%(asctime)s] %(message)s"
 23 | logging.basicConfig(level=logging.DEBUG, format=_LOG_FORMAT, datefmt=_LOG_DATE_FORMAT)
 24 | 
 25 | class RandomSchedule(simdag.StaticScheduler):
 26 |   def get_schedule(self, simulation):
 27 |     schedule = {host: [] for host in simulation.hosts}
 28 |     graph = simulation.get_task_graph()
 29 |     for task in networkx.topological_sort(graph):
 30 |       schedule[random.choice(simulation.hosts)].append(task)
 31 |     return schedule
 32 | 
 33 | 
 34 | class SimpleDynamic(simdag.DynamicScheduler):
 35 |   def prepare(self, simulation):
 36 |     for h in simulation.hosts:
 37 |       h.data = {}
 38 | 
 39 |   def schedule(self, simulation, changed):
 40 |     for h in simulation.hosts:
 41 |       h.data["free"] = True
 42 |     for task in simulation.tasks[simdag.TaskState.TASK_STATE_RUNNING, simdag.TaskState.TASK_STATE_SCHEDULED]:
 43 |       task.hosts[0].data["free"] = False
 44 |     for t in simulation.tasks[simdag.TaskState.TASK_STATE_SCHEDULABLE]:
 45 |       free_hosts = simulation.hosts.by_data("free", True).sorted(lambda h: t.get_eet(h))
 46 |       if free_hosts:
 47 |         t.schedule(free_hosts[0])
 48 |         free_hosts[0].data["free"] = False
 49 |       else:
 50 |         break
 51 | 
 52 | 
 53 | _SCHEDULERS = {
 54 |   "MinMin": algorithms.BatchMin,
 55 |   "MaxMin": algorithms.BatchMax,
 56 |   "Sufferage": algorithms.BatchSufferage,
 57 |   "DLS": algorithms.DLS,
 58 |   "RandomSchedule": algorithms.RandomStatic,
 59 |   "SimpleDynamic": SimpleDynamic,
 60 |   "MCT": algorithms.MCT,
 61 |   "OLB": algorithms.OLB,
 62 |   "HCPT": algorithms.HCPT,
 63 |   "HEFT": algorithms.HEFT,
 64 |   "Lookahead": algorithms.Lookahead,
 65 |   "PEFT": algorithms.PEFT
 66 | }
 67 | 
 68 | 
 69 | def run_simulation(scheduler):
 70 |   scheduler_class = _SCHEDULERS[scheduler]
 71 |   with simdag.Simulation("test/data/pl_4hosts_master.xml", "dag/tasks_exp2/testg0.6.dot") as simulation:
 72 |     print("Scheduler:", scheduler, scheduler_class)
 73 |     scheduler = scheduler_class(simulation)
 74 |     scheduler.run()
 75 |     print("Scheduler time:", scheduler.scheduler_time)
 76 | 
 77 | 
 78 | def main():
 79 |   # single run in current process mode, used for profiling
 80 |   if True:
 81 |     #with simdag.Simulation("test/data/pl_4hosts.xml", "test/data/basic_graph.dot") as simulation:
 82 |     with simdag.Simulation("dag/plat_exp1/cluster_5_1-4_100_100_0.xml", "dag/tasks_exp1/Inspiral_100.xml") as simulation:
 83 |     #with simdag.Simulation("dag/plat_exp1/cluster_20_1-4_100_100_0.xml", "dag/tasks_exp2/testg0.6.dot") as simulation:
 84 |       #graph = simulation.get_task_graph()
 85 |       #scheduler = algorithms.HEFT(simulation)
 86 |       #scheduler = algorithms.DLS(simulation)
 87 |       #scheduler = algorithms.HCPT(simulation)
 88 |       scheduler = algorithms.Lookahead(simulation)
 89 |       #scheduler = algorithms.OLB(simulation)
 90 |       #scheduler = algorithms.BatchMin(simulation)
 91 |       #scheduler = algorithms.PEFT(simulation)
 92 |       #scheduler = algorithms.SimHEFT(simulation)
 93 |       scheduler.run()
 94 |       for t in simulation.tasks.sorted(lambda t: t.start_time):
 95 |         print(t.name, t.start_time, t.finish_time, t.hosts[0].name)
 96 |       print(scheduler.scheduler_time, scheduler.total_time)
 97 |       print("EXEC", sum([(t.finish_time - t.start_time) for t in simulation.tasks]))
 98 |       print("COMM", sum([(t.finish_time - t.start_time) for t in simulation.connections]))
 99 |     return
100 |   # example: how to run multiple simulations in a single script (circumventing SimGrid limitation of 'non-restartable' simulator state)
101 |   for scheduler in _SCHEDULERS.keys():
102 |     p = multiprocessing.Process(target=run_simulation, args=(scheduler,))
103 |     p.start()
104 |     p.join()
105 | 
106 | 
107 | if __name__ == '__main__':
108 |   main()
109 | 


--------------------------------------------------------------------------------
/pysimgrid/simdag/algorithms/peft.py:
--------------------------------------------------------------------------------
  1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
  2 | #
  3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
  4 | #
  5 | # This library is free software: you can redistribute it and/or modify
  6 | # it under the terms of the GNU Lesser General Public License as published by
  7 | # the Free Software Foundation, either version 3 of the License, or
  8 | # (at your option) any later version.
  9 | #
 10 | # This library is distributed in the hope that it will be useful,
 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13 | # GNU Lesser General Public License for more details.
 14 | #
 15 | # You should have received a copy of the GNU Lesser General Public
 16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
 17 | #
 18 | 
 19 | import networkx
 20 | import numpy
 21 | 
 22 | from ... import cscheduling
 23 | from ..scheduler import StaticScheduler
 24 | 
 25 | 
 26 | class PEFT(StaticScheduler):
 27 |   """
 28 |   Predicted Earliest Finish Time (PEFT) scheduler.
 29 | 
 30 |   PEFT aims to get advantages of Lookahead approach while avoiding the high computational complexity.
 31 | 
 32 |   Instead of actual lookahead scheduling, PEFT pre-computes an Optimistic Cost Table (OCT),
 33 |   each element of which contains time to finish all the task descendants on their best hosts,
 34 |   disregarding the host availability.
 35 | 
 36 |   For more details and rationale please refer to the original publication:
 37 | 
 38 |     H. Arabnejad and J. G. Barbosa, "List Scheduling Algorithm for Heterogeneous Systems by
 39 |     an Optimistic Cost Table", IEEE Transactions on Parallel and Distributed Systems,
 40 |     Vol 25, No 3, 2014, pp. 682-694
 41 |   """
 42 |   def get_schedule(self, simulation):
 43 |     """
 44 |     Overriden.
 45 |     """
 46 |     nxgraph = simulation.get_task_graph()
 47 |     platform_model = cscheduling.PlatformModel(simulation)
 48 |     state = cscheduling.SchedulerState(simulation)
 49 | 
 50 |     oct_dict = self.oct_dict(nxgraph, platform_model)
 51 |     oct_rank = {task: by_host.mean() for (task, by_host) in oct_dict.items()}
 52 | 
 53 |     ordered_tasks = cscheduling.schedulable_order(nxgraph, oct_rank)
 54 | 
 55 |     for task in ordered_tasks:
 56 |       if cscheduling.try_schedule_boundary_task(task, nxgraph, platform_model, state):
 57 |         continue
 58 |       current_min = cscheduling.MinSelector()
 59 |       for host, timesheet in state.timetable.items():
 60 |         if cscheduling.is_master_host(host):
 61 |           continue
 62 |         est = platform_model.est(host, dict(nxgraph.pred[task]), state)
 63 |         eet = platform_model.eet(task, host)
 64 |         pos, start, finish = cscheduling.timesheet_insertion(timesheet, est, eet)
 65 |         # key order to ensure stable sorting:
 66 |         #  first sort by ECT + OCT (as PEFT requires)
 67 |         #  if equal - sort by host speed
 68 |         #  if equal - sort by host name (guaranteed to be unique)
 69 |         current_min.update((finish + oct_dict[task][platform_model.host_idx(host)], host.speed, host.name), (host, pos, start, finish))
 70 |       host, pos, start, finish = current_min.value
 71 |       state.update(task, host, pos, start, finish)
 72 |     expected_makespan = max([state["ect"] for state in state.task_states.values()])
 73 |     return state.schedule, expected_makespan
 74 | 
 75 |   @classmethod
 76 |   def oct_dict(cls, nxgraph, platform_model):
 77 |     """
 78 |     Build optimistic cost table as an dict task->array.
 79 | 
 80 |     Args:
 81 |       nxgraph: networkx representation of task graph
 82 |       platform_model: platform linear model
 83 |     """
 84 |     result = dict()
 85 |     for task in list(reversed(list(networkx.topological_sort(nxgraph)))):
 86 |       oct_row = numpy.zeros(platform_model.host_count)
 87 |       if not nxgraph[task]:
 88 |         result[task] = oct_row
 89 |         continue
 90 |       for host, idx in platform_model.host_map.items():
 91 |         child_results = []
 92 |         for child, edge_dict in nxgraph[task].items():
 93 |           row = result[child].copy()
 94 |           row += child.amount / platform_model.speed
 95 |           comm_cost = numpy.ones(platform_model.host_count) * edge_dict["weight"] / platform_model.mean_bandwidth
 96 |           comm_cost += platform_model.mean_latency
 97 |           comm_cost[idx] = 0
 98 |           row += comm_cost
 99 |           child_results.append(row.min())
100 |         oct_row[idx] = max(child_results)
101 |       result[task] = oct_row
102 |     return result
103 | 


--------------------------------------------------------------------------------
/run_tests.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Very quick&dirty solution to discover tests and then run them in separate processes.
  3 | Essentially, it emulates CMake/CTest approach.
  4 | 
  5 | Typical 'pythonic' approaches doesn't work well:
  6 | * nosetests still tries to execute multiple test cases in a single process
  7 | * multiprocessing is unable to intercept/redirect native stdout from C library
  8 | 
  9 | Zero flexibility, runs everything on a working copy for now.
 10 | Can be vastly improved if necessary.
 11 | """
 12 | 
 13 | from __future__ import print_function
 14 | 
 15 | import os
 16 | import sys
 17 | import re
 18 | import unittest
 19 | import subprocess
 20 | import time
 21 | import argparse
 22 | 
 23 | DEFAULT_TIMEOUT = 10
 24 | POLL_INTERVAL = 0.01
 25 | PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__))
 26 | TESTS_PACKAGE = "test"
 27 | TESTS_ROOT = os.path.join(PROJECT_ROOT, TESTS_PACKAGE)
 28 | TEST_MODULE_REGEX = re.compile("^test.*\.py$")
 29 | 
 30 | if PROJECT_ROOT not in sys.path:
 31 |   sys.path.insert(0, PROJECT_ROOT)
 32 | 
 33 | def collect_tests():
 34 |   tests = []
 35 |   for root, _, files in os.walk(TESTS_ROOT):
 36 |     for f in files:
 37 |       if TEST_MODULE_REGEX.match(f):
 38 |         test_file = os.path.join(root, f)
 39 |         module_name, _ = os.path.splitext(os.path.relpath(test_file, PROJECT_ROOT))
 40 |         module_name = module_name.replace("/", ".")
 41 |         module = __import__(module_name)
 42 |         for namepart in module_name.split(".")[1:]:
 43 |           module = getattr(module, namepart)
 44 |         for test_class in unittest.findTestCases(module)._tests:
 45 |           for test_function in test_class._tests:
 46 |             test_full_name = ".".join([module_name, type(test_function).__name__, test_function._testMethodName])
 47 |             tests.append((test_file, module_name, test_full_name))
 48 |   return tests
 49 | 
 50 | def run_tests(test_list, show_output):
 51 |   any_failed = False
 52 |   reports = []
 53 |   for filename, module, test in test_list:
 54 |     print("* ", "Starting", test, "...")
 55 |     # Python 2.7 compatibility kludges:
 56 |     #  * use open(os.devnull,...) instead of subprocess.DEVNULL
 57 |     #  * use POpen.poll
 58 |     with open(os.devnull, "w") as null_output:
 59 |       if show_output:
 60 |         output_config = {channel: None for channel in ["stdout", "stderr"]}
 61 |       else:
 62 |         output_config = {channel: null_output for channel in ["stdout", "stderr"]}
 63 |       start_time = time.time()
 64 |       process = subprocess.Popen([sys.executable, "-B", "-m", "unittest", test], cwd=PROJECT_ROOT, **output_config)
 65 |       while (time.time() - start_time) < DEFAULT_TIMEOUT:
 66 |         retcode = process.poll()
 67 |         if retcode is not None:
 68 |           break
 69 |       retcode = process.poll()
 70 |       test_failed = False
 71 |       execution_time_str = "{:.2f}s".format(time.time() - start_time)
 72 |       if retcode is None:
 73 |         process.kill()
 74 |         test_failed = True
 75 |         execution_time_str = "TIMEOUT ({} seconds)".format(DEFAULT_TIMEOUT)
 76 |       else:
 77 |         test_failed = retcode != 0
 78 |     report = " ".join(["  ", "FAILED" if test_failed else "PASSED", execution_time_str, test])
 79 |     reports.append(report)
 80 |     if not show_output:
 81 |       print(report)
 82 |     any_failed = any_failed or test_failed
 83 |   if show_output:
 84 |     print()
 85 |     print("=" * 30)
 86 |     print("Summary:")
 87 |     for report in reports:
 88 |       print(report)
 89 |   if any_failed:
 90 |     print("Some tests failed, exiting with non-zero code")
 91 |   else:
 92 |     print("All tests passed!")
 93 |   return int(any_failed)
 94 | 
 95 | def main():
 96 |   parser = argparse.ArgumentParser()
 97 |   parser.add_argument("-N", "--collect-only", action="store_true", default=False, help="don't run tests, only collect the list")
 98 |   parser.add_argument("-V", "--verbose", action="store_true", default=False, help="show verbose output")
 99 |   parser.add_argument("tests_to_run", nargs="*", help="optional: run tests by (any of passed) regex")
100 |   args = parser.parse_args()
101 |   test_list = collect_tests()
102 |   if args.tests_to_run:
103 |     filtered = []
104 |     regexes = [re.compile(expr) for expr in args.tests_to_run]
105 |     for test_data in test_list:
106 |       _, _, test = test_data
107 |       if any([r.search(test) for r in regexes]):
108 |         filtered.append(test_data)
109 |     test_list = filtered
110 |   if args.collect_only:
111 |     print("Tests found:")
112 |     for filename, module, test in test_list:
113 |       print("  ", test)
114 |     return 0
115 |   return run_tests(test_list, args.verbose)
116 | 
117 | if __name__ == '__main__':
118 |   sys.exit(main())
119 | 


--------------------------------------------------------------------------------
/test/test_capi.py:
--------------------------------------------------------------------------------
  1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
  2 | #
  3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
  4 | #
  5 | # License:  Standard 3-clause BSD; see "license.txt" for full license terms
  6 | #           and contributor agreement.
  7 | 
  8 | import os
  9 | import unittest
 10 | import random
 11 | import pysimgrid
 12 | # PY 2/3 compatibility tools
 13 | from pysimgrid.six import print_
 14 | 
 15 | DATA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
 16 | def _data_path(*relpath):
 17 |   return os.path.join(DATA_PATH, *relpath)
 18 | 
 19 | class TestSimDAG(unittest.TestCase):
 20 |   def setUp(self):
 21 |     """
 22 |     Initialize SimDAG library.
 23 |     """
 24 |     pysimgrid.csimdag.initialize()
 25 |     pysimgrid.csimdag.config("network/model", "LV08")
 26 | 
 27 |   def tearDown(self):
 28 |     """
 29 |     Initialize SimDAG library.
 30 |     """
 31 |     pysimgrid.csimdag.exit()
 32 | 
 33 |   def test_hosts(self):
 34 |     """
 35 |     Test platfrom C API - host/route properties.
 36 |     """
 37 |     hosts = pysimgrid.csimdag.load_platform(_data_path("pl_4hosts.xml"))
 38 |     static_hosts = pysimgrid.cplatform.get_hosts()
 39 |     self.assertSequenceEqual([h.native for h in hosts], [h.native for h in static_hosts], "host lists from load and current platform state must match")
 40 |     self.assertEqual(len(hosts), 4)
 41 |     for h in hosts:
 42 |       self.assertIsInstance(h, pysimgrid.cplatform.Host)
 43 |       self.assertNotEqual(h.native, 0)
 44 |       self.assertEqual(h.available_speed, 1.)
 45 |       self.assertEqual(h.speed, 2e9)
 46 |       self.assertEqual(h.native, pysimgrid.cplatform.host_by_name(h.name).native)
 47 |     with self.assertRaises(Exception):
 48 |       pysimgrid.cplatform.host_by_name("nicht exist")
 49 |     for h1 in hosts:
 50 |       for h2 in hosts:
 51 |         self.assertNotEqual(pysimgrid.cplatform.route(h1, h2), [])
 52 |         if h1 != h2:
 53 |           self.assertEqual(pysimgrid.cplatform.route_bandwidth(h1, h2), 1e8)
 54 |         else:
 55 |           self.assertEqual(pysimgrid.cplatform.route_bandwidth(h1, h2), 4.98e8)
 56 |         self.assertGreater(pysimgrid.cplatform.route_latency(h1, h2), 0)
 57 |     for h in hosts:
 58 |       h.dump()
 59 | 
 60 |   def test_random_schedule(self):
 61 |     """
 62 |     Test static random scheduling.
 63 |     """
 64 |     hosts = pysimgrid.csimdag.load_platform(_data_path("pl_4hosts.xml"))
 65 |     tasks = pysimgrid.csimdag.load_tasks(_data_path("basic_graph.dot"))
 66 |     print_("Static random scheduling...")
 67 |     for t in tasks:
 68 |       if t.kind == pysimgrid.csimdag.TASK_KIND_COMP_SEQ:
 69 |         t.schedule(random.choice(hosts))
 70 |         print("  Scheduled {} on host {}".format(t.name, t.hosts[0].name))
 71 |     print_("Starting the simulation...")
 72 |     pysimgrid.csimdag.simulate()
 73 |     for t in tasks:
 74 |       self.assertEqual(t.state, pysimgrid.csimdag.TASK_STATE_DONE)
 75 |     print_("Simulation time:", pysimgrid.csimdag.get_clock())
 76 | 
 77 |   def test_primitive_schedule(self):
 78 |     """
 79 |     Test simple dynamic schedule: send ready tasks to empty hosts.
 80 |     """
 81 |     hosts = pysimgrid.csimdag.load_platform(_data_path("pl_4hosts.xml"))
 82 |     tasks = pysimgrid.csimdag.load_tasks(_data_path("basic_graph.dot"))
 83 | 
 84 |     free_hosts = list(hosts)
 85 |     def schedule_all_schedulable():
 86 |       for t in filter(lambda t: t.kind == pysimgrid.csimdag.TASK_KIND_COMP_SEQ and t.state == pysimgrid.csimdag.TASK_STATE_SCHEDULABLE, tasks):
 87 |         if free_hosts:
 88 |           t.schedule(free_hosts.pop())
 89 |           print_("  Scheduled {} on host {}".format(t.name, t.hosts[0].name))
 90 |         else:
 91 |           break
 92 | 
 93 |     print_("Setting up watchpoints...")
 94 |     for t in filter(lambda t: t.kind == pysimgrid.csimdag.TASK_KIND_COMP_SEQ, tasks):
 95 |       t.watch(pysimgrid.csimdag.TASK_STATE_DONE)
 96 | 
 97 |     print_("First scheduling iteration:")
 98 |     schedule_all_schedulable()
 99 |     print_("Starting the simulation...")
100 |     changed = pysimgrid.csimdag.simulate()
101 |     while changed:
102 |       print_("Watchpoint reached, changed tasks:", [t.name for t in changed])
103 |       print_("  Schedulable:", [t.name for t in tasks if t.state == pysimgrid.csimdag.TASK_STATE_SCHEDULABLE])
104 |       print_("  Free hosts:", [h.name for h in hosts])
105 |       schedule_all_schedulable()
106 |       for t in filter(lambda t: t.kind == pysimgrid.csimdag.TASK_KIND_COMP_SEQ and t.state == pysimgrid.csimdag.TASK_STATE_DONE, changed):
107 |         free_hosts.append(t.hosts[0])
108 |       changed = pysimgrid.csimdag.simulate()
109 | 
110 |     print_("\nFinal state:")
111 |     for t in filter(lambda t: t.kind == pysimgrid.csimdag.TASK_KIND_COMP_SEQ, tasks):
112 |       print_(" ", t.name, t.hosts[0].name, t.start_time, t.finish_time)
113 | 
114 |     for t in tasks:
115 |       self.assertEqual(t.state, pysimgrid.csimdag.TASK_STATE_DONE)
116 |     print_("Simulation time:", pysimgrid.csimdag.get_clock())
117 | 
118 | 
119 | if __name__ == '__main__':
120 |   unittest.main()
121 | 


--------------------------------------------------------------------------------
/pysimgrid/cplatform.pyx:
--------------------------------------------------------------------------------
  1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
  2 | #
  3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
  4 | #
  5 | # This library is free software: you can redistribute it and/or modify
  6 | # it under the terms of the GNU Lesser General Public License as published by
  7 | # the Free Software Foundation, either version 3 of the License, or
  8 | # (at your option) any later version.
  9 | #
 10 | # This library is distributed in the hope that it will be useful,
 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13 | # GNU Lesser General Public License for more details.
 14 | #
 15 | # You should have received a copy of the GNU Lesser General Public
 16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
 17 | #
 18 | 
 19 | cimport cplatform
 20 | cimport common
 21 | 
 22 | 
 23 | def host_by_name(name):
 24 |   """
 25 |   Get a host instance by name.
 26 |   """
 27 |   cdef bytes utf8name = common.utf8_string(name)
 28 |   cdef cplatform.sg_host_t host = cplatform.sg_host_by_name(utf8name)
 29 |   if not host:
 30 |     raise Exception(u"no host '{}' found".format(utf8name.decode("utf-8")))
 31 |   return Host.wrap(host)
 32 | 
 33 | 
 34 | def get_hosts():
 35 |   """
 36 |   Get all hosts in a platform as a list.
 37 |   """
 38 |   cdef int hosts_count = cplatform.sg_host_count()
 39 |   if hosts_count:
 40 |     return Host.wrap_batch(cplatform.sg_host_list(), hosts_count)
 41 |   return []
 42 | 
 43 | 
 44 | def route(Host src not None, Host dst not None):
 45 |   """
 46 |   Get list of link names in a route.
 47 |   """
 48 |   if (not src.impl) or (not dst.impl):
 49 |     raise Exception("Cannot build route, one of hosts is invalid")
 50 |   cdef int links_count = cplatform.SD_route_get_size(src.impl, dst.impl)
 51 |   if not links_count:
 52 |     return []
 53 |   cdef cplatform.SD_link_t* links = cplatform.SD_route_get_list(src.impl, dst.impl)
 54 |   cdef list result = []
 55 |   for idx in range(links_count):
 56 |     result.append(cplatform.sg_link_name(links[idx]).decode("utf-8"))
 57 |   return result
 58 | 
 59 | 
 60 | def route_latency(Host src not None, Host dst not None):
 61 |   """
 62 |   Get route total latency.
 63 |   """
 64 |   if (not src.impl) or (not dst.impl):
 65 |     raise Exception("Cannot build route, one of hosts is invalid")
 66 |   return cplatform.SD_route_get_latency(src.impl, dst.impl)
 67 | 
 68 | 
 69 | def route_bandwidth(Host src not None, Host dst not None):
 70 |   """
 71 |   Get route bandwidth.
 72 |   """
 73 |   if (not src.impl) or (not dst.impl):
 74 |     raise Exception("Cannot build route, one of hosts is invalid")
 75 |   return cplatform.SD_route_get_bandwidth(src.impl, dst.impl)
 76 | 
 77 | 
 78 | cdef class Host:
 79 |   """
 80 |   Representation of a platform's host.
 81 | 
 82 |   See declaration in cplatform.pxd.
 83 |   """
 84 | 
 85 |   @staticmethod
 86 |   cdef Host wrap(cplatform.sg_host_t impl):
 87 |     """
 88 |     Wrap a native host handle.
 89 |     """
 90 |     cdef Host host = Host()
 91 |     host.impl = impl
 92 |     return host
 93 | 
 94 |   @staticmethod
 95 |   cdef list wrap_batch(sg_host_t* hosts, int count):
 96 |     """
 97 |     Wrap an array of host handles.
 98 |     """
 99 |     cdef list result = []
100 |     for idx in range(count):
101 |       result.append(Host.wrap(hosts[idx]))
102 |     return result
103 | 
104 |   def dump(self):
105 |     """
106 |     Dump host info to stdout as SimGrid pleases.
107 |     """
108 |     self.__check_impl()
109 |     cplatform.sg_host_dump(self.impl)
110 | 
111 |   @property
112 |   def native(self):
113 |     """
114 |     Get/set host native handle as intptr_t.
115 | 
116 |     Warning:
117 |       Be very careful with setting this property, wrong values will lead to SEGFAULT and no amout of try/excepts will save you.
118 |     """
119 |     return <common.intptr>self.impl
120 | 
121 |   @native.setter
122 |   def native(self, common.intptr value):
123 |     self.impl = <cplatform.sg_host_t>value
124 | 
125 |   @property
126 |   def name(self):
127 |     """
128 |     Get host name.
129 |     """
130 |     self.__check_impl()
131 |     cdef const char* cname = cplatform.sg_host_get_name(self.impl)
132 |     return cname.decode("utf-8")
133 | 
134 |   @property
135 |   def speed(self):
136 |     """
137 |     Get host speed in flops.
138 |     """
139 |     self.__check_impl()
140 |     return cplatform.sg_host_speed(self.impl)
141 | 
142 |   @property
143 |   def available_speed(self):
144 |     """
145 |     Get fraction of host speed avaiable (0 <= r <= 1).
146 |     """
147 |     self.__check_impl()
148 |     return cplatform.sg_host_get_available_speed(self.impl)
149 | 
150 |   @property
151 |   def data(self):
152 |     """
153 |     Get/set user data associated with this host.
154 |     """
155 |     self.__check_impl()
156 |     return self.user_data
157 | 
158 |   @data.setter
159 |   def data(self, object value):
160 |     self.__check_impl()
161 |     self.user_data = value
162 | 
163 |   @data.deleter
164 |   def data(self):
165 |     self.user_data = None
166 | 
167 |   def __cinit__(self):
168 |     """
169 |     Basic initialization.
170 |     """
171 |     self.impl = NULL
172 |     self.user_data = None
173 | 
174 |   def __check_impl(self):
175 |     """
176 |     Validate the impl pointer.
177 |     """
178 |     if self.impl == NULL:
179 |       raise RuntimeError("Host instance is NULL")
180 | 


--------------------------------------------------------------------------------
/pysimgrid/csimdag.pxd:
--------------------------------------------------------------------------------
  1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
  2 | #
  3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
  4 | #
  5 | # This library is free software: you can redistribute it and/or modify
  6 | # it under the terms of the GNU Lesser General Public License as published by
  7 | # the Free Software Foundation, either version 3 of the License, or
  8 | # (at your option) any later version.
  9 | #
 10 | # This library is distributed in the hope that it will be useful,
 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13 | # GNU Lesser General Public License for more details.
 14 | #
 15 | # You should have received a copy of the GNU Lesser General Public
 16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
 17 | #
 18 | 
 19 | cimport xbt
 20 | cimport cplatform
 21 | 
 22 | cdef extern from "simgrid/simdag.h":
 23 |   ctypedef void* SD_task_t
 24 |   ctypedef void* SD_link_t
 25 | 
 26 |   ctypedef enum e_SD_task_state_t:
 27 |     SD_NOT_SCHEDULED = 0
 28 |     SD_SCHEDULABLE = 0x0001
 29 |     SD_SCHEDULED = 0x0002
 30 |     SD_RUNNABLE = 0x0004
 31 |     SD_RUNNING = 0x0008
 32 |     SD_DONE = 0x0010
 33 |     SD_FAILED = 0x0020
 34 | 
 35 |   ctypedef enum e_SD_task_kind_t:
 36 |     SD_TASK_NOT_TYPED = 0
 37 |     SD_TASK_COMM_E2E = 1
 38 |     SD_TASK_COMP_SEQ = 2
 39 |     SD_TASK_COMP_PAR_AMDAHL = 3
 40 |     SD_TASK_COMM_PAR_MXN_1D_BLOCK = 4
 41 | 
 42 |   # funky stuff - expose some defines as static variables
 43 |   cdef int SIMGRID_VERSION_MAJOR
 44 |   cdef int SIMGRID_VERSION_MINOR
 45 |   cdef int SIMGRID_VERSION_PATCH
 46 |   cdef char* SIMGRID_VERSION_STRING
 47 | 
 48 |   ##########################################################
 49 |   # Top-level API - lifetime, plaform/dag import, simulation
 50 |   ##########################################################
 51 |   void SD_init(int* argc, char** argv);
 52 |   void SD_config(char* key, char* value);
 53 |   void SD_create_environment(const char* cplatform_file);
 54 |   xbt.xbt_dynar_t SD_simulate(double how_long);
 55 |   double SD_get_clock();
 56 |   void SD_exit();
 57 |   xbt.xbt_dynar_t SD_daxload(const char* filename);
 58 |   xbt.xbt_dynar_t SD_dotload(const char* filename);
 59 |   xbt.xbt_dynar_t SD_PTG_dotload(const char* filename);
 60 |   xbt.xbt_dynar_t SD_dotload_with_sched(const char* filename);
 61 |   void uniq_transfer_task_name(SD_task_t task);
 62 | 
 63 | 
 64 |   ##########################################################
 65 |   # Task API
 66 |   ##########################################################
 67 |   SD_task_t SD_task_create(const char* name, void* data, double amount);
 68 |   void* SD_task_get_data(SD_task_t task);
 69 |   void SD_task_set_data(SD_task_t task, void* data);
 70 |   e_SD_task_state_t SD_task_get_state(SD_task_t task);
 71 |   const char* SD_task_get_name(SD_task_t task);
 72 |   void SD_task_set_name(SD_task_t task, const char* name);
 73 |   void SD_task_set_rate(SD_task_t task, double rate);
 74 | 
 75 |   void SD_task_watch(SD_task_t task, e_SD_task_state_t state);
 76 |   void SD_task_unwatch(SD_task_t task, e_SD_task_state_t state);
 77 |   double SD_task_get_amount(SD_task_t task);
 78 |   void SD_task_set_amount(SD_task_t task, double amount);
 79 |   double SD_task_get_alpha(SD_task_t task);
 80 |   double SD_task_get_remaining_amount(SD_task_t task);
 81 |   double SD_task_get_execution_time(SD_task_t task, int workstation_nb, const cplatform.sg_host_t* workstation_list,
 82 |                                                 const double* flops_amount, const double* bytes_amount);
 83 |   e_SD_task_kind_t SD_task_get_kind(SD_task_t task);
 84 |   void SD_task_schedule(SD_task_t task, int workstation_nb, const cplatform.sg_host_t* workstation_list,
 85 |                                     const double* flops_amount, const double* bytes_amount, double rate);
 86 |   void SD_task_unschedule(SD_task_t task);
 87 |   double SD_task_get_start_time(SD_task_t task);
 88 |   double SD_task_get_finish_time(SD_task_t task);
 89 |   xbt.xbt_dynar_t SD_task_get_parents(SD_task_t task);
 90 |   xbt.xbt_dynar_t SD_task_get_children(SD_task_t task);
 91 |   int SD_task_get_workstation_count(SD_task_t task);
 92 |   cplatform.sg_host_t* SD_task_get_workstation_list(SD_task_t task);
 93 |   void SD_task_destroy(SD_task_t task);
 94 |   void SD_task_dump(SD_task_t task);
 95 |   void SD_task_dotty(SD_task_t task, void* out_FILE);
 96 | 
 97 |   void SD_task_schedulev(SD_task_t task, int count, const cplatform.sg_host_t* list);
 98 | 
 99 |   SD_task_t SD_task_create_comp_seq(const char* name, void* data, double amount);
100 |   SD_task_t SD_task_create_comp_par_amdahl(const char* name, void* data, double amount, double alpha);
101 |   SD_task_t SD_task_create_comm_e2e(const char* name, void* data, double amount);
102 |   SD_task_t SD_task_create_comm_par_mxn_1d_block(const char* name, void* data, double amount);
103 |   void SD_task_distribute_comp_amdahl(SD_task_t task, int ws_count);
104 |   #void SD_task_schedulel(SD_task_t task, int count, ...); # disable vararg signature
105 | 
106 |   void SD_task_dependency_add(const char* name, void* data, SD_task_t src, SD_task_t dst);
107 | 
108 | cdef class Task:
109 |   cdef SD_task_t impl
110 |   cdef double amount_estimate
111 |   cdef object user_data
112 | 
113 |   @staticmethod
114 |   cdef Task wrap(SD_task_t impl)
115 | 
116 |   @staticmethod
117 |   cdef list wrap_batch(SD_task_t* tasks, int count)
118 | 


--------------------------------------------------------------------------------
/doc/source/conf.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python3
  2 | # -*- coding: utf-8 -*-
  3 | #
  4 | # pysimgrid documentation build configuration file, created by
  5 | # sphinx-quickstart on Sun Jan  8 09:53:16 2017.
  6 | #
  7 | # This file is execfile()d with the current directory set to its
  8 | # containing dir.
  9 | #
 10 | # Note that not all possible configuration values are present in this
 11 | # autogenerated file.
 12 | #
 13 | # All configuration values have a default; values that are commented out
 14 | # serve to show the default.
 15 | 
 16 | # If extensions (or modules to document with autodoc) are in another directory,
 17 | # add these directories to sys.path here. If the directory is relative to the
 18 | # documentation root, use os.path.abspath to make it absolute, like shown here.
 19 | #
 20 | import os
 21 | import sys
 22 | sys.path.insert(0, os.path.abspath("../.."))
 23 | 
 24 | 
 25 | # -- General configuration ------------------------------------------------
 26 | 
 27 | # If your documentation needs a minimal Sphinx version, state it here.
 28 | #
 29 | # needs_sphinx = '1.0'
 30 | 
 31 | # Add any Sphinx extension module names here, as strings. They can be
 32 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 33 | # ones.
 34 | extensions = ['sphinx.ext.autodoc',
 35 |               'sphinx.ext.autosummary',
 36 |               'sphinx.ext.napoleon',
 37 |               'sphinx.ext.mathjax',
 38 |               'sphinx.ext.githubpages']
 39 | 
 40 | # Add any paths that contain templates here, relative to this directory.
 41 | templates_path = ['_templates']
 42 | 
 43 | # The suffix(es) of source filenames.
 44 | # You can specify multiple suffix as a list of string:
 45 | #
 46 | # source_suffix = ['.rst', '.md']
 47 | source_suffix = '.rst'
 48 | 
 49 | # The master toctree document.
 50 | master_doc = 'index'
 51 | 
 52 | # General information about the project.
 53 | project = 'pysimgrid'
 54 | copyright = '2017, Alexey Nazarenko'
 55 | author = 'Alexey Nazarenko'
 56 | 
 57 | # The version info for the project you're documenting, acts as replacement for
 58 | # |version| and |release|, also used in various other places throughout the
 59 | # built documents.
 60 | #
 61 | # The short X.Y version.
 62 | version = '1.0'
 63 | # The full version, including alpha/beta/rc tags.
 64 | release = '1.0'
 65 | 
 66 | # The language for content autogenerated by Sphinx. Refer to documentation
 67 | # for a list of supported languages.
 68 | #
 69 | # This is also used if you do content translation via gettext catalogs.
 70 | # Usually you set "language" from the command line for these cases.
 71 | language = None
 72 | 
 73 | # List of patterns, relative to source directory, that match files and
 74 | # directories to ignore when looking for source files.
 75 | # This patterns also effect to html_static_path and html_extra_path
 76 | exclude_patterns = []
 77 | 
 78 | # The name of the Pygments (syntax highlighting) style to use.
 79 | pygments_style = 'sphinx'
 80 | 
 81 | # If true, `todo` and `todoList` produce output, else they produce nothing.
 82 | todo_include_todos = False
 83 | 
 84 | 
 85 | # -- Options for HTML output ----------------------------------------------
 86 | 
 87 | # The theme to use for HTML and HTML Help pages.  See the documentation for
 88 | # a list of builtin themes.
 89 | #
 90 | html_theme = "sphinx_rtd_theme"
 91 | html_theme_path = ["_themes", ]
 92 | 
 93 | # Theme options are theme-specific and customize the look and feel of a theme
 94 | # further.  For a list of options available for each theme, see the
 95 | # documentation.
 96 | #
 97 | # html_theme_options = {}
 98 | 
 99 | # Add any paths that contain custom static files (such as style sheets) here,
100 | # relative to this directory. They are copied after the builtin static files,
101 | # so a file named "default.css" will overwrite the builtin "default.css".
102 | html_static_path = ['_static']
103 | 
104 | # disable source link
105 | html_show_sourcelink = False
106 | 
107 | 
108 | 
109 | # -- Options for HTMLHelp output ------------------------------------------
110 | 
111 | # Output file base name for HTML help builder.
112 | htmlhelp_basename = 'pysimgriddoc'
113 | 
114 | 
115 | # -- Options for LaTeX output ---------------------------------------------
116 | 
117 | latex_elements = {
118 |     # The paper size ('letterpaper' or 'a4paper').
119 |     #
120 |     # 'papersize': 'letterpaper',
121 | 
122 |     # The font size ('10pt', '11pt' or '12pt').
123 |     #
124 |     # 'pointsize': '10pt',
125 | 
126 |     # Additional stuff for the LaTeX preamble.
127 |     #
128 |     # 'preamble': '',
129 | 
130 |     # Latex figure (float) alignment
131 |     #
132 |     # 'figure_align': 'htbp',
133 | }
134 | 
135 | # Grouping the document tree into LaTeX files. List of tuples
136 | # (source start file, target name, title,
137 | #  author, documentclass [howto, manual, or own class]).
138 | latex_documents = [
139 |     (master_doc, 'pysimgrid.tex', 'pysimgrid Documentation',
140 |      'Alexey Nazarenko', 'manual'),
141 | ]
142 | 
143 | 
144 | # -- Options for manual page output ---------------------------------------
145 | 
146 | # One entry per manual page. List of tuples
147 | # (source start file, name, description, authors, manual section).
148 | man_pages = [
149 |     (master_doc, 'pysimgrid', 'pysimgrid Documentation',
150 |      [author], 1)
151 | ]
152 | 
153 | 
154 | # -- Options for Texinfo output -------------------------------------------
155 | 
156 | # Grouping the document tree into Texinfo files. List of tuples
157 | # (source start file, target name, title, author,
158 | #  dir menu entry, description, category)
159 | texinfo_documents = [
160 |     (master_doc, 'pysimgrid', 'pysimgrid Documentation',
161 |      author, 'pysimgrid', 'Python interface for SimGrid/SimDAG library.',
162 |      'Miscellaneous'),
163 | ]
164 | 


--------------------------------------------------------------------------------
/pysimgrid/simdag/algorithms/hcpt.py:
--------------------------------------------------------------------------------
  1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
  2 | #
  3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
  4 | #
  5 | # This library is free software: you can redistribute it and/or modify
  6 | # it under the terms of the GNU Lesser General Public License as published by
  7 | # the Free Software Foundation, either version 3 of the License, or
  8 | # (at your option) any later version.
  9 | #
 10 | # This library is distributed in the hope that it will be useful,
 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13 | # GNU Lesser General Public License for more details.
 14 | #
 15 | # You should have received a copy of the GNU Lesser General Public
 16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
 17 | #
 18 | 
 19 | from collections import deque
 20 | 
 21 | import networkx
 22 | import numpy
 23 | 
 24 | from ... import cscheduling
 25 | from ..scheduler import StaticScheduler
 26 | 
 27 | 
 28 | class HCPT(StaticScheduler):
 29 |   """
 30 |   Heterogeneous Critical Parent Trees (HCPT) scheduler.
 31 | 
 32 |   HCPT is based on heuristics to find a critical path in worflow and schedule it with
 33 |   the highest possible priority.
 34 | 
 35 |   The key part is 'guided' topological sort which finds a schedulable order selecting cricial
 36 |   nodes as early as possible.
 37 | 
 38 |   For more details and rationale please refer to the original publication:
 39 | 
 40 |     T. Hagras and J. Janecek, "A Simple Scheduling Heuristic for Heterogeneous Computing Environments",
 41 |     Proceedings of the Second International Symposium on Parallel and Distributed Computing (ISPDC’03),
 42 |     2003, pp. 104-110
 43 |   """
 44 |   def get_schedule(self, simulation):
 45 |     """
 46 |     Overridden.
 47 |     """
 48 |     nxgraph = simulation.get_task_graph()
 49 |     platform_model = cscheduling.PlatformModel(simulation)
 50 |     state = cscheduling.SchedulerState(simulation)
 51 |     tasks_aest, tasks_alst = self.get_tasks_aest_alst(nxgraph, platform_model)
 52 | 
 53 |     # All nodes in the critical path must have AEST=ALST
 54 |     critical_path = sorted(
 55 |       [(t, tasks_aest[t]) for t in tasks_aest if numpy.isclose(tasks_aest[t], tasks_alst[t])],
 56 |       key=lambda x: x[1]
 57 |     )
 58 |     critical_path.reverse()
 59 | 
 60 |     stack = deque([elem[0] for elem in critical_path])
 61 |     queue = deque()
 62 |     while len(stack):
 63 |       task = stack.pop()
 64 |       parents = nxgraph.pred[task]
 65 |       untracked_parents = sorted(
 66 |         set(parents) - set(queue),
 67 |         key=lambda x: tasks_aest[x]
 68 |       )
 69 |       if untracked_parents:
 70 |         stack.append(task)
 71 |         for parent in untracked_parents:
 72 |           stack.append(parent)
 73 |       else:
 74 |         queue.append(task)
 75 | 
 76 |     scheduled = set()
 77 |     while len(queue):
 78 |       task = queue.popleft()
 79 |       if task in scheduled:
 80 |         continue
 81 |       else:
 82 |         scheduled.add(task)
 83 |       if cscheduling.try_schedule_boundary_task(task, nxgraph, platform_model, state):
 84 |         continue
 85 |       current_min = cscheduling.MinSelector()
 86 |       for host, timesheet in state.timetable.items():
 87 |         if cscheduling.is_master_host(host):
 88 |           continue
 89 |         est = platform_model.est(host, dict(nxgraph.pred[task]), state)
 90 |         eet = platform_model.eet(task, host)
 91 |         pos, start, finish = cscheduling.timesheet_insertion(timesheet, est, eet)
 92 |         # strange key order to ensure stable sorting:
 93 |         #  first sort by ECT
 94 |         #  if equal - sort by host speed
 95 |         #  if equal - sort by host name (guaranteed to be unique)
 96 |         current_min.update((finish, host.speed, host.name), (host, pos, start, finish))
 97 |       host, pos, start, finish = current_min.value
 98 |       state.update(task, host, pos, start, finish)
 99 | 
100 |     expected_makespan = max([state["ect"] for state in state.task_states.values()])
101 |     return state.schedule, expected_makespan
102 | 
103 |   @classmethod
104 |   def get_tasks_aest_alst(cls, nxgraph, platform_model):
105 |     """
106 |     Return AEST and ALST of tasks.
107 | 
108 |     Args:
109 |       nxgraph: full task graph as networkx.DiGraph
110 |       platform_model: cscheduling.PlatformModel object
111 | 
112 |     Returns:
113 |       tuple containg 2 dictionaries
114 |         aest: task->aest_value
115 |         alst: task->alst_value
116 |     """
117 |     mean_speed = platform_model.mean_speed
118 |     mean_bandwidth = platform_model.mean_bandwidth
119 |     mean_latency = platform_model.mean_latency
120 |     topological_order = list(networkx.topological_sort(nxgraph))
121 | 
122 |     # Average execution cost
123 |     aec = {task: float(task.amount) / mean_speed for task in nxgraph}
124 | 
125 |     # Average earliest start time
126 |     aest = {}
127 |     # TODO: Check several roots and ends!
128 |     root = topological_order[0]
129 |     end = topological_order[-1]
130 |     aest[root] = 0.
131 |     for task in topological_order:
132 |       parents = nxgraph.pred[task]
133 |       if not parents:
134 |         aest[task] = 0
135 |         continue
136 |       aest[task] = max([
137 |         aest[parent] + aec[parent] + (nxgraph[parent][task]["weight"] / mean_bandwidth + mean_latency)
138 |         for parent in parents
139 |       ])
140 | 
141 |     topological_order.reverse()
142 | 
143 |     # Average latest start time
144 |     alst = {}
145 |     alst[end] = aest[end]
146 |     for task in topological_order:
147 |       if not nxgraph[task]:
148 |         alst[task] = aest[task]
149 |         continue
150 |       alst[task] = min([
151 |         alst[child] - (edge["weight"] / mean_bandwidth + mean_latency)
152 |         for child, edge in nxgraph[task].items()
153 |       ]) - aec[task]
154 | 
155 |     return aest, alst
156 | 


--------------------------------------------------------------------------------
/pysimgrid/tools/bot_gen.py:
--------------------------------------------------------------------------------
  1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
  2 | #
  3 | # Copyright 2015-2018 Alexey Nazarenko and contributors
  4 | #
  5 | # This library is free software: you can redistribute it and/or modify
  6 | # it under the terms of the GNU Lesser General Public License as published by
  7 | # the Free Software Foundation, either version 3 of the License, or
  8 | # (at your option) any later version.
  9 | #
 10 | # This library is distributed in the hope that it will be useful,
 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13 | # GNU Lesser General Public License for more details.
 14 | #
 15 | # You should have received a copy of the GNU Lesser General Public
 16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
 17 | #
 18 | 
 19 | """
 20 | Bag of tasks (BOT) applications generator.
 21 | 
 22 | Usage::
 23 | 
 24 |     python -m pysimgrid.tools.bot_gen [-h]
 25 |                                       output_dir num_graphs seed
 26 |                                       num_tasks input_size
 27 |                                       comp_size output_size
 28 | 
 29 |     Generator of synthetic graphs for bag-of-tasks applications
 30 | 
 31 |     positional arguments:
 32 |       output_dir   output directory
 33 |       num_graphs   number of generated graphs
 34 |       seed         random seed
 35 |       num_tasks    number of tasks
 36 |       input_size   task input size in bytes
 37 |       comp_size    task computational size in flops
 38 |       output_size  task output size in bytes
 39 |       output_dir   output directory
 40 |       num_graphs   number of generated graphs
 41 | 
 42 |     optional arguments:
 43 |       -h, --help   show this help message and exit
 44 | """
 45 | 
 46 | import argparse
 47 | import os
 48 | import random
 49 | import sys
 50 | 
 51 | 
 52 | def generate_tasks(num_tasks, input_size, comp_size, output_size):
 53 |     tasks = []
 54 |     for i in range(0, num_tasks):
 55 |         task = {
 56 |             'name': "task%d" % i
 57 |         }
 58 |         tasks.append(task)
 59 | 
 60 |     # populate task input sizes
 61 |     input_sizes = generate_values(input_size, num_tasks)
 62 |     for i in range(0, num_tasks):
 63 |         tasks[i]['input_size'] = input_sizes[i]
 64 | 
 65 |     # populate task computational sizes
 66 |     comp_sizes = generate_values(comp_size, num_tasks, input_sizes)
 67 |     for i in range(0, num_tasks):
 68 |         tasks[i]['comp_size'] = comp_sizes[i]
 69 | 
 70 |     # populate task output sizes
 71 |     output_sizes = generate_values(output_size, num_tasks, input_sizes)
 72 |     for i in range(0, num_tasks):
 73 |         tasks[i]['output_size'] = output_sizes[i]
 74 | 
 75 |     return tasks
 76 | 
 77 | 
 78 | def generate_values(spec, num, inputs=None):
 79 |     try:
 80 |         # fixed value
 81 |         fixed = float(spec)
 82 |         values = [fixed] * num
 83 | 
 84 |     except ValueError:
 85 |         parts = spec.split(':')
 86 |         dist_type = parts[0]
 87 |         params = parts[1:]
 88 | 
 89 |         # uniform distribution: u:min:max
 90 |         if dist_type == "u":
 91 |             min_value = float(params[0])
 92 |             max_value = float(params[1])
 93 |             values = [random.uniform(min_value, max_value) for _ in range(0, num)]
 94 | 
 95 |         # normal distribution: n:mean:std_dev
 96 |         elif dist_type == "n":
 97 |             mean = float(params[0])
 98 |             std_dev = float(params[1])
 99 |             values = [random.normalvariate(mean, std_dev) for _ in range(0, num)]
100 | 
101 |         # scaled values: x:factor
102 |         elif dist_type == "x":
103 |             factor = float(params[0])
104 |             if inputs is not None:
105 |                 values = [inputs[i] * factor for i in range(0, num)]
106 |             else:
107 |                 print("Inputs are not specified")
108 |                 sys.exit(-1)
109 | 
110 |         else:
111 |             print("Unknown distribution")
112 |             sys.exit(-1)
113 | 
114 |     return values
115 | 
116 | 
117 | def save_as_dot_file(tasks, output_path):
118 |     with open(output_path, "w") as f:
119 |         f.write("digraph G {\n")
120 |         for task in tasks:
121 |             f.write('  root -> %s [size="%e"]\n' % (task['name'], task['input_size']))
122 |             f.write('  %s [size="%e"]\n' % (task['name'], task['comp_size']))
123 |             f.write('  %s -> end [size="%e"]\n' % (task['name'], task['output_size']))
124 |         f.write("}\n")
125 | 
126 | 
127 | def main(output_dir, num_graphs, seed, num_tasks, input_size, comp_size, output_size):
128 |     random.seed(seed)
129 |     os.mkdir(output_dir)
130 |     for i in range(0, num_graphs):
131 |         tasks = generate_tasks(num_tasks, input_size, comp_size, output_size)
132 |         file_name = 'bot_%d_%s_%s_%s_%d.dot' % (num_tasks, input_size, comp_size, output_size, i)
133 |         file_path = output_dir + '/' + file_name
134 |         save_as_dot_file(tasks, file_path)
135 |         print('Generated file: %s' % file_path)
136 |     return 0
137 | 
138 | 
139 | def _cli():
140 |     parser = argparse.ArgumentParser(description="Generator of synthetic graphs for bag-of-tasks applications")
141 |     parser.add_argument("output_dir", type=str, help="output directory")
142 |     parser.add_argument("num_graphs", type=int, help="number of generated graphs")
143 |     parser.add_argument("seed", type=int, help="random seed")
144 |     parser.add_argument("num_tasks", type=int, help="number of tasks")
145 |     parser.add_argument("input_size", type=str, help="task input size in bytes")
146 |     parser.add_argument("comp_size", type=str, help="task computational size in flops")
147 |     parser.add_argument("output_size", type=str, help="task output size in bytes")
148 |     args = parser.parse_args()
149 |     return vars(args)
150 | 
151 | 
152 | if __name__ == '__main__':
153 |     main(**_cli())
154 | 


--------------------------------------------------------------------------------
/bot/gen/sys_gen.py:
--------------------------------------------------------------------------------
  1 | import argparse
  2 | import os
  3 | import random
  4 | 
  5 | 
  6 | def generate_cluster(num_hosts, host_speed, host_bandwidth, host_latency, master_bandwidth, master_latency):
  7 |     hosts = []
  8 |     links = []
  9 |     routes = []
 10 | 
 11 |     # master host
 12 |     master = {
 13 |         'id': 'master',
 14 |         'speed': 1
 15 |     }
 16 |     hosts.append(master)
 17 |     master_link = {
 18 |         'id': "link_master",
 19 |         'bandwidth': generate_values(master_bandwidth, 1)[0],
 20 |         'latency': generate_values(master_latency, 1)[0],
 21 |     }
 22 |     links.append(master_link)
 23 |     master_route = {
 24 |         'src': 'master',
 25 |         'dst': 'router',
 26 |         'links': [
 27 |             master_link['id']
 28 |         ]
 29 |     }
 30 |     routes.append(master_route)
 31 | 
 32 |     # worker hosts
 33 |     host_speeds = generate_values(host_speed, num_hosts)
 34 |     link_bandwidths = generate_values(host_bandwidth, num_hosts)
 35 |     link_latencies = generate_values(host_latency, num_hosts)
 36 |     for i in xrange(0, num_hosts):
 37 |         host = {
 38 |             'id': "host%d" % i,
 39 |             'speed': host_speeds[i]
 40 |         }
 41 |         hosts.append(host)
 42 |         link = {
 43 |             'id': "link%d" % i,
 44 |             'bandwidth': link_bandwidths[i],
 45 |             'latency': link_latencies[i]
 46 |         }
 47 |         links.append(link)
 48 |         route = {
 49 |             'src': host['id'],
 50 |             'dst': 'router',
 51 |             'links': [
 52 |                 link['id']
 53 |             ]
 54 |         }
 55 |         routes.append(route)
 56 | 
 57 |     system = {
 58 |         'hosts': hosts,
 59 |         'links': links,
 60 |         'routes': routes
 61 |     }
 62 |     return system
 63 | 
 64 | 
 65 | def generate_values(spec, num):
 66 |     try:
 67 |         # fixed value
 68 |         fixed = float(spec)
 69 |         values = [fixed] * num
 70 | 
 71 |     except ValueError:
 72 |         # uniform distribution: min-max
 73 |         parts = spec.split('-')
 74 |         min = float(parts[0])
 75 |         max = float(parts[1])
 76 |         values = [random.uniform(min, max) for _ in xrange(0, num)]
 77 | 
 78 |     return values
 79 | 
 80 | 
 81 | def save_as_xml_file(system, output_path):
 82 |     with open(output_path, "w") as f:
 83 |         f.write("<?xml version='1.0'?>\n")
 84 |         f.write('<!DOCTYPE platform SYSTEM "http://simgrid.gforge.inria.fr/simgrid/simgrid.dtd">\n')
 85 |         f.write('<platform version="4">\n')
 86 |         f.write('  <AS id="AS0" routing="Floyd">\n')
 87 | 
 88 |         for host in system['hosts']:
 89 |             f.write('    <host id="%s" core="1" speed="%fGf"/>\n' % (host['id'], host['speed']))
 90 |         f.write("\n")
 91 | 
 92 |         for link in system['links']:
 93 |             f.write('    <link id="%s" bandwidth="%fMBps" latency="%fus"/>\n' % (
 94 |             link['id'], link['bandwidth'], link['latency']))
 95 |         f.write("\n")
 96 | 
 97 |         f.write('    <router id="router"/>\n')
 98 |         for route in system['routes']:
 99 |             f.write('    <route src="%s" dst="%s">\n' % (route['src'], route['dst']))
100 |             for link in route['links']:
101 |                 f.write('      <link_ctn id="%s"/>\n' % link)
102 |             f.write('    </route>\n')
103 | 
104 |         f.write("  </AS>\n")
105 |         f.write("</platform>\n")
106 | 
107 | 
108 | def main():
109 |     parser = argparse.ArgumentParser(description="Generator of synthetic systems")
110 |     parser.add_argument("output_dir", type=str, help="output directory")
111 |     parser.add_argument("num_systems", type=int, help="number of generated systems")
112 |     subparsers = parser.add_subparsers(dest='system_type', help="system type")
113 | 
114 |     # cluster
115 |     parser_cluster = subparsers.add_parser("cluster", help="collection of hosts with a flat topology")
116 |     parser_cluster.add_argument("num_hosts", type=int, help="number of hosts (excluding master)")
117 |     parser_cluster.add_argument("host_speed", type=str, help="host speed in GFLOPS (e.g. '1', '1-10')")
118 |     parser_cluster.add_argument("link_bandwidth", type=str,
119 |                                 help="link bandwidth in MBps as 'bandwidth[:master_bandwidth]' (e.g. '125', '10-100:100')")
120 |     parser_cluster.add_argument("link_latency", type=str,
121 |                                 help="link latency in us as 'latency[:master_latency]' (e.g. '10', '10-100:10')")
122 | 
123 |     args = parser.parse_args()
124 | 
125 |     if not os.path.exists(args.output_dir):
126 |         os.makedirs(args.output_dir)
127 | 
128 |     for i in xrange(0, args.num_systems):
129 |         # cluster
130 |         if args.system_type == 'cluster':
131 |             # parse host/master bandwidth and latency
132 |             if ':' in args.link_bandwidth:
133 |                 parts = args.link_bandwidth.split(':')
134 |                 host_bandwidth = parts[0]
135 |                 master_bandwidth = parts[1]
136 |             else:
137 |                 host_bandwidth = args.link_bandwidth
138 |                 master_bandwidth = args.link_bandwidth
139 |             if ':' in args.link_latency:
140 |                 parts = args.link_latency.split(':')
141 |                 host_latency = parts[0]
142 |                 master_latency = parts[1]
143 |             else:
144 |                 host_latency = args.link_latency
145 |                 master_latency = args.link_latency
146 | 
147 |             # generate cluster
148 |             system = generate_cluster(args.num_hosts, args.host_speed, host_bandwidth, host_latency, master_bandwidth,
149 |                                       master_latency)
150 |             file_name = 'cluster_%d_%s_%s_%s_%d.xml' % (
151 |             args.num_hosts, args.host_speed, args.link_bandwidth, args.link_latency, i)
152 | 
153 |         file_path = args.output_dir + '/' + file_name
154 |         save_as_xml_file(system, file_path)
155 |         print('Generated file: %s' % file_path)
156 | 
157 |     return 0
158 | 
159 | 
160 | if __name__ == '__main__':
161 |     main()


--------------------------------------------------------------------------------
/pysimgrid/simdag/algorithms/simheft.py:
--------------------------------------------------------------------------------
  1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
  2 | #
  3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
  4 | #
  5 | # This library is free software: you can redistribute it and/or modify
  6 | # it under the terms of the GNU Lesser General Public License as published by
  7 | # the Free Software Foundation, either version 3 of the License, or
  8 | # (at your option) any later version.
  9 | #
 10 | # This library is distributed in the hope that it will be useful,
 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13 | # GNU Lesser General Public License for more details.
 14 | #
 15 | # You should have received a copy of the GNU Lesser General Public
 16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
 17 | #
 18 | 
 19 | """
 20 | TODO: algorithm is WIP.
 21 | """
 22 | 
 23 | import itertools
 24 | import multiprocessing
 25 | import tempfile
 26 | 
 27 | import networkx
 28 | 
 29 | from .. import scheduler
 30 | from ... import cscheduling
 31 | 
 32 | 
 33 | class _ExtrenalSchedule(scheduler.StaticScheduler):
 34 |   def __init__(self, simulation, schedule):
 35 |     super(_ExtrenalSchedule, self).__init__(simulation)
 36 |     self.__schedule = schedule
 37 | 
 38 |   def get_schedule(self, simulation):
 39 |     return self.__schedule
 40 | 
 41 | 
 42 | def _update_subgraph(full, subgraph, task):
 43 |   parents = full.pred[task]
 44 |   subgraph.add_node(task)
 45 |   for parent, edge_dict in parents.items():
 46 |     subgraph.add_edge(parent, task, weight=edge_dict["weight"])
 47 | 
 48 | 
 49 | def _serialize_graph(graph, output_file):
 50 |   output_file.write("digraph G {\n")
 51 |   for task in graph:
 52 |     output_file.write('  "%s" [size="%f"];\n' % (task.name, task.amount))
 53 |   output_file.write("\n")
 54 |   for src, dst, data in graph.edges(data='weight'):
 55 |     output_file.write('  "%s" -> "%s" [size="%f"];\n' % (src.name, dst.name, data))
 56 |   output_file.write("}\n")
 57 |   output_file.flush()
 58 | 
 59 | 
 60 | def _serialize_schedule(timetable):
 61 |   result = {}
 62 |   for host, timesheet in timetable.items():
 63 |     result[host.name] = [task.name for (task, _, _) in timesheet]
 64 |   return result
 65 | 
 66 | 
 67 | def _restore_state(simulation, serialized):
 68 |   tasks = {t.name: t for t in simulation.tasks}
 69 |   hosts = {h.name: h for h in simulation.hosts}
 70 |   state = cscheduling.SchedulerState(simulation)
 71 |   for hostname, timesheet in serialized.items():
 72 |     for pos, (taskname, start, finish) in enumerate(timesheet):
 73 |       state.update(tasks[taskname], hosts[hostname], pos, start, finish)
 74 |   return state
 75 | 
 76 | 
 77 | def _restore_schedule(simulation, serialized):
 78 |   tasks = {t.name: t for t in simulation.tasks}
 79 |   hosts = {h.name: h for h in simulation.hosts}
 80 |   result = {}
 81 |   end_scheduled = False
 82 |   for hostname, tasknames in serialized.items():
 83 |     tasklist = []
 84 |     for taskname in tasknames:
 85 |       end_scheduled = end_scheduled or taskname == "end"
 86 |       tasklist.append(tasks[taskname])
 87 |     result[hosts[hostname]] = tasklist
 88 |   if not end_scheduled:
 89 |     for host_schedule in result.values():
 90 |       host_schedule.append(tasks["end"])
 91 |       break
 92 |   return result, end_scheduled
 93 | 
 94 | 
 95 | def _run_simulation(platform_path, tasks_path, schedule_by_name):
 96 |   import collections
 97 |   import logging
 98 |   from .. import simulation
 99 |   logging.getLogger().setLevel(logging.WARNING)
100 |   with simulation.Simulation(platform_path, tasks_path, log_config="root.threshold:WARNING") as simulation:
101 |     restored_schedule_state, final = _restore_schedule(simulation, schedule_by_name)
102 |     scheduler = _ExtrenalSchedule(simulation, restored_schedule_state)
103 |     scheduler.run()
104 |     result = collections.defaultdict(list)
105 |     for t in sorted(simulation.tasks, key=lambda t: t.start_time):
106 |       if not final and t.name == "end":
107 |         continue
108 |       result[t.hosts[0].name].append((t.name, t.start_time, t.finish_time))
109 |     return result
110 | 
111 | 
112 | class SimHEFT(scheduler.StaticScheduler):
113 |   """
114 |   Modified version of HEFT scheduler using simulation estimates instead of linear models.
115 |   """
116 | 
117 |   def get_schedule(self, simulation):
118 |     """
119 |     Overriden.
120 |     """
121 |     nxgraph = simulation.get_task_graph()
122 |     platform_model = cscheduling.PlatformModel(simulation)
123 |     state = cscheduling.SchedulerState(simulation)
124 | 
125 |     ordered_tasks = cscheduling.heft_order(nxgraph, platform_model)
126 | 
127 |     subgraph = networkx.DiGraph()
128 | 
129 |     # fork context is incompatible with SimGrid static variables
130 |     ctx = multiprocessing.get_context("spawn")
131 |     for task in ordered_tasks:
132 |       _update_subgraph(nxgraph, subgraph, task)
133 |       if cscheduling.try_schedule_boundary_task(task, nxgraph, platform_model, state):
134 |         continue
135 |       current_min = cscheduling.MinSelector()
136 |       for host, timesheet in state.timetable.items():
137 |         if cscheduling.is_master_host(host):
138 |           continue
139 |         current_state = state.copy()
140 |         est = platform_model.est(host, dict(nxgraph.pred[task]), current_state)
141 |         eet = platform_model.eet(task, host)
142 |         # 'correct' way
143 |         pos, start, finish = cscheduling.timesheet_insertion(timesheet, est, eet)
144 |         # TODO: try aggressive inserts
145 |         current_state.update(task, host, pos, start, finish)
146 |         with tempfile.NamedTemporaryFile("w", suffix=".dot") as temp_file:
147 |           _serialize_graph(subgraph, temp_file)
148 |           subschedule = _serialize_schedule(current_state.timetable)
149 |           with ctx.Pool(1) as process:
150 |             serialized_state = process.apply(_run_simulation, (simulation.platform_path, temp_file.name, subschedule))
151 |           current_state = _restore_state(simulation, serialized_state)
152 |           current_min.update((current_state.max_time, host.speed, host.name), current_state)
153 |       state = current_min.value
154 |     expected_makespan = max([state["ect"] for state in state.task_states.values()])
155 |     return state.schedule, expected_makespan


--------------------------------------------------------------------------------
/doc/source/_themes/sphinx_rtd_theme/static/js/theme.js:
--------------------------------------------------------------------------------
  1 | require=(function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error("Cannot find module '"+o+"'");throw f.code="MODULE_NOT_FOUND",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o<r.length;o++)s(r[o]);return s})({"sphinx-rtd-theme":[function(require,module,exports){
  2 | var jQuery = (typeof(window) != 'undefined') ? window.jQuery : require('jquery');
  3 | 
  4 | // Sphinx theme nav state
  5 | function ThemeNav () {
  6 | 
  7 |     var nav = {
  8 |         navBar: null,
  9 |         win: null,
 10 |         winScroll: false,
 11 |         winResize: false,
 12 |         linkScroll: false,
 13 |         winPosition: 0,
 14 |         winHeight: null,
 15 |         docHeight: null,
 16 |         isRunning: false
 17 |     };
 18 | 
 19 |     nav.enable = function () {
 20 |         var self = this;
 21 | 
 22 |         if (!self.isRunning) {
 23 |             self.isRunning = true;
 24 |             jQuery(function ($) {
 25 |                 self.init($);
 26 | 
 27 |                 self.reset();
 28 |                 self.win.on('hashchange', self.reset);
 29 | 
 30 |                 // Set scroll monitor
 31 |                 self.win.on('scroll', function () {
 32 |                     if (!self.linkScroll) {
 33 |                         self.winScroll = true;
 34 |                     }
 35 |                 });
 36 |                 setInterval(function () { if (self.winScroll) self.onScroll(); }, 25);
 37 | 
 38 |                 // Set resize monitor
 39 |                 self.win.on('resize', function () {
 40 |                     self.winResize = true;
 41 |                 });
 42 |                 setInterval(function () { if (self.winResize) self.onResize(); }, 25);
 43 |                 self.onResize();
 44 |             });
 45 |         };
 46 |     };
 47 | 
 48 |     nav.init = function ($) {
 49 |         var doc = $(document),
 50 |             self = this;
 51 | 
 52 |         this.navBar = $('div.wy-side-scroll:first');
 53 |         this.win = $(window);
 54 | 
 55 |         // Set up javascript UX bits
 56 |         $(document)
 57 |             // Shift nav in mobile when clicking the menu.
 58 |             .on('click', "[data-toggle='wy-nav-top']", function() {
 59 |                 $("[data-toggle='wy-nav-shift']").toggleClass("shift");
 60 |                 $("[data-toggle='rst-versions']").toggleClass("shift");
 61 |             })
 62 | 
 63 |             // Nav menu link click operations
 64 |             .on('click', ".wy-menu-vertical .current ul li a", function() {
 65 |                 var target = $(this);
 66 |                 // Close menu when you click a link.
 67 |                 $("[data-toggle='wy-nav-shift']").removeClass("shift");
 68 |                 $("[data-toggle='rst-versions']").toggleClass("shift");
 69 |                 // Handle dynamic display of l3 and l4 nav lists
 70 |                 self.toggleCurrent(target);
 71 |                 self.hashChange();
 72 |             })
 73 |             .on('click', "[data-toggle='rst-current-version']", function() {
 74 |                 $("[data-toggle='rst-versions']").toggleClass("shift-up");
 75 |             })
 76 | 
 77 |         // Make tables responsive
 78 |         $("table.docutils:not(.field-list)")
 79 |             .wrap("<div class='wy-table-responsive'></div>");
 80 | 
 81 |         // Add expand links to all parents of nested ul
 82 |         $('.wy-menu-vertical ul').not('.simple').siblings('a').each(function () {
 83 |             var link = $(this);
 84 |                 expand = $('<span class="toctree-expand"></span>');
 85 |             expand.on('click', function (ev) {
 86 |                 self.toggleCurrent(link);
 87 |                 ev.stopPropagation();
 88 |                 return false;
 89 |             });
 90 |             link.prepend(expand);
 91 |         });
 92 |     };
 93 | 
 94 |     nav.reset = function () {
 95 |         // Get anchor from URL and open up nested nav
 96 |         var anchor = encodeURI(window.location.hash);
 97 |         if (anchor) {
 98 |             try {
 99 |                 var link = $('.wy-menu-vertical')
100 |                     .find('[href="' + anchor + '"]');
101 |                 $('.wy-menu-vertical li.toctree-l1 li.current')
102 |                     .removeClass('current');
103 |                 link.closest('li.toctree-l2').addClass('current');
104 |                 link.closest('li.toctree-l3').addClass('current');
105 |                 link.closest('li.toctree-l4').addClass('current');
106 |             }
107 |             catch (err) {
108 |                 console.log("Error expanding nav for anchor", err);
109 |             }
110 |         }
111 |     };
112 | 
113 |     nav.onScroll = function () {
114 |         this.winScroll = false;
115 |         var newWinPosition = this.win.scrollTop(),
116 |             winBottom = newWinPosition + this.winHeight,
117 |             navPosition = this.navBar.scrollTop(),
118 |             newNavPosition = navPosition + (newWinPosition - this.winPosition);
119 |         if (newWinPosition < 0 || winBottom > this.docHeight) {
120 |             return;
121 |         }
122 |         this.navBar.scrollTop(newNavPosition);
123 |         this.winPosition = newWinPosition;
124 |     };
125 | 
126 |     nav.onResize = function () {
127 |         this.winResize = false;
128 |         this.winHeight = this.win.height();
129 |         this.docHeight = $(document).height();
130 |     };
131 | 
132 |     nav.hashChange = function () {
133 |         this.linkScroll = true;
134 |         this.win.one('hashchange', function () {
135 |             this.linkScroll = false;
136 |         });
137 |     };
138 | 
139 |     nav.toggleCurrent = function (elem) {
140 |         var parent_li = elem.closest('li');
141 |         parent_li.siblings('li.current').removeClass('current');
142 |         parent_li.siblings().find('li.current').removeClass('current');
143 |         parent_li.find('> ul li.current').removeClass('current');
144 |         parent_li.toggleClass('current');
145 |     }
146 | 
147 |     return nav;
148 | };
149 | 
150 | module.exports.ThemeNav = ThemeNav();
151 | 
152 | if (typeof(window) != 'undefined') {
153 |     window.SphinxRtdTheme = { StickyNav: module.exports.ThemeNav };
154 | }
155 | 
156 | },{"jquery":"jquery"}]},{},["sphinx-rtd-theme"]);
157 | 


--------------------------------------------------------------------------------
/pysimgrid/simdag/algorithms/batch_dyn.py:
--------------------------------------------------------------------------------
  1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
  2 | #
  3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
  4 | #
  5 | # This library is free software: you can redistribute it and/or modify
  6 | # it under the terms of the GNU Lesser General Public License as published by
  7 | # the Free Software Foundation, either version 3 of the License, or
  8 | # (at your option) any later version.
  9 | #
 10 | # This library is distributed in the hope that it will be useful,
 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13 | # GNU Lesser General Public License for more details.
 14 | #
 15 | # You should have received a copy of the GNU Lesser General Public
 16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
 17 | #
 18 | 
 19 | import logging
 20 | import numpy as np
 21 | 
 22 | from .. import scheduler
 23 | from ... import csimdag
 24 | 
 25 | 
 26 | class DynamicBatchScheduler(scheduler.DynamicScheduler):
 27 |     """
 28 |     Batch-mode heuristic base implementation.
 29 | 
 30 |     Schedules all currently schedulable tasks to a best host by ECT in a single batch.
 31 | 
 32 |     The order in a batch is determined by a heuristic:
 33 | 
 34 |     * MinMin prioritizes the tasks with minimum ECT on a best host
 35 | 
 36 |     * MaxMin prioritizes the tasks with maximum ECT on a best host
 37 | 
 38 |     * Sufferage prioritizes tasks with maximum difference between ECT on 2 best hosts
 39 |     """
 40 | 
 41 |     def prepare(self, simulation):
 42 |         for h in simulation.hosts:
 43 |             h.data = {
 44 |                 "est": 0.
 45 |             }
 46 |         master_hosts = simulation.hosts.by_prop("name", self.MASTER_HOST_NAME)
 47 |         self._master_host = master_hosts[0] if master_hosts else None
 48 | 
 49 |         self.host_tasks = {}
 50 |         self._exec_hosts = simulation.hosts.by_prop("name", self.MASTER_HOST_NAME, True)
 51 |         for host in self._exec_hosts:
 52 |             self.host_tasks[host.name] = []
 53 | 
 54 |         if self._master_host:
 55 |             for task in simulation.tasks.by_func(lambda t: t.name in self.BOUNDARY_TASKS):
 56 |                 task.schedule(self._master_host)
 57 |         self._started_tasks = set()
 58 |         self._estimate_cache = {}
 59 | 
 60 |     def schedule(self, simulation, changed):
 61 |         clock = simulation.clock
 62 | 
 63 |         free_hosts = set(self._exec_hosts)
 64 |         for task in simulation.tasks[csimdag.TaskState.TASK_STATE_RUNNING, csimdag.TaskState.TASK_STATE_SCHEDULED]:
 65 |             host = task.hosts[0]
 66 |             free_hosts.discard(host)
 67 |             if task.start_time > 0 and task not in self._started_tasks:
 68 |                 self._started_tasks.add(task)
 69 |                 host.data["est"] = task.start_time + task.get_eet(host)
 70 | 
 71 |         host_est = {}
 72 |         for h in self._exec_hosts:
 73 |             host_est[h] = h.data["est"]
 74 | 
 75 |         tasks = simulation.tasks[csimdag.TaskState.TASK_STATE_SCHEDULABLE]
 76 |         num_tasks = len(tasks)
 77 | 
 78 |         # build ECT matrix
 79 |         ECT = np.zeros((num_tasks, len(self._exec_hosts)))
 80 |         for t, task in enumerate(tasks):
 81 |             for h, host in enumerate(self._exec_hosts):
 82 |                 ECT[t][h] = self.get_ect(host_est[host], clock, task, host)
 83 | 
 84 |         # build schedule
 85 |         task_idx = np.arange(num_tasks)
 86 |         for _ in range(0, num_tasks):
 87 |             min_hosts = np.argmin(ECT, axis=1)
 88 |             min_times = ECT[np.arange(ECT.shape[0]), min_hosts]
 89 | 
 90 |             if ECT.shape[1] > 1:
 91 |                 min2_times = np.partition(ECT, 1)[:, 1]
 92 |                 sufferages = min2_times - min_times
 93 |             else:
 94 |                 sufferages = -min_times
 95 | 
 96 |             possible_schedules = []
 97 |             for i in range(0, len(task_idx)):
 98 |                 best_host_idx = int(min_hosts[i])
 99 |                 best_ect = min_times[i]
100 |                 sufferage = sufferages[i]
101 |                 possible_schedules.append((i, best_host_idx, best_ect, sufferage))
102 | 
103 |             t, h, ect = self.batch_heuristic(possible_schedules)
104 |             task = tasks[int(task_idx[t])]
105 |             host = self._exec_hosts[h]
106 | 
107 |             task_time = ect - host_est[host]
108 |             host_est[host] = ect
109 | 
110 |             if host in free_hosts:
111 |                 task.schedule(host)
112 |                 # logging.info("%s -> %s" % (task.name, host.name))
113 |                 host.data["est"] = ect
114 |                 free_hosts.remove(host)
115 |                 if len(free_hosts) == 0:
116 |                     break
117 | 
118 |             task_idx = np.delete(task_idx, t)
119 |             ECT = np.delete(ECT, t, 0)
120 |             ECT[:,h] += task_time
121 | 
122 |     def get_ect(self, est, clock, task, host):
123 |         if (task, host) in self._estimate_cache:
124 |             task_time = self._estimate_cache[(task, host)]
125 |         else:
126 |             parent_connections = [p for p in task.parents if p.kind == csimdag.TaskKind.TASK_KIND_COMM_E2E]
127 |             comm_times = [conn.get_ecomt(conn.parents[0].hosts[0], host) for conn in parent_connections]
128 |             task_time = (max(comm_times) if comm_times else 0.) + task.get_eet(host)
129 |             self._estimate_cache[(task, host)] = task_time
130 |         return max(est, clock) + task_time
131 | 
132 | 
133 | class DynamicBatchMin(DynamicBatchScheduler):
134 |     """
135 |     Batch-mode MinMin scheduler.
136 | 
137 |     Schedules all currently schedulable tasks to a best host by ECT in a single batch.
138 | 
139 |     The order in a batch is determined by a heuristic:
140 |     MinMin prioritizes the tasks with minimum ECT on a best host.
141 |     """
142 | 
143 |     def batch_heuristic(self, possible_schedules):
144 |         return min(possible_schedules, key=lambda s: (s[2], s[0]))[:-1]
145 | 
146 | 
147 | class DynamicBatchMax(DynamicBatchScheduler):
148 |     """
149 |     Batch-mode MaxMin scheduler.
150 | 
151 |     Schedules all currently schedulable tasks to a best host by ECT in a single batch.
152 | 
153 |     The order in a batch is determined by a heuristic:
154 |     MaxMin prioritizes the tasks with maximum ECT on a best host
155 |     """
156 | 
157 |     def batch_heuristic(self, possible_schedules):
158 |         return max(possible_schedules, key=lambda s: (s[2], s[0]))[:-1]
159 | 
160 | 
161 | class DynamicBatchSufferage(DynamicBatchScheduler):
162 |     """
163 |     Batch-mode Sufferage scheduler.
164 | 
165 |     Schedules all currently schedulable tasks to a best host by ECT in a single batch.
166 | 
167 |     The order in a batch is determined by a heuristic:
168 |     Sufferage prioritizes tasks with maximum difference between ECT on 2 best hosts
169 |     """
170 | 
171 |     def batch_heuristic(self, possible_schedules):
172 |         return max(possible_schedules, key=lambda s: (s[3], s[0]))[:-1]
173 | 


--------------------------------------------------------------------------------
/dag/analyse.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Experiment results analysis tool.
  3 | 
  4 | Not yet generic. Should be included in pysimgrid.tools when is.
  5 | """
  6 | 
  7 | from __future__ import print_function
  8 | 
  9 | import argparse
 10 | import collections
 11 | import json
 12 | import numpy
 13 | import os
 14 | import textwrap
 15 | 
 16 | 
 17 | def groupby(results, condition, asitems=True):
 18 |     groups = collections.defaultdict(list)
 19 |     for data in results:
 20 |         key = condition(data)
 21 |         groups[key].append(data)
 22 |     return groups.items() if asitems else groups
 23 | 
 24 | 
 25 | def get_app_filename(item):
 26 |     return str(os.path.basename(item["tasks"]).rsplit(".", 1)[0])
 27 | 
 28 | 
 29 | def get_app_group(item):
 30 |     return "_".join(get_app_filename(item).split("_")[:2])
 31 | 
 32 | 
 33 | def get_fat(item):
 34 |     return float(get_app_filename(item).split("_")[2])
 35 | 
 36 | 
 37 | def get_ccr(item):
 38 |     return float(get_app_filename(item).split("_")[8])
 39 | 
 40 | 
 41 | def get_platform_name(item):
 42 |     return os.path.basename(item["platform"])
 43 | 
 44 | 
 45 | def get_host_count(item):
 46 |     return int(get_platform_name(item).split("_")[1])
 47 | 
 48 | 
 49 | def get_algorithm(item):
 50 |     return item["algorithm"]["name"]
 51 | 
 52 | 
 53 | def main():
 54 |     algorithms = ["OLB", "MCT", "MinMin", "MaxMin", "Sufferage", "DLS", "HEFT", "HCPT", "Lookahead", "PEFT"]
 55 | 
 56 |     metrics = [
 57 |         "makespan",
 58 |         "norm_makespan",
 59 |         "norm_exp_makespan"
 60 |     ]
 61 | 
 62 |     groups = {
 63 |         "app": {"label": "Application", "cond": get_app_group},
 64 |         "hcount": {"label": "Hosts", "cond": get_host_count},
 65 |         "fat": {"label": "Fat", "cond": get_fat},
 66 |         "ccr": {"label": "CCR", "cond": get_ccr}
 67 |     }
 68 | 
 69 |     output_modes = ["plain", "latex"]
 70 | 
 71 |     parser = argparse.ArgumentParser(description="Experiment results analysis")
 72 |     parser.add_argument("input_file", type=str, help="experiment results")
 73 |     parser.add_argument("-a", "--algorithms", type=str, default=",".join(algorithms), help="algorithms")
 74 |     parser.add_argument("-m", "--metric", type=str, default="norm_makespan", choices=list(metrics), help="metric")
 75 |     parser.add_argument("-g1", "--group1", type=str, default="app", choices=list(groups), help="group 1")
 76 |     parser.add_argument("-g2", "--group2", type=str, default="hcount", choices=list(groups), help="group 2")
 77 |     parser.add_argument("-o", "--output", type=str, default="plain", choices=list(output_modes), help="output mode")
 78 |     parser.add_argument("--std", action='store_true', help="print standard deviation")
 79 | 
 80 |     args = parser.parse_args()
 81 |     algorithms = args.algorithms.split(',')
 82 |     group1 = groups.get(args.group1)
 83 |     group2 = groups.get(args.group2)
 84 | 
 85 |     with open(args.input_file) as input_file:
 86 |         results = json.load(input_file)
 87 | 
 88 |     compute_metric(results, args.metric)
 89 |     grouped = group_results(results, algorithms, group1.get("cond"), group2.get("cond"))
 90 | 
 91 |     if args.output == 'plain':
 92 |         output_plain(grouped, algorithms, group1.get("label"), group2.get("label"), args.std)
 93 |     elif args.output == 'latex':
 94 |         output_latex(grouped, algorithms, group1.get("label"), group2.get("label"), args.std)
 95 | 
 96 | 
 97 | def compute_metric(results, metric):
 98 |     for task, bytask in groupby(results, get_app_filename):
 99 |         for platform, byplat in groupby(bytask, get_platform_name):
100 |             algorithm_results = groupby(byplat, get_algorithm, False)
101 |             for algorithm, byalg in algorithm_results.items():
102 |                 if metric == "makespan":
103 |                     byalg[0]["result"] = byalg[0]["makespan"]
104 |                 if metric == "norm_makespan":
105 |                     reference_algo = "OLB"
106 |                     assert len(algorithm_results[reference_algo]) == 1
107 |                     reference = algorithm_results[reference_algo][0]
108 |                     byalg[0]["result"] = byalg[0]["makespan"] / reference["makespan"]
109 |                 elif metric == "norm_exp_makespan":
110 |                     byalg[0]["result"] = byalg[0]["expected_makespan"] / byalg[0]["makespan"]
111 | 
112 | 
113 | def group_results(results, algorithms, cond1, cond2):
114 |     grouped_results = []
115 |     for c1, bycond1 in sorted(groupby(results, cond1)):
116 |         res1 = {'group': c1, 'results': []}
117 |         for c2, bycond2 in sorted(groupby(bycond1, cond2)):
118 |             res2 = {'group': c2, 'results': []}
119 |             for algorithm, byalg in sorted(groupby(bycond2, get_algorithm), key=lambda pair: algorithms.index(pair[0])):
120 |                 mean = numpy.mean([r["result"] for r in byalg])
121 |                 std = numpy.std([r["result"] for r in byalg])
122 |                 res2['results'].append({'mean': mean, 'std': std})
123 |             res1['results'].append(res2)
124 |         grouped_results.append(res1)
125 |     return grouped_results
126 | 
127 | 
128 | def output_plain(results, algorithms, label1, label2, std):
129 |     print("".ljust(16), end="")
130 |     for alg in algorithms:
131 |         if std is True:
132 |             print(alg.ljust(15), end="")
133 |         else:
134 |             print(alg[:6].ljust(8), end="")
135 |     print("")
136 |     for group1_res in results:
137 |         print("%s=%s" % (label1, str(group1_res['group'])))
138 |         for group2_res in group1_res['results']:
139 |             print(("  %s=%s" % (label2, str(group2_res['group']))).ljust(16), end="")
140 |             for res in group2_res['results']:
141 |                 if std is True:
142 |                     print(("%5.3f (%5.3f)" % (res['mean'], res['std'])).ljust(15), end="")
143 |                 else:
144 |                     print(("%5.3f" % res['mean']).ljust(8), end="")
145 |             print("")
146 | 
147 | 
148 | def par(string):
149 |     return textwrap.dedent(string).strip()
150 | 
151 | 
152 | def output_latex(results, algorithms, label1, label2, std):
153 |     print(par(r"""
154 |   \begin{table}
155 |   \caption{TODO}
156 |   \begin{center}
157 |      \small\begin{tabular}{*{%d}{l}}
158 |   \toprule
159 |   %s & %s \\ \midrule
160 |   """) % (len(algorithms) + 1, label2, " &  ".join(algorithms),))
161 |     for group1_res in results:
162 |         print(par(r"""
163 |     \multicolumn{%d}{l}{%s=%s} \\ \midrule
164 |     """) % (len(algorithms) + 1, label1, str(group1_res['group'])))
165 |         for group2_res in group1_res['results']:
166 |             print("%-15s" % str(group2_res['group']), end="")
167 |             for res in group2_res['results']:
168 |                 if std is True:
169 |                     print(" & %5.3f (%5.3f)" % (res['mean'], res['std']), end="")
170 |                 else:
171 |                     print(" & %5.3f" % res['mean'], end="")
172 |             print(r" \\")
173 |         print(r"\midrule")
174 |     print(par(r"""
175 |   \bottomrule
176 |   \end{tabular}
177 |   \end{center}
178 |   \label{tab:TODO}
179 |   \end{table}
180 |   """))
181 | 
182 | 
183 | if __name__ == "__main__":
184 |     main()
185 | 


--------------------------------------------------------------------------------
/pysimgrid/simdag/algorithms/batch.py:
--------------------------------------------------------------------------------
  1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
  2 | #
  3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
  4 | #
  5 | # This library is free software: you can redistribute it and/or modify
  6 | # it under the terms of the GNU Lesser General Public License as published by
  7 | # the Free Software Foundation, either version 3 of the License, or
  8 | # (at your option) any later version.
  9 | #
 10 | # This library is distributed in the hope that it will be useful,
 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13 | # GNU Lesser General Public License for more details.
 14 | #
 15 | # You should have received a copy of the GNU Lesser General Public
 16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
 17 | #
 18 | 
 19 | import logging
 20 | 
 21 | import numpy as np
 22 | 
 23 | from .. import scheduler
 24 | from ... import csimdag
 25 | 
 26 | 
 27 | class BatchScheduler(scheduler.DynamicScheduler):
 28 |     """
 29 |     Batch-mode heuristic base implementation.
 30 | 
 31 |     Schedules all currently schedulable tasks to a best host by ECT in a single batch.
 32 | 
 33 |     The order in a batch is determined by a heuristic:
 34 | 
 35 |     * MinMin prioritizes the tasks with minimum ECT on a best host
 36 | 
 37 |     * MaxMin prioritizes the tasks with maximum ECT on a best host
 38 | 
 39 |     * Sufferage prioritizes tasks with maximum difference between ECT on 2 best hosts
 40 |     """
 41 | 
 42 |     def prepare(self, simulation):
 43 |         master_hosts = simulation.hosts.by_prop("name", self.MASTER_HOST_NAME)
 44 |         self.master_host = master_hosts[0] if master_hosts else None
 45 |         self.exec_hosts = simulation.hosts.by_prop("name", self.MASTER_HOST_NAME, True)
 46 |         for h in self.exec_hosts:
 47 |             h.data = {
 48 |                 "est": 0.,
 49 |                 "is_free": True,
 50 |                 "tasks": []
 51 |             }
 52 | 
 53 |         self.tasks = simulation.tasks
 54 |         for t in self.tasks:
 55 |             t.data = {
 56 |                 "target_host": None
 57 |             }
 58 | 
 59 |         # schedule root/end tasks to master
 60 |         if self.master_host:
 61 |             for task in self.tasks.by_func(lambda t: t.name in self.BOUNDARY_TASKS):
 62 |                 # logging.info("%s -> %s" % (task.name, self.master_host.name))
 63 |                 task.schedule(self.master_host)
 64 | 
 65 |     def schedule(self, simulation, changed):
 66 |         clock = simulation.clock
 67 | 
 68 |         # mark freed hosts
 69 |         for task in changed:
 70 |             if task.kind == csimdag.TaskKind.TASK_KIND_COMP_SEQ and task.state == csimdag.TaskState.TASK_STATE_DONE:
 71 |                 host = task.hosts[0]
 72 |                 if host != self.master_host:
 73 |                     task.hosts[0].data["is_free"] = True
 74 | 
 75 |         # schedule unscheduled ready tasks
 76 |         unscheduled = self.tasks[csimdag.TaskState.TASK_STATE_SCHEDULABLE]\
 77 |                           .by_func(lambda task: task.data["target_host"] is None)
 78 |         num_unscheduled = len(unscheduled)
 79 |         if num_unscheduled > 0:
 80 | 
 81 |             # build ECT matrix
 82 |             ECT = np.zeros((num_unscheduled, len(self.exec_hosts)))
 83 |             for t, task in enumerate(unscheduled):
 84 |                 parents = [(p, p.parents[0].hosts[0]) for p in task.parents if p.kind == csimdag.TaskKind.TASK_KIND_COMM_E2E]
 85 |                 for h, host in enumerate(self.exec_hosts):
 86 |                     comm_times = [p_comm.get_ecomt(p_host, host) for (p_comm, p_host) in parents]
 87 |                     ECT[t][h] = max(host.data["est"], clock) + task.get_eet(host) + (max(comm_times) if comm_times else 0.)
 88 | 
 89 |             # build schedule
 90 |             task_idx = np.arange(num_unscheduled)
 91 |             for _ in range(0, num_unscheduled):
 92 |                 min_hosts = np.argmin(ECT, axis=1)
 93 |                 min_times = ECT[np.arange(ECT.shape[0]), min_hosts]
 94 | 
 95 |                 if ECT.shape[1] > 1:
 96 |                     min2_times = np.partition(ECT, 1)[:, 1]
 97 |                     # round sufferage values to eliminate the influence of floating point errors
 98 |                     sufferages = np.round(min2_times - min_times, decimals=2)
 99 |                 else:
100 |                     # use min time for a single host case
101 |                     sufferages = min_times
102 | 
103 |                 possible_schedules = []
104 |                 for i in range(0, len(task_idx)):
105 |                     best_host_idx = int(min_hosts[i])
106 |                     best_ect = min_times[i]
107 |                     sufferage = sufferages[i]
108 |                     possible_schedules.append((i, best_host_idx, best_ect, sufferage))
109 | 
110 |                 t, h, ect = self.batch_heuristic(possible_schedules)
111 |                 task = unscheduled[int(task_idx[t])]
112 |                 host = self.exec_hosts[h]
113 |                 # logging.info("%s -> %s" % (task.name, host.name))
114 | 
115 |                 task.data["target_host"] = host.name
116 |                 host.data["tasks"].append(task)
117 |                 task_time = ect - max(host.data["est"], clock)
118 |                 host.data["est"] = ect
119 | 
120 |                 task_idx = np.delete(task_idx, t)
121 |                 ECT = np.delete(ECT, t, 0)
122 |                 ECT[:,h] += task_time
123 | 
124 |         for host in self.exec_hosts:
125 |             if host.data["is_free"]:
126 |                 tasks = host.data["tasks"]
127 |                 if len(tasks) > 0:
128 |                     task = tasks.pop(0)
129 |                     task.schedule(host)
130 |                     host.data["is_free"] = False
131 |                     # logging.info("%s -> %s" % (task.name, host.name))
132 | 
133 | 
134 | class BatchMin(BatchScheduler):
135 |     """
136 |     Batch-mode MinMin scheduler.
137 | 
138 |     Schedules all currently schedulable tasks to a best host by ECT in a single batch.
139 | 
140 |     The order in a batch is determined by a heuristic:
141 |     MinMin prioritizes the tasks with minimum ECT on a best host.
142 |     """
143 | 
144 |     def batch_heuristic(self, possible_schedules):
145 |         return min(possible_schedules, key=lambda s: (s[2], s[0]))[:-1]
146 | 
147 | 
148 | class BatchMax(BatchScheduler):
149 |     """
150 |     Batch-mode MaxMin scheduler.
151 | 
152 |     Schedules all currently schedulable tasks to a best host by ECT in a single batch.
153 | 
154 |     The order in a batch is determined by a heuristic:
155 |     MaxMin prioritizes the tasks with maximum ECT on a best host
156 |     """
157 | 
158 |     def batch_heuristic(self, possible_schedules):
159 |         return max(possible_schedules, key=lambda s: (s[2], -s[0]))[:-1]
160 | 
161 | 
162 | class BatchSufferage(BatchScheduler):
163 |     """
164 |     Batch-mode Sufferage scheduler.
165 | 
166 |     Schedules all currently schedulable tasks to a best host by ECT in a single batch.
167 | 
168 |     The order in a batch is determined by a heuristic:
169 |     Sufferage prioritizes tasks with maximum difference between ECT on 2 best hosts
170 |     """
171 | 
172 |     def batch_heuristic(self, possible_schedules):
173 |         return max(possible_schedules, key=lambda s: (s[3], -s[0]))[:-1]
174 |         # TODO: use ECT on a best host for breaking ties?
175 |         # return max(possible_schedules, key=lambda s: (s[3], s[2], -s[0]))[:-1]
176 | 


--------------------------------------------------------------------------------
/doc/source/_themes/sphinx_rtd_theme/layout.html:
--------------------------------------------------------------------------------
  1 | {# TEMPLATE VAR SETTINGS #}
  2 | {%- set url_root = pathto('', 1) %}
  3 | {%- if url_root == '#' %}{% set url_root = '' %}{% endif %}
  4 | {%- if not embedded and docstitle %}
  5 |   {%- set titlesuffix = " &mdash; "|safe + docstitle|e %}
  6 | {%- else %}
  7 |   {%- set titlesuffix = "" %}
  8 | {%- endif %}
  9 | 
 10 | <!DOCTYPE html>
 11 | <!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
 12 | <!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
 13 | <head>
 14 |   <meta charset="utf-8">
 15 |   {{ metatags }}
 16 |   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 17 |   {% block htmltitle %}
 18 |   <title>{{ title|striptags|e }}{{ titlesuffix }}</title>
 19 |   {% endblock %}
 20 | 
 21 |   {# FAVICON #}
 22 |   {% if favicon %}
 23 |     <link rel="shortcut icon" href="{{ pathto('_static/' + favicon, 1) }}"/>
 24 |   {% endif %}
 25 | 
 26 |   {# CSS #}
 27 | 
 28 |   {# OPENSEARCH #}
 29 |   {% if not embedded %}
 30 |     {% if use_opensearch %}
 31 |       <link rel="search" type="application/opensearchdescription+xml" title="{% trans docstitle=docstitle|e %}Search within {{ docstitle }}{% endtrans %}" href="{{ pathto('_static/opensearch.xml', 1) }}"/>
 32 |     {% endif %}
 33 | 
 34 |   {% endif %}
 35 | 
 36 |   {# RTD hosts this file, so just load on non RTD builds #}
 37 |   {% if not READTHEDOCS %}
 38 |     <link rel="stylesheet" href="{{ pathto('_static/' + style, 1) }}" type="text/css" />
 39 |   {% endif %}
 40 | 
 41 |   {% for cssfile in css_files %}
 42 |     <link rel="stylesheet" href="{{ pathto(cssfile, 1) }}" type="text/css" />
 43 |   {% endfor %}
 44 | 
 45 |   {% for cssfile in extra_css_files %}
 46 |     <link rel="stylesheet" href="{{ pathto(cssfile, 1) }}" type="text/css" />
 47 |   {% endfor %}
 48 | 
 49 |   {%- block linktags %}
 50 |     {%- if hasdoc('about') %}
 51 |         <link rel="author" title="{{ _('About these documents') }}"
 52 |               href="{{ pathto('about') }}"/>
 53 |     {%- endif %}
 54 |     {%- if hasdoc('genindex') %}
 55 |         <link rel="index" title="{{ _('Index') }}"
 56 |               href="{{ pathto('genindex') }}"/>
 57 |     {%- endif %}
 58 |     {%- if hasdoc('search') %}
 59 |         <link rel="search" title="{{ _('Search') }}" href="{{ pathto('search') }}"/>
 60 |     {%- endif %}
 61 |     {%- if hasdoc('copyright') %}
 62 |         <link rel="copyright" title="{{ _('Copyright') }}" href="{{ pathto('copyright') }}"/>
 63 |     {%- endif %}
 64 |     <link rel="top" title="{{ docstitle|e }}" href="{{ pathto('index') }}"/>
 65 |     {%- if parents %}
 66 |         <link rel="up" title="{{ parents[-1].title|striptags|e }}" href="{{ parents[-1].link|e }}"/>
 67 |     {%- endif %}
 68 |     {%- if next %}
 69 |         <link rel="next" title="{{ next.title|striptags|e }}" href="{{ next.link|e }}"/>
 70 |     {%- endif %}
 71 |     {%- if prev %}
 72 |         <link rel="prev" title="{{ prev.title|striptags|e }}" href="{{ prev.link|e }}"/>
 73 |     {%- endif %}
 74 |   {%- endblock %}
 75 |   {%- block extrahead %} {% endblock %}
 76 | 
 77 |   {# Keep modernizr in head - http://modernizr.com/docs/#installing #}
 78 |   <script src="{{ pathto('_static/js/modernizr.min.js', 1) }}"></script>
 79 | 
 80 | </head>
 81 | 
 82 | <body class="wy-body-for-nav" role="document">
 83 | 
 84 |   {% block extrabody %} {% endblock %}
 85 |   <div class="wy-grid-for-nav">
 86 | 
 87 |     {# SIDE NAV, TOGGLES ON MOBILE #}
 88 |     <nav data-toggle="wy-nav-shift" class="wy-nav-side">
 89 |       <div class="wy-side-scroll">
 90 |         <div class="wy-side-nav-search">
 91 |           {% block sidebartitle %}
 92 | 
 93 |           {% if logo and theme_logo_only %}
 94 |             <a href="{{ pathto(master_doc) }}">
 95 |           {% else %}
 96 |             <a href="{{ pathto(master_doc) }}" class="icon icon-home"> {{ project }}
 97 |           {% endif %}
 98 | 
 99 |           {% if logo %}
100 |             {# Not strictly valid HTML, but it's the only way to display/scale it properly, without weird scripting or heaps of work #}
101 |             <img src="{{ pathto('_static/' + logo, 1) }}" class="logo" />
102 |           {% endif %}
103 |           </a>
104 | 
105 |           {% if theme_display_version %}
106 |             {%- set nav_version = version %}
107 |             {% if READTHEDOCS and current_version %}
108 |               {%- set nav_version = current_version %}
109 |             {% endif %}
110 |             {% if nav_version %}
111 |               <div class="version">
112 |                 {{ nav_version }}
113 |               </div>
114 |             {% endif %}
115 |           {% endif %}
116 | 
117 |           {% include "searchbox.html" %}
118 | 
119 |           {% endblock %}
120 |         </div>
121 | 
122 |         <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
123 |           {% block menu %}
124 |             {% set toctree = toctree(maxdepth=theme_navigation_depth|int, collapse=theme_collapse_navigation, includehidden=True) %}
125 |             {% if toctree %}
126 |                 {{ toctree }}
127 |             {% else %}
128 |                 <!-- Local TOC -->
129 |                 <div class="local-toc">{{ toc }}</div>
130 |             {% endif %}
131 |           {% endblock %}
132 |         </div>
133 |       </div>
134 |     </nav>
135 | 
136 |     <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
137 | 
138 |       {# MOBILE NAV, TRIGGLES SIDE NAV ON TOGGLE #}
139 |       <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
140 |         {% block mobile_nav %}
141 |           <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
142 |           <a href="{{ pathto(master_doc) }}">{{ project }}</a>
143 |         {% endblock %}
144 |       </nav>
145 | 
146 | 
147 |       {# PAGE CONTENT #}
148 |       <div class="wy-nav-content">
149 |         <div class="rst-content">
150 |           {% include "breadcrumbs.html" %}
151 |           <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
152 |            <div itemprop="articleBody">
153 |             {% block body %}{% endblock %}
154 |            </div>
155 |            <div class="articleComments">
156 |             {% block comments %}{% endblock %}
157 |            </div>
158 |           </div>
159 |           {% include "footer.html" %}
160 |         </div>
161 |       </div>
162 | 
163 |     </section>
164 | 
165 |   </div>
166 |   {% include "versions.html" %}
167 | 
168 |   {% if not embedded %}
169 | 
170 |     <script type="text/javascript">
171 |         var DOCUMENTATION_OPTIONS = {
172 |             URL_ROOT:'{{ url_root }}',
173 |             VERSION:'{{ release|e }}',
174 |             COLLAPSE_INDEX:false,
175 |             FILE_SUFFIX:'{{ '' if no_search_suffix else file_suffix }}',
176 |             HAS_SOURCE:  {{ has_source|lower }},
177 |             SOURCELINK_SUFFIX: '{{ sourcelink_suffix }}'
178 |         };
179 |     </script>
180 |     {%- for scriptfile in script_files %}
181 |       <script type="text/javascript" src="{{ pathto(scriptfile, 1) }}"></script>
182 |     {%- endfor %}
183 | 
184 |   {% endif %}
185 | 
186 |   {# RTD hosts this file, so just load on non RTD builds #}
187 |   {% if not READTHEDOCS %}
188 |     <script type="text/javascript" src="{{ pathto('_static/js/theme.js', 1) }}"></script>
189 |   {% endif %}
190 | 
191 |   {# STICKY NAVIGATION #}
192 |   {% if theme_sticky_navigation %}
193 |   <script type="text/javascript">
194 |       jQuery(function () {
195 |           SphinxRtdTheme.StickyNav.enable();
196 |       });
197 |   </script>
198 |   {% endif %}
199 | 
200 |   {%- block footer %} {% endblock %}
201 | 
202 | </body>
203 | </html>
204 | 


--------------------------------------------------------------------------------
/pysimgrid/simdag/algorithms/dls.py:
--------------------------------------------------------------------------------
  1 | # This file is part of pysimgrid, a Python interface to the SimGrid library.
  2 | #
  3 | # Copyright 2015-2016 Alexey Nazarenko and contributors
  4 | #
  5 | # This library is free software: you can redistribute it and/or modify
  6 | # it under the terms of the GNU Lesser General Public License as published by
  7 | # the Free Software Foundation, either version 3 of the License, or
  8 | # (at your option) any later version.
  9 | #
 10 | # This library is distributed in the hope that it will be useful,
 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13 | # GNU Lesser General Public License for more details.
 14 | #
 15 | # You should have received a copy of the GNU Lesser General Public
 16 | # along with this library.  If not, see <http://www.gnu.org/licenses/>.
 17 | #
 18 | 
 19 | from collections import deque
 20 | 
 21 | import networkx
 22 | import numpy
 23 | import operator
 24 | 
 25 | from ... import cscheduling
 26 | from ..scheduler import StaticScheduler
 27 | 
 28 | 
 29 | class DLS(StaticScheduler):
 30 |   """
 31 |   Heterogeneous Dynamic Level Scheduler.
 32 | 
 33 |   Algorithm is based on heuristics which use Dynamic Level to chose pair - host and task to assign task to this host.
 34 |   DL(Task_X, Host_Y, Timetable) = SL (Task_X) + D (Task_X, Host_Y) - max( DA(Task_X, Host_Y, Timetable), TF(Host_Y, Timetable) )
 35 | 
 36 |   SL(Task_X) (Task Static Level) is the largest sum of executions times (calculated using mean speed among all hosts)
 37 |   along any directed path from Task_X to final node (including Task_X execution time)
 38 | 
 39 |   TF(Host_Y, Timetable) - time when the last assigned finish execution on Host_Y.
 40 | 
 41 |   DA(Task_X, Host_Y, Timetable) - time when all parent tasks of Task_X will be finished and transferred to the Host_Y
 42 | 
 43 |   It is more easier to understand  "max( DA(Task_X, Host_Y, Timetable), TF(Host_Y, Timetable) )" as minimal time
 44 |   when we could start Task_X on Host_Y.
 45 | 
 46 |   D (Task_X, Host_Y) - difference between time execution of Task_X using mean speed and using Host_Y speed
 47 | 
 48 |   Desc:
 49 |   At the beginning of the algorithm find all of tasks which don't have undone parents so they are ready to execute.
 50 | 
 51 |   Lets define this as set of tasks as RT and WT as a ser of all other tasks.
 52 | 
 53 |   First calculate DL for every pair (Task, Host) where Task from RT and Host from all Hosts.
 54 | 
 55 |   Then on every step:
 56 | 
 57 |    1. Choose Task and Host with maximal DL and assign this Task to Host (choosing minimal available time to start execution)
 58 | 
 59 |    2. Delete assigned Task from RT.
 60 | 
 61 |    3. Recalculate DL for every pait (Task, Host) where Task from RT and Host is Host which we choose on the step one
 62 | 
 63 |    4. Find new Tasks from WT which don't have undone parents and delete them from WT and add them to RT.
 64 | 
 65 |    5. Calculated DL for all pairs : (new Task, Host) new Task - from Tasks added on prev. step, Host from all Hosts.
 66 | 
 67 | 
 68 | 
 69 |   For more details and rationale please refer to the original publication:
 70 | 
 71 |     Gilbert C. Sih, Edward A. Lee "A Compile-Time Scheduling Heuristic for Interconnection-Constrained Heterogeneous Processor Architectures",
 72 |     Transactions on Parallel and Distributed Systems (IEEE) 1993
 73 | 
 74 |   """
 75 |   def get_schedule(self, simulation):
 76 |     """
 77 |     Overridden.
 78 |     """
 79 |     nxgraph = simulation.get_task_graph()
 80 |     platform_model = cscheduling.PlatformModel(simulation)
 81 |     state = cscheduling.SchedulerState(simulation)
 82 | 
 83 |     mean_speed = platform_model.mean_speed
 84 |     aec, sl = self.get_tasks_sl_aec(nxgraph, platform_model)
 85 |     # unreal dynamic level - used to mark deleted on not set values in a queue
 86 |     unreal_dl = 1 + max(sl.items(), key=operator.itemgetter(1))[1] + max(aec.items(), key=operator.itemgetter(1))[1]
 87 |     dl = {host: {task: unreal_dl for task in nxgraph} for host in simulation.hosts}
 88 |     undone_parents = {task: len(nxgraph.pred[task]) for task in nxgraph}
 89 |     waiting_tasks = set(nxgraph)
 90 |     queue_tasks = set()
 91 |     for task in nxgraph:
 92 |       if undone_parents[task] == 0:
 93 |         for host in simulation.hosts:
 94 |           dl[host][task] = sl[task] + (task.amount / mean_speed - task.amount / host.speed)
 95 |         waiting_tasks.remove(task)
 96 |         queue_tasks.add(task)
 97 | 
 98 |     for iterations in range(len(nxgraph)):
 99 |       cur_max = unreal_dl
100 |       task_to_schedule = -1
101 |       host_to_schedule = -1
102 |       for host in simulation.hosts:
103 |         if cscheduling.is_master_host(host):
104 |           continue
105 |         for task in queue_tasks:
106 |           if dl[host][task] == unreal_dl:
107 |             continue
108 |           if cur_max == unreal_dl or dl[host][task] > cur_max:
109 |             cur_max = dl[host][task]
110 |             host_to_schedule = host
111 |             task_to_schedule = task
112 | 
113 |       assert (cur_max != unreal_dl)
114 | 
115 |       if cscheduling.try_schedule_boundary_task(task_to_schedule, nxgraph, platform_model, state) == False:
116 |         est = platform_model.est(host_to_schedule, dict(nxgraph.pred[task_to_schedule]), state)
117 |         eet = platform_model.eet(task_to_schedule, host_to_schedule)
118 |         timesheet = state.timetable[host_to_schedule]
119 |         pos, start, finish = cscheduling.timesheet_insertion(timesheet, est, eet)
120 |         state.update(task_to_schedule, host_to_schedule, pos, start, finish)
121 | 
122 |       new_tasks = set()
123 |       for child, edge in nxgraph[task_to_schedule].items():
124 |         undone_parents[child] -= 1
125 |         if undone_parents[child] == 0:
126 |           new_tasks.add(child)
127 |           for host in simulation.hosts:
128 |             dl[host][child] = self.calculate_dl(nxgraph, platform_model, state, sl, aec, child, host)
129 | 
130 |       for host in simulation.hosts:
131 |         dl[host][task_to_schedule] = unreal_dl
132 | 
133 |       queue_tasks.remove(task_to_schedule)
134 | 
135 |       for task in queue_tasks:
136 |         if undone_parents[task] == 0:
137 |           dl[host_to_schedule][task] = self.calculate_dl(nxgraph, platform_model, state, sl, aec, task,
138 |                                                          host_to_schedule)
139 | 
140 |       for task in new_tasks:
141 |         waiting_tasks.remove(task)
142 |         queue_tasks.add(task)
143 | 
144 |     expected_makespan = max([state["ect"] for state in state.task_states.values()])
145 |     return state.schedule, expected_makespan
146 | 
147 |   @classmethod
148 |   def calculate_dl(cls, nxgraph, platform_model, state, sl, aec, task, host):
149 |     est = platform_model.est(host, dict(nxgraph.pred[task]), state)
150 |     eet = platform_model.eet(task, host)
151 |     timesheet = state.timetable[host]
152 |     pos, start, finish = cscheduling.timesheet_insertion(timesheet, est, eet)
153 |     return sl[task] + (aec[task] - task.amount / host.speed) - start
154 | 
155 |   @classmethod
156 |   def get_tasks_sl_aec(cls, nxgraph, platform_model):
157 |     """
158 |     Return Average Execution Cost and Static Level for every task.
159 | 
160 |     Args:
161 |       nxgraph: full task graph as networkx.DiGraph
162 |       platform_model: cscheduling.PlatformModel object
163 | 
164 |     Returns:
165 |         aec: task->aec_value
166 |         sl: task->static_level_value
167 |     """
168 |     mean_speed = platform_model.mean_speed
169 |     topological_order = list(reversed(list(networkx.topological_sort(nxgraph))))
170 | 
171 |     # Average execution cost
172 |     aec = {task: float(task.amount) / mean_speed for task in nxgraph}
173 | 
174 |     sl = {task: aec[task] for task in nxgraph}
175 | 
176 |     # Static Level
177 |     for task in topological_order:
178 |       for parent in nxgraph.pred[task]:
179 |         sl[parent] = max(sl[parent], sl[task] + aec[parent])
180 | 
181 |     return aec, sl
182 | 


--------------------------------------------------------------------------------
/pysimgrid/tools/dax_to_dot.py:
--------------------------------------------------------------------------------
  1 | import argparse
  2 | import networkx as nx
  3 | 
  4 | from collections import defaultdict
  5 | 
  6 | try:
  7 |     import xml.etree.cElementTree as ET
  8 | except ImportError:
  9 |     import xml.etree.ElementTree as ET
 10 | 
 11 | 
 12 | def strip_namespace(tag):
 13 |     if '}' in tag:
 14 |         tag = tag.split('}', 1)[1]
 15 |     return tag
 16 | 
 17 | 
 18 | class Task:
 19 |     def __init__(self, id, name, runtime, input_files={}, output_files={}):
 20 |         self.id = id
 21 |         self.name = name
 22 |         self.runtime = runtime
 23 |         self.input_files = input_files
 24 |         self.output_files = output_files
 25 |         self.parents = set()
 26 | 
 27 | 
 28 | class File:
 29 |     def __init__(self, name, size, transfer):
 30 |         self.name = name
 31 |         self.size = size
 32 |         self.transfer = transfer
 33 | 
 34 | 
 35 | def main():
 36 |     parser = argparse.ArgumentParser(description="DAX to DOT converter")
 37 |     parser.add_argument("input_file", type=str, help="path to DAX file")
 38 |     parser.add_argument("output_file", type=str, help="path to DOT file")
 39 |     parser.add_argument("-s", "--speed", type=int, default=4200000000,
 40 |                         help="host speed in flop/s used to convert task runtime to flop, "
 41 |                              "default value is 4200000000 (see simdag/sd_daxloader.cpp)")
 42 |     parser.add_argument("--no-boundary", action="store_true", default=False, help="omit root and end tasks")
 43 | 
 44 |     args = parser.parse_args()
 45 | 
 46 |     tree = ET.parse(args.input_file)
 47 |     tasks = {}
 48 |     file_producers = defaultdict(list)
 49 |     file_consumers = defaultdict(list)
 50 | 
 51 |     root = Task('root', 'root', 0.)
 52 |     end = Task('end', 'end', 0.)
 53 |     tasks[root.id] = root
 54 |     tasks[end.id] = end
 55 | 
 56 |     for el in tree.getroot():
 57 |         tag = strip_namespace(el.tag)
 58 |         if tag == 'job':
 59 |             task_id = el.attrib['id']
 60 |             task_name = el.attrib['name']
 61 |             runtime = float(el.attrib['runtime'])
 62 |             input_files = {}
 63 |             output_files = {}
 64 |             for sub_el in el:
 65 |                 sub_tag = strip_namespace(sub_el.tag)
 66 |                 if sub_tag == 'uses':
 67 |                     file_name = sub_el.attrib['file']
 68 |                     file_size = float(sub_el.attrib['size'])
 69 |                     link = sub_el.attrib['link']
 70 |                     transfer = sub_el.attrib['transfer']
 71 | 
 72 |                     file = File(file_name, file_size, transfer)
 73 | 
 74 |                     if link == 'input':
 75 |                         input_files[file_name] = file
 76 |                         file_consumers[file_name].append(task_id)
 77 |                     elif link == 'output':
 78 |                         output_files[file_name] = file
 79 |                         # assert file_name not in file_producers, \
 80 |                         #     "Multiple producers for file %s: %s, %s" % (file_name, file_producers[file_name], task_id)
 81 |                         file_producers[file_name].append(task_id)
 82 |             task = Task(task_id, task_name, runtime, input_files, output_files)
 83 |             assert task_id not in tasks, "Duplicate task id"
 84 |             tasks[task_id] = task
 85 |             # dag.add_node(task, weight=task.runtime)
 86 |         elif tag == 'child':
 87 |             task = tasks[el.attrib['ref']]
 88 |             for sub_el in el:
 89 |                 sub_tag = strip_namespace(sub_el.tag)
 90 |                 if sub_tag == 'parent':
 91 |                     parent = tasks[sub_el.attrib['ref']]
 92 |                     task.parents.add(parent.id)
 93 | 
 94 |     dag = nx.DiGraph()
 95 | 
 96 |     for task in tasks.values():
 97 |         weight = task.runtime * args.speed
 98 |         dag.add_node(task, weight=weight)
 99 | 
100 |     for task in tasks.values():
101 |         for file in task.input_files.values():
102 |             if file.name in file_producers:
103 |                 # assert file.transfer == 'false'
104 |                 producers = file_producers[file.name]
105 |             else:
106 |                 # input files not produced in DAG are outputs of the end task
107 |                 # assert file.transfer == 'true', 'Wrong file transfer for file %s in task %s?' % (file.name, task.id)
108 |                 root = tasks['root']
109 |                 if file.name not in root.output_files:
110 |                     root.output_files[file.name] = file
111 |                 producers = [root.id]
112 |                 task.parents.add(root.id)
113 |             for producer_id in producers:
114 |                 if producer_id in task.parents:
115 |                     parent = tasks[producer_id]
116 |                     parent_file = parent.output_files[file.name]
117 |                     weight = file.size
118 |                     try:
119 |                         assert parent_file.size == file.size, \
120 |                             "File %s sizes differ: producer %s %f, consumer %s %f)" % \
121 |                             (file.name, parent.id, parent_file.size, task.id, file.size)
122 |                     except AssertionError as e:
123 |                         print(e)
124 |                         weight = parent_file.size
125 | 
126 |                     # pysimgrid does not support multigraphs, i.e. multiple data transfers between same tasks
127 |                     # (see simulation.get_task_graph())
128 |                     # therefore multiple data transfers are converted to a single edge with total data size
129 |                     if not dag.has_edge(parent, task):
130 |                         dag.add_edge(parent, task, weight=weight)
131 |                     else:
132 |                         print("!!! Duplicate edge: %s -> %s" % (parent.id, task.id))
133 |                         dag[parent][task]['weight'] += weight
134 | 
135 |         # output files not consumed in DAG are inputs to the end task
136 |         for file in task.output_files.values():
137 |             if file.name not in file_consumers or len(file_consumers[file.name]) == 0:
138 |                 assert file.transfer == 'true'
139 |                 if not dag.has_edge(task, tasks['end']):
140 |                     dag.add_edge(task, tasks['end'], weight=file.size)
141 |                     tasks['end'].parents.add(task.id)
142 |                 else:
143 |                     dag[task][tasks['end']]['weight'] += file.size
144 |             # else:
145 |             #     assert file.transfer == 'false', 'Wrong file transfer for file %s in task %s?' % (file.name, task.id)
146 | 
147 |         # check that edges for all parent tasks exist
148 |         for parent_id in task.parents:
149 |             parent = tasks[parent_id]
150 |             assert dag.has_edge(parent, task), "Non-data dependency: %s -> %s" % (parent_id, task.id)
151 | 
152 |         # check that no edges for non-parent tasks exist
153 |         for src, dst in dag.edges():
154 |             assert src.id in dst.parents, "Wrong edge, %s is not parent of %s" % (src.id, dst.id)
155 | 
156 |     with open(args.output_file, 'w') as out:
157 |         out.write('digraph DAG {\n')
158 |         out.write('  ranksep=5.0\n')
159 |         out.write('  node [style=filled,color="#444444",fillcolor="#ffed6f"]\n')
160 |         out.write('  edge [arrowhead=normal,arrowsize=1.0]\n')
161 |         out.write("\n")
162 |         for task, data in dag.nodes(True):
163 |             if not args.no_boundary or task.id not in ['root', 'end']:
164 |                 if task.name != task.id:
165 |                     task_label = task.name + "_" + task.id
166 |                 else:
167 |                     task_label = task.name
168 |                 out.write('  %s [label="%s",size="%e"];\n' % (task.id, task_label, data["weight"]))
169 |         out.write("\n")
170 |         for src, dst, data in dag.edges(data='weight'):
171 |             if not args.no_boundary or (src.id not in ['root', 'end'] and dst.id not in ['root', 'end']):
172 |                 out.write('  %s -> %s [size="%e"];\n' % (src.id, dst.id, data))
173 |         out.write("}\n")
174 | 
175 | 
176 | if __name__ == "__main__":
177 |     main()
178 | 


--------------------------------------------------------------------------------
/LICENSE.LGPL:
--------------------------------------------------------------------------------
  1 |                    GNU LESSER GENERAL PUBLIC LICENSE
  2 |                        Version 3, 29 June 2007
  3 | 
  4 |  Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
  5 |  Everyone is permitted to copy and distribute verbatim copies
  6 |  of this license document, but changing it is not allowed.
  7 | 
  8 | 
  9 |   This version of the GNU Lesser General Public License incorporates
 10 | the terms and conditions of version 3 of the GNU General Public
 11 | License, supplemented by the additional permissions listed below.
 12 | 
 13 |   0. Additional Definitions.
 14 | 
 15 |   As used herein, "this License" refers to version 3 of the GNU Lesser
 16 | General Public License, and the "GNU GPL" refers to version 3 of the GNU
 17 | General Public License.
 18 | 
 19 |   "The Library" refers to a covered work governed by this License,
 20 | other than an Application or a Combined Work as defined below.
 21 | 
 22 |   An "Application" is any work that makes use of an interface provided
 23 | by the Library, but which is not otherwise based on the Library.
 24 | Defining a subclass of a class defined by the Library is deemed a mode
 25 | of using an interface provided by the Library.
 26 | 
 27 |   A "Combined Work" is a work produced by combining or linking an
 28 | Application with the Library.  The particular version of the Library
 29 | with which the Combined Work was made is also called the "Linked
 30 | Version".
 31 | 
 32 |   The "Minimal Corresponding Source" for a Combined Work means the
 33 | Corresponding Source for the Combined Work, excluding any source code
 34 | for portions of the Combined Work that, considered in isolation, are
 35 | based on the Application, and not on the Linked Version.
 36 | 
 37 |   The "Corresponding Application Code" for a Combined Work means the
 38 | object code and/or source code for the Application, including any data
 39 | and utility programs needed for reproducing the Combined Work from the
 40 | Application, but excluding the System Libraries of the Combined Work.
 41 | 
 42 |   1. Exception to Section 3 of the GNU GPL.
 43 | 
 44 |   You may convey a covered work under sections 3 and 4 of this License
 45 | without being bound by section 3 of the GNU GPL.
 46 | 
 47 |   2. Conveying Modified Versions.
 48 | 
 49 |   If you modify a copy of the Library, and, in your modifications, a
 50 | facility refers to a function or data to be supplied by an Application
 51 | that uses the facility (other than as an argument passed when the
 52 | facility is invoked), then you may convey a copy of the modified
 53 | version:
 54 | 
 55 |    a) under this License, provided that you make a good faith effort to
 56 |    ensure that, in the event an Application does not supply the
 57 |    function or data, the facility still operates, and performs
 58 |    whatever part of its purpose remains meaningful, or
 59 | 
 60 |    b) under the GNU GPL, with none of the additional permissions of
 61 |    this License applicable to that copy.
 62 | 
 63 |   3. Object Code Incorporating Material from Library Header Files.
 64 | 
 65 |   The object code form of an Application may incorporate material from
 66 | a header file that is part of the Library.  You may convey such object
 67 | code under terms of your choice, provided that, if the incorporated
 68 | material is not limited to numerical parameters, data structure
 69 | layouts and accessors, or small macros, inline functions and templates
 70 | (ten or fewer lines in length), you do both of the following:
 71 | 
 72 |    a) Give prominent notice with each copy of the object code that the
 73 |    Library is used in it and that the Library and its use are
 74 |    covered by this License.
 75 | 
 76 |    b) Accompany the object code with a copy of the GNU GPL and this license
 77 |    document.
 78 | 
 79 |   4. Combined Works.
 80 | 
 81 |   You may convey a Combined Work under terms of your choice that,
 82 | taken together, effectively do not restrict modification of the
 83 | portions of the Library contained in the Combined Work and reverse
 84 | engineering for debugging such modifications, if you also do each of
 85 | the following:
 86 | 
 87 |    a) Give prominent notice with each copy of the Combined Work that
 88 |    the Library is used in it and that the Library and its use are
 89 |    covered by this License.
 90 | 
 91 |    b) Accompany the Combined Work with a copy of the GNU GPL and this license
 92 |    document.
 93 | 
 94 |    c) For a Combined Work that displays copyright notices during
 95 |    execution, include the copyright notice for the Library among
 96 |    these notices, as well as a reference directing the user to the
 97 |    copies of the GNU GPL and this license document.
 98 | 
 99 |    d) Do one of the following:
100 | 
101 |        0) Convey the Minimal Corresponding Source under the terms of this
102 |        License, and the Corresponding Application Code in a form
103 |        suitable for, and under terms that permit, the user to
104 |        recombine or relink the Application with a modified version of
105 |        the Linked Version to produce a modified Combined Work, in the
106 |        manner specified by section 6 of the GNU GPL for conveying
107 |        Corresponding Source.
108 | 
109 |        1) Use a suitable shared library mechanism for linking with the
110 |        Library.  A suitable mechanism is one that (a) uses at run time
111 |        a copy of the Library already present on the user's computer
112 |        system, and (b) will operate properly with a modified version
113 |        of the Library that is interface-compatible with the Linked
114 |        Version.
115 | 
116 |    e) Provide Installation Information, but only if you would otherwise
117 |    be required to provide such information under section 6 of the
118 |    GNU GPL, and only to the extent that such information is
119 |    necessary to install and execute a modified version of the
120 |    Combined Work produced by recombining or relinking the
121 |    Application with a modified version of the Linked Version. (If
122 |    you use option 4d0, the Installation Information must accompany
123 |    the Minimal Corresponding Source and Corresponding Application
124 |    Code. If you use option 4d1, you must provide the Installation
125 |    Information in the manner specified by section 6 of the GNU GPL
126 |    for conveying Corresponding Source.)
127 | 
128 |   5. Combined Libraries.
129 | 
130 |   You may place library facilities that are a work based on the
131 | Library side by side in a single library together with other library
132 | facilities that are not Applications and are not covered by this
133 | License, and convey such a combined library under terms of your
134 | choice, if you do both of the following:
135 | 
136 |    a) Accompany the combined library with a copy of the same work based
137 |    on the Library, uncombined with any other library facilities,
138 |    conveyed under the terms of this License.
139 | 
140 |    b) Give prominent notice with the combined library that part of it
141 |    is a work based on the Library, and explaining where to find the
142 |    accompanying uncombined form of the same work.
143 | 
144 |   6. Revised Versions of the GNU Lesser General Public License.
145 | 
146 |   The Free Software Foundation may publish revised and/or new versions
147 | of the GNU Lesser General Public License from time to time. Such new
148 | versions will be similar in spirit to the present version, but may
149 | differ in detail to address new problems or concerns.
150 | 
151 |   Each version is given a distinguishing version number. If the
152 | Library as you received it specifies that a certain numbered version
153 | of the GNU Lesser General Public License "or any later version"
154 | applies to it, you have the option of following the terms and
155 | conditions either of that published version or of any later version
156 | published by the Free Software Foundation. If the Library as you
157 | received it does not specify a version number of the GNU Lesser
158 | General Public License, you may choose any version of the GNU Lesser
159 | General Public License ever published by the Free Software Foundation.
160 | 
161 |   If the Library as you received it specifies that a proxy can decide
162 | whether future versions of the GNU Lesser General Public License shall
163 | apply, that proxy's public statement of acceptance of any version is
164 | permanent authorization for you to choose that version for the
165 | Library.
166 | 


--------------------------------------------------------------------------------