├── neat ├── locals │ ├── __init__.py │ ├── overload │ │ ├── __init__.py │ │ ├── mhod │ │ │ ├── __init__.py │ │ │ ├── l_2_states.py │ │ │ ├── nlp.py │ │ │ └── bruteforce.py │ │ ├── otf.py │ │ └── trivial.py │ ├── underload │ │ ├── __init__.py │ │ └── trivial.py │ └── vm_selection │ │ ├── __init__.py │ │ └── algorithms.py ├── globals │ ├── __init__.py │ ├── vm_placement │ │ ├── __init__.py │ │ └── bin_packing.py │ └── db_cleaner.py ├── contracts_primitive.py ├── __init__.py ├── contracts_extra.py ├── db_utils.py └── config.py ├── tests ├── globals │ ├── __init__.py │ ├── vm_placement │ │ └── __init__.py │ └── test_db_cleaner.py ├── locals │ ├── __init__.py │ ├── overload │ │ ├── __init__.py │ │ ├── mhod │ │ │ ├── __init__.py │ │ │ ├── test_l_2_states.py │ │ │ ├── test_nlp.py │ │ │ ├── test_bruteforce.py │ │ │ └── test_core.py │ │ ├── test_trivial.py │ │ ├── test_otf.py │ │ └── test_statistics.py │ ├── underload │ │ ├── __init__.py │ │ └── test_trivial.py │ ├── vm_selection │ │ ├── __init__.py │ │ └── test_algorithms.py │ └── test_manager.py ├── resources │ └── vms │ │ ├── e615c450-e5d0-11e1-aff1-0800200c9a66 │ │ ├── ec452be0-e5d0-11e1-aff1-0800200c9a66 │ │ └── f3e142d0-e5d0-11e1-aff1-0800200c9a66 ├── __init__.py ├── test_db_utils.py ├── test_config.py └── test_common.py ├── doc └── blueprint │ ├── src │ ├── thanks.tex │ ├── build.sh │ ├── openstack-neat-local-manager.png │ ├── openstack-neat-sequence-diagram.png │ ├── openstack-neat-deployment-diagram.png │ ├── build-epub.sh │ ├── affiliation.tex │ ├── build-html.sh │ ├── build-pdf.sh │ ├── build-rst.sh │ ├── readme-intro.md │ ├── template.tex │ └── bibliography.bib │ ├── openstack-neat-blueprint.epub │ └── openstack-neat-blueprint.pdf ├── nosetests.sh ├── all-clean-logs.sh ├── all-sync-time.sh ├── all-update.sh ├── .gitignore ├── install-rpm.sh ├── MANIFEST.in ├── all-stop.sh ├── all-start.sh ├── setup ├── update-chkconfig.sh ├── README.md ├── deps-arch.sh └── deps-centos.sh ├── stats.sh ├── .travis.yml ├── TODO ├── start-global-manager.py ├── start-local-manager.py ├── start-data-collector.py ├── compute-clean-logs.py ├── compute-sync-time.py ├── compute-install-deps.py ├── compute-local-manager-stop.py ├── compute-data-collector-start.py ├── compute-data-collector-stop.py ├── compute-local-manager-start.py ├── compute-local-manager-status.py ├── compute-data-collector-status.py ├── compute-install-neat.py ├── compute-update.py ├── compute-clone-neat.py ├── compute-copy-conf.py ├── utils ├── vm-migrations.py ├── idle-time-fraction.py ├── overload-time-fraction.py └── db.py ├── NOTICE ├── init.d ├── openstack-neat-db-cleaner ├── openstack-neat-global-manager ├── openstack-neat-data-collector └── openstack-neat-local-manager ├── vm-placement.py ├── setup.py ├── neat.conf └── README.md /neat/locals/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /neat/globals/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/globals/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/locals/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /neat/locals/overload/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /neat/locals/underload/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/locals/overload/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /neat/globals/vm_placement/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /neat/locals/overload/mhod/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /neat/locals/vm_selection/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/globals/vm_placement/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/locals/overload/mhod/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/locals/underload/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/locals/vm_selection/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /doc/blueprint/src/thanks.tex: -------------------------------------------------------------------------------- 1 | -- A Blueprint 2 | -------------------------------------------------------------------------------- /tests/resources/vms/e615c450-e5d0-11e1-aff1-0800200c9a66: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/resources/vms/ec452be0-e5d0-11e1-aff1-0800200c9a66: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/resources/vms/f3e142d0-e5d0-11e1-aff1-0800200c9a66: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /nosetests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | nosetests --nocapture $1 4 | -------------------------------------------------------------------------------- /all-clean-logs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ./compute-clean-logs.py 4 | rm -rf /var/log/neat/* 5 | -------------------------------------------------------------------------------- /all-sync-time.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ./compute-sync-time.py 4 | service ntpdate restart 5 | -------------------------------------------------------------------------------- /all-update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ./compute-update.py 4 | 5 | git pull 6 | python setup.py install 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .ropeproject 3 | openstack_neat.egg-info 4 | build 5 | dist 6 | distribute-* 7 | .idea/ 8 | -------------------------------------------------------------------------------- /install-rpm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ./build-rpm.sh 4 | yum reinstall -y dist/openstack-neat-0.1-1.noarch.rpm 5 | -------------------------------------------------------------------------------- /doc/blueprint/src/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ./build-rst.sh 4 | ./build-pdf.sh 5 | ./build-epub.sh 6 | ./build-html.sh 7 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | include LICENSE 3 | include NOTICE 4 | include distribute_setup.py 5 | include init.d/* 6 | include neat.conf 7 | -------------------------------------------------------------------------------- /doc/blueprint/openstack-neat-blueprint.epub: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beloglazov/openstack-neat/HEAD/doc/blueprint/openstack-neat-blueprint.epub -------------------------------------------------------------------------------- /doc/blueprint/openstack-neat-blueprint.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beloglazov/openstack-neat/HEAD/doc/blueprint/openstack-neat-blueprint.pdf -------------------------------------------------------------------------------- /doc/blueprint/src/openstack-neat-local-manager.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beloglazov/openstack-neat/HEAD/doc/blueprint/src/openstack-neat-local-manager.png -------------------------------------------------------------------------------- /doc/blueprint/src/openstack-neat-sequence-diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beloglazov/openstack-neat/HEAD/doc/blueprint/src/openstack-neat-sequence-diagram.png -------------------------------------------------------------------------------- /doc/blueprint/src/openstack-neat-deployment-diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beloglazov/openstack-neat/HEAD/doc/blueprint/src/openstack-neat-deployment-diagram.png -------------------------------------------------------------------------------- /all-stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | service openstack-neat-global-manager stop 4 | service openstack-neat-db-cleaner stop 5 | ./compute-local-manager-stop.py 6 | ./compute-data-collector-stop.py 7 | -------------------------------------------------------------------------------- /all-start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ./compute-data-collector-start.py 4 | service openstack-neat-global-manager start 5 | service openstack-neat-db-cleaner start 6 | 7 | sleep 2 8 | 9 | ./compute-local-manager-start.py 10 | -------------------------------------------------------------------------------- /doc/blueprint/src/build-epub.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | pandoc \ 4 | --number-sections \ 5 | --bibliography=bibliography.bib --csl=ieee.csl \ 6 | --output=../openstack-neat-blueprint.epub \ 7 | openstack-neat-blueprint.md 8 | -------------------------------------------------------------------------------- /setup/update-chkconfig.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | chkconfig --add openstack-neat-data-collector 4 | chkconfig --add openstack-neat-db-cleaner 5 | chkconfig --add openstack-neat-global-manager 6 | chkconfig --add openstack-neat-local-manager 7 | -------------------------------------------------------------------------------- /doc/blueprint/src/affiliation.tex: -------------------------------------------------------------------------------- 1 | Cloud Computing and Distributed Systems (CLOUDS) Laboratory\\ 2 | Department of Computing and Information Systems\\ 3 | The University of Melbourne, Australia\\ 4 | a.beloglazov@student.unimelb.edu.au\\ 5 | rbuyya@unimelb.edu.au 6 | -------------------------------------------------------------------------------- /setup/README.md: -------------------------------------------------------------------------------- 1 | 1. Create a MySQL database and user for OpenStack Neat: 2 | 3 | ``` 4 | CREATE DATABASE neat; 5 | GRANT ALL ON neat.* TO 'neat'@'controller' IDENTIFIED BY 'neatpassword'; 6 | GRANT ALL ON neat.* TO 'neat'@'%' IDENTIFIED BY 'neatpassword'; 7 | ``` 8 | -------------------------------------------------------------------------------- /doc/blueprint/src/build-html.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | pandoc \ 4 | --standalone \ 5 | --smart \ 6 | --self-contained \ 7 | --table-of-contents \ 8 | --number-sections \ 9 | --bibliography=bibliography.bib --csl=ieee.csl \ 10 | --output=../openstack-neat-blueprint.html \ 11 | openstack-neat-blueprint.md 12 | -------------------------------------------------------------------------------- /stats.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ $# -ne 2 ] 4 | then 5 | echo "You must specify 2 arguments: start and end time as YYYY-MM-DD HH:MM:SS" 6 | exit 1 7 | fi 8 | 9 | ./utils/overload-time-fraction.py root $MYSQL_ROOT_PASSWORD "$1" "$2" 10 | ./utils/idle-time-fraction.py root $MYSQL_ROOT_PASSWORD "$1" "$2" 11 | ./utils/vm-migrations.py root $MYSQL_ROOT_PASSWORD "$1" "$2" 12 | -------------------------------------------------------------------------------- /doc/blueprint/src/build-pdf.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | pandoc \ 4 | --smart \ 5 | --table-of-contents \ 6 | --number-sections \ 7 | --template=template.tex \ 8 | --variable=geometry:textwidth=365pt \ 9 | --variable=affilation:"`cat affiliation.tex`" \ 10 | --variable=thanks:"`cat thanks.tex`" \ 11 | --bibliography=bibliography.bib --csl=ieee.csl \ 12 | --output=../openstack-neat-blueprint.pdf \ 13 | openstack-neat-blueprint.md 14 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - 2.7 4 | install: 5 | - sudo apt-get update 6 | - sudo apt-get install -qq python-libvirt python-numpy python-scipy 7 | - cp /usr/lib/python2.7/dist-packages/libvirt* ~/virtualenv/python2.7/lib/python2.7/site-packages/ 8 | - cp -r /usr/lib/python2.7/dist-packages/numpy* ~/virtualenv/python2.7/lib/python2.7/site-packages/ 9 | - cp -r /usr/lib/python2.7/dist-packages/scipy* ~/virtualenv/python2.7/lib/python2.7/site-packages/ 10 | - pip install --use-mirrors pyqcy mocktest PyContracts nose SQLAlchemy bottle requests python-novaclient 11 | script: nosetests 12 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | RPM package 2 | 3 | 1. python2 setup.py bdist_rpm 4 | 2. Added #!/usr/bin/python2 to start-*.py 5 | 3. cp start-data-collector.py /usr/bin/neat-data-collector 6 | 4. cp initscripts/* /etc/init.d/ 7 | 5. cp neat.conf /etc/neat/neat.conf 8 | 9 | RPM manuals: 10 | 11 | https://fedoraproject.org/wiki/How_to_create_an_RPM_package 12 | https://fedoraproject.org/wiki/Packaging:Guidelines 13 | https://fedoraproject.org/wiki/Packaging:Python 14 | http://fedoraproject.org/wiki/Packaging:SysVInitScript 15 | http://docs.python.org/distutils/builtdist.html 16 | http://stackoverflow.com/questions/2324933/creating-python-rpm 17 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /start-global-manager.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import neat.globals.manager as manager 16 | 17 | 18 | manager.start() 19 | -------------------------------------------------------------------------------- /start-local-manager.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import neat.locals.manager as manager 16 | 17 | 18 | manager.start() 19 | -------------------------------------------------------------------------------- /start-data-collector.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import neat.locals.collector as collector 16 | 17 | 18 | collector.start() 19 | -------------------------------------------------------------------------------- /doc/blueprint/src/build-rst.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | pandoc \ 4 | --smart \ 5 | --bibliography=bibliography.bib --csl=ieee.csl \ 6 | --output=../../../README.rst \ 7 | readme-intro.md \ 8 | openstack-neat-blueprint.md 9 | 10 | sed -i 's/openstack-neat-deployment-diagram.png/\/beloglazov\/openstack-neat\/raw\/master\/doc\/blueprint\/src\/openstack-neat-deployment-diagram.png/g' ../../../README.rst 11 | sed -i 's/openstack-neat-local-manager.png/\/beloglazov\/openstack-neat\/raw\/master\/doc\/blueprint\/src\/openstack-neat-local-manager.png/g' ../../../README.rst 12 | sed -i 's/openstack-neat-sequence-diagram.png/\/beloglazov\/openstack-neat\/raw\/master\/doc\/blueprint\/src\/openstack-neat-sequence-diagram.png/g' ../../../README.rst 13 | -------------------------------------------------------------------------------- /setup/deps-arch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright 2012 Anton Beloglazov 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | sudo pip2 install --upgrade pyqcy mocktest PyContracts SQLAlchemy bottle requests Sphinx python-novaclient 18 | sudo pacman -S python2-numpy python2-scipy 19 | -------------------------------------------------------------------------------- /neat/contracts_primitive.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from contracts import new_contract 16 | 17 | 18 | new_contract('long', lambda x: isinstance(x, (int, long))) 19 | new_contract('function', lambda x: hasattr(x, '__call__')) 20 | -------------------------------------------------------------------------------- /neat/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | OpenStack Neat :: an add-on to OpenStack implementing energy and performance efficient dynamic consolidation of virtual machines 17 | """ 18 | __version__ = "0.1" 19 | __author__ = "Anton Beloglazov" 20 | -------------------------------------------------------------------------------- /setup/deps-centos.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright 2012 Anton Beloglazov 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | sudo yum install -y python-pip numpy scipy libvirt-python 18 | sudo pip install --upgrade pyqcy PyContracts SQLAlchemy bottle requests Sphinx python-novaclient 19 | sudo pip install mocktest 20 | -------------------------------------------------------------------------------- /compute-clean-logs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2 2 | 3 | # Copyright 2012 Anton Beloglazov 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from neat.config import * 18 | import neat.common as common 19 | 20 | config = read_and_validate_config([DEFAILT_CONFIG_PATH, CONFIG_PATH], 21 | REQUIRED_FIELDS) 22 | compute_hosts = common.parse_compute_hosts(config['compute_hosts']) 23 | 24 | common.execute_on_hosts( 25 | compute_hosts, 26 | ['rm -rf /var/log/neat/*']) 27 | -------------------------------------------------------------------------------- /compute-sync-time.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2 2 | 3 | # Copyright 2012 Anton Beloglazov 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from neat.config import * 18 | import neat.common as common 19 | 20 | config = read_and_validate_config([DEFAILT_CONFIG_PATH, CONFIG_PATH], 21 | REQUIRED_FIELDS) 22 | compute_hosts = common.parse_compute_hosts(config['compute_hosts']) 23 | 24 | common.execute_on_hosts( 25 | compute_hosts, 26 | ['service ntpdate restart']) 27 | -------------------------------------------------------------------------------- /compute-install-deps.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2 2 | 3 | # Copyright 2012 Anton Beloglazov 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from neat.config import * 18 | import neat.common as common 19 | 20 | config = read_and_validate_config([DEFAILT_CONFIG_PATH, CONFIG_PATH], 21 | REQUIRED_FIELDS) 22 | compute_hosts = common.parse_compute_hosts(config['compute_hosts']) 23 | 24 | common.execute_on_hosts( 25 | compute_hosts, 26 | ['./openstack-neat/setup/deps-centos.sh']) 27 | -------------------------------------------------------------------------------- /compute-local-manager-stop.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2 2 | 3 | # Copyright 2012 Anton Beloglazov 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from neat.config import * 18 | import neat.common as common 19 | 20 | config = read_and_validate_config([DEFAILT_CONFIG_PATH, CONFIG_PATH], 21 | REQUIRED_FIELDS) 22 | compute_hosts = common.parse_compute_hosts(config['compute_hosts']) 23 | 24 | common.execute_on_hosts( 25 | compute_hosts, 26 | ['service openstack-neat-local-manager stop']) 27 | -------------------------------------------------------------------------------- /compute-data-collector-start.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2 2 | 3 | # Copyright 2012 Anton Beloglazov 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from neat.config import * 18 | import neat.common as common 19 | 20 | config = read_and_validate_config([DEFAILT_CONFIG_PATH, CONFIG_PATH], 21 | REQUIRED_FIELDS) 22 | compute_hosts = common.parse_compute_hosts(config['compute_hosts']) 23 | 24 | common.execute_on_hosts( 25 | compute_hosts, 26 | ['service openstack-neat-data-collector start']) 27 | -------------------------------------------------------------------------------- /compute-data-collector-stop.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2 2 | 3 | # Copyright 2012 Anton Beloglazov 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from neat.config import * 18 | import neat.common as common 19 | 20 | config = read_and_validate_config([DEFAILT_CONFIG_PATH, CONFIG_PATH], 21 | REQUIRED_FIELDS) 22 | compute_hosts = common.parse_compute_hosts(config['compute_hosts']) 23 | 24 | common.execute_on_hosts( 25 | compute_hosts, 26 | ['service openstack-neat-data-collector stop']) 27 | -------------------------------------------------------------------------------- /compute-local-manager-start.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2 2 | 3 | # Copyright 2012 Anton Beloglazov 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from neat.config import * 18 | import neat.common as common 19 | 20 | config = read_and_validate_config([DEFAILT_CONFIG_PATH, CONFIG_PATH], 21 | REQUIRED_FIELDS) 22 | compute_hosts = common.parse_compute_hosts(config['compute_hosts']) 23 | 24 | common.execute_on_hosts( 25 | compute_hosts, 26 | ['service openstack-neat-local-manager start']) 27 | -------------------------------------------------------------------------------- /compute-local-manager-status.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2 2 | 3 | # Copyright 2012 Anton Beloglazov 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from neat.config import * 18 | import neat.common as common 19 | 20 | config = read_and_validate_config([DEFAILT_CONFIG_PATH, CONFIG_PATH], 21 | REQUIRED_FIELDS) 22 | compute_hosts = common.parse_compute_hosts(config['compute_hosts']) 23 | 24 | common.execute_on_hosts( 25 | compute_hosts, 26 | ['service openstack-neat-local-manager status']) 27 | -------------------------------------------------------------------------------- /compute-data-collector-status.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2 2 | 3 | # Copyright 2012 Anton Beloglazov 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from neat.config import * 18 | import neat.common as common 19 | 20 | config = read_and_validate_config([DEFAILT_CONFIG_PATH, CONFIG_PATH], 21 | REQUIRED_FIELDS) 22 | compute_hosts = common.parse_compute_hosts(config['compute_hosts']) 23 | 24 | common.execute_on_hosts( 25 | compute_hosts, 26 | ['service openstack-neat-data-collector status']) 27 | -------------------------------------------------------------------------------- /doc/blueprint/src/readme-intro.md: -------------------------------------------------------------------------------- 1 | # OpenStack Neat: A Framework for Dynamic Consolidation of Virtual Machines in OpenStack Clouds 2 | 3 | ![Build Status](https://secure.travis-ci.org/beloglazov/openstack-neat.png) 4 | 5 | [View the build history](http://travis-ci.org/beloglazov/openstack-neat) 6 | 7 | A discussion group / mailing list: http://groups.google.com/group/openstack-neat 8 | 9 | This blueprint has been submitted to OpenStack Compute (Nova) Blueprints: 10 | 11 | https://blueprints.launchpad.net/nova/+spec/dynamic-consolidation-of-virtual-machines 12 | 13 | The blueprint is also available in the following formats: 14 | 15 | - [PDF](https://github.com/beloglazov/openstack-neat/raw/master/doc/blueprint/openstack-neat-blueprint.pdf 16 | "Download this blueprint in the PDF format") 17 | - [EPUB](https://github.com/beloglazov/openstack-neat/raw/master/doc/blueprint/openstack-neat-blueprint.epub 18 | "Download this blueprint in the EPUB format") 19 | - [HTML](https://raw.github.com/beloglazov/openstack-neat/master/doc/blueprint/openstack-neat-blueprint.html 20 | "Download this blueprint in the HTML format") 21 | -------------------------------------------------------------------------------- /neat/contracts_extra.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from contracts import new_contract 16 | 17 | 18 | import collections 19 | new_contract('deque', collections.deque) 20 | 21 | import datetime 22 | new_contract('datetime', datetime.datetime) 23 | 24 | import libvirt 25 | new_contract('virConnect', libvirt.virConnect) 26 | new_contract('virDomain', libvirt.virDomain) 27 | 28 | import sqlalchemy 29 | new_contract('Table', sqlalchemy.Table) 30 | 31 | import neat.db 32 | new_contract('Database', neat.db.Database) 33 | -------------------------------------------------------------------------------- /compute-install-neat.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2 2 | 3 | # Copyright 2012 Anton Beloglazov 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from neat.config import * 18 | import neat.common as common 19 | 20 | config = read_and_validate_config([DEFAILT_CONFIG_PATH, CONFIG_PATH], 21 | REQUIRED_FIELDS) 22 | compute_hosts = common.parse_compute_hosts(config['compute_hosts']) 23 | 24 | common.execute_on_hosts( 25 | compute_hosts, 26 | ['rm -f /etc/init.d/openstack-neat-*', 27 | 'cd openstack-neat', 28 | 'python2 setup.py install -f']) 29 | -------------------------------------------------------------------------------- /compute-update.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2 2 | 3 | # Copyright 2012 Anton Beloglazov 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import subprocess 18 | 19 | python = 'python' 20 | scripts = ['compute-clone-neat.py', 'compute-install-neat.py'] 21 | 22 | for script in scripts: 23 | command = python + ' ' + script 24 | print '[' + command + ']' 25 | print 26 | output = subprocess.Popen( 27 | command, 28 | stdout=subprocess.PIPE, 29 | stderr=subprocess.STDOUT, 30 | shell=True).communicate()[0] 31 | if output: 32 | print output 33 | -------------------------------------------------------------------------------- /compute-clone-neat.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2 2 | 3 | # Copyright 2012 Anton Beloglazov 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from neat.config import * 18 | import neat.common as common 19 | 20 | config = read_and_validate_config([DEFAILT_CONFIG_PATH, CONFIG_PATH], 21 | REQUIRED_FIELDS) 22 | compute_hosts = common.parse_compute_hosts(config['compute_hosts']) 23 | 24 | common.execute_on_hosts( 25 | compute_hosts, 26 | ['git clone git@github.com:beloglazov/openstack-neat.git', 27 | 'cd openstack-neat', 28 | 'git pull origin master']) 29 | -------------------------------------------------------------------------------- /compute-copy-conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2 2 | 3 | # Copyright 2012 Anton Beloglazov 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import subprocess 18 | 19 | from neat.config import * 20 | import neat.common as common 21 | 22 | 23 | config = read_and_validate_config([DEFAILT_CONFIG_PATH, CONFIG_PATH], 24 | REQUIRED_FIELDS) 25 | compute_hosts = common.parse_compute_hosts(config['compute_hosts']) 26 | 27 | for host in compute_hosts: 28 | subprocess.call('ssh ' + host + ' "mkdir -p /etc/neat"', shell=True) 29 | command = 'scp /etc/neat/neat.conf ' + host + ':/etc/neat' 30 | print '$ ' + command 31 | output = subprocess.Popen( 32 | command, 33 | stdout=subprocess.PIPE, 34 | stderr=subprocess.STDOUT, 35 | shell=True).communicate()[0] 36 | if output: 37 | print output 38 | -------------------------------------------------------------------------------- /tests/locals/overload/mhod/test_l_2_states.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from mocktest import * 16 | from pyqcy import * 17 | 18 | import neat.locals.overload.mhod.l_2_states as l 19 | 20 | import logging 21 | logging.disable(logging.CRITICAL) 22 | 23 | 24 | class L2States(TestCase): 25 | 26 | def test_l0(self): 27 | p = [[0.4, 0.6], 28 | [0.9, 0.1]] 29 | p0 = [1, 0] 30 | 31 | self.assertAlmostEqual(l.l0(p0, p, [0.2, 0.8]), 1.690, 3) 32 | self.assertAlmostEqual(l.l0(p0, p, [0.62, 0.38]), 1.404, 3) 33 | 34 | def test_l1(self): 35 | p = [[0.4, 0.6], 36 | [0.9, 0.1]] 37 | p0 = [1, 0] 38 | 39 | self.assertAlmostEqual(l.l1(p0, p, [0.2, 0.8]), 0.828, 3) 40 | self.assertAlmostEqual(l.l1(p0, p, [0.62, 0.38]), 0.341, 3) 41 | -------------------------------------------------------------------------------- /utils/vm-migrations.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2 2 | 3 | # Copyright 2012 Anton Beloglazov 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import sys 18 | import os 19 | import random 20 | import shutil 21 | import time 22 | from datetime import datetime 23 | from db import init_db 24 | 25 | if len(sys.argv) < 5: 26 | print 'You must specify 4 arguments:' 27 | print '1. The MySQL DB user name' 28 | print '2. The MySQL DB password' 29 | print '3. The start datetime in the format: %Y-%m-%d %H:%M:%S' 30 | print '4. The finish datetime in the format: %Y-%m-%d %H:%M:%S' 31 | sys.exit(1) 32 | 33 | db = init_db('mysql://' + sys.argv[1] + ':' + sys.argv[2] + '@localhost/spe') 34 | start_time = datetime.fromtimestamp( 35 | time.mktime(time.strptime(sys.argv[3], '%Y-%m-%d %H:%M:%S'))) 36 | finish_time = datetime.fromtimestamp( 37 | time.mktime(time.strptime(sys.argv[4], '%Y-%m-%d %H:%M:%S'))) 38 | 39 | #print "Start time: " + str(start_time) 40 | #print "Finish time: " + str(finish_time) 41 | print "VM migrations: " + str(len(db.select_vm_migrations(start_time, finish_time))) 42 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | OpenStack Neat 2 | Copyright 2012 Anton Beloglazov 3 | 4 | This product includes software developed at the Cloud Computing and 5 | Distributed Systems (CLOUDS) Laboratory, Department of Computing and 6 | Information Systems, The University of Melbourne, Australia 7 | (http://www.cloudbus.org/). 8 | 9 | 10 | This software uses distribute, a library for working with Python 11 | module distributions developed by Tarek Ziadé, released under the 12 | Python Software Foundation License, and available from 13 | https://bitbucket.org/tarek/distribute 14 | 15 | This software uses pyqcy, a QuickCheck-like testing framework for 16 | Python developed by Karol Kuczmarski, released under the FreeBSD 17 | License, and available from https://github.com/Xion/pyqcy 18 | 19 | This software uses PyContracts, a Python library for supporting Design 20 | by Contract (DbC) developed by Andrea Censi, released under the GNU 21 | Lesser General Public License, and available from 22 | https://github.com/AndreaCensi/contracts 23 | 24 | This software uses SQLAlchemy, a Python SQL toolkit and Object 25 | Relational Mapper developed by the SQLAlchemy (trademark of Michael 26 | Bayer) authors and contributors, released under the MIT License, and 27 | available from http://www.sqlalchemy.org/ 28 | 29 | This software uses Bottle, a micro web-framework for Python developed 30 | by Marcel Hellkamp, released under the MIT License, and available from 31 | http://bottlepy.org/ 32 | 33 | This software uses Requests, a Python HTTP client library developed by 34 | Kenneth Reitz, released under the ISC License, and available from 35 | http://python-requests.org. 36 | 37 | This software uses libvirt, a virtualization toolkit with Python 38 | bindings developed by Red Hat, released under the LGPL licensem and 39 | available from http://libvirt.org/ 40 | 41 | This software uses python-novaclient, a Python Nova API client 42 | implementation developed by Jacob Kaplan-Moss and Rackspace - 43 | OpenStack extensions, released under the Apache 2.0 License, and 44 | available from https://github.com/openstack/python-novaclient 45 | 46 | This software uses Sphinx, a documentation generator for Python 47 | developed by Georg Brandl, released under the BSD License, and 48 | available from http://sphinx.pocoo.org/ 49 | -------------------------------------------------------------------------------- /utils/idle-time-fraction.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2 2 | 3 | # Copyright 2012 Anton Beloglazov 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import sys 18 | import os 19 | import random 20 | import shutil 21 | import time 22 | from datetime import datetime 23 | from db import init_db 24 | 25 | if len(sys.argv) < 5: 26 | print 'You must specify 4 arguments:' 27 | print '1. The MySQL DB user name' 28 | print '2. The MySQL DB password' 29 | print '3. The start datetime in the format: %Y-%m-%d %H:%M:%S' 30 | print '4. The finish datetime in the format: %Y-%m-%d %H:%M:%S' 31 | sys.exit(1) 32 | 33 | db = init_db('mysql://' + sys.argv[1] + ':' + sys.argv[2] + '@localhost/spe') 34 | start_time = datetime.fromtimestamp( 35 | time.mktime(time.strptime(sys.argv[3], '%Y-%m-%d %H:%M:%S'))) 36 | finish_time = datetime.fromtimestamp( 37 | time.mktime(time.strptime(sys.argv[4], '%Y-%m-%d %H:%M:%S'))) 38 | 39 | #print "Start time: " + str(start_time) 40 | #print "Finish time: " + str(finish_time) 41 | 42 | def total_seconds(delta): 43 | return (delta.microseconds + 44 | (delta.seconds + delta.days * 24 * 3600) * 1000000) / 1000000 45 | 46 | total_time = 0 47 | total_idle_time = 0 48 | for hostname, host_id in db.select_host_ids().items(): 49 | prev_timestamp = start_time 50 | prev_state = 1 51 | states = {0: [], 1: []} 52 | for timestamp, state in db.select_host_states(host_id, start_time, finish_time): 53 | if prev_timestamp: 54 | states[prev_state].append(total_seconds(timestamp - prev_timestamp)) 55 | prev_timestamp = timestamp 56 | prev_state = state 57 | states[prev_state].append(total_seconds(finish_time - prev_timestamp)) 58 | #print states 59 | off_time = sum(states[0]) 60 | on_time = sum(states[1]) 61 | total_time += off_time + on_time 62 | total_idle_time += off_time 63 | 64 | print "Total time: " + str(total_time) 65 | print "Total idle time: " + str(total_idle_time) 66 | print "Idle time fraction: " + str(float(total_idle_time) / total_time) 67 | -------------------------------------------------------------------------------- /tests/test_db_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from mocktest import * 16 | from pyqcy import * 17 | 18 | from sqlalchemy import * 19 | 20 | import neat.db 21 | import neat.db_utils as db_utils 22 | 23 | import logging 24 | logging.disable(logging.CRITICAL) 25 | 26 | 27 | class DbUtils(TestCase): 28 | 29 | @qc(1) 30 | def init_db(): 31 | db = db_utils.init_db('sqlite:///:memory:') 32 | assert type(db) is neat.db.Database 33 | assert isinstance(db.hosts, Table) 34 | assert isinstance(db.vms, Table) 35 | assert isinstance(db.vm_resource_usage, Table) 36 | assert isinstance(db.host_states, Table) 37 | assert db.hosts.c.keys() == \ 38 | ['id', 'hostname', 'cpu_mhz', 'cpu_cores', 'ram'] 39 | assert db.host_resource_usage.c.keys() == \ 40 | ['id', 'host_id', 'timestamp', 'cpu_mhz'] 41 | assert list(db.host_resource_usage.foreign_keys)[0].target_fullname \ 42 | == 'hosts.id' 43 | assert db.vms.c.keys() == \ 44 | ['id', 'uuid'] 45 | assert db.vm_resource_usage.c.keys() == \ 46 | ['id', 'vm_id', 'timestamp', 'cpu_mhz'] 47 | assert list(db.vm_resource_usage.foreign_keys)[0].target_fullname \ 48 | == 'vms.id' 49 | assert db.vm_migrations.c.keys() == \ 50 | ['id', 'vm_id', 'host_id', 'timestamp'] 51 | keys = set([list(db.vm_migrations.foreign_keys)[0].target_fullname, 52 | list(db.vm_migrations.foreign_keys)[1].target_fullname]) 53 | assert keys == set(['vms.id', 'hosts.id']) 54 | assert db.host_states.c.keys() == \ 55 | ['id', 'host_id', 'timestamp', 'state'] 56 | assert list(db.host_states.foreign_keys)[0].target_fullname \ 57 | == 'hosts.id' 58 | assert db.host_overload.c.keys() == \ 59 | ['id', 'host_id', 'timestamp', 'overload'] 60 | assert list(db.host_overload.foreign_keys)[0].target_fullname \ 61 | == 'hosts.id' 62 | -------------------------------------------------------------------------------- /tests/locals/overload/mhod/test_nlp.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from mocktest import * 16 | from pyqcy import * 17 | 18 | import operator 19 | 20 | import neat.locals.overload.mhod.nlp as nlp 21 | 22 | import logging 23 | logging.disable(logging.CRITICAL) 24 | 25 | 26 | class Nlp(TestCase): 27 | 28 | def test_build_objective(self): 29 | with MockTransaction: 30 | state_vector = [1, 0] 31 | p = [[-0.1, 0.1], 32 | [0.3, -0.3]] 33 | m1 = mock('m1') 34 | m2 = mock('m2') 35 | m = [m1, m2] 36 | container = mock('function container') 37 | expect(container).l0(state_vector, p, m).and_return(2).once() 38 | expect(container).l1(state_vector, p, m).and_return(3).once() 39 | ls = [container.l0, container.l1] 40 | 41 | objective = nlp.build_objective(ls, state_vector, p) 42 | 43 | self.assertTrue(hasattr(objective, '__call__')) 44 | self.assertEqual(objective(m1, m2), 5) 45 | 46 | def test_build_constraint(self): 47 | with MockTransaction: 48 | otf = 0.05 49 | migration_time = 20. 50 | state_vector = [1, 0] 51 | p = [[-0.1, 0.1], 52 | [0.3, -0.3]] 53 | m1 = mock('m1') 54 | m2 = mock('m2') 55 | m = [m1, m2] 56 | container = mock('function container') 57 | expect(container).l0(state_vector, p, m).and_return(2).once() 58 | expect(container).l1(state_vector, p, m). \ 59 | and_return(3).exactly(2).times() 60 | ls = [container.l0, container.l1] 61 | 62 | constraint = nlp.build_constraint(otf, migration_time, 63 | ls, state_vector, p, 0, 0) 64 | 65 | self.assertTrue(hasattr(constraint[0], '__call__')) 66 | assert constraint[1] is operator.le 67 | self.assertEqual(constraint[2], otf) 68 | self.assertEqual(constraint[0](m1, m2), 69 | float(migration_time + 3) / 70 | (migration_time + 5)) 71 | -------------------------------------------------------------------------------- /neat/locals/overload/mhod/l_2_states.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ L functions for the 2 state configuration of the MHOD algorithm. 16 | """ 17 | 18 | from contracts import contract 19 | from neat.contracts_primitive import * 20 | from neat.contracts_extra import * 21 | 22 | import logging 23 | log = logging.getLogger(__name__) 24 | 25 | 26 | @contract 27 | def l0(p_initial, p_matrix, m): 28 | """ Compute the L0 function. 29 | 30 | :param p_initial: The initial state distribution. 31 | :type p_initial: list(number) 32 | 33 | :param p_matrix: A matrix of transition probabilities. 34 | :type p_matrix: list(list(number)) 35 | 36 | :param m: The m values. 37 | :type m: list(number) 38 | 39 | :return: The value of the L0 function. 40 | :rtype: number 41 | """ 42 | p0 = p_initial[0] 43 | p1 = p_initial[1] 44 | p00 = p_matrix[0][0] 45 | p01 = p_matrix[0][1] 46 | p10 = p_matrix[1][0] 47 | p11 = p_matrix[1][1] 48 | m0 = m[0] 49 | m1 = m[1] 50 | return ((p0 * (-1 * m1 * p11 + p11 - 1) + (m1 * p1 - p1) * p10) / 51 | (p00 * (m1 * (p11 - m0 * p11) - p11 + m0 * (p11 - 1) + 1) - 52 | m1 * p11 + p11 + (m1 * (m0 * p01 - p01) - m0 * p01 + p01) * 53 | p10 - 1)) 54 | 55 | 56 | @contract 57 | def l1(p_initial, p_matrix, m): 58 | """ Compute the L1 function. 59 | 60 | :param p_initial: The initial state distribution. 61 | :type p_initial: list(number) 62 | 63 | :param p_matrix: A matrix of transition probabilities. 64 | :type p_matrix: list(list(number)) 65 | 66 | :param m: The m values. 67 | :type m: list(number) 68 | 69 | :return: The value of the L1 function. 70 | :rtype: number 71 | """ 72 | p0 = p_initial[0] 73 | p1 = p_initial[1] 74 | p00 = p_matrix[0][0] 75 | p01 = p_matrix[0][1] 76 | p10 = p_matrix[1][0] 77 | p11 = p_matrix[1][1] 78 | m0 = m[0] 79 | m1 = m[1] 80 | return (-1 * (p00 * (m0 * p1 - p1) + p1 + p0 * (p01 - m0 * p01)) / 81 | (p00 * (m1 * (p11 - m0 * p11) - p11 + m0 * (p11 - 1) + 1) - 82 | m1 * p11 + p11 + (m1 * (m0 * p01 - p01) - m0 * p01 + p01) * 83 | p10 - 1)) 84 | 85 | 86 | ls = [l0, l1] 87 | -------------------------------------------------------------------------------- /init.d/openstack-neat-db-cleaner: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # openstack-neat-db-cleaner OpenStack Neat Database Cleaner 4 | # 5 | # chkconfig: - 99 01 6 | # description: The database cleaner periodically cleans up \ 7 | # the data on resource usage by VMs stored in \ 8 | # the database. This is requried to avoid excess \ 9 | # growth of the database size. 10 | 11 | ### BEGIN INIT INFO 12 | # Provides: openstack_neat_db_cleaner 13 | # Required-Start: $remote_fs $network $syslog 14 | # Required-Stop: $remote_fs $network $syslog 15 | # Short-Description: OpenStack Neat Database Cleaner 16 | # Description: The database cleaner periodically cleans up 17 | # the data on resource usage by VMs stored in 18 | # the database. This is requried to avoid excess 19 | # growth of the database size. 20 | ### END INIT INFO 21 | 22 | . /etc/rc.d/init.d/functions 23 | 24 | suffix=db-cleaner 25 | prog=openstack-neat-$suffix 26 | exec="/usr/bin/neat-$suffix" 27 | piddir="/var/run/neat" 28 | pidfile="$piddir/neat-$suffix.pid" 29 | logdir="/var/log/neat" 30 | logfile="$logdir/db-cleaner-service.log" 31 | 32 | [ -e /etc/sysconfig/$prog ] && . /etc/sysconfig/$prog 33 | 34 | lockfile=/var/lock/subsys/$prog 35 | 36 | start() { 37 | [ -x $exec ] || exit 5 38 | [ -f $config ] || exit 6 39 | echo -n $"Starting $prog: " 40 | mkdir -p $piddir 41 | mkdir -p $logdir 42 | daemon --user root --pidfile $pidfile "$exec &>$logfile & echo \$! > $pidfile" 43 | retval=$? 44 | echo 45 | [ $retval -eq 0 ] && touch $lockfile 46 | return $retval 47 | } 48 | 49 | stop() { 50 | echo -n $"Stopping $prog: " 51 | killproc -p $pidfile $prog 52 | retval=$? 53 | echo 54 | [ $retval -eq 0 ] && rm -f $lockfile 55 | return $retval 56 | } 57 | 58 | restart() { 59 | stop 60 | start 61 | } 62 | 63 | reload() { 64 | restart 65 | } 66 | 67 | force_reload() { 68 | restart 69 | } 70 | 71 | rh_status() { 72 | status -p $pidfile $prog 73 | } 74 | 75 | rh_status_q() { 76 | rh_status >/dev/null 2>&1 77 | } 78 | 79 | 80 | case "$1" in 81 | start) 82 | rh_status_q && exit 0 83 | $1 84 | ;; 85 | stop) 86 | rh_status_q || exit 0 87 | $1 88 | ;; 89 | restart) 90 | $1 91 | ;; 92 | reload) 93 | rh_status_q || exit 7 94 | $1 95 | ;; 96 | force-reload) 97 | force_reload 98 | ;; 99 | status) 100 | rh_status 101 | ;; 102 | condrestart|try-restart) 103 | rh_status_q || exit 0 104 | restart 105 | ;; 106 | *) 107 | echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}" 108 | exit 2 109 | esac 110 | exit $? 111 | -------------------------------------------------------------------------------- /neat/locals/overload/mhod/nlp.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ Functions for defing the NLP problem of the MHOD algorithm. 16 | """ 17 | 18 | from contracts import contract 19 | from neat.contracts_primitive import * 20 | from neat.contracts_extra import * 21 | 22 | import operator 23 | 24 | import logging 25 | log = logging.getLogger(__name__) 26 | 27 | 28 | @contract 29 | def build_objective(ls, state_vector, p): 30 | """ Creates an objective function, which is a sum of the L functions. 31 | 32 | :param ls: A list of L functions. 33 | :type ls: list(function) 34 | 35 | :param state_vector: A state vector. 36 | :type state_vector: list(int) 37 | 38 | :param p: A matrix of transition probabilities. 39 | :type p: list(list(number)) 40 | 41 | :return: An objective function. 42 | :rtype: function 43 | """ 44 | def objective(*m): 45 | return sum(l(state_vector, p, list(m)) for l in ls) 46 | return objective 47 | 48 | 49 | @contract 50 | def build_constraint(otf, migration_time, ls, state_vector, 51 | p, time_in_states, time_in_state_n): 52 | """ Creates an optimization constraint from the L functions. 53 | 54 | :param otf: The OTF parameter. 55 | :type otf: float 56 | 57 | :param migration_time: The VM migration time in time steps. 58 | :type migration_time: float,>=0 59 | 60 | :param ls: A list of L functions. 61 | :type ls: list(function) 62 | 63 | :param state_vector: A state vector. 64 | :type state_vector: list(int) 65 | 66 | :param p: A matrix of transition probabilities. 67 | :type p: list(list(number)) 68 | 69 | :param time_in_states: The total time in all the states in time steps. 70 | :type time_in_states: number,>=0 71 | 72 | :param time_in_state_n: The total time in the state N in time steps. 73 | :type time_in_state_n: number,>=0 74 | 75 | :return: The created constraint. 76 | :rtype: tuple(function, function, number) 77 | """ 78 | def constraint(*m): 79 | m_list = list(m) 80 | return float(migration_time + 81 | time_in_state_n + 82 | ls[-1](state_vector, p, m_list)) / \ 83 | (migration_time + 84 | time_in_states + 85 | sum(l(state_vector, p, m_list) for l in ls)) 86 | return (constraint, operator.le, otf) 87 | -------------------------------------------------------------------------------- /init.d/openstack-neat-global-manager: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # openstack-neat-global-manager OpenStack Neat Global Manager 4 | # 5 | # chkconfig: - 99 01 6 | # description: The global manager runs a REST web service \ 7 | # listening to requests from the local managers. \ 8 | # Once a request is received, the global manager \ 9 | # executes the actions corresponding to the type \ 10 | # of the request: underload, or overload of a host. 11 | 12 | ### BEGIN INIT INFO 13 | # Provides: openstack_neat_global_manager 14 | # Required-Start: $remote_fs $network $syslog 15 | # Required-Stop: $remote_fs $network $syslog 16 | # Short-Description: OpenStack Neat Global Manager 17 | # Description: The global manager runs a REST web service 18 | # listening to requests from the local managers. 19 | # Once a request is received, the global manager 20 | # executes the actions corresponding to the type 21 | # of the request: underload, or overload of a host. 22 | ### END INIT INFO 23 | 24 | . /etc/rc.d/init.d/functions 25 | 26 | suffix=global-manager 27 | prog=openstack-neat-$suffix 28 | exec="/usr/bin/neat-$suffix" 29 | piddir="/var/run/neat" 30 | pidfile="$piddir/neat-$suffix.pid" 31 | logdir="/var/log/neat" 32 | logfile="$logdir/global-manager-service.log" 33 | 34 | [ -e /etc/sysconfig/$prog ] && . /etc/sysconfig/$prog 35 | 36 | lockfile=/var/lock/subsys/$prog 37 | 38 | start() { 39 | [ -x $exec ] || exit 5 40 | [ -f $config ] || exit 6 41 | echo -n $"Starting $prog: " 42 | mkdir -p $piddir 43 | mkdir -p $logdir 44 | daemon --user root --pidfile $pidfile "$exec &>$logfile & echo \$! > $pidfile" 45 | retval=$? 46 | echo 47 | [ $retval -eq 0 ] && touch $lockfile 48 | return $retval 49 | } 50 | 51 | stop() { 52 | echo -n $"Stopping $prog: " 53 | killproc -p $pidfile $prog 54 | retval=$? 55 | echo 56 | [ $retval -eq 0 ] && rm -f $lockfile 57 | return $retval 58 | } 59 | 60 | restart() { 61 | stop 62 | start 63 | } 64 | 65 | reload() { 66 | restart 67 | } 68 | 69 | force_reload() { 70 | restart 71 | } 72 | 73 | rh_status() { 74 | status -p $pidfile $prog 75 | } 76 | 77 | rh_status_q() { 78 | rh_status >/dev/null 2>&1 79 | } 80 | 81 | 82 | case "$1" in 83 | start) 84 | rh_status_q && exit 0 85 | $1 86 | ;; 87 | stop) 88 | rh_status_q || exit 0 89 | $1 90 | ;; 91 | restart) 92 | $1 93 | ;; 94 | reload) 95 | rh_status_q || exit 7 96 | $1 97 | ;; 98 | force-reload) 99 | force_reload 100 | ;; 101 | status) 102 | rh_status 103 | ;; 104 | condrestart|try-restart) 105 | rh_status_q || exit 0 106 | restart 107 | ;; 108 | *) 109 | echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}" 110 | exit 2 111 | esac 112 | exit $? 113 | -------------------------------------------------------------------------------- /init.d/openstack-neat-data-collector: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # openstack-neat-data-collector OpenStack Neat Data Collector 4 | # 5 | # chkconfig: - 99 01 6 | # description: The data collector monitors the virtual machines \ 7 | # running on the host and collects the data on the \ 8 | # CPU utilization. The collected data are stored \ 9 | # locally in files and in the central database to \ 10 | # be used by the local and global managers. 11 | 12 | ### BEGIN INIT INFO 13 | # Provides: openstack_neat_data_collector 14 | # Required-Start: $remote_fs $network $syslog 15 | # Required-Stop: $remote_fs $network $syslog 16 | # Short-Description: OpenStack Neat Data Collector 17 | # Description: The data collector monitors the virtual machines 18 | # running on the host and collects the data on the 19 | # CPU utilization. The collected data are stored 20 | # locally in files and in the central database to 21 | # be used by the local and global managers. 22 | ### END INIT INFO 23 | 24 | . /etc/rc.d/init.d/functions 25 | 26 | suffix=data-collector 27 | prog=openstack-neat-$suffix 28 | exec="/usr/bin/neat-$suffix" 29 | piddir="/var/run/neat" 30 | pidfile="$piddir/neat-$suffix.pid" 31 | logdir="/var/log/neat" 32 | logfile="$logdir/data-collector-service.log" 33 | 34 | [ -e /etc/sysconfig/$prog ] && . /etc/sysconfig/$prog 35 | 36 | lockfile=/var/lock/subsys/$prog 37 | 38 | start() { 39 | [ -x $exec ] || exit 5 40 | [ -f $config ] || exit 6 41 | echo -n $"Starting $prog: " 42 | mkdir -p $piddir 43 | mkdir -p $logdir 44 | daemon --user root --pidfile $pidfile "$exec &>$logfile & echo \$! > $pidfile" 45 | retval=$? 46 | echo 47 | [ $retval -eq 0 ] && touch $lockfile 48 | return $retval 49 | } 50 | 51 | stop() { 52 | echo -n $"Stopping $prog: " 53 | killproc -p $pidfile $prog 54 | retval=$? 55 | echo 56 | [ $retval -eq 0 ] && rm -f $lockfile 57 | return $retval 58 | } 59 | 60 | restart() { 61 | stop 62 | start 63 | } 64 | 65 | reload() { 66 | restart 67 | } 68 | 69 | force_reload() { 70 | restart 71 | } 72 | 73 | rh_status() { 74 | status -p $pidfile $prog 75 | } 76 | 77 | rh_status_q() { 78 | rh_status >/dev/null 2>&1 79 | } 80 | 81 | 82 | case "$1" in 83 | start) 84 | rh_status_q && exit 0 85 | $1 86 | ;; 87 | stop) 88 | rh_status_q || exit 0 89 | $1 90 | ;; 91 | restart) 92 | $1 93 | ;; 94 | reload) 95 | rh_status_q || exit 7 96 | $1 97 | ;; 98 | force-reload) 99 | force_reload 100 | ;; 101 | status) 102 | rh_status 103 | ;; 104 | condrestart|try-restart) 105 | rh_status_q || exit 0 106 | restart 107 | ;; 108 | *) 109 | echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}" 110 | exit 2 111 | esac 112 | exit $? 113 | -------------------------------------------------------------------------------- /init.d/openstack-neat-local-manager: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # openstack-neat-local-manager OpenStack Neat Local Manager 4 | # 5 | # chkconfig: - 99 01 6 | # description: The local manager periodically reads the data \ 7 | # stored by the data collector and using the \ 8 | # specified in the configuration algorithms \ 9 | # determines whether the host is underloaded or \ 10 | # overloaded. Then it sends the appropriate \ 11 | # messages to the global manager. 12 | 13 | ### BEGIN INIT INFO 14 | # Provides: openstack_neat_local_manager 15 | # Required-Start: $remote_fs $network $syslog openstack_neat_data_collector 16 | # Required-Stop: $remote_fs $network $syslog 17 | # Short-Description: OpenStack Neat Local Manager 18 | # Description: The local manager periodically reads the data 19 | # stored by the data collector and using the 20 | # specified in the configuration algorithms 21 | # determines whether the host is underloaded or 22 | # overloaded. Then it sends the appropriate 23 | # messages to the global manager. 24 | ### END INIT INFO 25 | 26 | . /etc/rc.d/init.d/functions 27 | 28 | suffix=local-manager 29 | prog=openstack-neat-$suffix 30 | exec="/usr/bin/neat-$suffix" 31 | piddir="/var/run/neat" 32 | pidfile="$piddir/neat-$suffix.pid" 33 | logdir="/var/log/neat" 34 | logfile="$logdir/local-manager-service.log" 35 | 36 | [ -e /etc/sysconfig/$prog ] && . /etc/sysconfig/$prog 37 | 38 | lockfile=/var/lock/subsys/$prog 39 | 40 | start() { 41 | [ -x $exec ] || exit 5 42 | [ -f $config ] || exit 6 43 | echo -n $"Starting $prog: " 44 | mkdir -p $piddir 45 | mkdir -p $logdir 46 | daemon --user root --pidfile $pidfile "$exec &>$logfile & echo \$! > $pidfile" 47 | retval=$? 48 | echo 49 | [ $retval -eq 0 ] && touch $lockfile 50 | return $retval 51 | } 52 | 53 | stop() { 54 | echo -n $"Stopping $prog: " 55 | killproc -p $pidfile $prog 56 | retval=$? 57 | echo 58 | [ $retval -eq 0 ] && rm -f $lockfile 59 | return $retval 60 | } 61 | 62 | restart() { 63 | stop 64 | start 65 | } 66 | 67 | reload() { 68 | restart 69 | } 70 | 71 | force_reload() { 72 | restart 73 | } 74 | 75 | rh_status() { 76 | status -p $pidfile $prog 77 | } 78 | 79 | rh_status_q() { 80 | rh_status >/dev/null 2>&1 81 | } 82 | 83 | 84 | case "$1" in 85 | start) 86 | rh_status_q && exit 0 87 | $1 88 | ;; 89 | stop) 90 | rh_status_q || exit 0 91 | $1 92 | ;; 93 | restart) 94 | $1 95 | ;; 96 | reload) 97 | rh_status_q || exit 7 98 | $1 99 | ;; 100 | force-reload) 101 | force_reload 102 | ;; 103 | status) 104 | rh_status 105 | ;; 106 | condrestart|try-restart) 107 | rh_status_q || exit 0 108 | restart 109 | ;; 110 | *) 111 | echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}" 112 | exit 2 113 | esac 114 | exit $? 115 | -------------------------------------------------------------------------------- /tests/locals/overload/mhod/test_bruteforce.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from mocktest import * 16 | from pyqcy import * 17 | 18 | from operator import le 19 | 20 | import neat.locals.overload.mhod.bruteforce as b 21 | import neat.locals.overload.mhod.nlp as nlp 22 | 23 | import logging 24 | logging.disable(logging.CRITICAL) 25 | 26 | 27 | class Bruteforce(TestCase): 28 | 29 | def test_solve2(self): 30 | def fn1(x, y): 31 | return x + y 32 | 33 | def fn2(x, y): 34 | return 2 * x + y 35 | 36 | def fn3(x, y): 37 | return x - y 38 | 39 | def fn4(x, y): 40 | return x / y 41 | 42 | self.assertEqual([round(x, 1) 43 | for x in b.solve2(fn1, (fn1, le, 10), 0.1, 1.0)], 44 | [1.0, 1.0]) 45 | self.assertEqual([round(x, 1) 46 | for x in b.solve2(fn1, (fn1, le, 0.5), 0.1, 1.0)], 47 | [0.0, 0.5]) 48 | self.assertEqual([round(x, 1) 49 | for x in b.solve2(fn2, (fn1, le, 0.5), 0.1, 1.0)], 50 | [0.5, 0.0]) 51 | self.assertEqual([round(x, 1) 52 | for x in b.solve2(fn3, (fn3, le, 10), 0.1, 1.0)], 53 | [1.0, 0.0]) 54 | self.assertEqual([round(x, 1) 55 | for x in b.solve2(fn4, (fn4, le, 10), 0.1, 1.0)], 56 | [1.0, 0.1]) 57 | 58 | def test_optimize(self): 59 | with MockTransaction: 60 | step = 0.1 61 | limit = 1 62 | otf = 0.3 63 | migration_time = 20. 64 | ls = [lambda x: x, lambda x: x] 65 | p = [[0, 1]] 66 | state_vector = [0, 1] 67 | time_in_states = 10 68 | time_in_state_n = 5 69 | objective = mock('objective') 70 | constraint = mock('constraint') 71 | solution = [1, 2, 3] 72 | expect(nlp).build_objective(ls, state_vector, p). \ 73 | and_return(objective).once() 74 | expect(nlp).build_constraint( 75 | otf, migration_time, ls, state_vector, 76 | p, time_in_states, time_in_state_n). \ 77 | and_return(constraint).once() 78 | expect(b).solve2(objective, constraint, step, limit). \ 79 | and_return(solution).once() 80 | self.assertEqual( 81 | b.optimize(step, limit, otf, migration_time, ls, 82 | p, state_vector, time_in_states, time_in_state_n), 83 | solution) 84 | -------------------------------------------------------------------------------- /utils/overload-time-fraction.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2 2 | 3 | # Copyright 2012 Anton Beloglazov 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import sys 18 | import os 19 | import random 20 | import shutil 21 | import time 22 | from datetime import datetime 23 | from db import init_db 24 | 25 | if len(sys.argv) < 5: 26 | print 'You must specify 4 arguments:' 27 | print '1. The MySQL DB user name' 28 | print '2. The MySQL DB password' 29 | print '3. The start datetime in the format: %Y-%m-%d %H:%M:%S' 30 | print '4. The finish datetime in the format: %Y-%m-%d %H:%M:%S' 31 | sys.exit(1) 32 | 33 | db = init_db('mysql://' + sys.argv[1] + ':' + sys.argv[2] + '@localhost/neat') 34 | start_time = datetime.fromtimestamp( 35 | time.mktime(time.strptime(sys.argv[3], '%Y-%m-%d %H:%M:%S'))) 36 | finish_time = datetime.fromtimestamp( 37 | time.mktime(time.strptime(sys.argv[4], '%Y-%m-%d %H:%M:%S'))) 38 | 39 | #print "Start time: " + str(start_time) 40 | #print "Finish time: " + str(finish_time) 41 | 42 | def total_seconds(delta): 43 | return (delta.microseconds + 44 | (delta.seconds + delta.days * 24 * 3600) * 1000000) / 1000000 45 | 46 | total_idle_time = 0 47 | for hostname, host_id in db.select_host_ids().items(): 48 | prev_timestamp = start_time 49 | prev_state = 1 50 | states = {0: [], 1: []} 51 | for timestamp, state in db.select_host_states(host_id, start_time, finish_time): 52 | if prev_timestamp: 53 | states[prev_state].append(total_seconds(timestamp - prev_timestamp)) 54 | prev_timestamp = timestamp 55 | prev_state = state 56 | states[prev_state].append(total_seconds(finish_time - prev_timestamp)) 57 | #print states 58 | off_time = sum(states[0]) 59 | total_idle_time += off_time 60 | 61 | total_time = 0 62 | total_overload_time = 0 63 | for hostname, host_id in db.select_host_ids().items(): 64 | prev_timestamp = start_time 65 | prev_state = 0 66 | states = {0: [], 1: []} 67 | for timestamp, state in db.select_host_overload(host_id, start_time, finish_time): 68 | if prev_timestamp: 69 | states[prev_state].append(total_seconds(timestamp - prev_timestamp)) 70 | prev_timestamp = timestamp 71 | prev_state = state 72 | states[prev_state].append(total_seconds(finish_time - prev_timestamp)) 73 | #print states 74 | nonoverload_time = sum(states[0]) 75 | overload_time = sum(states[1]) 76 | total_time += nonoverload_time + overload_time 77 | total_overload_time += overload_time 78 | 79 | print "Total time: " + str(total_time) 80 | print "Overload time: " + str(total_overload_time) 81 | print "Overload time fraction: " + str(float(total_overload_time) / (total_time - total_idle_time)) 82 | -------------------------------------------------------------------------------- /tests/test_config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from mocktest import * 16 | from pyqcy import * 17 | 18 | import neat.config as config 19 | 20 | import logging 21 | logging.disable(logging.CRITICAL) 22 | 23 | 24 | class Config(TestCase): 25 | 26 | @qc 27 | def read_default_config(): 28 | paths = [config.DEFAILT_CONFIG_PATH] 29 | test_config = config.read_config(paths) 30 | assert config.validate_config(test_config, config.REQUIRED_FIELDS) 31 | 32 | @qc 33 | def read_config(): 34 | paths = [config.DEFAILT_CONFIG_PATH, config.CONFIG_PATH] 35 | test_config = config.read_config(paths) 36 | assert config.validate_config(test_config, config.REQUIRED_FIELDS) 37 | 38 | @qc 39 | def validate_valid_config( 40 | x=list_(of=str_(of='abc123_', max_length=20), 41 | min_length=0, max_length=10) 42 | ): 43 | test_config = dict(zip(x, x)) 44 | assert config.validate_config(test_config, x) 45 | 46 | @qc 47 | def validate_invalid_config( 48 | x=list_(of=str_(of='abc123_', max_length=20), 49 | min_length=0, max_length=5), 50 | y=list_(of=str_(of='abc123_', max_length=20), 51 | min_length=6, max_length=10) 52 | ): 53 | test_config = dict(zip(x, x)) 54 | assert not config.validate_config(test_config, y) 55 | 56 | @qc(10) 57 | def read_and_validate_valid_config( 58 | x=list_(of=str_(of='abc123_', max_length=20), 59 | min_length=0, max_length=10) 60 | ): 61 | with MockTransaction: 62 | test_config = dict(zip(x, x)) 63 | paths = ['path1', 'path2'] 64 | expect(config).read_config(paths).and_return(test_config).once() 65 | expect(config).validate_config(test_config, x). \ 66 | and_return(True).once() 67 | assert config.read_and_validate_config(paths, x) == test_config 68 | 69 | @qc(10) 70 | def read_and_validate_invalid_config( 71 | x=list_(of=str_(of='abc123_', max_length=20), 72 | min_length=0, max_length=5), 73 | y=list_(of=str_(of='abc123_', max_length=20), 74 | min_length=6, max_length=10) 75 | ): 76 | with MockTransaction: 77 | test_config = dict(zip(x, x)) 78 | paths = [config.DEFAILT_CONFIG_PATH, config.CONFIG_PATH] 79 | expect(config).read_config(paths).and_return(test_config).once() 80 | expect(config).validate_config(test_config, y). \ 81 | and_return(False).once() 82 | try: 83 | config.read_and_validate_config(paths, y) 84 | except KeyError: 85 | assert True 86 | else: 87 | assert False 88 | -------------------------------------------------------------------------------- /tests/globals/test_db_cleaner.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from mocktest import * 16 | from pyqcy import * 17 | 18 | import datetime 19 | 20 | import neat.globals.db_cleaner as cleaner 21 | import neat.common as common 22 | import neat.db_utils as db_utils 23 | 24 | import logging 25 | logging.disable(logging.CRITICAL) 26 | 27 | 28 | class DbCleaner(TestCase): 29 | 30 | @qc(10) 31 | def start( 32 | iterations=int_(min=0, max=10), 33 | time_interval=int_(min=0) 34 | ): 35 | with MockTransaction: 36 | state = {'property': 'value'} 37 | config = { 38 | 'log_directory': 'dir', 39 | 'log_level': 2, 40 | 'db_cleaner_interval': str(time_interval)} 41 | paths = [cleaner.DEFAILT_CONFIG_PATH, cleaner.CONFIG_PATH] 42 | fields = cleaner.REQUIRED_FIELDS 43 | expect(cleaner).read_and_validate_config(paths, fields). \ 44 | and_return(config).once() 45 | expect(common).init_logging('dir', 'db-cleaner.log', 2).once() 46 | expect(common).start(cleaner.init_state, 47 | cleaner.execute, 48 | config, 49 | time_interval).and_return(state).once() 50 | assert cleaner.start() == state 51 | 52 | @qc(1) 53 | def init_state(): 54 | with MockTransaction: 55 | db = mock('db') 56 | expect(cleaner).init_db('db'). \ 57 | and_return(db).once() 58 | config = {'sql_connection': 'db', 59 | 'db_cleaner_interval': 7200} 60 | state = cleaner.init_state(config) 61 | assert state['db'] == db 62 | assert state['time_delta'] == datetime.timedelta(0, 7200) 63 | 64 | @qc(1) 65 | def execute( 66 | uuid=str_(of='abc123-', min_length=36, max_length=36) 67 | ): 68 | with MockTransaction: 69 | db = db_utils.init_db('sqlite:///:memory:') 70 | result = db.vms.insert().execute(uuid=uuid) 71 | vm_id = result.inserted_primary_key[0] 72 | time = datetime.datetime.today() 73 | for i in range(10): 74 | db.vm_resource_usage.insert().execute( 75 | vm_id=1, 76 | cpu_mhz=i, 77 | timestamp=time.replace(second=i)) 78 | state = { 79 | 'db': db, 80 | 'time_delta': datetime.timedelta(seconds=5)} 81 | expect(cleaner).today(). \ 82 | and_return(time.replace(second=10)).once() 83 | assert db.select_cpu_mhz_for_vm(uuid, 100) == range(10) 84 | cleaner.execute({}, state) 85 | assert db.select_cpu_mhz_for_vm(uuid, 100) == range(5, 10) 86 | 87 | -------------------------------------------------------------------------------- /neat/globals/db_cleaner.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ The database cleaner module. 16 | 17 | The database cleaner periodically cleans up the data on resource usage 18 | by VMs stored in the database. This is requried to avoid excess growth 19 | of the database size. 20 | """ 21 | 22 | from contracts import contract 23 | from neat.contracts_primitive import * 24 | from neat.contracts_extra import * 25 | 26 | import datetime 27 | 28 | import neat.common as common 29 | from neat.config import * 30 | from neat.db_utils import * 31 | 32 | import logging 33 | log = logging.getLogger(__name__) 34 | 35 | 36 | @contract 37 | def start(): 38 | """ Start the database cleaner loop. 39 | 40 | :return: The final state. 41 | :rtype: dict(str: *) 42 | """ 43 | config = read_and_validate_config([DEFAILT_CONFIG_PATH, CONFIG_PATH], 44 | REQUIRED_FIELDS) 45 | 46 | common.init_logging( 47 | config['log_directory'], 48 | 'db-cleaner.log', 49 | int(config['log_level'])) 50 | 51 | interval = config['db_cleaner_interval'] 52 | if log.isEnabledFor(logging.INFO): 53 | log.info('Starting the database cleaner, ' + 54 | 'iterations every %s seconds', interval) 55 | return common.start( 56 | init_state, 57 | execute, 58 | config, 59 | int(interval)) 60 | 61 | 62 | @contract 63 | def init_state(config): 64 | """ Initialize a dict for storing the state of the database cleaner. 65 | 66 | :param config: A config dictionary. 67 | :type config: dict(str: *) 68 | 69 | :return: A dictionary containing the initial state of the database cleaner. 70 | :rtype: dict 71 | """ 72 | return { 73 | 'db': init_db(config['sql_connection']), 74 | 'time_delta': datetime.timedelta( 75 | seconds=int(config['db_cleaner_interval']))} 76 | 77 | 78 | @contract 79 | def execute(config, state): 80 | """ Execute an iteration of the database cleaner. 81 | 82 | :param config: A config dictionary. 83 | :type config: dict(str: *) 84 | 85 | :param state: A state dictionary. 86 | :type state: dict(str: *) 87 | 88 | :return: The updated state dictionary. 89 | :rtype: dict(str: *) 90 | """ 91 | datetime_threshold = today() - state['time_delta'] 92 | state['db'].cleanup_vm_resource_usage(datetime_threshold) 93 | state['db'].cleanup_host_resource_usage(datetime_threshold) 94 | if log.isEnabledFor(logging.INFO): 95 | log.info('Cleaned up data older than %s', 96 | datetime_threshold.strftime('%Y-%m-%d %H:%M:%S')) 97 | return state 98 | 99 | 100 | @contract 101 | def today(): 102 | """ Return the today's datetime. 103 | 104 | :return: A datetime object representing current date and time. 105 | :rtype: datetime 106 | """ 107 | return datetime.datetime.today() 108 | -------------------------------------------------------------------------------- /vm-placement.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2 2 | 3 | # Copyright 2012 Anton Beloglazov 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from novaclient.v2 import client 18 | import neat.common as common 19 | import neat.globals.manager as manager 20 | from neat.config import * 21 | import sys 22 | 23 | 24 | def vm_hostname(vm): 25 | return str(getattr(vm, 'OS-EXT-SRV-ATTR:host')) 26 | 27 | 28 | config = read_and_validate_config([DEFAILT_CONFIG_PATH, CONFIG_PATH], 29 | REQUIRED_FIELDS) 30 | db = manager.init_db(config['sql_connection']) 31 | nova = client.Client(config['os_admin_user'], 32 | config['os_admin_password'], 33 | config['os_admin_tenant_name'], 34 | config['os_auth_url'], 35 | service_type="compute") 36 | hosts = common.parse_compute_hosts(config['compute_hosts']) 37 | 38 | hosts_cpu_total, hosts_cpu_cores, hosts_ram_total = \ 39 | db.select_host_characteristics() 40 | hosts_cpu_core = \ 41 | dict((host, int(hosts_cpu_total[host] / hosts_cpu_cores[host])) 42 | for host in hosts_cpu_total.keys()) 43 | hosts_to_vms = manager.vms_by_hosts(nova, hosts) 44 | vms = [item for sublist in hosts_to_vms.values() for item in sublist] 45 | 46 | vms_names = [] 47 | vms_status = {} 48 | for uuid in vms: 49 | vm = nova.servers.get(uuid) 50 | vms_names.append((vm.human_id, uuid)) 51 | vms_status[uuid] = str(vm.status) 52 | vms_names = sorted(vms_names) 53 | 54 | vms_cpu_usage = db.select_last_cpu_mhz_for_vms() 55 | for vm in vms: 56 | if not vm in vms_cpu_usage: 57 | vms_cpu_usage[vm] = 0 58 | vms_ram_usage = manager.vms_ram_limit(nova, vms) 59 | 60 | hosts_cpu_usage_hypervisor = db.select_last_cpu_mhz_for_hosts() 61 | 62 | hosts_cpu_usage = {} 63 | hosts_ram_usage = {} 64 | for host, vms in hosts_to_vms.items(): 65 | hosts_cpu_usage[host] = hosts_cpu_usage_hypervisor[host] + \ 66 | sum(vms_cpu_usage[x] for x in vms) 67 | hosts_ram_usage[host] = manager.host_used_ram(nova, host) 68 | 69 | 70 | first = True 71 | for host in sorted(hosts_to_vms.keys()): 72 | if not first: 73 | print "-----------------------------------------------------------" 74 | first = False 75 | vms = hosts_to_vms[host] 76 | print '{0:24} {1:5d} / {2:5d} MHz {3:5d} / {4:5d} MB'. \ 77 | format(host, 78 | hosts_cpu_usage[host], 79 | hosts_cpu_total[host], 80 | hosts_ram_usage[host], 81 | hosts_ram_total[host]) 82 | for vm, uuid in vms_names: 83 | if uuid in vms: 84 | if uuid not in vms_ram_usage: 85 | vms_ram = 0 86 | else: 87 | vms_ram = vms_ram_usage[uuid] 88 | print ' {0:10} {1:9} {2:5d} / {3:5d} MHz {4:5d} / {5:5d} MB'. \ 89 | format(vm, 90 | vms_status[uuid], 91 | vms_cpu_usage[uuid], 92 | hosts_cpu_core[host], 93 | vms_ram, 94 | vms_ram) 95 | -------------------------------------------------------------------------------- /neat/locals/overload/mhod/bruteforce.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ Functions for solving NLP problems using the bruteforce method. 16 | """ 17 | 18 | from contracts import contract 19 | from neat.contracts_primitive import * 20 | from neat.contracts_extra import * 21 | 22 | import nlp 23 | from neat.common import frange 24 | 25 | import logging 26 | log = logging.getLogger(__name__) 27 | 28 | 29 | @contract 30 | def solve2(objective, constraint, step, limit): 31 | """ Solve a maximization problem for 2 states. 32 | 33 | :param objective: The objective function. 34 | :type objective: function 35 | 36 | :param constraint: A tuple representing the constraint. 37 | :type constraint: tuple(function, function, number) 38 | 39 | :param step: The step size. 40 | :type step: number,>0 41 | 42 | :param limit: The maximum value of the variables. 43 | :type limit: number,>0 44 | 45 | :return: The problem solution. 46 | :rtype: list(number) 47 | """ 48 | res_best = 0 49 | solution = [] 50 | for x in frange(0, limit, step): 51 | for y in frange(0, limit, step): 52 | try: 53 | res = objective(x, y) 54 | if res > res_best and \ 55 | constraint[1](constraint[0](x, y), constraint[2]): 56 | res_best = res 57 | solution = [x, y] 58 | except ZeroDivisionError: 59 | pass 60 | return solution 61 | 62 | 63 | @contract 64 | def optimize(step, limit, otf, migration_time, ls, p, state_vector, 65 | time_in_states, time_in_state_n): 66 | """ Solve a MHOD optimization problem. 67 | 68 | :param step: The step size for the bruteforce algorithm. 69 | :type step: number,>0 70 | 71 | :param limit: The maximum value of the variables. 72 | :type limit: number,>0 73 | 74 | :param otf: The OTF parameter. 75 | :type otf: number,>=0,<=1 76 | 77 | :param migration_time: The VM migration time in time steps. 78 | :type migration_time: float,>=0 79 | 80 | :param ls: L functions. 81 | :type ls: list(function) 82 | 83 | :param p: A matrix of transition probabilities. 84 | :type p: list(list(number)) 85 | 86 | :param state_vector: A state vector. 87 | :type state_vector: list(int) 88 | 89 | :param time_in_states: The total time in all the states in time steps. 90 | :type time_in_states: number,>=0 91 | 92 | :param time_in_state_n: The total time in the state N in time steps. 93 | :type time_in_state_n: number,>=0 94 | 95 | :return: The solution of the problem. 96 | :rtype: list(number) 97 | """ 98 | objective = nlp.build_objective(ls, state_vector, p) 99 | constraint = nlp.build_constraint(otf, migration_time, ls, state_vector, 100 | p, time_in_states, time_in_state_n) 101 | return solve2(objective, constraint, step, limit) 102 | -------------------------------------------------------------------------------- /neat/db_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from contracts import contract 16 | from neat.contracts_primitive import * 17 | from neat.contracts_extra import * 18 | 19 | from sqlalchemy import * 20 | from sqlalchemy.sql import func 21 | 22 | from neat.db import Database 23 | 24 | import logging 25 | log = logging.getLogger(__name__) 26 | 27 | 28 | @contract 29 | def init_db(sql_connection): 30 | """ Initialize the database. 31 | 32 | :param sql_connection: A database connection URL. 33 | :type sql_connection: str 34 | 35 | :return: The initialized database. 36 | :rtype: Database 37 | """ 38 | engine = create_engine(sql_connection) # 'sqlite:///:memory:' 39 | metadata = MetaData() 40 | metadata.bind = engine 41 | 42 | hosts = Table('hosts', metadata, 43 | Column('id', Integer, primary_key=True), 44 | Column('hostname', String(255), nullable=False), 45 | Column('cpu_mhz', Integer, nullable=False), 46 | Column('cpu_cores', Integer, nullable=False), 47 | Column('ram', Integer, nullable=False)) 48 | 49 | host_resource_usage = \ 50 | Table('host_resource_usage', metadata, 51 | Column('id', Integer, primary_key=True), 52 | Column('host_id', Integer, ForeignKey('hosts.id'), nullable=False), 53 | Column('timestamp', DateTime, default=func.now()), 54 | Column('cpu_mhz', Integer, nullable=False)) 55 | 56 | vms = Table('vms', metadata, 57 | Column('id', Integer, primary_key=True), 58 | Column('uuid', String(36), nullable=False)) 59 | 60 | vm_resource_usage = \ 61 | Table('vm_resource_usage', metadata, 62 | Column('id', Integer, primary_key=True), 63 | Column('vm_id', Integer, ForeignKey('vms.id'), nullable=False), 64 | Column('timestamp', DateTime, default=func.now()), 65 | Column('cpu_mhz', Integer, nullable=False)) 66 | 67 | vm_migrations = \ 68 | Table('vm_migrations', metadata, 69 | Column('id', Integer, primary_key=True), 70 | Column('vm_id', Integer, ForeignKey('vms.id'), nullable=False), 71 | Column('host_id', Integer, ForeignKey('hosts.id'), nullable=False), 72 | Column('timestamp', DateTime, default=func.now())) 73 | 74 | host_states = \ 75 | Table('host_states', metadata, 76 | Column('id', Integer, primary_key=True), 77 | Column('host_id', Integer, ForeignKey('hosts.id'), nullable=False), 78 | Column('timestamp', DateTime, default=func.now()), 79 | Column('state', Integer, nullable=False)) 80 | 81 | host_overload = \ 82 | Table('host_overload', metadata, 83 | Column('id', Integer, primary_key=True), 84 | Column('host_id', Integer, ForeignKey('hosts.id'), nullable=False), 85 | Column('timestamp', DateTime, default=func.now()), 86 | Column('overload', Integer, nullable=False)) 87 | 88 | metadata.create_all() 89 | connection = engine.connect() 90 | db = Database(connection, hosts, host_resource_usage, vms, 91 | vm_resource_usage, vm_migrations, host_states, host_overload) 92 | 93 | log.debug('Initialized a DB connection to %s', sql_connection) 94 | return db 95 | -------------------------------------------------------------------------------- /neat/locals/overload/otf.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ OTF threshold based algorithms. 16 | """ 17 | 18 | from contracts import contract 19 | from neat.contracts_primitive import * 20 | from neat.contracts_extra import * 21 | 22 | import logging 23 | log = logging.getLogger(__name__) 24 | 25 | 26 | @contract 27 | def otf_factory(time_step, migration_time, params): 28 | """ Creates the OTF algorithm with limiting and migration time. 29 | 30 | :param time_step: The length of the simulation time step in seconds. 31 | :type time_step: int,>=0 32 | 33 | :param migration_time: The VM migration time in time seconds. 34 | :type migration_time: float,>=0 35 | 36 | :param params: A dictionary containing the algorithm's parameters. 37 | :type params: dict(str: *) 38 | 39 | :return: A function implementing the OTF algorithm. 40 | :rtype: function 41 | """ 42 | migration_time_normalized = float(migration_time) / time_step 43 | def otf_wrapper(utilization, state=None): 44 | if state is None or state == {}: 45 | state = {'overload': 0, 46 | 'total': 0} 47 | return otf(params['otf'], 48 | params['threshold'], 49 | params['limit'], 50 | migration_time_normalized, 51 | utilization, 52 | state) 53 | 54 | return otf_wrapper 55 | 56 | 57 | @contract 58 | def otf(otf, threshold, limit, migration_time, utilization, state): 59 | """ The OTF threshold algorithm with limiting and migration time. 60 | 61 | :param otf: The threshold on the OTF value. 62 | :type otf: float,>=0 63 | 64 | :param threshold: The utilization overload threshold. 65 | :type threshold: float,>=0 66 | 67 | :param limit: The minimum number of values in the utilization history. 68 | :type limit: int,>=0 69 | 70 | :param migration_time: The VM migration time in time steps. 71 | :type migration_time: float,>=0 72 | 73 | :param utilization: The history of the host's CPU utilization. 74 | :type utilization: list(float) 75 | 76 | :param state: The state dictionary. 77 | :type state: dict(str: *) 78 | 79 | :return: The decision of the algorithm and updated state. 80 | :rtype: tuple(bool, dict(*: *)) 81 | """ 82 | state['total'] += 1 83 | overload = (utilization[-1] >= threshold) 84 | if overload: 85 | state['overload'] += 1 86 | 87 | if log.isEnabledFor(logging.DEBUG): 88 | log.debug('OTF overload:' + str(overload)) 89 | log.debug('OTF overload steps:' + str(state['overload'])) 90 | log.debug('OTF total steps:' + str(state['total'])) 91 | log.debug('OTF:' + str(float(state['overload']) / state['total'])) 92 | log.debug('OTF migration time:' + str(migration_time)) 93 | log.debug('OTF + migration time:' + 94 | str((migration_time + state['overload']) / \ 95 | (migration_time + state['total']))) 96 | log.debug('OTF decision:' + 97 | str(overload and (migration_time + state['overload']) / \ 98 | (migration_time + state['total']) >= otf)) 99 | 100 | if not overload or len(utilization) < limit: 101 | decision = False 102 | else: 103 | decision = (migration_time + state['overload']) / \ 104 | (migration_time + state['total']) >= otf 105 | 106 | return (decision, state) 107 | -------------------------------------------------------------------------------- /tests/locals/underload/test_trivial.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from mocktest import * 16 | from pyqcy import * 17 | 18 | import neat.locals.underload.trivial as trivial 19 | 20 | import logging 21 | logging.disable(logging.CRITICAL) 22 | 23 | 24 | class Trivial(TestCase): 25 | 26 | @qc(10) 27 | def always_underloaded_factory( 28 | time_step=int_(min=0, max=10), 29 | migration_time=float_(min=0, max=10), 30 | utilization=list_(of=float) 31 | ): 32 | alg = trivial.always_underloaded_factory(time_step, migration_time, {}) 33 | assert alg(utilization) == (True, {}) 34 | 35 | def test_threshold_factory(self): 36 | alg = trivial.threshold_factory(300, 20., {'threshold': 0.5}) 37 | self.assertEqual(alg([]), (False, {})) 38 | self.assertEqual(alg([0.0, 0.0]), (True, {})) 39 | self.assertEqual(alg([0.0, 0.4]), (True, {})) 40 | self.assertEqual(alg([0.0, 0.5]), (True, {})) 41 | self.assertEqual(alg([0.0, 0.6]), (False, {})) 42 | self.assertEqual(alg([0.0, 1.0]), (False, {})) 43 | 44 | def test_last_n_average_threshold_factory(self): 45 | alg = trivial.last_n_average_threshold_factory( 46 | 300, 20., {'threshold': 0.5, 47 | 'n': 2}) 48 | self.assertEqual(alg([]), (False, {})) 49 | self.assertEqual(alg([0.0, 0.0]), (True, {})) 50 | self.assertEqual(alg([0.0, 0.4]), (True, {})) 51 | self.assertEqual(alg([0.0, 0.5]), (True, {})) 52 | self.assertEqual(alg([0.0, 0.6]), (True, {})) 53 | self.assertEqual(alg([0.0, 1.0]), (True, {})) 54 | self.assertEqual(alg([0.2, 1.0]), (False, {})) 55 | self.assertEqual(alg([0.0, 0.2, 1.0]), (False, {})) 56 | self.assertEqual(alg([0.0, 1.0, 1.0]), (False, {})) 57 | self.assertEqual(alg([0.0, 0.6, 0.6]), (False, {})) 58 | 59 | alg = trivial.last_n_average_threshold_factory( 60 | 300, 20., {'threshold': 0.5, 61 | 'n': 3}) 62 | self.assertEqual(alg([0.0, 0.6, 0.6]), (True, {})) 63 | 64 | def test_threshold(self): 65 | self.assertEqual(trivial.threshold(0.5, []), False) 66 | self.assertEqual(trivial.threshold(0.5, [0.0, 0.0]), True) 67 | self.assertEqual(trivial.threshold(0.5, [0.0, 0.4]), True) 68 | self.assertEqual(trivial.threshold(0.5, [0.0, 0.5]), True) 69 | self.assertEqual(trivial.threshold(0.5, [0.0, 0.6]), False) 70 | self.assertEqual(trivial.threshold(0.5, [0.0, 1.0]), False) 71 | 72 | def test_last_n_average_threshold(self): 73 | self.assertEqual(trivial.last_n_average_threshold( 74 | 0.5, 2, []), False) 75 | self.assertEqual(trivial.last_n_average_threshold( 76 | 0.5, 2, [0.0, 0.0]), True) 77 | self.assertEqual(trivial.last_n_average_threshold( 78 | 0.5, 2, [0.0, 0.4]), True) 79 | self.assertEqual(trivial.last_n_average_threshold( 80 | 0.5, 2, [0.0, 0.5]), True) 81 | self.assertEqual(trivial.last_n_average_threshold( 82 | 0.5, 2, [0.0, 0.6]), True) 83 | self.assertEqual(trivial.last_n_average_threshold( 84 | 0.5, 2, [0.0, 1.0]), True) 85 | self.assertEqual(trivial.last_n_average_threshold( 86 | 0.5, 2, [0.2, 1.0]), False) 87 | self.assertEqual(trivial.last_n_average_threshold( 88 | 0.5, 2, [0.0, 0.2, 1.0]), False) 89 | self.assertEqual(trivial.last_n_average_threshold( 90 | 0.5, 2, [0.0, 1.0, 1.0]), False) 91 | self.assertEqual(trivial.last_n_average_threshold( 92 | 0.5, 2, [0.0, 0.6, 0.6]), False) 93 | self.assertEqual(trivial.last_n_average_threshold( 94 | 0.5, 3, [0.0, 0.6, 0.6]), True) 95 | -------------------------------------------------------------------------------- /tests/locals/overload/test_trivial.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from mocktest import * 16 | from pyqcy import * 17 | 18 | import neat.locals.overload.trivial as trivial 19 | 20 | import logging 21 | logging.disable(logging.CRITICAL) 22 | 23 | 24 | class Trivial(TestCase): 25 | 26 | @qc(10) 27 | def never_overloaded_factory( 28 | time_step=int_(min=0, max=10), 29 | migration_time=float_(min=0, max=10), 30 | utilization=list_(of=float) 31 | ): 32 | alg = trivial.never_overloaded_factory(time_step, migration_time, {}) 33 | assert alg(utilization) == (False, {}) 34 | 35 | def test_threshold_factory(self): 36 | alg = trivial.threshold_factory(300, 20., {'threshold': 0.5}) 37 | self.assertEquals(alg([0.9, 0.8, 1.1, 1.2, 1.3]), (True, {})) 38 | self.assertEquals(alg([0.9, 0.8, 1.1, 1.2, 0.6]), (True, {})) 39 | self.assertEquals(alg([0.9, 0.8, 1.1, 1.2, 0.5]), (False, {})) 40 | self.assertEquals(alg([0.9, 0.8, 1.1, 1.2, 0.3]), (False, {})) 41 | self.assertEquals(alg([]), (False, {})) 42 | 43 | def test_last_n_average_threshold_factory(self): 44 | alg = trivial.last_n_average_threshold_factory( 45 | 300, 20., {'threshold': 0.5, 'n': 1}) 46 | self.assertEquals(alg([0.9, 0.8, 1.1, 1.2, 1.3]), (True, {})) 47 | self.assertEquals(alg([0.9, 0.8, 1.1, 1.2, 0.6]), (True, {})) 48 | self.assertEquals(alg([0.9, 0.8, 1.1, 1.2, 0.5]), (False, {})) 49 | self.assertEquals(alg([0.9, 0.8, 1.1, 1.2, 0.3]), (False, {})) 50 | self.assertEquals(alg([]), (False, {})) 51 | 52 | alg = trivial.last_n_average_threshold_factory( 53 | 300, 20., {'threshold': 0.5, 'n': 2}) 54 | self.assertEquals(alg([0.9, 0.8, 1.1, 1.2, 1.3]), (True, {})) 55 | self.assertEquals(alg([0.9, 0.8, 1.1, 1.2, 0.6]), (True, {})) 56 | self.assertEquals(alg([0.9, 0.8, 1.1, 1.2, 0.4]), (True, {})) 57 | self.assertEquals(alg([0.9, 0.8, 1.1, 0.4, 0.5]), (False, {})) 58 | self.assertEquals(alg([0.9, 0.8, 1.1, 0.2, 0.3]), (False, {})) 59 | self.assertEquals(alg([]), (False, {})) 60 | 61 | def test_threshold(self): 62 | self.assertTrue(trivial.threshold(0.5, [0.9, 0.8, 1.1, 1.2, 1.3])) 63 | self.assertTrue(trivial.threshold(0.5, [0.9, 0.8, 1.1, 1.2, 0.6])) 64 | self.assertFalse(trivial.threshold(0.5, [0.9, 0.8, 1.1, 1.2, 0.5])) 65 | self.assertFalse(trivial.threshold(0.5, [0.9, 0.8, 1.1, 1.2, 0.3])) 66 | self.assertFalse(trivial.threshold(0.5, [])) 67 | 68 | def test_last_n_average_threshold(self): 69 | self.assertTrue(trivial.last_n_average_threshold( 70 | 0.5, 1, [0.9, 0.8, 1.1, 1.2, 1.3])) 71 | self.assertTrue(trivial.last_n_average_threshold( 72 | 0.5, 1, [0.9, 0.8, 1.1, 1.2, 0.6])) 73 | self.assertFalse(trivial.last_n_average_threshold( 74 | 0.5, 1, [0.9, 0.8, 1.1, 1.2, 0.5])) 75 | self.assertFalse(trivial.last_n_average_threshold( 76 | 0.5, 1, [0.9, 0.8, 1.1, 1.2, 0.3])) 77 | self.assertFalse(trivial.last_n_average_threshold( 78 | 0.5, 1, [])) 79 | 80 | self.assertTrue(trivial.last_n_average_threshold( 81 | 0.5, 2, [0.9, 0.8, 1.1, 1.2, 1.3])) 82 | self.assertTrue(trivial.last_n_average_threshold( 83 | 0.5, 2, [0.9, 0.8, 1.1, 1.2, 0.6])) 84 | self.assertTrue(trivial.last_n_average_threshold( 85 | 0.5, 2, [0.9, 0.8, 1.1, 1.2, 0.4])) 86 | self.assertFalse(trivial.last_n_average_threshold( 87 | 0.5, 2, [0.9, 0.8, 1.1, 0.4, 0.5])) 88 | self.assertFalse(trivial.last_n_average_threshold( 89 | 0.5, 2, [0.9, 0.8, 1.1, 0.2, 0.3])) 90 | self.assertFalse(trivial.last_n_average_threshold(0.5, 2, [])) 91 | -------------------------------------------------------------------------------- /neat/config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ A set of functions for reading configuration options from files. 16 | 17 | """ 18 | 19 | from contracts import contract 20 | import os 21 | import ConfigParser 22 | 23 | import logging 24 | log = logging.getLogger(__name__) 25 | 26 | 27 | # This is the default config, which should not be modified 28 | DEFAILT_CONFIG_PATH = os.path.join(os.path.dirname(__file__), 29 | '..', 30 | 'neat.conf') 31 | 32 | # This is the custom config, which may override the defaults 33 | CONFIG_PATH = "/etc/neat/neat.conf" 34 | # The following value is used for testing purposes 35 | #CONFIG_PATH = os.path.join(os.path.dirname(__file__), 36 | # '..', 37 | # 'neat.conf') 38 | 39 | # These fields must present in the configuration file 40 | REQUIRED_FIELDS = [ 41 | 'log_directory', 42 | 'log_level', 43 | 'vm_instance_directory', 44 | 'sql_connection', 45 | 'os_admin_tenant_name', 46 | 'os_admin_user', 47 | 'os_admin_password', 48 | 'os_auth_url', 49 | 'compute_hosts', 50 | 'global_manager_host', 51 | 'global_manager_port', 52 | 'db_cleaner_interval', 53 | 'local_data_directory', 54 | 'local_manager_interval', 55 | 'data_collector_interval', 56 | 'data_collector_data_length', 57 | 'host_cpu_overload_threshold', 58 | 'host_cpu_usable_by_vms', 59 | 'compute_user', 60 | 'compute_password', 61 | 'sleep_command', 62 | 'ether_wake_interface', 63 | 'block_migration', 64 | 'network_migration_bandwidth', 65 | 'algorithm_underload_detection_factory', 66 | 'algorithm_underload_detection_parameters', 67 | 'algorithm_overload_detection_factory', 68 | 'algorithm_overload_detection_parameters', 69 | 'algorithm_vm_selection_factory', 70 | 'algorithm_vm_selection_parameters', 71 | 'algorithm_vm_placement_factory', 72 | 'algorithm_vm_placement_parameters', 73 | ] 74 | 75 | 76 | @contract 77 | def read_config(paths): 78 | """ Read the configuration files and return the options. 79 | 80 | :param paths: A list of required configuration file paths. 81 | :type paths: list(str) 82 | 83 | :return: A dictionary of the configuration options. 84 | :rtype: dict(str: str) 85 | """ 86 | configParser = ConfigParser.ConfigParser() 87 | for path in paths: 88 | configParser.read(path) 89 | return dict(configParser.items("DEFAULT")) 90 | 91 | 92 | @contract 93 | def validate_config(config, required_fields): 94 | """ Check that the config contains all the required fields. 95 | 96 | :param config: A config dictionary to check. 97 | :type config: dict(str: str) 98 | 99 | :param required_fields: A list of required fields. 100 | :type required_fields: list(str) 101 | 102 | :return: Whether the config is valid. 103 | :rtype: bool 104 | """ 105 | for field in required_fields: 106 | if not field in config: 107 | return False 108 | return True 109 | 110 | 111 | @contract 112 | def read_and_validate_config(paths, required_fields): 113 | """ Read the configuration files, validate and return the options. 114 | 115 | :param paths: A list of required configuration file paths. 116 | :type paths: list(str) 117 | 118 | :param required_fields: A list of required fields. 119 | :type required_fields: list(str) 120 | 121 | :return: A dictionary of the configuration options. 122 | :rtype: dict(str: str) 123 | """ 124 | config = read_config(paths) 125 | if not validate_config(config, required_fields): 126 | message = 'The config dictionary does not contain ' + \ 127 | 'all the required fields' 128 | log.critical(message) 129 | raise KeyError(message) 130 | return config 131 | -------------------------------------------------------------------------------- /neat/locals/overload/trivial.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ Trivial overload detection algorithms. 16 | """ 17 | 18 | from contracts import contract 19 | from neat.contracts_primitive import * 20 | from neat.contracts_extra import * 21 | 22 | import logging 23 | log = logging.getLogger(__name__) 24 | 25 | 26 | @contract 27 | def never_overloaded_factory(time_step, migration_time, params): 28 | """ Creates an algorithm that never considers the host overloaded. 29 | 30 | :param time_step: The length of the simulation time step in seconds. 31 | :type time_step: int,>=0 32 | 33 | :param migration_time: The VM migration time in time seconds. 34 | :type migration_time: float,>=0 35 | 36 | :param params: A dictionary containing the algorithm's parameters. 37 | :type params: dict(str: *) 38 | 39 | :return: A function implementing the algorithm. 40 | :rtype: function 41 | """ 42 | return lambda utilization, state=None: (False, {}) 43 | 44 | 45 | @contract 46 | def threshold_factory(time_step, migration_time, params): 47 | """ Creates the static CPU utilization threshold algorithm. 48 | 49 | :param time_step: The length of the simulation time step in seconds. 50 | :type time_step: int,>=0 51 | 52 | :param migration_time: The VM migration time in time seconds. 53 | :type migration_time: float,>=0 54 | 55 | :param params: A dictionary containing the algorithm's parameters. 56 | :type params: dict(str: *) 57 | 58 | :return: A function implementing the static threshold algorithm. 59 | :rtype: function 60 | """ 61 | return lambda utilization, state=None: (threshold(params['threshold'], 62 | utilization), 63 | {}) 64 | 65 | 66 | @contract 67 | def last_n_average_threshold_factory(time_step, migration_time, params): 68 | """ Creates the averaging CPU utilization threshold algorithm. 69 | 70 | :param time_step: The length of the simulation time step in seconds. 71 | :type time_step: int,>=0 72 | 73 | :param migration_time: The VM migration time in time seconds. 74 | :type migration_time: float,>=0 75 | 76 | :param params: A dictionary containing the algorithm's parameters. 77 | :type params: dict(str: *) 78 | 79 | :return: A function implementing the averaging threshold algorithm. 80 | :rtype: function 81 | """ 82 | return lambda utilization, state=None: ( 83 | last_n_average_threshold(params['threshold'], 84 | params['n'], 85 | utilization), 86 | {}) 87 | 88 | 89 | @contract 90 | def threshold(threshold, utilization): 91 | """ The static CPU utilization threshold algorithm. 92 | 93 | :param threshold: The threshold on the CPU utilization. 94 | :type threshold: float,>=0 95 | 96 | :param utilization: The history of the host's CPU utilization. 97 | :type utilization: list(float) 98 | 99 | :return: The decision of the algorithm. 100 | :rtype: bool 101 | """ 102 | if utilization: 103 | return utilization[-1] > threshold 104 | return False 105 | 106 | 107 | @contract 108 | def last_n_average_threshold(threshold, n, utilization): 109 | """ The averaging CPU utilization threshold algorithm. 110 | 111 | :param threshold: The threshold on the CPU utilization. 112 | :type threshold: float,>=0 113 | 114 | :param n: The number of last CPU utilization values to average. 115 | :type n: int,>0 116 | 117 | :param utilization: The history of the host's CPU utilization. 118 | :type utilization: list(float) 119 | 120 | :return: The decision of the algorithm. 121 | :rtype: bool 122 | """ 123 | if utilization: 124 | utilization = utilization[-n:] 125 | return sum(utilization) / len(utilization) > threshold 126 | return False 127 | -------------------------------------------------------------------------------- /tests/locals/overload/test_otf.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from mocktest import * 16 | from pyqcy import * 17 | 18 | import neat.locals.overload.otf as otf 19 | 20 | import logging 21 | logging.disable(logging.CRITICAL) 22 | 23 | 24 | class Otf(TestCase): 25 | 26 | def test_otf(self): 27 | state = {'overload': 0, 'total': 0} 28 | 29 | decision, state = otf.otf(0.5, 1.0, 4, 1., 30 | [0.9], state) 31 | self.assertEqual(state, {'overload': 0, 'total': 1}) 32 | self.assertFalse(decision) 33 | 34 | decision, state = otf.otf(0.5, 1.0, 4, 1., 35 | [0.9, 1.3], state) 36 | self.assertEqual(state, {'overload': 1, 'total': 2}) 37 | self.assertFalse(decision) 38 | 39 | decision, state = otf.otf(0.5, 1.0, 4, 1., 40 | [0.9, 1.3, 1.1], state) 41 | self.assertEqual(state, {'overload': 2, 'total': 3}) 42 | self.assertFalse(decision) 43 | 44 | decision, state = otf.otf(0.5, 1.0, 4, 1., 45 | [0.9, 1.3, 1.1, 1.2], state) 46 | self.assertEqual(state, {'overload': 3, 'total': 4}) 47 | self.assertTrue(decision) 48 | 49 | decision, state = otf.otf(0.5, 1.0, 4, 100., 50 | [0.9, 1.3, 1.1, 1.2, 0.3], state) 51 | self.assertEqual(state, {'overload': 3, 'total': 5}) 52 | self.assertFalse(decision) 53 | 54 | decision, state = otf.otf(0.5, 1.0, 4, 1., 55 | [0.9, 1.3, 1.1, 1.2, 1.3], state) 56 | self.assertEqual(state, {'overload': 4, 'total': 6}) 57 | self.assertTrue(decision) 58 | 59 | decision, state = otf.otf(0.5, 1.0, 4, 1., 60 | [0.9, 1.3, 1.1, 1.2, 0.3, 0.2], state) 61 | self.assertEqual(state, {'overload': 4, 'total': 7}) 62 | self.assertFalse(decision) 63 | 64 | decision, state = otf.otf(0.5, 1.0, 4, 0., 65 | [0.9, 1.3, 1.1, 1.2, 0.3, 0.2, 0.1], state) 66 | self.assertEqual(state, {'overload': 4, 'total': 8}) 67 | self.assertFalse(decision) 68 | 69 | decision, state = otf.otf(0.5, 1.0, 4, 0., 70 | [0.9, 1.3, 1.1, 1.2, 0.3, 0.2, 0.1, 0.1], state) 71 | self.assertEqual(state, {'overload': 4, 'total': 9}) 72 | self.assertFalse(decision) 73 | 74 | 75 | def test_otf_factory(self): 76 | alg = otf.otf_factory(30, 0., 77 | {'otf': 0.5, 'threshold': 1.0, 'limit': 4}) 78 | 79 | decision, state = alg([0.9], None) 80 | self.assertEqual(state, {'overload': 0, 'total': 1}) 81 | self.assertFalse(decision) 82 | 83 | decision, state = alg([0.9, 1.3], state) 84 | self.assertEqual(state, {'overload': 1, 'total': 2}) 85 | self.assertFalse(decision) 86 | 87 | decision, state = alg([0.9, 1.3, 1.1], state) 88 | self.assertEqual(state, {'overload': 2, 'total': 3}) 89 | self.assertFalse(decision) 90 | 91 | decision, state = alg([0.9, 1.3, 1.1, 1.2], state) 92 | self.assertEqual(state, {'overload': 3, 'total': 4}) 93 | self.assertTrue(decision) 94 | 95 | decision, state = alg([0.9, 1.3, 1.1, 1.2, 0.3], state) 96 | self.assertEqual(state, {'overload': 3, 'total': 5}) 97 | self.assertFalse(decision) 98 | 99 | decision, state = alg([0.9, 1.3, 1.1, 1.2, 1.3], state) 100 | self.assertEqual(state, {'overload': 4, 'total': 6}) 101 | self.assertTrue(decision) 102 | 103 | decision, state = alg([0.9, 1.3, 1.1, 1.2, 0.3, 0.2], state) 104 | self.assertEqual(state, {'overload': 4, 'total': 7}) 105 | self.assertFalse(decision) 106 | 107 | decision, state = alg([0.9, 1.3, 1.1, 1.2, 0.3, 0.2, 0.1], state) 108 | self.assertEqual(state, {'overload': 4, 'total': 8}) 109 | self.assertFalse(decision) 110 | 111 | decision, state = alg([0.9, 1.3, 1.1, 1.2, 0.3, 0.2, 0.1, 0.1], state) 112 | self.assertEqual(state, {'overload': 4, 'total': 9}) 113 | self.assertFalse(decision) 114 | -------------------------------------------------------------------------------- /neat/locals/underload/trivial.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ Trivial underload detection algorithms. 16 | """ 17 | 18 | from contracts import contract 19 | from neat.contracts_primitive import * 20 | from neat.contracts_extra import * 21 | 22 | import logging 23 | log = logging.getLogger(__name__) 24 | 25 | 26 | @contract 27 | def always_underloaded_factory(time_step, migration_time, params): 28 | """ Creates an algorithm that always considers the host underloaded. 29 | 30 | :param time_step: The length of the simulation time step in seconds. 31 | :type time_step: int,>=0 32 | 33 | :param migration_time: The VM migration time in time seconds. 34 | :type migration_time: float,>=0 35 | 36 | :param params: A dictionary containing the algorithm's parameters. 37 | :type params: dict(str: *) 38 | 39 | :return: A function implementing the algorithm. 40 | :rtype: function 41 | """ 42 | return lambda utilization, state=None: (True, {}) 43 | 44 | 45 | @contract 46 | def threshold_factory(time_step, migration_time, params): 47 | """ Creates the threshold underload detection algorithm. 48 | 49 | :param time_step: The length of the simulation time step in seconds. 50 | :type time_step: int,>=0 51 | 52 | :param migration_time: The VM migration time in time seconds. 53 | :type migration_time: float,>=0 54 | 55 | :param params: A dictionary containing the algorithm's parameters. 56 | :type params: dict(str: *) 57 | 58 | :return: A function implementing the OTF algorithm. 59 | :rtype: function 60 | """ 61 | return lambda utilization, state=None: (threshold(params['threshold'], 62 | utilization), 63 | {}) 64 | 65 | @contract 66 | def last_n_average_threshold_factory(time_step, migration_time, params): 67 | """ Creates the averaging threshold underload detection algorithm. 68 | 69 | :param time_step: The length of the simulation time step in seconds. 70 | :type time_step: int,>=0 71 | 72 | :param migration_time: The VM migration time in time seconds. 73 | :type migration_time: float,>=0 74 | 75 | :param params: A dictionary containing the algorithm's parameters. 76 | :type params: dict(str: *) 77 | 78 | :return: A function implementing the averaging underload detection. 79 | :rtype: function 80 | """ 81 | return lambda utilization, state=None: ( 82 | last_n_average_threshold(params['threshold'], 83 | params['n'], 84 | utilization), 85 | {}) 86 | 87 | 88 | @contract 89 | def threshold(threshold, utilization): 90 | """ Static threshold-based underload detection algorithm. 91 | 92 | The algorithm returns True, if the last value of the host's 93 | CPU utilization is lower than the specified threshold. 94 | 95 | :param threshold: The static underload CPU utilization threshold. 96 | :type threshold: float,>=0,<=1 97 | 98 | :param utilization: The history of the host's CPU utilization. 99 | :type utilization: list(float) 100 | 101 | :return: A decision of whether the host is underloaded. 102 | :rtype: bool 103 | """ 104 | if utilization: 105 | return utilization[-1] <= threshold 106 | return False 107 | 108 | 109 | @contract 110 | def last_n_average_threshold(threshold, n, utilization): 111 | """ Averaging static threshold-based underload detection algorithm. 112 | 113 | The algorithm returns True, if the average of the last n values of 114 | the host's CPU utilization is lower than the specified threshold. 115 | 116 | :param threshold: The static underload CPU utilization threshold. 117 | :type threshold: float,>=0,<=1 118 | 119 | :param n: The number of last values to average. 120 | :type n: int,>0 121 | 122 | :param utilization: The history of the host's CPU utilization. 123 | :type utilization: list(float) 124 | 125 | :return: A decision of whether the host is underloaded. 126 | :rtype: bool 127 | """ 128 | if utilization: 129 | utilization = utilization[-n:] 130 | return sum(utilization) / len(utilization) <= threshold 131 | return False 132 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | The OpenStack Neat Project 17 | ========================== 18 | 19 | OpenStack Neat is a project intended to provide an extension to 20 | OpenStack implementing dynamic consolidation of Virtual Machines (VMs) 21 | using live migration. The major objective of dynamic VM consolidation 22 | is to improve the utilization of physical resources and reduce energy 23 | consumption by re-allocating VMs using live migration according to 24 | their real-time resource demand and switching idle hosts to the sleep 25 | mode. Apart from consolidating VMs, the system should be able to react 26 | to increases in the resource demand and deconsolidate VMs when 27 | necessary to avoid performance degradation. In general, the problem of 28 | dynamic VM consolidation includes 4 sub-problems: host underload / 29 | overload detection, VM selection, and VM placement. 30 | 31 | This work is conducted within the Cloud Computing and Distributed 32 | Systems (CLOUDS) Laboratory (http://www.cloudbus.org/) at the 33 | University of Melbourne. The problem of dynamic VM consolidation 34 | considering Quality of Service (QoS) constraints has been studied from 35 | the theoretical perspective and algorithms addressing the sub-problems 36 | listed above have been proposed [1], [2]. The algorithms have been 37 | evaluated using CloudSim (http://code.google.com/p/cloudsim/) and 38 | real-world workload traces collected from more than a thousand 39 | PlanetLab VMs hosted on servers located in more than 500 places around 40 | the world. 41 | 42 | The aim of the OpenStack Neat project is to provide an extensible 43 | framework for dynamic consolidation of VMs based on the OpenStack 44 | platform. The framework should provide an infrastructure enabling the 45 | interaction of components implementing the decision-making algorithms. 46 | The framework should allow configuration-driven switching of different 47 | implementations of the decision-making algorithms. The implementation 48 | of the framework will include the algorithms proposed in our previous 49 | works [1], [2]. 50 | 51 | [1] Anton Beloglazov and Rajkumar Buyya, "Optimal Online Deterministic 52 | Algorithms and Adaptive Heuristics for Energy and Performance 53 | Efficient Dynamic Consolidation of Virtual Machines in Cloud Data 54 | Centers", Concurrency and Computation: Practice and Experience (CCPE), 55 | Volume 24, Issue 13, Pages: 1397-1420, John Wiley & Sons, Ltd, New 56 | York, USA, 2012. Download: 57 | http://beloglazov.info/papers/2012-optimal-algorithms-ccpe.pdf 58 | 59 | [2] Anton Beloglazov and Rajkumar Buyya, "Managing Overloaded Hosts 60 | for Dynamic Consolidation of Virtual Machines in Cloud Data Centers 61 | Under Quality of Service Constraints", IEEE Transactions on Parallel 62 | and Distributed Systems (TPDS), IEEE CS Press, USA, 2012 (in press, 63 | accepted on August 2, 2012). Download: 64 | http://beloglazov.info/papers/2012-host-overload-detection-tpds.pdf 65 | """ 66 | 67 | import distribute_setup 68 | distribute_setup.use_setuptools() 69 | 70 | from setuptools import setup, find_packages 71 | 72 | 73 | setup( 74 | name='openstack-neat', 75 | version='0.1', 76 | description='The OpenStack Neat Project', 77 | long_description=__doc__, 78 | author='Anton Beloglazov', 79 | author_email='anton.beloglazov@gmail.com', 80 | url='https://github.com/beloglazov/openstack-neat', 81 | platforms='any', 82 | include_package_data=True, 83 | license='LICENSE', 84 | packages=find_packages(), 85 | test_suite='tests', 86 | tests_require=['pyqcy', 'mocktest', 'PyContracts'], 87 | entry_points = { 88 | 'console_scripts': [ 89 | 'neat-data-collector = neat.locals.collector:start', 90 | 'neat-local-manager = neat.locals.manager:start', 91 | 'neat-global-manager = neat.globals.manager:start', 92 | 'neat-db-cleaner = neat.globals.db_cleaner:start', 93 | ] 94 | }, 95 | data_files = [('/etc/init.d', ['init.d/openstack-neat-data-collector', 96 | 'init.d/openstack-neat-local-manager', 97 | 'init.d/openstack-neat-global-manager', 98 | 'init.d/openstack-neat-db-cleaner']), 99 | ('/etc/neat', ['neat.conf'])], 100 | ) 101 | -------------------------------------------------------------------------------- /doc/blueprint/src/template.tex: -------------------------------------------------------------------------------- 1 | \documentclass[$if(fontsize)$$fontsize$,$endif$$if(lang)$$lang$,$endif$]{$documentclass$} 2 | \usepackage[affil-it]{authblk} 3 | \usepackage[T1]{fontenc} 4 | \usepackage{lmodern} 5 | \usepackage{amssymb,amsmath} 6 | \usepackage{ifxetex,ifluatex} 7 | \usepackage{fixltx2e} % provides \textsubscript 8 | % use microtype if available 9 | \IfFileExists{microtype.sty}{\usepackage{microtype}}{} 10 | \ifnum 0\ifxetex 1\fi\ifluatex 1\fi=0 % if pdftex 11 | \usepackage[utf8]{inputenc} 12 | $if(euro)$ 13 | \usepackage{eurosym} 14 | $endif$ 15 | \else % if luatex or xelatex 16 | \usepackage{fontspec} 17 | \ifxetex 18 | \usepackage{xltxtra,xunicode} 19 | \fi 20 | \defaultfontfeatures{Mapping=tex-text,Scale=MatchLowercase} 21 | \newcommand{\euro}{€} 22 | $if(mainfont)$ 23 | \setmainfont{$mainfont$} 24 | $endif$ 25 | $if(sansfont)$ 26 | \setsansfont{$sansfont$} 27 | $endif$ 28 | $if(monofont)$ 29 | \setmonofont{$monofont$} 30 | $endif$ 31 | $if(mathfont)$ 32 | \setmathfont{$mathfont$} 33 | $endif$ 34 | \fi 35 | $if(geometry)$ 36 | \usepackage[$for(geometry)$$geometry$$sep$,$endfor$]{geometry} 37 | $endif$ 38 | $if(natbib)$ 39 | \usepackage{natbib} 40 | \bibliographystyle{plainnat} 41 | $endif$ 42 | $if(biblatex)$ 43 | \usepackage{biblatex} 44 | $if(biblio-files)$ 45 | \bibliography{$biblio-files$} 46 | $endif$ 47 | $endif$ 48 | $if(listings)$ 49 | \usepackage{listings} 50 | $endif$ 51 | $if(lhs)$ 52 | \lstnewenvironment{code}{\lstset{language=Haskell,basicstyle=\small\ttfamily}}{} 53 | $endif$ 54 | $if(highlighting-macros)$ 55 | $highlighting-macros$ 56 | $endif$ 57 | $if(verbatim-in-note)$ 58 | \usepackage{fancyvrb} 59 | $endif$ 60 | $if(fancy-enums)$ 61 | % Redefine labelwidth for lists; otherwise, the enumerate package will cause 62 | % markers to extend beyond the left margin. 63 | \makeatletter\AtBeginDocument{% 64 | \renewcommand{\@listi} 65 | {\setlength{\labelwidth}{4em}} 66 | }\makeatother 67 | \usepackage{enumerate} 68 | $endif$ 69 | $if(tables)$ 70 | \usepackage{ctable} 71 | \usepackage{float} % provides the H option for float placement 72 | $endif$ 73 | $if(graphics)$ 74 | \usepackage{graphicx} 75 | % We will generate all images so they have a width \maxwidth. This means 76 | % that they will get their normal width if they fit onto the page, but 77 | % are scaled down if they would overflow the margins. 78 | \makeatletter 79 | \def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth 80 | \else\Gin@nat@width\fi} 81 | \makeatother 82 | \let\Oldincludegraphics\includegraphics 83 | \renewcommand{\includegraphics}[1]{\Oldincludegraphics[width=\maxwidth]{#1}} 84 | $endif$ 85 | \ifxetex 86 | \usepackage[setpagesize=false, % page size defined by xetex 87 | unicode=false, % unicode breaks when used with xetex 88 | xetex]{hyperref} 89 | \else 90 | \usepackage[unicode=true]{hyperref} 91 | \fi 92 | \hypersetup{breaklinks=true, 93 | bookmarks=true, 94 | pdfauthor={$author-meta$}, 95 | pdftitle={$title-meta$}, 96 | colorlinks=true, 97 | urlcolor=$if(urlcolor)$$urlcolor$$else$blue$endif$, 98 | linkcolor=$if(linkcolor)$$linkcolor$$else$magenta$endif$, 99 | pdfborder={0 0 0}} 100 | $if(links-as-notes)$ 101 | % Make links footnotes instead of hotlinks: 102 | \renewcommand{\href}[2]{#2\footnote{\url{#1}}} 103 | $endif$ 104 | $if(strikeout)$ 105 | \usepackage[normalem]{ulem} 106 | % avoid problems with \sout in headers with hyperref: 107 | \pdfstringdefDisableCommands{\renewcommand{\sout}{}} 108 | $endif$ 109 | \setlength{\parindent}{0pt} 110 | \setlength{\parskip}{6pt plus 2pt minus 1pt} 111 | \setlength{\emergencystretch}{3em} % prevent overfull lines 112 | $if(numbersections)$ 113 | $else$ 114 | \setcounter{secnumdepth}{0} 115 | $endif$ 116 | $if(verbatim-in-note)$ 117 | \VerbatimFootnotes % allows verbatim text in footnotes 118 | $endif$ 119 | $if(lang)$ 120 | \ifxetex 121 | \usepackage{polyglossia} 122 | \setmainlanguage{$mainlang$} 123 | \else 124 | \usepackage[$lang$]{babel} 125 | \fi 126 | $endif$ 127 | $for(header-includes)$ 128 | $header-includes$ 129 | $endfor$ 130 | 131 | $if(title)$ 132 | \title{$title$$thanks$} 133 | $endif$ 134 | %\author{$for(author)$$author$$sep$ \and $endfor$} 135 | $for(author)$\author{$author$}$endfor$ 136 | \affil{$affilation$} 137 | \date{$date$} 138 | 139 | \begin{document} 140 | $if(title)$ 141 | \maketitle 142 | $endif$ 143 | 144 | $for(include-before)$ 145 | $include-before$ 146 | 147 | $endfor$ 148 | $if(toc)$ 149 | { 150 | \hypersetup{linkcolor=black} 151 | \tableofcontents 152 | } 153 | $endif$ 154 | $body$ 155 | 156 | $if(natbib)$ 157 | $if(biblio-files)$ 158 | $if(biblio-title)$ 159 | $if(book-class)$ 160 | \renewcommand\bibname{$biblio-title$} 161 | $else$ 162 | \renewcommand\refname{$biblio-title$} 163 | $endif$ 164 | $endif$ 165 | \bibliography{$biblio-files$} 166 | 167 | $endif$ 168 | $endif$ 169 | $if(biblatex)$ 170 | \printbibliography$if(biblio-title)$[title=$biblio-title$]$endif$ 171 | 172 | $endif$ 173 | $for(include-after)$ 174 | $include-after$ 175 | 176 | $endfor$ 177 | \end{document} 178 | -------------------------------------------------------------------------------- /doc/blueprint/src/bibliography.bib: -------------------------------------------------------------------------------- 1 | @article{beloglazov2012optimal, 2 | author = {Anton Beloglazov and Rajkumar Buyya}, 3 | title = {Optimal online deterministic algorithms and adaptive heuristics for energy and performance efficient dynamic consolidation of virtual machines in Cloud data centers}, 4 | journal = {Concurrency and Computation: Practice and Experience ({CCPE})}, 5 | volume = {24}, 6 | number = {13}, 7 | pages = {1397--1420}, 8 | publisher = {John Wiley & Sons, Ltd}, 9 | year = {2012} 10 | }, 11 | 12 | @article{beloglazov2012overload, 13 | author = {Anton Beloglazov and Rajkumar Buyya}, 14 | title = {Managing Overloaded Hosts for Dynamic Consolidation of Virtual Machines in Cloud Data Centers Under Quality of Service Constraints}, 15 | journal = {IEEE Transactions on Parallel and Distributed Systems ({TPDS})}, 16 | publisher = {IEEE Press}, 17 | year = {2012 (in press, accepted on August 2, 2012)}, 18 | note = {(in press, accepted on August 2, 2012)} 19 | }, 20 | 21 | @article{beloglazov2012openstack, 22 | author = {Anton Beloglazov and Sareh Fotuhi Piraghaj and Mohammed Alrokayan and Rajkumar Buyya}, 23 | title = {Deploying OpenStack on CentOS Using the KVM Hypervisor and GlusterFS Distributed File System}, 24 | journal = {Technical Report CLOUDS-TR-2012-3, Cloud Computing and Distributed Systems Laboratory, The University of Melbourne}, 25 | year = {August 14, 2012} 26 | }, 27 | 28 | @book{koomey2011growth, 29 | author = {Jonathan Koomey}, 30 | title = {Growth in data center electricity use 2005 to 2010}, 31 | publisher = {Oakland, CA: Analytics Press}, 32 | year = {2011} 33 | }, 34 | 35 | @book{gartner2007co2, 36 | author = {Gartner Inc.}, 37 | title = {Gartner estimates {ICT} industry accounts for 2 percent of global {CO2} emissions}, 38 | publisher = {Gartner Press Release {(April} 2007)} 39 | }, 40 | 41 | @article{nathuji2007virtualpower, 42 | author = {R. Nathuji and K. Schwan}, 43 | title = {VirtualPower: Coordinated power management in virtualized enterprise systems}, 44 | volume = {41}, 45 | number = {6}, 46 | journal = {{ACM} {SIGOPS} Operating Systems Review}, 47 | year = {2007}, 48 | pages = {265--278} 49 | }, 50 | 51 | @inproceedings{verma2008pmapper, 52 | author = {A. Verma and P. Ahuja and A. Neogi}, 53 | title = {{pMapper}: Power and migration cost aware application placement in virtualized systems}, 54 | booktitle = {Proc. of the 9th {ACM/IFIP/USENIX} Intl. Conf. on Middleware}, 55 | year = {2008}, 56 | pages = {243--264} 57 | }, 58 | 59 | @inproceedings{zhu20081000, 60 | author = {X. Zhu and D. Young and B. J Watson and Z. Wang and J. Rolia and S. Singhal and B. {McKee} and C. Hyser and others}, 61 | title = {{1000 Islands}: Integrated capacity and workload management for the next generation data center}, 62 | shorttitle = {1000 Islands}, 63 | booktitle = {Proc. of the 5th Intl. Conf. on Autonomic Computing (ICAC)}, 64 | year = {2008}, 65 | pages = {172--181} 66 | }, 67 | 68 | @inproceedings{gmach2008integrated, 69 | author={Daniel Gmach and Jerry Rolia and Ludmila Cherkasova and Guillaume Belrose and Tom Turicchi and Alfons Kemper}, 70 | title={An integrated approach to resource pool management: Policies, efficiency and quality metrics}, 71 | booktitle={Proc. of the 38th IEEE Intl. Conf. on Dependable Systems and Networks (DSN)}, 72 | pages={326--335}, 73 | year={2008} 74 | }, 75 | 76 | @article{gmach2009resource, 77 | author = {Daniel Gmach and Jerry Rolia and Ludmila Cherkasova and Alfons Kemper}, 78 | title = {Resource pool management: Reactive versus proactive or lets be friends}, 79 | shorttitle = {Resource pool management}, 80 | journal = {Computer Networks}, 81 | volume={53}, 82 | number={17}, 83 | pages={2905--2922}, 84 | year = {2009} 85 | }, 86 | 87 | @article{vmware2010distributed, 88 | author = {{VMware Inc.}}, 89 | title = {{VMware} Distributed Power Management Concepts and Use}, 90 | journal = {Information Guide}, 91 | year = {2010} 92 | }, 93 | 94 | @inproceedings{jung2010mistral, 95 | author={Gueyoung Jung and Matti A. Hiltunen and Kaustubh R. Joshi and Richard D. Schlichting and Calton Pu}, 96 | title={Mistral: Dynamically Managing Power, Performance, and Adaptation Cost in {Cloud} Infrastructures}, 97 | booktitle={Proc. of the 30th Intl. Conf. on Distributed Computing Systems (ICDCS)}, 98 | pages={62--73}, 99 | year={2010} 100 | }, 101 | 102 | 103 | @inproceedings{zheng2009justrunit, 104 | author={Zheng, W. and Bianchini, R. and Janakiraman, G.J. and Santos, J.R. and Turner, Y.}, 105 | title={{JustRunIt}: Experiment-based management of virtualized data centers}, 106 | booktitle = {Proc. of the 2009 {USENIX} Annual Technical Conf.}, 107 | pages={18--33}, 108 | year={2009} 109 | }, 110 | 111 | @inproceedings{kumar2009vmanage, 112 | author = {S. Kumar and V. Talwar and V. Kumar and P. Ranganathan and K. Schwan}, 113 | title = {{vManage}: Loosely coupled platform and virtualization management in data centers}, 114 | shorttitle = {{vManage}}, 115 | booktitle = {Proc. of the 6th Intl. Conf. on Autonomic Computing (ICAC)}, 116 | year = {2009}, 117 | pages = {127--136} 118 | }, 119 | 120 | @inproceedings{guenter2011managing, 121 | author = {Brian Guenter and Navendu Jain and Charles Williams}, 122 | title = {Managing Cost, Performance, and Reliability Tradeoffs for Energy-Aware Server Provisioning}, 123 | booktitle = {Proc. of the 30st Annual {IEEE} Intl. Conf. on Computer Communications (INFOCOM)}, 124 | pages={1332--1340}, 125 | year = {2011} 126 | }, 127 | 128 | @inproceedings{bobroff2007dynamic, 129 | author={Bobroff, N. and Kochut, A. and Beaty, K.}, 130 | title={Dynamic placement of virtual machines for managing {SLA} violations}, 131 | booktitle={Proc. of the 10th IFIP/IEEE Intl. Symp. on Integrated Network Management (IM)}, 132 | pages={119--128}, 133 | year={2007} 134 | }, 135 | 136 | @article{beloglazov2011taxonomy, 137 | author = {Anton Beloglazov and Rajkumar Buyya and Young Choon Lee and Albert Zomaya}, 138 | title = {A Taxonomy and Survey of Energy-Efficient Data Centers and Cloud Computing Systems}, 139 | journal={Advances in Computers, M. Zelkowitz (ed.)}, 140 | volume={82}, 141 | pages={47--111}, 142 | year={2011}, 143 | publisher={Elsevier} 144 | } -------------------------------------------------------------------------------- /neat/locals/vm_selection/algorithms.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ VM selection algorithms. 16 | """ 17 | 18 | from contracts import contract 19 | from neat.contracts_primitive import * 20 | from neat.contracts_extra import * 21 | 22 | from random import choice 23 | import operator 24 | 25 | import logging 26 | log = logging.getLogger(__name__) 27 | 28 | 29 | @contract 30 | def random_factory(time_step, migration_time, params): 31 | """ Creates the random VM selection algorithm. 32 | 33 | :param time_step: The length of the simulation time step in seconds. 34 | :type time_step: int,>=0 35 | 36 | :param migration_time: The VM migration time in time seconds. 37 | :type migration_time: float,>=0 38 | 39 | :param params: A dictionary containing the algorithm's parameters. 40 | :type params: dict(str: *) 41 | 42 | :return: A function implementing the random VM selection algorithm. 43 | :rtype: function 44 | """ 45 | return lambda vms_cpu, vms_ram, state=None: ([random(vms_cpu)], {}) 46 | 47 | 48 | @contract 49 | def minimum_utilization_factory(time_step, migration_time, params): 50 | """ Creates the minimum utilization VM selection algorithm. 51 | 52 | :param time_step: The length of the simulation time step in seconds. 53 | :type time_step: int,>=0 54 | 55 | :param migration_time: The VM migration time in time seconds. 56 | :type migration_time: float,>=0 57 | 58 | :param params: A dictionary containing the algorithm's parameters. 59 | :type params: dict(str: *) 60 | 61 | :return: A function implementing the minimum utilization VM selection. 62 | :rtype: function 63 | """ 64 | return lambda vms_cpu, vms_ram, state=None: \ 65 | ([minimum_utilization(vms_cpu)], {}) 66 | 67 | 68 | @contract 69 | def minimum_migration_time_factory(time_step, migration_time, params): 70 | """ Creates the minimum migration time VM selection algorithm. 71 | 72 | :param time_step: The length of the simulation time step in seconds. 73 | :type time_step: int,>=0 74 | 75 | :param migration_time: The VM migration time in time seconds. 76 | :type migration_time: float,>=0 77 | 78 | :param params: A dictionary containing the algorithm's parameters. 79 | :type params: dict(str: *) 80 | 81 | :return: A function implementing the minimum migration time VM selection. 82 | :rtype: function 83 | """ 84 | return lambda vms_cpu, vms_ram, state=None: \ 85 | ([minimum_migration_time(vms_ram)], {}) 86 | 87 | 88 | @contract 89 | def minimum_migration_time_max_cpu_factory(time_step, migration_time, params): 90 | """ Creates the minimum migration time / max CPU usage VM selection algorithm. 91 | 92 | :param time_step: The length of the simulation time step in seconds. 93 | :type time_step: int,>=0 94 | 95 | :param migration_time: The VM migration time in time seconds. 96 | :type migration_time: float,>=0 97 | 98 | :param params: A dictionary containing the algorithm's parameters. 99 | :type params: dict(str: *) 100 | 101 | :return: A function implementing the minimum migration time / max CPU VM selection. 102 | :rtype: function 103 | """ 104 | return lambda vms_cpu, vms_ram, state=None: \ 105 | ([minimum_migration_time_max_cpu(params['last_n'], 106 | vms_cpu, 107 | vms_ram)], {}) 108 | 109 | 110 | @contract 111 | def minimum_migration_time(vms_ram): 112 | """ Selects the VM with the minimum RAM usage. 113 | 114 | :param vms_ram: A map of VM UUID and their RAM usage data. 115 | :type vms_ram: dict(str: number) 116 | 117 | :return: A VM to migrate from the host. 118 | :rtype: str 119 | """ 120 | min_index, min_value = min(enumerate(vms_ram.values()), 121 | key=operator.itemgetter(1)) 122 | return vms_ram.keys()[min_index] 123 | 124 | 125 | @contract 126 | def minimum_utilization(vms_cpu): 127 | """ Selects the VM with the minimum CPU utilization. 128 | 129 | :param vms_cpu: A map of VM UUID and their CPU utilization histories. 130 | :type vms_cpu: dict(str: list) 131 | 132 | :return: A VM to migrate from the host. 133 | :rtype: str 134 | """ 135 | last_utilization = [x[-1] for x in vms_cpu.values()] 136 | min_index, min_value = min(enumerate(last_utilization), 137 | key=operator.itemgetter(1)) 138 | return vms_cpu.keys()[min_index] 139 | 140 | 141 | @contract 142 | def random(vms_cpu): 143 | """ Selects a random VM. 144 | 145 | :param vms_cpu: A map of VM UUID and their CPU utilization histories. 146 | :type vms_cpu: dict(str: list) 147 | 148 | :return: A VM to migrate from the host. 149 | :rtype: str 150 | """ 151 | return choice(vms_cpu.keys()) 152 | 153 | 154 | @contract 155 | def minimum_migration_time_max_cpu(last_n, vms_cpu, vms_ram): 156 | """ Selects the VM with the minimum RAM and maximum CPU usage. 157 | 158 | :param last_n: The number of last CPU utilization values to average. 159 | :type last_n: int,>0 160 | 161 | :param vms_cpu: A map of VM UUID and their CPU utilization histories. 162 | :type vms_cpu: dict(str: list) 163 | 164 | :param vms_ram: A map of VM UUID and their RAM usage data. 165 | :type vms_ram: dict(str: number) 166 | 167 | :return: A VM to migrate from the host. 168 | :rtype: str 169 | """ 170 | min_ram = min(vms_ram.values()) 171 | max_cpu = 0 172 | selected_vm = None 173 | for vm, cpu in vms_cpu.items(): 174 | if vms_ram[vm] > min_ram: 175 | continue 176 | vals = cpu[-last_n:] 177 | avg = float(sum(vals)) / len(vals) 178 | if max_cpu < avg: 179 | max_cpu = avg 180 | selected_vm = vm 181 | return selected_vm 182 | -------------------------------------------------------------------------------- /neat/globals/vm_placement/bin_packing.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ Bin Packing based VM placement algorithms. 16 | """ 17 | 18 | from contracts import contract 19 | from neat.contracts_primitive import * 20 | from neat.contracts_extra import * 21 | 22 | import logging 23 | log = logging.getLogger(__name__) 24 | 25 | 26 | @contract 27 | def best_fit_decreasing_factory(time_step, migration_time, params): 28 | """ Creates the Best Fit Decreasing (BFD) heuristic for VM placement. 29 | 30 | :param time_step: The length of the simulation time step in seconds. 31 | :type time_step: int,>=0 32 | 33 | :param migration_time: The VM migration time in time seconds. 34 | :type migration_time: float,>=0 35 | 36 | :param params: A dictionary containing the algorithm's parameters. 37 | :type params: dict(str: *) 38 | 39 | :return: A function implementing the BFD algorithm. 40 | :rtype: function 41 | """ 42 | return lambda hosts_cpu_usage, hosts_cpu_total, \ 43 | hosts_ram_usage, hosts_ram_total, \ 44 | inactive_hosts_cpu, inactive_hosts_ram, \ 45 | vms_cpu, vms_ram, state=None: \ 46 | (best_fit_decreasing( 47 | params['last_n_vm_cpu'], 48 | get_available_resources( 49 | params['cpu_threshold'], 50 | hosts_cpu_usage, 51 | hosts_cpu_total), 52 | get_available_resources( 53 | params['ram_threshold'], 54 | hosts_ram_usage, 55 | hosts_ram_total), 56 | inactive_hosts_cpu, 57 | inactive_hosts_ram, 58 | vms_cpu, 59 | vms_ram), 60 | {}) 61 | 62 | 63 | @contract 64 | def get_available_resources(threshold, usage, total): 65 | """ Get a map of the available resource capacity. 66 | 67 | :param threshold: A threshold on the maximum allowed resource usage. 68 | :type threshold: float,>=0 69 | 70 | :param usage: A map of hosts to the resource usage. 71 | :type usage: dict(str: number) 72 | 73 | :param total: A map of hosts to the total resource capacity. 74 | :type total: dict(str: number) 75 | 76 | :return: A map of hosts to the available resource capacity. 77 | :rtype: dict(str: int) 78 | """ 79 | return dict((host, int(threshold * total[host] - resource)) 80 | for host, resource in usage.items()) 81 | 82 | 83 | @contract 84 | def best_fit_decreasing(last_n_vm_cpu, hosts_cpu, hosts_ram, 85 | inactive_hosts_cpu, inactive_hosts_ram, 86 | vms_cpu, vms_ram): 87 | """ The Best Fit Decreasing (BFD) heuristic for placing VMs on hosts. 88 | 89 | :param last_n_vm_cpu: The last n VM CPU usage values to average. 90 | :type last_n_vm_cpu: int 91 | 92 | :param hosts_cpu: A map of host names and their available CPU in MHz. 93 | :type hosts_cpu: dict(str: int) 94 | 95 | :param hosts_ram: A map of host names and their available RAM in MB. 96 | :type hosts_ram: dict(str: int) 97 | 98 | :param inactive_hosts_cpu: A map of inactive hosts and available CPU MHz. 99 | :type inactive_hosts_cpu: dict(str: int) 100 | 101 | :param inactive_hosts_ram: A map of inactive hosts and available RAM MB. 102 | :type inactive_hosts_ram: dict(str: int) 103 | 104 | :param vms_cpu: A map of VM UUID and their CPU utilization in MHz. 105 | :type vms_cpu: dict(str: list(int)) 106 | 107 | :param vms_ram: A map of VM UUID and their RAM usage in MB. 108 | :type vms_ram: dict(str: int) 109 | 110 | :return: A map of VM UUIDs to host names, or {} if cannot be solved. 111 | :rtype: dict(str: str) 112 | """ 113 | if log.isEnabledFor(logging.DEBUG): 114 | log.debug('last_n_vm_cpu: %s', str(last_n_vm_cpu)) 115 | log.debug('hosts_cpu: %s', str(hosts_cpu)) 116 | log.debug('hosts_ram: %s', str(hosts_ram)) 117 | log.debug('inactive_hosts_cpu: %s', str(inactive_hosts_cpu)) 118 | log.debug('inactive_hosts_ram: %s', str(inactive_hosts_ram)) 119 | log.debug('vms_cpu: %s', str(vms_cpu)) 120 | log.debug('vms_ram: %s', str(vms_ram)) 121 | vms_tmp = [] 122 | for vm, cpu in vms_cpu.items(): 123 | if cpu: 124 | last_n_cpu = cpu[-last_n_vm_cpu:] 125 | vms_tmp.append((sum(last_n_cpu) / len(last_n_cpu), 126 | vms_ram[vm], 127 | vm)) 128 | else: 129 | log.warning('No CPU data for VM: %s - skipping', vm) 130 | 131 | vms = sorted(vms_tmp, reverse=True) 132 | hosts = sorted(((v, hosts_ram[k], k) 133 | for k, v in hosts_cpu.items())) 134 | inactive_hosts = sorted(((v, inactive_hosts_ram[k], k) 135 | for k, v in inactive_hosts_cpu.items())) 136 | mapping = {} 137 | for vm_cpu, vm_ram, vm_uuid in vms: 138 | mapped = False 139 | while not mapped: 140 | for _, _, host in hosts: 141 | if hosts_cpu[host] >= vm_cpu and \ 142 | hosts_ram[host] >= vm_ram: 143 | mapping[vm_uuid] = host 144 | hosts_cpu[host] -= vm_cpu 145 | hosts_ram[host] -= vm_ram 146 | mapped = True 147 | break 148 | else: 149 | if inactive_hosts: 150 | activated_host = inactive_hosts.pop(0) 151 | hosts.append(activated_host) 152 | hosts = sorted(hosts) 153 | hosts_cpu[activated_host[2]] = activated_host[0] 154 | hosts_ram[activated_host[2]] = activated_host[1] 155 | else: 156 | break 157 | 158 | if len(vms) == len(mapping): 159 | return mapping 160 | return {} 161 | -------------------------------------------------------------------------------- /neat.conf: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # This is the default configuration file for OpenStack Neat 16 | 17 | [DEFAULT] 18 | 19 | # The directory, where log files will be created by the Neat services 20 | log_directory = /var/log/neat 21 | 22 | # The level of emitted log messages: 23 | # 0 -- no logging 24 | # 1 -- errors and warnings 25 | # 2 -- errors, warnings, and information messages 26 | # 3 -- errors, warnings, information messages, and debug messages 27 | log_level = 3 28 | 29 | # The directory, where the VM instance data are stored 30 | vm_instance_directory = /var/lib/nova/instances 31 | 32 | # The host name and credentials for connecting to the MySQL database 33 | # specified in the format supported by SQLAlchemy 34 | sql_connection = mysql://neat:neatpassword@controller/neat 35 | 36 | # The admin tenant name for authentication with Nova using Keystone 37 | os_admin_tenant_name = admin 38 | 39 | # The admin user name for authentication with Nova using Keystone 40 | os_admin_user = admin 41 | 42 | # The admin password for authentication with Nova using Keystone 43 | os_admin_password = adminpassword 44 | 45 | # The OpenStack authentication URL 46 | os_auth_url = http://controller:5000/v2.0/ 47 | 48 | # A coma-separated list of compute host names 49 | compute_hosts = compute1, compute2, compute3, compute4 50 | 51 | # The name of the host running the global manager 52 | global_manager_host = controller 53 | 54 | # The port of the REST web service exposed by the global manager 55 | global_manager_port = 60080 56 | 57 | # The time interval between subsequent invocations of the database 58 | # cleaner in seconds 59 | db_cleaner_interval = 7200 60 | 61 | # The directory used by the data collector to store the data on the 62 | # resource usage by the VMs running on the host 63 | local_data_directory = /var/lib/neat 64 | 65 | # The time interval between subsequent invocations of the local 66 | # manager in seconds 67 | local_manager_interval = 300 68 | 69 | # The time interval between subsequent invocations of the data 70 | # collector in seconds 71 | data_collector_interval = 300 72 | 73 | # The number of the latest data values stored locally by the data 74 | # collector and passed to the underload / overload detection and VM 75 | # placement algorithms 76 | data_collector_data_length = 100 77 | 78 | # The threshold on the overall (all cores) utilization of the physical 79 | # CPU of a host, above which the host is considered to be overloaded. 80 | # This is used for logging host overloads into the database. 81 | host_cpu_overload_threshold = 0.8 82 | 83 | # The threshold on the overall (all cores) utilization of the physical 84 | # CPU of a host that can be allocated to VMs. 85 | host_cpu_usable_by_vms = 1.0 86 | 87 | # The user name for connecting to the compute hosts to switch them 88 | # into the sleep mode 89 | compute_user = neat 90 | 91 | # The password of the user account used for connecting to the compute 92 | # hosts to switch them into the sleep mode 93 | compute_password = neatpassword 94 | 95 | # Whether to use block migration (includes disk migration) 96 | block_migration = False 97 | 98 | # The network bandwidth in MB/s available for VM migration 99 | network_migration_bandwidth = 10 100 | 101 | # A shell command used to switch a host into the sleep mode, the 102 | # compute_user must have permissions to execute this command 103 | sleep_command = pm-suspend 104 | 105 | # The network interface to send a magic packet from using ether-wake 106 | ether_wake_interface = eth0 107 | 108 | # The fully qualified name of a Python factory function that returns a 109 | # function implementing an underload detection algorithm 110 | #algorithm_underload_detection_factory = neat.locals.underload.trivial.threshold_factory 111 | algorithm_underload_detection_factory = neat.locals.underload.trivial.last_n_average_threshold_factory 112 | 113 | # A JSON encoded parameters, which will be parsed and passed to the 114 | # specified underload detection algorithm factory 115 | #algorithm_underload_detection_parameters = {"threshold": 0.3} 116 | algorithm_underload_detection_parameters = {"threshold": 0.5, "n": 2} 117 | 118 | # The fully qualified name of a Python factory function that returns a 119 | # function implementing an overload detection algorithm 120 | #algorithm_overload_detection_factory = neat.locals.overload.trivial.threshold_factory 121 | algorithm_overload_detection_factory = neat.locals.overload.mhod.core.mhod_factory 122 | #algorithm_overload_detection_factory = neat.locals.overload.trivial.last_n_average_threshold_factory 123 | #algorithm_overload_detection_factory = neat.locals.overload.statistics.loess_factory 124 | #algorithm_overload_detection_factory = neat.locals.overload.otf.otf_factory 125 | 126 | # A JSON encoded parameters, which will be parsed and passed to the 127 | # specified overload detection algorithm factory 128 | #algorithm_overload_detection_parameters = {"threshold": 0.9} 129 | algorithm_overload_detection_parameters = {"state_config": [0.8], "otf": 0.1, "history_size": 500, "window_sizes": [30, 40, 50, 60, 70, 80, 90, 100], "bruteforce_step": 0.5, "learning_steps": 10} 130 | #algorithm_overload_detection_parameters = {"threshold": 0.95, "n": 2} 131 | #algorithm_overload_detection_parameters = {"threshold": 0.8, "param": 1.0, "length": 30} 132 | #algorithm_overload_detection_parameters = {"otf": 0.2, "threshold": 0.8, "limit": 10} 133 | 134 | # The fully qualified name of a Python factory function that returns a 135 | # function implementing a VM selection algorithm 136 | #algorithm_vm_selection_factory = neat.locals.vm_selection.algorithms.minimum_migration_time_factory 137 | algorithm_vm_selection_factory = neat.locals.vm_selection.algorithms.minimum_migration_time_max_cpu_factory 138 | 139 | # A JSON encoded parameters, which will be parsed and passed to the 140 | # specified VM selection algorithm factory 141 | #algorithm_vm_selection_parameters = {} 142 | algorithm_vm_selection_parameters = {"last_n": 2} 143 | 144 | # The fully qualified name of a Python factory function that returns a 145 | # function implementing a VM placement algorithm 146 | algorithm_vm_placement_factory = neat.globals.vm_placement.bin_packing.best_fit_decreasing_factory 147 | 148 | # A JSON encoded parameters, which will be parsed and passed to the 149 | # specified VM placement algorithm factory 150 | algorithm_vm_placement_parameters = {"cpu_threshold": 0.8, "ram_threshold": 0.95, "last_n_vm_cpu": 2} 151 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OpenStack Neat: A Framework for Dynamic Consolidation of Virtual Machines in Openstack Clouds 2 | 3 | [![Build Status](https://travis-ci.org/beloglazov/openstack-neat.svg?branch=master)](https://travis-ci.org/beloglazov/openstack-neat) 4 | 5 | 6 | OpenStack Neat is an extension to OpenStack implementing dynamic consolidation 7 | of Virtual Machines (VMs) using live migration. The major objective of dynamic 8 | VM consolidation is to improve the utilization of physical resources and reduce 9 | energy consumption by re-allocating VMs using live migration according to their 10 | real-time resource demand and switching idle hosts to the sleep mode. 11 | 12 | For example, assume that two VMs are placed on two different hosts, but the 13 | combined resource capacity required by the VMs to serve the current load can be 14 | provided by just one of the hosts. Then, one of the VMs can be migrated to the 15 | host serving the other VM, and the idle host can be switched to a low power mode 16 | to save energy. When the resource demand of either of the VMs increases, they 17 | get deconsolidated to avoid performance degradation. This process is dynamically 18 | managed by OpenStack Neat. 19 | 20 | In general, the problem of dynamic VM consolidation can be split into 4 21 | sub-problems: 22 | 23 | - Deciding when a host is considered to be underloaded, so that all the VMs 24 | should be migrated from it, and the host should be switched to a low power 25 | mode, such as the sleep mode. 26 | - Deciding when a host is considered to be overloaded, so that some VMs should 27 | be migrated from the host to other hosts to avoid performance degradation. 28 | - Selecting VMs to migrate from an overloaded host out of the full set of the 29 | VMs currently served by the host. 30 | - Placing VMs selected for migration to other active or re-activated hosts. 31 | 32 | The aim of the OpenStack Neat project is to provide an extensible framework for 33 | dynamic consolidation of VMs based on the OpenStack platform. The framework 34 | provides an infrastructure enabling the interaction of components implementing 35 | the 4 decision-making algorithms listed above. The framework allows 36 | configuration-driven switching of different implementations of the 37 | decision-making algorithms. The implementation of the framework includes the 38 | algorithms proposed in our publications [1], [2]. 39 | 40 | 41 | ## More details 42 | 43 | For more information please refer to the 44 | [paper](http://beloglazov.info/papers/2014-ccpe-openstack-neat.pdf) describing 45 | the architecture and implementation of OpenStack Neat and Chapter 6 of Anton 46 | Beloglazov's PhD thesis: http://beloglazov.info/thesis.pdf 47 | 48 | 49 | ## Installation 50 | 51 | Unfortunately, there is no clear installation and usage guide yet. However, the 52 | basic installation steps are the following: 53 | 54 | - Clone the repository on every compute and controller node. 55 | - Adjust the configuration by modifying neat.conf file in the repo directory on 56 | every node. In particular, you have to specify the names of your compute nodes 57 | in the `compute_hosts` field. Then, update the following three fields: 58 | `admin_tenant_name`, `admin_user` and `admin_password` according to your OpenStack 59 | configuration. If you have the default values, the correct settings are: 60 | ``` 61 | admin_tenant_name: service 62 | admin_user: nova 63 | admin_password: NOVA_PASS 64 | ``` 65 | - [UBUNTU] Edit all the `openstack-*` files present in `/etc/init.d/` 66 | replacing `/etc/rc.d/init.d/functions` with `/lib/lsb/init-functions` and 67 | `exec="/usr/bin/neat-$suffix"` with `exec="/usr/local/bin/neat-$suffix"` 68 | on every neat node. 69 | - [UBUNTU] Install all the dependencies with (while doing this, DO NOT remove 70 | any python related package previously installed by OpenStack, it will break 71 | your OpenStack install!): 72 | ``` 73 | apt-get install python-pip numpy scipy libvirt-python 74 | sudo pip install --upgrade pyqcy PyContracts SQLAlchemy bottle requests Sphinx python-novaclient 75 | sudo pip install mocktest 76 | ``` 77 | - Install the package by running the following command from the repo directory 78 | on every node: 79 | ``` 80 | sudo python setup.py install 81 | ``` 82 | - Start the Neat services by running the following commands on every compute node: 83 | ``` 84 | python2 start-data-collector.py 85 | python2 start-local-manager.py 86 | ``` 87 | - Start the global manager service by running the following command on the controller: 88 | ``` 89 | python2 start-global-manager.py 90 | ``` 91 | 92 | You can monitor the current VM placement using the `./vm-placement.py` script. 93 | 94 | Some information about running experiments on the system can be found in the 95 | following thread: 96 | https://groups.google.com/forum/#!topic/openstack-neat/PKz2vpKPMcA 97 | 98 | 99 | ## Who we are 100 | 101 | This work is conducted within the Cloud Computing and Distributed Systems 102 | (CLOUDS) Laboratory at the University of Melbourne: http://www.cloudbus.org/ 103 | 104 | The problem of dynamic VM consolidation considering Quality of Service (QoS) 105 | constraints has been studied from the theoretical perspective and algorithms 106 | addressing the sub-problems listed above have been proposed [1], [2]. The 107 | algorithms have been evaluated using CloudSim and real-world workload traces 108 | collected from more than a thousand PlanetLab VMs hosted on servers located in 109 | more than 500 places around the world. 110 | 111 | 112 | ## Discussion group / mailing list 113 | 114 | Please feel free to post any questions or suggestions in the project's 115 | discussion group: http://groups.google.com/group/openstack-neat 116 | 117 | 118 | ## Issues / bugs 119 | 120 | Please submit any bugs you encounter or suggestions for improvements to our 121 | issue tracker: http://github.com/beloglazov/openstack-neat/issues 122 | 123 | 124 | ## Publications 125 | 126 | [1] Anton Beloglazov and Rajkumar Buyya, "OpenStack Neat: A Framework for 127 | Dynamic and Energy-Efficient Consolidation of Virtual Machines in OpenStack 128 | Clouds", Concurrency and Computation: Practice and Experience (CCPE), John Wiley 129 | & Sons, Ltd, USA, 2014 (in press, accepted on 19/05/2014). 130 | 131 | Download: http://beloglazov.info/papers/2014-ccpe-openstack-neat.pdf 132 | 133 | [2] Anton Beloglazov, "Energy-Efficient Management of Virtual Machines in 134 | Data Centers for Cloud Computing", PhD Thesis, The University of Melbourne, 135 | 2013. 136 | 137 | Download: http://beloglazov.info/thesis.pdf 138 | 139 | [3] Anton Beloglazov and Rajkumar Buyya, "Managing Overloaded Hosts for 140 | Dynamic Consolidation of Virtual Machines in Cloud Data Centers Under Quality of 141 | Service Constraints", IEEE Transactions on Parallel and Distributed Systems 142 | (TPDS), Volume 24, Issue 7, Pages 1366-1379, IEEE CS Press, USA, 2013. 143 | 144 | Download: http://beloglazov.info/papers/2013-tpds-managing-overloaded-hosts.pdf 145 | 146 | [4] Anton Beloglazov and Rajkumar Buyya, "Optimal Online Deterministic 147 | Algorithms and Adaptive Heuristics for Energy and Performance Efficient Dynamic 148 | Consolidation of Virtual Machines in Cloud Data Centers", Concurrency and 149 | Computation: Practice and Experience (CCPE), Volume 24, Issue 13, Pages: 150 | 1397-1420, John Wiley & Sons, Ltd, New York, USA, 2012. 151 | 152 | Download: http://beloglazov.info/papers/2012-ccpe-vm-consolidation-algorithms.pdf 153 | 154 | 155 | ## License 156 | 157 | The source code is distributed under the Apache 2.0 license. 158 | 159 | Copyright (C) 2012-2014 Anton Beloglazov, Google Inc. 160 | -------------------------------------------------------------------------------- /utils/db.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from contracts import contract 16 | import datetime 17 | from sqlalchemy import * 18 | from sqlalchemy.engine.base import Connection 19 | 20 | 21 | class Database(object): 22 | """ A class representing the database, where fields are tables. 23 | """ 24 | 25 | @contract(connection=Connection, 26 | hosts=Table, 27 | host_resource_usage=Table, 28 | vms=Table, 29 | vm_resource_usage=Table, 30 | vm_migrations=Table, 31 | host_states=Table, 32 | host_overload=Table) 33 | def __init__(self, connection, hosts, host_resource_usage, vms, 34 | vm_resource_usage, vm_migrations, host_states, host_overload): 35 | """ Initialize the database. 36 | 37 | :param connection: A database connection table. 38 | :param hosts: The hosts table. 39 | :param host_resource_usage: The host_resource_usage table. 40 | :param vms: The vms table. 41 | :param vm_resource_usage: The vm_resource_usage table. 42 | :param vm_migrations: The vm_migrations table. 43 | :param host_states: The host_states table. 44 | :param host_overload: The host_overload table. 45 | """ 46 | self.connection = connection 47 | self.hosts = hosts 48 | self.host_resource_usage = host_resource_usage 49 | self.vms = vms 50 | self.vm_resource_usage = vm_resource_usage 51 | self.vm_migrations = vm_migrations 52 | self.host_states = host_states 53 | self.host_overload = host_overload 54 | 55 | @contract 56 | def select_host_ids(self): 57 | """ Select the IDs of all the hosts. 58 | 59 | :return: A dict of host names to IDs. 60 | :rtype: dict(str: int) 61 | """ 62 | return dict((str(x[1]), int(x[0])) 63 | for x in self.hosts.select().execute().fetchall()) 64 | 65 | @contract 66 | def select_host_states(self, host_id, start_time, end_time): 67 | """ Select the states of a host. 68 | 69 | :param start_time: The start time to select host states. 70 | :type start_time: * 71 | 72 | :param end_time: The end time to select host states. 73 | :type end_time: * 74 | 75 | :return: A list of timestamps and host states. 76 | :rtype: list(tuple(*, int)) 77 | """ 78 | hs = self.host_states 79 | sel = select([hs.c.timestamp, hs.c.state]). \ 80 | where(and_(hs.c.host_id == host_id, 81 | hs.c.timestamp >= start_time, 82 | hs.c.timestamp <= end_time)). \ 83 | order_by(hs.c.id.asc()) 84 | return [(x[0], int(x[1])) 85 | for x in self.connection.execute(sel).fetchall()] 86 | 87 | @contract 88 | def select_host_overload(self, host_id, start_time, end_time): 89 | """ Select the overload of a host. 90 | 91 | :param start_time: The start time to select host overload. 92 | :type start_time: * 93 | 94 | :param end_time: The end time to select host states. 95 | :type end_time: * 96 | 97 | :return: A list of timestamps and overloads. 98 | :rtype: list(tuple(*, int)) 99 | """ 100 | ho = self.host_overload 101 | sel = select([ho.c.timestamp, ho.c.overload]). \ 102 | where(and_(ho.c.host_id == host_id, 103 | ho.c.timestamp >= start_time, 104 | ho.c.timestamp <= end_time)). \ 105 | order_by(ho.c.id.asc()) 106 | return [(x[0], int(x[1])) 107 | for x in self.connection.execute(sel).fetchall()] 108 | 109 | @contract 110 | def select_vm_migrations(self, start_time, end_time): 111 | """ Select VM migrations. 112 | 113 | :param start_time: The start time to select data. 114 | :type start_time: * 115 | 116 | :param end_time: The end time to select data. 117 | :type end_time: * 118 | 119 | :return: A list of timestamps and VM IDs. 120 | :rtype: list(tuple(*, int)) 121 | """ 122 | vm = self.vm_migrations 123 | sel = select([vm.c.timestamp, vm.c.vm_id]). \ 124 | where(and_(vm.c.timestamp >= start_time, 125 | vm.c.timestamp <= end_time)). \ 126 | order_by(vm.c.id.asc()) 127 | return [(x[0], int(x[1])) 128 | for x in self.connection.execute(sel).fetchall()] 129 | 130 | 131 | @contract 132 | def init_db(sql_connection): 133 | """ Initialize the database. 134 | 135 | :param sql_connection: A database connection URL. 136 | :type sql_connection: str 137 | 138 | :return: The initialized database. 139 | :rtype: * 140 | """ 141 | engine = create_engine(sql_connection) # 'sqlite:///:memory:' 142 | metadata = MetaData() 143 | metadata.bind = engine 144 | 145 | hosts = Table('hosts', metadata, 146 | Column('id', Integer, primary_key=True), 147 | Column('hostname', String(255), nullable=False), 148 | Column('cpu_mhz', Integer, nullable=False), 149 | Column('cpu_cores', Integer, nullable=False), 150 | Column('ram', Integer, nullable=False)) 151 | 152 | host_resource_usage = \ 153 | Table('host_resource_usage', metadata, 154 | Column('id', Integer, primary_key=True), 155 | Column('host_id', Integer, ForeignKey('hosts.id'), nullable=False), 156 | Column('timestamp', DateTime, default=func.now()), 157 | Column('cpu_mhz', Integer, nullable=False)) 158 | 159 | vms = Table('vms', metadata, 160 | Column('id', Integer, primary_key=True), 161 | Column('uuid', String(36), nullable=False)) 162 | 163 | vm_resource_usage = \ 164 | Table('vm_resource_usage', metadata, 165 | Column('id', Integer, primary_key=True), 166 | Column('vm_id', Integer, ForeignKey('vms.id'), nullable=False), 167 | Column('timestamp', DateTime, default=func.now()), 168 | Column('cpu_mhz', Integer, nullable=False)) 169 | 170 | vm_migrations = \ 171 | Table('vm_migrations', metadata, 172 | Column('id', Integer, primary_key=True), 173 | Column('vm_id', Integer, ForeignKey('vms.id'), nullable=False), 174 | Column('host_id', Integer, ForeignKey('hosts.id'), nullable=False), 175 | Column('timestamp', DateTime, default=func.now())) 176 | 177 | host_states = \ 178 | Table('host_states', metadata, 179 | Column('id', Integer, primary_key=True), 180 | Column('host_id', Integer, ForeignKey('hosts.id'), nullable=False), 181 | Column('timestamp', DateTime, default=func.now()), 182 | Column('state', Integer, nullable=False)) 183 | 184 | host_overload = \ 185 | Table('host_overload', metadata, 186 | Column('id', Integer, primary_key=True), 187 | Column('host_id', Integer, ForeignKey('hosts.id'), nullable=False), 188 | Column('timestamp', DateTime, default=func.now()), 189 | Column('overload', Integer, nullable=False)) 190 | 191 | metadata.create_all() 192 | connection = engine.connect() 193 | db = Database(connection, hosts, host_resource_usage, vms, 194 | vm_resource_usage, vm_migrations, host_states, host_overload) 195 | 196 | return db 197 | -------------------------------------------------------------------------------- /tests/test_common.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from mocktest import * 16 | from pyqcy import * 17 | 18 | import os 19 | import shutil 20 | import libvirt 21 | 22 | import neat.common as common 23 | 24 | import logging 25 | logging.disable(logging.CRITICAL) 26 | 27 | 28 | class Common(TestCase): 29 | 30 | @qc(10) 31 | def start(iterations=int_(0, 10)): 32 | with MockTransaction: 33 | config = {'option': 'value'} 34 | state = {'property': 'value'} 35 | fn = mock('function container') 36 | expect(fn).init_state(any_dict).and_return(state).once() 37 | expect(fn).execute(any_dict, any_dict). \ 38 | and_return(state).exactly(iterations).times() 39 | assert common.start(fn.init_state, 40 | fn.execute, 41 | config, 42 | 0, 43 | iterations) == state 44 | 45 | @qc(10) 46 | def build_local_vm_path( 47 | x=str_(of='abc123_-/') 48 | ): 49 | assert common.build_local_vm_path(x) == os.path.join(x, 'vms') 50 | 51 | @qc(10) 52 | def build_local_host_path( 53 | x=str_(of='abc123_-/') 54 | ): 55 | assert common.build_local_host_path(x) == os.path.join(x, 'host') 56 | 57 | @qc(10) 58 | def physical_cpu_count(x=int_(min=0, max=8)): 59 | with MockTransaction: 60 | connection = libvirt.virConnect() 61 | expect(connection).getInfo().and_return([0, 0, x]).once() 62 | assert common.physical_cpu_count(connection) == x 63 | 64 | @qc(10) 65 | def physical_cpu_mhz(x=int_(min=0, max=8)): 66 | with MockTransaction: 67 | connection = libvirt.virConnect() 68 | expect(connection).getInfo().and_return([0, 0, 0, x]).once() 69 | assert common.physical_cpu_mhz(connection) == x 70 | 71 | @qc(10) 72 | def physical_cpu_mhz_total(x=int_(min=0, max=8), y=int_(min=0, max=8)): 73 | with MockTransaction: 74 | connection = libvirt.virConnect() 75 | expect(common).physical_cpu_count(connection). \ 76 | and_return(x).once() 77 | expect(common).physical_cpu_mhz(connection). \ 78 | and_return(y).once() 79 | assert common.physical_cpu_mhz_total(connection) == x * y 80 | 81 | def test_frange(self): 82 | self.assertEqual([round(x, 1) for x in common.frange(0, 1.0, 0.5)], 83 | [0.0, 0.5, 1.0]) 84 | self.assertEqual([round(x, 1) for x in common.frange(0, 1.0, 0.2)], 85 | [0.0, 0.2, 0.4, 0.6, 0.8, 1.0]) 86 | 87 | def test_init_logging(self): 88 | log_dir = os.path.join( 89 | os.path.dirname(__file__), 'resources', 'log') 90 | log_file = 'test.log' 91 | log_path = os.path.join(log_dir, log_file) 92 | 93 | with MockTransaction: 94 | logging.root = mock('root') 95 | expect(logging).disable(logging.CRITICAL).once() 96 | expect(logging.root).setLevel.never() 97 | expect(logging.root).addHandler.never() 98 | assert common.init_logging(log_dir, log_file, 0) 99 | 100 | with MockTransaction: 101 | shutil.rmtree(log_dir, True) 102 | logging.root = mock('root') 103 | expect(logging).disable.never() 104 | expect(logging.root).setLevel(logging.WARNING).once() 105 | handler = mock('handler') 106 | expect(logging).FileHandler(log_path).and_return(handler).once() 107 | expect(handler).setFormatter.and_return(True).once() 108 | expect(logging).Formatter( 109 | '%(asctime)s %(levelname)-8s %(name)s %(message)s').once() 110 | expect(logging.root).addHandler.once() 111 | assert common.init_logging(log_dir, log_file, 1) 112 | assert os.access(log_dir, os.W_OK) 113 | 114 | with MockTransaction: 115 | logging.root = mock('root') 116 | expect(logging).disable.never() 117 | expect(logging.root).setLevel(logging.INFO).once() 118 | handler = mock('handler') 119 | expect(logging).FileHandler(log_path).and_return(handler).once() 120 | expect(handler).setFormatter.and_return(True).once() 121 | expect(logging).Formatter( 122 | '%(asctime)s %(levelname)-8s %(name)s %(message)s').once() 123 | expect(logging.root).addHandler.once() 124 | assert common.init_logging(log_dir, log_file, 2) 125 | assert os.access(log_dir, os.W_OK) 126 | 127 | with MockTransaction: 128 | logging.root = mock('root') 129 | expect(logging).disable.never() 130 | expect(logging.root).setLevel(logging.DEBUG).once() 131 | handler = mock('handler') 132 | expect(logging).FileHandler(log_path).and_return(handler).once() 133 | expect(handler).setFormatter.and_return(True).once() 134 | expect(logging).Formatter( 135 | '%(asctime)s %(levelname)-8s %(name)s %(message)s').once() 136 | expect(logging.root).addHandler.once() 137 | assert common.init_logging(log_dir, log_file, 3) 138 | assert os.access(log_dir, os.W_OK) 139 | 140 | shutil.rmtree(log_dir, True) 141 | 142 | def test_call_function_by_name(self): 143 | with MockTransaction: 144 | arg1 = 'a' 145 | arg2 = 'b' 146 | expect(common).func_to_call(arg1, arg2).and_return('res').once() 147 | assert common.call_function_by_name('neat.common.func_to_call', 148 | [arg1, arg2]) == 'res' 149 | 150 | def test_parse_parameters(self): 151 | params = '{"param1": 0.56, "param2": "abc"}' 152 | self.assertEqual(common.parse_parameters(params), {'param1': 0.56, 153 | 'param2': 'abc'}) 154 | 155 | def test_parse_compute_hosts(self): 156 | assert common.parse_compute_hosts('') == [] 157 | assert common.parse_compute_hosts('test1, test2') == \ 158 | ['test1', 'test2'] 159 | assert common.parse_compute_hosts('test-1, test_2') == \ 160 | ['test-1', 'test_2'] 161 | assert common.parse_compute_hosts('t1,, t2 , t3') == \ 162 | ['t1', 't2', 't3'] 163 | 164 | @qc(10) 165 | def calculate_migration_time( 166 | data=dict_( 167 | keys=str_(of='abc123-', min_length=36, max_length=36), 168 | values=int_(min=1, max=1000), 169 | min_length=1, max_length=10 170 | ), 171 | bandwidth=float_(min=1., max=100.) 172 | ): 173 | ram = data.values() 174 | migration_time = float(sum(ram)) / len(ram) / bandwidth 175 | assert common.calculate_migration_time(data, bandwidth) == \ 176 | migration_time 177 | 178 | @qc(10) 179 | def calculate_migration_time_long( 180 | data=dict_( 181 | keys=str_(of='abc123-', min_length=36, max_length=36), 182 | values=int_(min=1, max=1000), 183 | min_length=1, max_length=10 184 | ), 185 | bandwidth=float_(min=1., max=100.) 186 | ): 187 | data = dict([(k, long(v)) for (k, v) in data.iteritems()]) 188 | ram = data.values() 189 | migration_time = float(sum(ram)) / len(ram) / bandwidth 190 | assert common.calculate_migration_time(data, bandwidth) == \ 191 | migration_time 192 | -------------------------------------------------------------------------------- /tests/locals/overload/mhod/test_core.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from mocktest import * 16 | from pyqcy import * 17 | 18 | import neat.locals.overload.mhod.multisize_estimation as estimation 19 | import neat.locals.overload.mhod.core as c 20 | 21 | import logging 22 | logging.disable(logging.CRITICAL) 23 | 24 | 25 | class Core(TestCase): 26 | 27 | def test_init_state(self): 28 | state = c.init_state(100, [20, 40], 2) 29 | self.assertEquals(state['previous_state'], 0) 30 | self.assertEquals(state['previous_utilization'], []) 31 | self.assertEquals(state['time_in_states'], 0) 32 | self.assertEquals(state['time_in_state_n'], 0) 33 | self.assertTrue('request_windows' in state) 34 | self.assertTrue('estimate_windows' in state) 35 | self.assertTrue('variances' in state) 36 | self.assertTrue('acceptable_variances' in state) 37 | 38 | def test_utilization_to_state(self): 39 | state_config = [0.4, 0.7] 40 | self.assertEqual(c.utilization_to_state(state_config, 0.0), 0) 41 | self.assertEqual(c.utilization_to_state(state_config, 0.1), 0) 42 | self.assertEqual(c.utilization_to_state(state_config, 0.2), 0) 43 | self.assertEqual(c.utilization_to_state(state_config, 0.3), 0) 44 | self.assertEqual(c.utilization_to_state(state_config, 0.4), 1) 45 | self.assertEqual(c.utilization_to_state(state_config, 0.5), 1) 46 | self.assertEqual(c.utilization_to_state(state_config, 0.6), 1) 47 | self.assertEqual(c.utilization_to_state(state_config, 0.7), 2) 48 | self.assertEqual(c.utilization_to_state(state_config, 0.8), 2) 49 | self.assertEqual(c.utilization_to_state(state_config, 0.9), 2) 50 | self.assertEqual(c.utilization_to_state(state_config, 1.0), 2) 51 | self.assertEqual(c.utilization_to_state(state_config, 1.1), 2) 52 | 53 | self.assertEqual(c.utilization_to_state([1.0], 0.0), 0) 54 | self.assertEqual(c.utilization_to_state([1.0], 0.1), 0) 55 | self.assertEqual(c.utilization_to_state([1.0], 0.2), 0) 56 | self.assertEqual(c.utilization_to_state([1.0], 0.3), 0) 57 | self.assertEqual(c.utilization_to_state([1.0], 0.4), 0) 58 | self.assertEqual(c.utilization_to_state([1.0], 0.5), 0) 59 | self.assertEqual(c.utilization_to_state([1.0], 0.6), 0) 60 | self.assertEqual(c.utilization_to_state([1.0], 0.7), 0) 61 | self.assertEqual(c.utilization_to_state([1.0], 0.8), 0) 62 | self.assertEqual(c.utilization_to_state([1.0], 0.9), 0) 63 | self.assertEqual(c.utilization_to_state([1.0], 1.0), 1) 64 | self.assertEqual(c.utilization_to_state([1.0], 1.1), 1) 65 | 66 | def test_build_state_vector(self): 67 | state_config = [0.4, 0.7] 68 | self.assertEqual(c.build_state_vector(state_config, [0.0, 0.1]), 69 | [1, 0, 0]) 70 | self.assertEqual(c.build_state_vector(state_config, [0.0, 0.2]), 71 | [1, 0, 0]) 72 | self.assertEqual(c.build_state_vector(state_config, [0.0, 0.3]), 73 | [1, 0, 0]) 74 | self.assertEqual(c.build_state_vector(state_config, [0.0, 0.4]), 75 | [0, 1, 0]) 76 | self.assertEqual(c.build_state_vector(state_config, [0.0, 0.5]), 77 | [0, 1, 0]) 78 | self.assertEqual(c.build_state_vector(state_config, [0.0, 0.6]), 79 | [0, 1, 0]) 80 | self.assertEqual(c.build_state_vector(state_config, [0.0, 0.7]), 81 | [0, 0, 1]) 82 | self.assertEqual(c.build_state_vector(state_config, [0.0, 0.8]), 83 | [0, 0, 1]) 84 | self.assertEqual(c.build_state_vector(state_config, [0.0, 0.9]), 85 | [0, 0, 1]) 86 | self.assertEqual(c.build_state_vector(state_config, [0.0, 1.0]), 87 | [0, 0, 1]) 88 | 89 | def test_get_current_state(self): 90 | self.assertEqual(c.get_current_state([1, 0, 0]), 0) 91 | self.assertEqual(c.get_current_state([0, 1, 0]), 1) 92 | self.assertEqual(c.get_current_state([0, 0, 1]), 2) 93 | 94 | def test_utilization_to_states(self): 95 | state_config = [0.4, 0.7] 96 | data = [0.25, 0.30, 0.62, 0.59, 0.67, 0.73, 0.85, 0.97, 0.73, 97 | 0.68, 0.69, 0.52, 0.51, 0.25, 0.38, 0.46, 0.52, 0.55, 98 | 0.58, 0.65, 0.70] 99 | states = [0, 0, 1, 1, 1, 2, 2, 2, 2, 1, 1, 100 | 1, 1, 0, 0, 1, 1, 1, 1, 1, 2] 101 | self.assertEqual(c.utilization_to_states(state_config, data), states) 102 | 103 | state_config = [1.0] 104 | data = [0.5, 0.5, 1.0, 1.0, 0.5] 105 | states = [0, 0, 1, 1, 0] 106 | self.assertEqual(c.utilization_to_states(state_config, data), states) 107 | 108 | def test_issue_command_deterministic(self): 109 | self.assertEqual(c.issue_command_deterministic([1]), False) 110 | self.assertEqual(c.issue_command_deterministic([]), True) 111 | 112 | def test_mhod(self): 113 | state_config = [1.0] 114 | otf = 0.1 115 | window_sizes = [30, 40] 116 | bruteforce_step = 0.5 117 | learning_steps = 0 118 | time_step = 300 119 | migration_time = 20. 120 | utilization = [1.0] 121 | state = c.init_state(10, window_sizes, 2) 122 | 123 | with MockTransaction: 124 | state['previous_utilization'] = [] 125 | expect(estimation).select_best_estimates.and_return([[0., 0.], [0., 0.]]) 126 | expect(c).get_current_state.and_return(1).once() 127 | decision, _ = c.mhod(state_config, otf, window_sizes, bruteforce_step, 128 | learning_steps, time_step, migration_time, utilization, state) 129 | self.assertFalse(decision) 130 | 131 | with MockTransaction: 132 | state['previous_utilization'] = [] 133 | expect(estimation).select_best_estimates.and_return([[0., 0.], [0., 0.]]) 134 | expect(c).get_current_state.and_return(0).once() 135 | decision, _ = c.mhod(state_config, otf, window_sizes, bruteforce_step, 136 | learning_steps, time_step, migration_time, utilization, state) 137 | self.assertFalse(decision) 138 | 139 | with MockTransaction: 140 | state['previous_utilization'] = [] 141 | expect(estimation).select_best_estimates.and_return([[0., 1.], [0., 1.]]) 142 | expect(c).get_current_state.and_return(0).once() 143 | decision, _ = c.mhod(state_config, otf, window_sizes, bruteforce_step, 144 | learning_steps, time_step, migration_time, utilization, state) 145 | self.assertFalse(decision) 146 | 147 | with MockTransaction: 148 | state['previous_utilization'] = [] 149 | expect(estimation).select_best_estimates.and_return([[0., 1.], [0., 1.]]) 150 | expect(c).get_current_state.and_return(1).once() 151 | decision, _ = c.mhod(state_config, otf, window_sizes, bruteforce_step, 152 | learning_steps, time_step, migration_time, utilization, state) 153 | self.assertTrue(decision) 154 | 155 | # with MockTransaction: 156 | # utilization = [1.0, 1.0] 157 | # state['previous_utilization'] = [1.0, 1.0] 158 | # state['time_in_states'] = 2 159 | # expect(estimation).select_best_estimates.never() 160 | # decision, _ = c.mhod(state_config, otf, window_sizes, bruteforce_step, 161 | # learning_steps, time_step, migration_time, utilization, state) 162 | # self.assertFalse(decision) 163 | 164 | 165 | def deque_maxlen(coll): 166 | return int(re.sub("\)$", "", re.sub(".*=", "", coll.__repr__()))) 167 | -------------------------------------------------------------------------------- /tests/locals/overload/test_statistics.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from mocktest import * 16 | from pyqcy import * 17 | 18 | import neat.locals.overload.statistics as stats 19 | 20 | import logging 21 | logging.disable(logging.CRITICAL) 22 | 23 | 24 | class Statistics(TestCase): 25 | 26 | def test_loess_factory(self): 27 | alg = stats.loess_factory( 28 | 300, 20., {'threshold': 1.0, 'param': 1.2, 'length': 3}) 29 | self.assertEqual(alg([]), (False, {})) 30 | 31 | data = [1.05, 1.09, 1.07, 1.12, 1.02, 1.18, 32 | 1.15, 1.04, 1.10, 1.16, 1.08] 33 | self.assertEqual(alg(data), (True, {})) 34 | 35 | data = [0.55, 0.60, 0.62, 0.59, 0.67, 0.73, 0.85, 0.97, 0.73, 36 | 0.68, 0.69, 0.52, 0.51, 0.55, 0.48, 0.46, 0.52, 0.55, 37 | 0.58, 0.65, 0.70] 38 | self.assertEqual(alg(data), (False, {})) 39 | 40 | def test_loess_robust_factory(self): 41 | alg = stats.loess_robust_factory( 42 | 300, 20., {'threshold': 1.0, 'param': 1.2, 'length': 3}) 43 | self.assertEqual(alg([]), (False, {})) 44 | 45 | data = [1.05, 1.09, 1.07, 1.12, 1.02, 1.18, 46 | 1.15, 1.04, 1.10, 1.16, 1.08] 47 | self.assertEqual(alg(data), (True, {})) 48 | 49 | data = [0.55, 0.60, 0.62, 0.59, 0.67, 0.73, 0.85, 0.97, 0.73, 50 | 0.68, 0.69, 0.52, 0.51, 0.55, 0.48, 0.46, 0.52, 0.55, 51 | 0.58, 0.65, 0.70] 52 | self.assertEqual(alg(data), (False, {})) 53 | 54 | def test_mad_threshold_factory(self): 55 | with MockTransaction: 56 | expect(stats).mad.and_return(0.125).exactly(6).times() 57 | alg = stats.mad_threshold_factory( 58 | 300, 20., {'threshold': 1.6, 'limit': 3}) 59 | self.assertEqual(alg([]), (False, {})) 60 | self.assertEqual(alg([0., 0., 0.0]), (False, {})) 61 | self.assertEqual(alg([0., 0., 0.5]), (False, {})) 62 | self.assertEqual(alg([0., 0., 0.6]), (False, {})) 63 | self.assertEqual(alg([0., 0., 0.8]), (True, {})) 64 | self.assertEqual(alg([0., 0., 0.9]), (True, {})) 65 | self.assertEqual(alg([0., 0., 1.0]), (True, {})) 66 | 67 | def test_iqr_threshold_factory(self): 68 | with MockTransaction: 69 | expect(stats).iqr.and_return(0.125).exactly(6).times() 70 | alg = stats.iqr_threshold_factory( 71 | 300, 20., {'threshold': 1.6, 'limit': 3}) 72 | self.assertEqual(alg([]), (False, {})) 73 | self.assertEqual(alg([0., 0., 0.0]), (False, {})) 74 | self.assertEqual(alg([0., 0., 0.5]), (False, {})) 75 | self.assertEqual(alg([0., 0., 0.6]), (False, {})) 76 | self.assertEqual(alg([0., 0., 0.8]), (True, {})) 77 | self.assertEqual(alg([0., 0., 0.9]), (True, {})) 78 | self.assertEqual(alg([0., 0., 1.0]), (True, {})) 79 | 80 | def test_loess(self): 81 | assert not stats.loess(1.0, 1.2, 3, 0.5, []) 82 | 83 | data = [1.05, 1.09, 1.07, 1.12, 1.02, 1.18, 84 | 1.15, 1.04, 1.10, 1.16, 1.08] 85 | assert stats.loess(1.0, 1.2, 3, 0.5, data) 86 | 87 | data = [0.55, 0.60, 0.62, 0.59, 0.67, 0.73, 0.85, 0.97, 0.73, 88 | 0.68, 0.69, 0.52, 0.51, 0.55, 0.48, 0.46, 0.52, 0.55, 89 | 0.58, 0.65, 0.70] 90 | assert not stats.loess(1.0, 1.2, 3, 0.5, data) 91 | 92 | def test_loess_robust(self): 93 | assert not stats.loess_robust(1.0, 1.2, 3, 0.5, []) 94 | 95 | data = [1.05, 1.09, 1.07, 1.12, 1.02, 1.18, 96 | 1.15, 1.04, 1.10, 1.16, 1.08] 97 | assert stats.loess_robust(1.0, 1.2, 3, 0.5, data) 98 | 99 | data = [0.55, 0.60, 0.62, 0.59, 0.67, 0.73, 0.85, 0.97, 0.73, 100 | 0.68, 0.69, 0.52, 0.51, 0.55, 0.48, 0.46, 0.52, 0.55, 101 | 0.58, 0.65, 0.70] 102 | assert not stats.loess_robust(1.0, 1.2, 3, 0.5, data) 103 | 104 | def test_mad_threshold(self): 105 | with MockTransaction: 106 | expect(stats).mad.and_return(0.125).exactly(6).times() 107 | assert not stats.mad_threshold(1., 3, []) 108 | assert not stats.mad_threshold(1., 3, [0., 0., 0.]) 109 | assert not stats.mad_threshold(1.6, 3, [0., 0., 0.5]) 110 | assert not stats.mad_threshold(1.6, 3, [0., 0., 0.6]) 111 | assert stats.mad_threshold(1.6, 3, [0., 0., 0.8]) 112 | assert stats.mad_threshold(1.6, 3, [0., 0., 0.9]) 113 | assert stats.mad_threshold(1.6, 3, [0., 0., 1.0]) 114 | 115 | def test_iqr_threshold(self): 116 | with MockTransaction: 117 | expect(stats).iqr.and_return(0.125).exactly(6).times() 118 | assert not stats.iqr_threshold(1., 3, []) 119 | assert not stats.iqr_threshold(1., 3, [0., 0., 0.]) 120 | assert not stats.iqr_threshold(1.6, 3, [0., 0., 0.5]) 121 | assert not stats.iqr_threshold(1.6, 3, [0., 0., 0.6]) 122 | assert stats.iqr_threshold(1.6, 3, [0., 0., 0.8]) 123 | assert stats.iqr_threshold(1.6, 3, [0., 0., 0.9]) 124 | assert stats.iqr_threshold(1.6, 3, [0., 0., 1.0]) 125 | 126 | def test_utilization_threshold_abstract(self): 127 | f = lambda x: 0.8 128 | assert not stats.utilization_threshold_abstract(f, 3, []) 129 | assert not stats.utilization_threshold_abstract(f, 3, [0., 0., 0.]) 130 | assert stats.utilization_threshold_abstract(f, 3, [0., 0., 1.0]) 131 | assert not stats.utilization_threshold_abstract( 132 | f, 3, [0., 0., 0., 0.]) 133 | assert not stats.utilization_threshold_abstract( 134 | f, 3, [0., 0., 0., 0.5]) 135 | assert not stats.utilization_threshold_abstract( 136 | f, 3, [0., 0., 0., 0.7]) 137 | assert stats.utilization_threshold_abstract(f, 3, [0., 0., 0., 0.8]) 138 | assert stats.utilization_threshold_abstract(f, 3, [0., 0., 0., 0.9]) 139 | assert stats.utilization_threshold_abstract(f, 3, [0., 0., 0., 1.0]) 140 | 141 | def test_mad(self): 142 | data = [1, 1, 2, 2, 4, 6, 9] 143 | assert stats.mad(data) == 1. 144 | 145 | def test_iqr(self): 146 | data = [105, 109, 107, 112, 102, 118, 115, 104, 110, 116, 108] 147 | assert stats.iqr(data) == 10. 148 | 149 | data = [2., 4., 7., -20., 22., -1., 0., -1., 7., 15., 8., 4., 150 | -4., 11., 11., 12., 3., 12., 18., 1.] 151 | assert stats.iqr(data) == 12. 152 | 153 | def test_loess_parameter_estimates(self): 154 | data = [2., 4., 7., -20., 22., -1., 0., -1., 7., 15., 8., 4., 155 | -4., 11., 11., 12., 3., 12., 18., 1.] 156 | estimates = stats.loess_parameter_estimates(data) 157 | self.assertAlmostEqual(estimates[0], 2.2639, 3) 158 | self.assertAlmostEqual(estimates[1], 0.3724, 3) 159 | 160 | def test_loess_robust_parameter_estimates(self): 161 | data = [2., 4., 7., -20., 22., -1., 0., -1., 7., 15., 8., 4., 162 | -4., 11., 11., 12., 3., 12., 18., 1.] 163 | estimates = stats.loess_robust_parameter_estimates(data) 164 | self.assertAlmostEqual(estimates[0], 2.4547, 3) 165 | self.assertAlmostEqual(estimates[1], 0.3901, 3) 166 | 167 | def test_tricube_weights(self): 168 | for actual, expected in zip( 169 | stats.tricube_weights(5), 170 | [0.669, 0.669, 0.669, 0.953, 1.0]): 171 | self.assertAlmostEqual(actual, expected, 2) 172 | 173 | for actual, expected in zip( 174 | stats.tricube_weights(10), 175 | [0.148, 0.148, 0.148, 0.348, 0.568, 0.759, 176 | 0.892, 0.967, 0.995, 1.0]): 177 | self.assertAlmostEqual(actual, expected, 2) 178 | 179 | def test_tricube_bisquare_weights(self): 180 | for actual, expected in zip( 181 | stats.tricube_bisquare_weights([1., 1., 2., 2., 4., 6., 9.]), 182 | [0.329, 0.329, 0.329, 0.633, 0.705, 0.554, 0.191]): 183 | self.assertAlmostEqual(actual, expected, 2) 184 | -------------------------------------------------------------------------------- /tests/locals/vm_selection/test_algorithms.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from mocktest import * 16 | from pyqcy import * 17 | 18 | import neat.locals.vm_selection.algorithms as selection 19 | 20 | import logging 21 | logging.disable(logging.CRITICAL) 22 | 23 | 24 | class Selection(TestCase): 25 | 26 | @qc(10) 27 | def minimum_migration_time_factory( 28 | x=dict_( 29 | keys=str_(of='abc123-', min_length=36, max_length=36), 30 | values=int_(min=0, max=3000), 31 | min_length=1, max_length=5 32 | ) 33 | ): 34 | alg = selection.minimum_migration_time_factory(300, 20., dict()) 35 | values = x.values() 36 | vm_index = values.index(min(values)) 37 | vm = x.keys()[vm_index] 38 | assert alg(dict(), x) == ([vm], {}) 39 | 40 | @qc(10) 41 | def minimum_utilization_factory( 42 | x=dict_( 43 | keys=str_(of='abc123-', min_length=36, max_length=36), 44 | values=list_(of=int_(min=0, max=3000), 45 | min_length=1, max_length=10), 46 | min_length=1, max_length=5 47 | ) 48 | ): 49 | alg = selection.minimum_utilization_factory(300, 20., dict()) 50 | last_utilization = [] 51 | for utilization in x.values(): 52 | last_utilization.append(utilization[-1]) 53 | vm_index = last_utilization.index(min(last_utilization)) 54 | vm = x.keys()[vm_index] 55 | assert alg(x, dict()) == ([vm], {}) 56 | 57 | @qc(10) 58 | def random_factory( 59 | x=dict_( 60 | keys=str_(of='abc123-', min_length=36, max_length=36), 61 | values=list_(of=int_(min=0, max=3000), 62 | min_length=0, max_length=10), 63 | min_length=1, max_length=3 64 | ) 65 | ): 66 | with MockTransaction: 67 | alg = selection.random_factory(300, 20., dict()) 68 | vm = x.keys()[random.randrange(len(x))] 69 | expect(selection).choice(x.keys()).and_return(vm).once() 70 | assert alg(x, dict()) == ([vm], {}) 71 | 72 | @qc(10) 73 | def minimum_migration_time_max_cpu_factory( 74 | x=dict_( 75 | keys=str_(of='abc123-', min_length=36, max_length=36), 76 | values=tuple_(list_(of=int_(min=0, max=3000), 77 | min_length=1, max_length=10), 78 | int_(min=0, max=3000)), 79 | min_length=1, max_length=5 80 | ), 81 | last_n=int_(min=1, max=10) 82 | ): 83 | alg = selection.minimum_migration_time_max_cpu_factory( 84 | 300, 20., {'last_n': last_n}) 85 | vms_cpu = dict((k, v[0]) for k, v in x.items()) 86 | vms_ram = dict((k, v[1]) for k, v in x.items()) 87 | min_ram = min(vms_ram.values()) 88 | min_ram_vms_cpu = dict((k, float(sum(v[-last_n:])) / len(v[-last_n:])) 89 | for k, v in vms_cpu.items() 90 | if vms_ram[k] == min_ram and len(v[-last_n:]) > 0) 91 | values = min_ram_vms_cpu.values() 92 | vm_index = values.index(max(values)) 93 | vm = min_ram_vms_cpu.keys()[vm_index] 94 | assert alg(vms_cpu, vms_ram) == ([vm], {}) 95 | 96 | @qc(10) 97 | def minimum_migration_time_max_cpu_factory_equal_ram( 98 | vms_cpu=dict_( 99 | keys=str_(of='abc123-', min_length=36, max_length=36), 100 | values=list_(of=int_(min=0, max=3000), 101 | min_length=1, max_length=10), 102 | min_length=1, max_length=5 103 | ), 104 | ram=int_(min=1000, max=3000), 105 | last_n=int_(min=1, max=10) 106 | ): 107 | alg = selection.minimum_migration_time_max_cpu_factory( 108 | 300, 20., {'last_n': last_n}) 109 | vms_ram = dict((k, ram) for k, _ in vms_cpu.items()) 110 | min_ram = min(vms_ram.values()) 111 | min_ram_vms_cpu = dict((k, float(sum(v[-last_n:])) / len(v[-last_n:])) 112 | for k, v in vms_cpu.items() 113 | if vms_ram[k] == min_ram and len(v[-last_n:]) > 0) 114 | values = min_ram_vms_cpu.values() 115 | vm_index = values.index(max(values)) 116 | vm = min_ram_vms_cpu.keys()[vm_index] 117 | assert alg(vms_cpu, vms_ram) == ([vm], {}) 118 | 119 | @qc(10) 120 | def minimum_migration_time( 121 | x=dict_( 122 | keys=str_(of='abc123-', min_length=36, max_length=36), 123 | values=int_(min=0, max=3000), 124 | min_length=1, max_length=5 125 | ) 126 | ): 127 | values = x.values() 128 | vm_index = values.index(min(values)) 129 | vm = x.keys()[vm_index] 130 | assert selection.minimum_migration_time(x) == vm 131 | 132 | @qc(10) 133 | def minimum_utilization( 134 | x=dict_( 135 | keys=str_(of='abc123-', min_length=36, max_length=36), 136 | values=list_(of=int_(min=0, max=3000), 137 | min_length=1, max_length=10), 138 | min_length=1, max_length=5 139 | ) 140 | ): 141 | last_utilization = [] 142 | for utilization in x.values(): 143 | last_utilization.append(utilization[-1]) 144 | vm_index = last_utilization.index(min(last_utilization)) 145 | vm = x.keys()[vm_index] 146 | assert selection.minimum_utilization(x) == vm 147 | 148 | @qc(10) 149 | def random( 150 | x=dict_( 151 | keys=str_(of='abc123-', min_length=36, max_length=36), 152 | values=list_(of=int_(min=0, max=3000), 153 | min_length=0, max_length=10), 154 | min_length=1, max_length=3 155 | ) 156 | ): 157 | with MockTransaction: 158 | vm = x.keys()[random.randrange(len(x))] 159 | expect(selection).choice(x.keys()).and_return(vm).once() 160 | assert selection.random(x) == vm 161 | 162 | @qc(10) 163 | def minimum_migration_time_max_cpu( 164 | x=dict_( 165 | keys=str_(of='abc123-', min_length=36, max_length=36), 166 | values=tuple_(list_(of=int_(min=0, max=3000), 167 | min_length=1, max_length=10), 168 | int_(min=0, max=3000)), 169 | min_length=1, max_length=5 170 | ), 171 | last_n=int_(min=1, max=10) 172 | ): 173 | vms_cpu = dict((k, v[0]) for k, v in x.items()) 174 | vms_ram = dict((k, v[1]) for k, v in x.items()) 175 | min_ram = min(vms_ram.values()) 176 | min_ram_vms_cpu = dict((k, float(sum(v[-last_n:])) / len(v[-last_n:])) 177 | for k, v in vms_cpu.items() 178 | if vms_ram[k] == min_ram and len(v[-last_n:]) > 0) 179 | values = min_ram_vms_cpu.values() 180 | vm_index = values.index(max(values)) 181 | vm = min_ram_vms_cpu.keys()[vm_index] 182 | assert selection.minimum_migration_time_max_cpu( 183 | last_n, vms_cpu, vms_ram) == vm 184 | 185 | @qc(10) 186 | def minimum_migration_time_max_cpu_equal_ram( 187 | vms_cpu=dict_( 188 | keys=str_(of='abc123-', min_length=36, max_length=36), 189 | values=list_(of=int_(min=0, max=3000), 190 | min_length=1, max_length=10), 191 | min_length=1, max_length=5 192 | ), 193 | ram=int_(min=1000, max=3000), 194 | last_n=int_(min=1, max=10) 195 | ): 196 | vms_ram = dict((k, ram) for k, _ in vms_cpu.items()) 197 | min_ram = min(vms_ram.values()) 198 | min_ram_vms_cpu = dict((k, float(sum(v[-last_n:])) / len(v[-last_n:])) 199 | for k, v in vms_cpu.items() 200 | if vms_ram[k] == min_ram and len(v[-last_n:]) > 0) 201 | values = min_ram_vms_cpu.values() 202 | vm_index = values.index(max(values)) 203 | vm = min_ram_vms_cpu.keys()[vm_index] 204 | assert selection.minimum_migration_time_max_cpu( 205 | last_n, vms_cpu, vms_ram) == vm 206 | -------------------------------------------------------------------------------- /tests/locals/test_manager.py: -------------------------------------------------------------------------------- 1 | # Copyright 2012 Anton Beloglazov 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from mocktest import * 16 | from pyqcy import * 17 | 18 | import shutil 19 | import libvirt 20 | from hashlib import sha1 21 | 22 | import neat.locals.manager as manager 23 | import neat.common as common 24 | import neat.locals.collector as collector 25 | 26 | import logging 27 | logging.disable(logging.CRITICAL) 28 | 29 | 30 | class LocalManager(TestCase): 31 | 32 | @qc(10) 33 | def start( 34 | iterations=int_(min=0, max=10), 35 | time_interval=int_(min=0) 36 | ): 37 | with MockTransaction: 38 | state = {'property': 'value'} 39 | config = { 40 | 'log_directory': 'dir', 41 | 'log_level': 2, 42 | 'local_manager_interval': str(time_interval)} 43 | paths = [manager.DEFAILT_CONFIG_PATH, manager.CONFIG_PATH] 44 | fields = manager.REQUIRED_FIELDS 45 | expect(manager).read_and_validate_config(paths, fields). \ 46 | and_return(config).once() 47 | expect(common).init_logging('dir', 'local-manager.log', 2).once() 48 | expect(common).start(manager.init_state, 49 | manager.execute, 50 | config, 51 | time_interval).and_return(state).once() 52 | assert manager.start() == state 53 | 54 | @qc(1) 55 | def init_state(): 56 | with MockTransaction: 57 | vir_connection = mock('virConnect') 58 | db = mock('db') 59 | mhz = 3000 60 | expect(libvirt).openReadOnly(None). \ 61 | and_return(vir_connection).once() 62 | expect(manager).init_db('db'). \ 63 | and_return(db).once() 64 | expect(common).physical_cpu_mhz_total(vir_connection). \ 65 | and_return(mhz) 66 | expect(vir_connection).getHostname().and_return('host').once() 67 | config = {'sql_connection': 'db', 68 | 'os_admin_user': 'user', 69 | 'os_admin_password': 'password', 70 | 'host_cpu_usable_by_vms': 0.75} 71 | state = manager.init_state(config) 72 | assert state['previous_time'] == 0 73 | assert state['vir_connection'] == vir_connection 74 | assert state['db'] == db 75 | assert state['physical_cpu_mhz_total'] == mhz * 0.75 76 | assert state['hostname'] == 'host' 77 | assert state['hashed_username'] == sha1('user').hexdigest() 78 | assert state['hashed_password'] == sha1('password').hexdigest() 79 | 80 | @qc(1) 81 | def get_local_vm_data( 82 | data=dict_( 83 | keys=str_(of='abc123-', min_length=36, max_length=36), 84 | values=list_(of=int_(min=1, max=3000), 85 | min_length=0, max_length=10), 86 | min_length=0, max_length=5 87 | ) 88 | ): 89 | path = os.path.join(os.path.dirname(__file__), 90 | '..', 'resources', 'vms', 'tmp') 91 | shutil.rmtree(path, True) 92 | os.mkdir(path) 93 | collector.write_vm_data_locally(path, data, 10) 94 | 95 | assert manager.get_local_vm_data(path) == data 96 | shutil.rmtree(path) 97 | 98 | @qc(1) 99 | def get_local_host_data( 100 | data=list_(of=int_(min=1, max=3000), 101 | min_length=0, max_length=10) 102 | ): 103 | path = os.path.join(os.path.dirname(__file__), 104 | '..', 'resources', 'host') 105 | assert manager.get_local_host_data(path) == [] 106 | 107 | with open(path, 'w') as f: 108 | f.write('\n'.join([str(x) 109 | for x in data]) + '\n') 110 | assert manager.get_local_host_data(path) == data 111 | os.remove(path) 112 | 113 | @qc(10) 114 | def cleanup_vm_data( 115 | data=dict_( 116 | keys=str_(of='abc123-', min_length=36, max_length=36), 117 | values=list_(of=int_(min=1, max=3000), 118 | min_length=0, max_length=10), 119 | min_length=0, max_length=5 120 | ) 121 | ): 122 | original_data = dict(data) 123 | uuids = data.keys() 124 | if data: 125 | n = random.randrange(len(data)) 126 | for _ in range(n): 127 | uuid = random.choice(uuids) 128 | del data[uuid] 129 | uuids.remove(uuid) 130 | 131 | assert manager.cleanup_vm_data(original_data, uuids) == data 132 | 133 | @qc(10) 134 | def get_ram( 135 | data=dict_( 136 | keys=str_(of='abc123-', min_length=36, max_length=36), 137 | values=int_(min=1, max=100), 138 | min_length=0, max_length=10 139 | ) 140 | ): 141 | with MockTransaction: 142 | def mock_get_max_ram(vir_connection, uuid): 143 | return data[uuid] 144 | connection = libvirt.virConnect() 145 | when(manager).get_max_ram(connection, any_string). \ 146 | then_call(mock_get_max_ram) 147 | assert manager.get_ram(connection, data.keys()) == data 148 | 149 | @qc(10) 150 | def get_ram_long( 151 | data=dict_( 152 | keys=str_(of='abc123-', min_length=36, max_length=36), 153 | values=int_(min=1, max=100), 154 | min_length=0, max_length=10 155 | ) 156 | ): 157 | data = dict([(k, long(v)) for (k, v) in data.iteritems()]) 158 | with MockTransaction: 159 | def mock_get_max_ram(vir_connection, uuid): 160 | return data[uuid] 161 | connection = libvirt.virConnect() 162 | when(manager).get_max_ram(connection, any_string). \ 163 | then_call(mock_get_max_ram) 164 | assert manager.get_ram(connection, data.keys()) == data 165 | 166 | @qc(10) 167 | def get_max_ram( 168 | uuid=str_(of='abc123-', min_length=36, max_length=36), 169 | x=int_(min=0) 170 | ): 171 | with MockTransaction: 172 | connection = libvirt.virConnect() 173 | domain = mock('domain') 174 | expect(connection).lookupByUUIDString(uuid). \ 175 | and_return(domain).once() 176 | expect(domain).maxMemory().and_return(x).once() 177 | assert manager.get_max_ram(connection, uuid) == int(x) / 1024 178 | 179 | @qc(10) 180 | def get_max_ram_long( 181 | uuid=str_(of='abc123-', min_length=36, max_length=36), 182 | x=int_(min=0) 183 | ): 184 | with MockTransaction: 185 | connection = libvirt.virConnect() 186 | domain = mock('domain') 187 | expect(connection).lookupByUUIDString(uuid). \ 188 | and_return(domain).once() 189 | expect(domain).maxMemory().and_return(long(x)).once() 190 | assert manager.get_max_ram(connection, uuid) == long(x) / 1024 191 | 192 | @qc(1) 193 | def get_max_ram_none( 194 | uuid=str_(of='abc123-', min_length=36, max_length=36) 195 | ): 196 | with MockTransaction: 197 | def raise_libvirt_error(): 198 | raise libvirt.libvirtError(None) 199 | connection = libvirt.virConnect() 200 | expect(connection).lookupByUUIDString(uuid). \ 201 | and_call(lambda _: raise_libvirt_error()) 202 | assert manager.get_max_ram(connection, uuid) is None 203 | 204 | def test_vm_mhz_to_percentage(self): 205 | self.assertEqual(manager.vm_mhz_to_percentage( 206 | [[100, 200, 300], 207 | [300, 100, 300, 200], 208 | [100, 100, 700]], 209 | [300, 0, 300], 210 | 3000), 211 | [0.1, 0.2, 0.2, 0.5]) 212 | 213 | self.assertEqual(manager.vm_mhz_to_percentage( 214 | [[100, 200, 300], 215 | [100, 300, 200], 216 | [100, 100, 700]], 217 | [0, 300], 218 | 3000), 219 | [0.1, 0.2, 0.5]) 220 | 221 | self.assertEqual(manager.vm_mhz_to_percentage( 222 | [[100, 200, 300], 223 | [300, 100, 300, 200], 224 | [100, 100, 700]], 225 | [300, 0, 300, 0, 300], 226 | 3000), 227 | [0.1, 0.2, 0.2, 0.5]) 228 | --------------------------------------------------------------------------------