├── .gitignore ├── LICENSE ├── README.md ├── cgroups ├── __init__.py ├── cgroup.py ├── common.py ├── user.py └── utils.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | *pyc 2 | *.*~ 3 | *.log 4 | *.coverage 5 | .Python 6 | 7 | bin/ 8 | include/ 9 | lib/ 10 | .ropeproject/ 11 | */htmlcov/ 12 | *.egg-info/ 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2014, Francis Bouvier 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | 1. Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 11 | 2. Redistributions in binary form must reproduce the above copyright notice, 12 | this list of conditions and the following disclaimer in the documentation 13 | and/or other materials provided with the distribution. 14 | 15 | 3. Neither the name of the copyright holder nor the names of its 16 | contributors may be used to endorse or promote products derived from 17 | this software without specific prior written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 23 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 25 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 26 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cgroups 2 | 3 | *cgroups* is a library for managing Linux kernel cgroups. 4 | 5 | `cgroups` is a great Linux kernel feature used to control process ressources by groups. 6 | 7 | For now the library can only handle `cpu` and `memory` cgroups. 8 | 9 | 10 | ## Quick start 11 | 12 | Let's say you have some workers and you want them to use no more than 50 % of the CPU and no more than 500 Mb of memory. 13 | 14 | ```python 15 | import os 16 | import subprocess 17 | 18 | from cgroups import Cgroup 19 | 20 | # First we create the cgroup 'charlie' and we set it's cpu and memory limits 21 | cg = Cgroup('charlie') 22 | cg.set_cpu_limit(50) 23 | cg.set_memory_limit(500) 24 | 25 | # Then we a create a function to add a process in the cgroup 26 | def in_my_cgroup(): 27 | pid = os.getpid() 28 | cg = Cgroup('charlie') 29 | cg.add(pid) 30 | 31 | # And we pass this function to the preexec_fn parameter of the subprocess call 32 | # in order to add the process to the cgroup 33 | p1 = subprocess.Popen(['worker_1'], preexec_fn=in_my_cgroup) 34 | p2 = subprocess.Popen(['worker_2'], preexec_fn=in_my_cgroup) 35 | p3 = subprocess.Popen(['worker_3'], preexec_fn=in_my_cgroup) 36 | 37 | # Processes worker_1, worker_2, and worker_3 are now in the cgroup 'charlie' 38 | # and all limits of this cgroup apply to them 39 | 40 | # We can change the cgroup limit while those process are still running 41 | cg.set_cpu_limit(80) 42 | 43 | # And of course we can add other applications to the cgroup 44 | # Let's say we have an application running with pid 27033 45 | cg.add(27033) 46 | ``` 47 | 48 | *Note*: You have to execute this add with root privilages or with sudo (see below **Root and non-root usage**). 49 | 50 | 51 | ## Installation 52 | 53 | ```bash 54 | pip install cgroups 55 | ``` 56 | 57 | 58 | ## Requirements 59 | 60 | **Linux and cgroups** 61 | 62 | The `cgroups` feature is only available on Linux systems with a recent kernel and with the cgroups filesystem mounted at `/sys/fs/cgroup` (which is the case of most Linux distributions since 2012). 63 | 64 | If the cgroups filesystem is mounted elsewhere you can change the `BASE_CGROUPS` value to accomidate: 65 | 66 | ```python 67 | from cgroups import BASE_CGROUPS 68 | 69 | BASE_CGROUPS = 'path_to_cgroups_filesystem' 70 | ``` 71 | 72 | **Root and non-root usage** 73 | 74 | To use *cgroups* the current user has to have root privileges **OR** existing cgroups sub-directories. 75 | 76 | In order to create those cgroups sub-directories you use the `user_cgroups` command as root. 77 | 78 | ```bash 79 | sudo user_cgroups USER 80 | ``` 81 | 82 | *N.B.*: This will only give the user permissions to manage cgroups in his or her own sub-directories and process. It wiil not give the user permissions on other cgroups, process, or system commands. 83 | 84 | *N.B.*: You only need to execute this script once. 85 | 86 | 87 | ## Usage 88 | 89 | **class Cgroup(name, hierarchies='all', user='current')** 90 | 91 | Create or load a cgroup. 92 | 93 | *name* is the name of the cgroup. 94 | 95 | *hierarchies* is a list of cgroup hierarchies you want to use. `all` will use all hierarchies supported by the library. 96 | This parameter will be ignored if the cgroup already exists (all existing hierarchies will be used). 97 | 98 | *user* is the cgroups sub-directories name to use. `current` will use the name of the current user. 99 | 100 | ```python 101 | from cgroups import Cgroup 102 | 103 | cg = Cgroup('charlie') 104 | ``` 105 | 106 | 107 | **Cgroup.add(pid)** 108 | 109 | Add the process to all hierarchies within the cgroup. 110 | 111 | *pid* is the pid of the process you want to add to the cgroup. 112 | 113 | ```python 114 | from cgroups import Cgroup 115 | 116 | cg = Cgroup('charlie') 117 | cg.add(27033) 118 | ``` 119 | 120 | If *pid* is already in cgroup hierarchy, this function will fail silently. 121 | 122 | *N.B*: For security reasons the process has to belong to user if you execute this code as a non-root user. 123 | 124 | 125 | **Cgroup.remove(pid)** 126 | 127 | Remove the process from all hierarchies within the cgroup. 128 | 129 | *pid* is the pid of the process you want to remove from the cgroup. 130 | 131 | ```python 132 | from cgroups import Cgroup 133 | 134 | cg = Cgroup('charlie') 135 | cg.remove(27033) 136 | ``` 137 | 138 | If *pid* is not in the cgroup hierarchy this function will fail silently. 139 | 140 | *N.B*: For security reasons the process has to belong to user if you execute this code as a non-root user. 141 | 142 | 143 | **Cgroup.set_cpu_limit(limit)** 144 | 145 | Set the cpu limit for the cgroup. 146 | This function uses the `cpu.shares` hierarchy. 147 | 148 | *limit* is the limit you want to set (as a percentage). 149 | If you don't provide an argument to this method, the menthod will set the cpu limit to the default cpu limit (ie. no limit). 150 | 151 | ```python 152 | from cgroups import Cgroup 153 | 154 | cg = Cgroup('charlie') 155 | 156 | # Give the cgroup 'charlie' 10% limit of the cpu capacity 157 | cg.set_cpu_limit(10) 158 | 159 | # Reset the limit 160 | cg.set_cpu_limit() 161 | ``` 162 | 163 | 164 | **Cgroup.cpu_limit** 165 | 166 | Get the cpu limit of the cgroup as a percentage. 167 | 168 | 169 | **Cgroup.set_memory_limit(limit, unit='megabytes')** 170 | 171 | Set the memory limit of the cgroup (including file cache but exluding swap). 172 | This function uses the `memory.limit_in_bytes` hierarchy. 173 | 174 | *limit* is the limit you want to set. 175 | If you don't provide an argument to this method, the menthod will set the memory limit to the default memory limit (ie. no limit) 176 | 177 | *unit* is the unit used for the limit. Available choices are 'bytes', 'kilobytes', 'megabytes' and 'gigabytes'. Default is 'megabytes'. 178 | 179 | 180 | ```python 181 | from cgroups import Cgroup 182 | 183 | cg = Cgroup('charlie') 184 | 185 | # Give the cgroup 'charlie' a maximum memory of 50 Mo. 186 | cg.set_memory_limit(50) 187 | 188 | # Reset the limit 189 | cg.set_memory_limit('charlie') 190 | ``` 191 | 192 | 193 | **Cgroup.memory_limit** 194 | 195 | Get the memory limit of the the cgroup in megabytes. 196 | 197 | 198 | **Cgroup.delete()** 199 | 200 | Delete the cgroup. 201 | 202 | *N.B*: If there are any processes in the cgroup, they will be moved into the user's cgroup sub-directories. 203 | 204 | ```python 205 | from cgroups import Cgroup 206 | 207 | cg = Cgroup('charlie') 208 | cg.delete() 209 | ``` 210 | -------------------------------------------------------------------------------- /cgroups/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | 4 | from .cgroup import Cgroup 5 | from .common import BASE_CGROUPS 6 | -------------------------------------------------------------------------------- /cgroups/cgroup.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | 4 | from __future__ import unicode_literals 5 | from __future__ import print_function 6 | from __future__ import division 7 | 8 | import os 9 | import getpass 10 | 11 | from cgroups.common import BASE_CGROUPS, CgroupsException 12 | from cgroups.user import create_user_cgroups 13 | 14 | HIERARCHIES = [ 15 | 'cpu', 16 | 'memory', 17 | ] 18 | MEMORY_DEFAULT = -1 19 | CPU_DEFAULT = 1024 20 | 21 | 22 | class Cgroup(object): 23 | 24 | def __init__(self, name, hierarchies='all', user='current'): 25 | self.name = name 26 | # Get user 27 | self.user = user 28 | if self.user == 'current': 29 | self.user = getpass.getuser() 30 | # Get hierarchies 31 | if hierarchies == 'all': 32 | hierachies = HIERARCHIES 33 | self.hierarchies = [h for h in hierachies if h in HIERARCHIES] 34 | # Get user cgroups 35 | self.user_cgroups = {} 36 | system_hierarchies = os.listdir(BASE_CGROUPS) 37 | for hierarchy in self.hierarchies: 38 | if hierarchy not in system_hierarchies: 39 | raise CgroupsException( 40 | "Hierarchy %s is not mounted" % hierarchy) 41 | user_cgroup = os.path.join(BASE_CGROUPS, hierarchy, self.user) 42 | self.user_cgroups[hierarchy] = user_cgroup 43 | create_user_cgroups(self.user, script=False) 44 | # Create name cgroups 45 | self.cgroups = {} 46 | for hierarchy, user_cgroup in self.user_cgroups.items(): 47 | cgroup = os.path.join(user_cgroup, self.name) 48 | if not os.path.exists(cgroup): 49 | os.mkdir(cgroup) 50 | self.cgroups[hierarchy] = cgroup 51 | 52 | def _get_cgroup_file(self, hierarchy, file_name): 53 | return os.path.join(self.cgroups[hierarchy], file_name) 54 | 55 | def _get_user_file(self, hierarchy, file_name): 56 | return os.path.join(self.user_cgroups[hierarchy], file_name) 57 | 58 | def delete(self): 59 | for hierarchy, cgroup in self.cgroups.items(): 60 | # Put all pids of name cgroup in user cgroup 61 | tasks_file = self._get_cgroup_file(hierarchy, 'tasks') 62 | with open(tasks_file, 'r+') as f: 63 | tasks = f.read().split('\n') 64 | user_tasks_file = self._get_user_file(hierarchy, 'tasks') 65 | with open(user_tasks_file, 'a+') as f: 66 | f.write('\n'.join(tasks)) 67 | os.rmdir(cgroup) 68 | 69 | # PIDS 70 | 71 | def add(self, pid): 72 | try: 73 | os.kill(pid, 0) 74 | except OSError: 75 | raise CgroupsException('Pid %s does not exists' % pid) 76 | for hierarchy, cgroup in self.cgroups.items(): 77 | tasks_file = self._get_cgroup_file(hierarchy, 'tasks') 78 | with open(tasks_file, 'r+') as f: 79 | cgroups_pids = f.read().split('\n') 80 | if not str(pid) in cgroups_pids: 81 | with open(tasks_file, 'a+') as f: 82 | f.write('%s\n' % pid) 83 | 84 | def remove(self, pid): 85 | try: 86 | os.kill(pid, 0) 87 | except OSError: 88 | raise CgroupsException('Pid %s does not exists' % pid) 89 | for hierarchy, cgroup in self.cgroups.items(): 90 | tasks_file = self._get_cgroup_file(hierarchy, 'tasks') 91 | with open(tasks_file, 'r+') as f: 92 | pids = f.read().split('\n') 93 | if str(pid) in pids: 94 | user_tasks_file = self._get_user_file(hierarchy, 'tasks') 95 | with open(user_tasks_file, 'a+') as f: 96 | f.write('%s\n' % pid) 97 | 98 | @property 99 | def pids(self): 100 | hierarchy = self.hierarchies[0] 101 | tasks_file = self._get_cgroup_file(hierarchy, 'tasks') 102 | with open(tasks_file, 'r+') as f: 103 | pids = f.read().split('\n')[:-1] 104 | pids = [int(pid) for pid in pids] 105 | return pids 106 | 107 | # CPU 108 | 109 | def _format_cpu_value(self, limit=None): 110 | if limit is None: 111 | value = CPU_DEFAULT 112 | else: 113 | try: 114 | limit = float(limit) 115 | except ValueError: 116 | raise CgroupsException('Limit must be convertible to a float') 117 | else: 118 | if limit <= float(0) or limit > float(100): 119 | raise CgroupsException('Limit must be between 0 and 100') 120 | else: 121 | limit = limit / 100 122 | value = int(round(CPU_DEFAULT * limit)) 123 | return value 124 | 125 | 126 | def set_cpu_limit(self, limit=None): 127 | if 'cpu' in self.cgroups: 128 | value = self._format_cpu_value(limit) 129 | cpu_shares_file = self._get_cgroup_file('cpu', 'cpu.shares') 130 | with open(cpu_shares_file, 'w+') as f: 131 | f.write('%s\n' % value) 132 | else: 133 | raise CgroupsException( 134 | 'CPU hierarchy not available in this cgroup') 135 | 136 | @property 137 | def cpu_limit(self): 138 | if 'cpu' in self.cgroups: 139 | cpu_shares_file = self._get_cgroup_file('cpu', 'cpu.shares') 140 | with open(cpu_shares_file, 'r+') as f: 141 | value = int(f.read().split('\n')[0]) 142 | value = int(round((value / CPU_DEFAULT) * 100)) 143 | return value 144 | else: 145 | return None 146 | 147 | # MEMORY 148 | 149 | def _format_memory_value(self, unit, limit=None): 150 | units = ('bytes', 'kilobytes', 'megabytes', 'gigabytes') 151 | if unit not in units: 152 | raise CgroupsException('Unit must be in %s' % units) 153 | if limit is None: 154 | value = MEMORY_DEFAULT 155 | else: 156 | try: 157 | limit = int(limit) 158 | except ValueError: 159 | raise CgroupsException('Limit must be convertible to an int') 160 | else: 161 | if unit == 'bytes': 162 | value = limit 163 | elif unit == 'kilobytes': 164 | value = limit * 1024 165 | elif unit == 'megabytes': 166 | value = limit * 1024 * 1024 167 | elif unit == 'gigabytes': 168 | value = limit * 1024 * 1024 * 1024 169 | return value 170 | 171 | 172 | def set_memory_limit(self, limit=None, unit='megabytes'): 173 | if 'memory' in self.cgroups: 174 | value = self._format_memory_value(unit, limit) 175 | memory_limit_file = self._get_cgroup_file( 176 | 'memory', 'memory.limit_in_bytes') 177 | with open(memory_limit_file, 'w+') as f: 178 | f.write('%s\n' % value) 179 | else: 180 | raise CgroupsException( 181 | 'MEMORY hierarchy not available in this cgroup') 182 | 183 | @property 184 | def memory_limit(self): 185 | if 'memory' in self.cgroups: 186 | memory_limit_file = self._get_cgroup_file( 187 | 'memory', 'memory.limit_in_bytes') 188 | with open(memory_limit_file, 'r+') as f: 189 | value = f.read().split('\n')[0] 190 | value = int(int(value) / 1024 / 1024) 191 | return value 192 | else: 193 | return None 194 | -------------------------------------------------------------------------------- /cgroups/common.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | 4 | BASE_CGROUPS = '/sys/fs/cgroup' 5 | 6 | 7 | class CgroupsException(Exception): 8 | pass 9 | -------------------------------------------------------------------------------- /cgroups/user.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | 4 | from __future__ import unicode_literals 5 | from __future__ import print_function 6 | 7 | import os 8 | import getpass 9 | import logging 10 | import argparse 11 | from pwd import getpwnam 12 | 13 | from cgroups.common import BASE_CGROUPS, CgroupsException 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | def get_user_info(user): 19 | try: 20 | user_system = getpwnam(user) 21 | except KeyError: 22 | raise CgroupsException("User %s doesn't exists" % user) 23 | else: 24 | uid = user_system.pw_uid 25 | gid = user_system.pw_gid 26 | return uid, gid 27 | 28 | 29 | def create_user_cgroups(user, script=True): 30 | logger.info('Creating cgroups sub-directories for user %s' % user) 31 | # Get hierarchies and create cgroups sub-directories 32 | try: 33 | hierarchies = os.listdir(BASE_CGROUPS) 34 | except OSError as e: 35 | if e.errno == 2: 36 | raise CgroupsException( 37 | "cgroups filesystem is not mounted on %s" % BASE_CGROUPS) 38 | else: 39 | raise OSError(e) 40 | logger.debug('Hierarchies availables: %s' % hierarchies) 41 | for hierarchy in hierarchies: 42 | user_cgroup = os.path.join(BASE_CGROUPS, hierarchy, user) 43 | if not os.path.exists(user_cgroup): 44 | try: 45 | os.mkdir(user_cgroup) 46 | except OSError as e: 47 | if e.errno == 13: 48 | if script: 49 | raise CgroupsException( 50 | "Permission denied, you don't have root privileges") 51 | else: 52 | raise CgroupsException( 53 | "Permission denied. If you want to use cgroups " + 54 | "without root priviliges, please execute first " + 55 | "the 'user_cgroups' command (as root or sudo).") 56 | elif e.errno == 17: 57 | pass 58 | else: 59 | raise OSError(e) 60 | else: 61 | uid, gid = get_user_info(user) 62 | os.chown(user_cgroup, uid, gid) 63 | logger.warn('cgroups sub-directories created for user %s' % user) 64 | 65 | 66 | def main(): 67 | 68 | # Arguments 69 | parser = argparse.ArgumentParser( 70 | description='Allow a non-root user to use cgroups') 71 | parser.add_argument( 72 | '-v', '--verbose', required=False, 73 | default='INFO', help='Logging level (default "INFO")' 74 | ) 75 | parser.add_argument( 76 | 'user', help='User to grant privileges to use cgroups') 77 | args = parser.parse_args() 78 | 79 | # Logging 80 | formatter = logging.Formatter( 81 | '%(asctime)s - %(name)s - %(levelname)s - %(message)s') 82 | logstream = logging.StreamHandler() 83 | logstream.setFormatter(formatter) 84 | logger.addHandler(logstream) 85 | if args.verbose == 'DEBUG': 86 | logger.setLevel(logging.DEBUG) 87 | elif args.verbose == 'INFO': 88 | logger.setLevel(logging.INFO) 89 | elif args.verbose == 'WARN': 90 | logger.setLevel(logging.WARN) 91 | else: 92 | logger.setLevel(logging.ERROR) 93 | logger.debug('Logging level: %s' % args.verbose) 94 | 95 | # Launch 96 | create_user_cgroups(args.user) 97 | 98 | 99 | if __name__ == '__main__': 100 | main() 101 | -------------------------------------------------------------------------------- /cgroups/utils.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | 4 | from __future__ import unicode_literals 5 | from __future__ import print_function 6 | 7 | import os 8 | import getpass 9 | 10 | BASE_CGROUPS_DIR = '/sys/fs/cgroup' 11 | CGROUPS_DIRS = [ 12 | 'cpu', 13 | 'memory', 14 | ] 15 | 16 | 17 | def get_user_cgroups(): 18 | user = getpass.getuser() 19 | user_cgroups = {} 20 | for cgroup in CGROUPS_DIRS: 21 | user_cgroup = os.path.join(BASE_CGROUPS_DIR, cgroup, user) 22 | # if not os.path.exists(user_cgroup): 23 | # os.mkdir(user_cgroup) 24 | user_cgroups[cgroup] = user_cgroup 25 | return user_cgroups 26 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from setuptools import setup, find_packages 5 | 6 | # List of python package dependancies availables on pypi repository 7 | apps_pypi = [] 8 | 9 | # Tests requirements 10 | # Note : not installed automatically 11 | tests_requires = ['coverage'] 12 | 13 | 14 | setup( 15 | name='cgroups', 16 | version='0.1.0', # 3 numbers notation major.minor.bugfix_or_security 17 | description='Python module to manage cgroups', 18 | author='Francis Bouvier - Cloud Orchestra', 19 | author_email='francis.bouvier@cloudorchestra.io', 20 | license='New BSD', 21 | url='http://www.xerus-technologies.fr', 22 | platforms='Linux', 23 | packages=find_packages(exclude=['ez_setup']), 24 | zip_safe=False, 25 | include_package_data=True, 26 | install_requires=apps_pypi, 27 | entry_points={ 28 | 'console_scripts': [ 29 | 'user_cgroups = cgroups.user:main', 30 | ] 31 | }, 32 | ) 33 | --------------------------------------------------------------------------------