├── .github └── workflows │ └── unittests.yml ├── .gitignore ├── .travis.yml ├── ChangeLog ├── LICENSE ├── MANIFEST.in ├── README.md ├── conf ├── cache │ └── README ├── models │ └── example.lmf ├── shine.conf ├── storage.conf └── tuning.conf.example ├── debian ├── README.debian ├── changelog ├── compat ├── control ├── copyright ├── rules ├── shine.default ├── shine.dirs ├── shine.docs ├── shine.examples ├── shine.init ├── shine.install ├── shine.manpages ├── shine.postrm ├── shine.yaml └── source │ └── format ├── doc ├── extras │ └── vim │ │ ├── ftdetect │ │ └── shine.vim │ │ └── syntax │ │ ├── shine.vim │ │ └── shinefs.vim ├── shine.1 └── shine.conf.5 ├── lib └── Shine │ ├── CLI │ ├── Display.py │ ├── TextTable.py │ └── __init__.py │ ├── Commands │ ├── Base │ │ ├── Command.py │ │ ├── CommandRCDefs.py │ │ ├── FSEventHandler.py │ │ ├── FSLiveCommand.py │ │ ├── RemoteCallEventHandler.py │ │ └── __init__.py │ ├── Config.py │ ├── Execute.py │ ├── Format.py │ ├── Fsck.py │ ├── Install.py │ ├── List.py │ ├── Mount.py │ ├── Remove.py │ ├── Show.py │ ├── Start.py │ ├── Status.py │ ├── Stop.py │ ├── Tune.py │ ├── Tunefs.py │ ├── Umount.py │ ├── Update.py │ └── __init__.py │ ├── Configuration │ ├── Backend │ │ ├── Backend.py │ │ ├── BackendRegistry.py │ │ ├── File.py │ │ └── __init__.py │ ├── Configuration.py │ ├── Exceptions.py │ ├── FileSystem.py │ ├── Globals.py │ ├── Model.py │ ├── ModelFile.py │ ├── TargetDevice.py │ ├── TuningModel.py │ └── __init__.py │ ├── Controller.py │ ├── FSUtils.py │ ├── Lustre │ ├── Actions │ │ ├── Action.py │ │ ├── Execute.py │ │ ├── Format.py │ │ ├── Fsck.py │ │ ├── Install.py │ │ ├── Modules.py │ │ ├── Proxy.py │ │ ├── StartClient.py │ │ ├── StartRouter.py │ │ ├── StartTarget.py │ │ ├── Status.py │ │ ├── StopClient.py │ │ ├── StopRouter.py │ │ ├── StopTarget.py │ │ ├── Tune.py │ │ └── __init__.py │ ├── Client.py │ ├── Component.py │ ├── Disk.py │ ├── EventHandler.py │ ├── FileSystem.py │ ├── Router.py │ ├── Server.py │ ├── Target.py │ └── __init__.py │ └── __init__.py ├── mkrelease.sh ├── scripts └── shine.init.redhat ├── setup.cfg ├── setup.py ├── shine.spec └── tests ├── CLI ├── DisplayTest.py ├── TextTableTest.py └── __init__.py ├── Configuration ├── BackendFileTest.py ├── ConfigFileSystemTest.py ├── ConfigurationTest.py ├── GlobalsTest.py ├── ModelFileTest.py ├── ModelTest.py ├── TuningModelTest.py └── __init__.py ├── FSUtilsTest.py ├── Lustre ├── Actions │ ├── CmdTest.py │ ├── DepsTest.py │ ├── LaunchTest.py │ ├── ProxyTest.py │ └── __init__.py ├── ClientTest.py ├── ComponentTest.py ├── DiskTest.py ├── FileSystemTest.py ├── ServerTest.py ├── TargetTest.py └── __init__.py └── Utils.py /.github/workflows/unittests.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Unit tests 3 | run-name: Shine Unit tests 4 | on: [workflow_dispatch, push, pull_request] 5 | 6 | permissions: 7 | contents: read 8 | pull-requests: read 9 | 10 | 11 | jobs: 12 | launch_tests: 13 | runs-on: ubuntu-latest 14 | strategy: 15 | matrix: 16 | python-version: ["3.9"] 17 | 18 | steps: 19 | - uses: actions/checkout@v4 20 | - name: Set up Python 21 | uses: actions/setup-python@v5 22 | with: 23 | python-version: ${{ matrix.python-version }} 24 | 25 | - name: Install dependencies 26 | run: | 27 | python -m pip install --upgrade pip 28 | #pip install -r requirements.txt 29 | pip install clustershell 30 | pip install nose 31 | pip install coverage 32 | 33 | - name: Set up passordless ssh connection 34 | run: | 35 | ssh-keygen -f ~/.ssh/id_rsa -N "" 36 | cp ~/.ssh/{id_rsa.pub,authorized_keys} 37 | # Avoid ssh "known hosts" warnings 38 | echo -e 'Host *\n StrictHostKeyChecking no\n LogLevel ERROR' \ 39 | >> ~/.ssh/config 40 | 41 | - name: Run nosetests 42 | run: | 43 | cd tests 44 | PYTHONPATH=$PYTHONPATH:$PWD/../lib nosetests \ 45 | -v --all-modules \ 46 | --with-coverage 47 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .gitignore 2 | 3 | # Compiled python modules. 4 | *.pyc 5 | 6 | # backup files 7 | *~ 8 | 9 | # Setuptools distribution folder. 10 | /dist/ 11 | /build/ 12 | 13 | # Python egg metadata, regenerated from source files by setuptools. 14 | /lib/*.egg-info 15 | /lib/*.egg 16 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.7" 4 | 5 | install: 6 | - pip install clustershell 7 | - pip install coverage 8 | - pip install codecov 9 | 10 | # Shine needs same names for getfqdn() and hostname() 11 | addons: 12 | hostname: shine-hostname.shine-domain.net 13 | hosts: 14 | - shine-hostname 15 | - shine-hostname.shine-domain.net 16 | 17 | before_script: 18 | # Allow us to SSH passwordless to local host 19 | - ssh-keygen -f ~/.ssh/id_rsa -N "" 20 | - cp ~/.ssh/{id_rsa.pub,authorized_keys} 21 | # Avoid ssh "known hosts" warnings 22 | - printf '%s\n %s\n %s\n' 'Host *' 'StrictHostKeyChecking no' 'LogLevel ERROR' >> ~/.ssh/config 23 | 24 | script: 25 | - cd tests; PYTHONPATH=$PYTHONPATH:$PWD/../lib nosetests -v --all-modules --with-coverage 26 | 27 | # Push the results back to codecov 28 | after_success: 29 | - codecov 30 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.md 3 | include ChangeLog 4 | include conf/models/example.lmf 5 | include conf/shine.conf 6 | include conf/storage.conf 7 | include conf/tuning.conf.example 8 | include conf/cache/README 9 | include scripts/shine.init.redhat 10 | include doc/shine.1 11 | include doc/shine.conf.5 12 | include doc/extras/vim/syntax/shine*.vim 13 | include doc/extras/vim/ftdetect/shine.vim 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Shine - Lustre file system administration utility 2 | ================================================= 3 | 4 | Requirements 5 | ------------ 6 | 7 | * Python: 2.7+ 8 | * ClusterShell: 1.5.1+ (http://github.com/cea-hpc/clustershell/) 9 | 10 | * Supports *Lustre* (http://lustre.org/) versions: 1.8, 2.1, 2.4 and above 11 | 12 | Installation 13 | ------------ 14 | 15 | When possible, please use the *RPM distribution* for an easy install. You can get 16 | it from http://github.com/cea-hpc/shine/releases/latest/. 17 | 18 | Shine is based on [ClusterShell](http://github.com/cea-hpc/clustershell/), a 19 | python library easily available, from [EPEL](https://fedoraproject.org/wiki/EPEL) 20 | by example. 21 | 22 | If you want to do it from source, type: 23 | 24 | # python setup.py install 25 | 26 | On RHEL-6 like systems, you may want to use the provided init script: 27 | 28 | # cp /var/share/shine/shine.init.redhat /etc/rc.d/init.d/shine 29 | 30 | Quick Start 31 | ----------- 32 | 33 | Make sure Shine is installed on all nodes. 34 | 35 | Edit the file `/etc/shine/shine.conf` and copy it on all nodes. 36 | 37 | To create **myfs** Lustre file system, copy the provided file system 38 | model file: 39 | 40 | # cd /etc/shine/models 41 | # cp example.lmf myfs.lmf 42 | 43 | Edit `myfs.lmf` to match your needs. This file describes the file system 44 | to be installed. 45 | 46 | 47 | Install the file system with: 48 | 49 | # shine install -m /etc/shine/models/myfs.lmf 50 | 51 | Then format the file system with: 52 | 53 | # shine format -f myfs 54 | 55 | Start servers with: 56 | 57 | # shine start -f myfs 58 | 59 | Mount clients with: 60 | 61 | # shine mount -f myfs 62 | 63 | 64 | Testing code 65 | ------------ 66 | 67 | If you modify Shine source code, do not forget to test it with the test suite 68 | available in `tests/` directory of the source code. 69 | 70 | `python-nose` is the recommended way to run the testsuite. However `unittest` 71 | provided with Python 2.7 and above should also works. 72 | 73 | $ export PYTHONPATH=$PWD/lib 74 | $ cd tests 75 | $ nosetests -v 76 | $ nosetests -v --all-modules 77 | 78 | Some tests expect being able to ssh into the current hostname without password, 79 | make sure `ssh $HOSTNAME echo ok` works. 80 | 81 | Some tests needs to launch real Lustre commands and so needs to have root permissions. 82 | These tests will be skipped if you do not have these permissions. 83 | 84 | # nosetests -v --all-modules 85 | -------------------------------------------------------------------------------- /conf/cache/README: -------------------------------------------------------------------------------- 1 | This is the shine file system configuration cache directory. 2 | Do not edit. 3 | -------------------------------------------------------------------------------- /conf/models/example.lmf: -------------------------------------------------------------------------------- 1 | # 2 | # This is an example of lustre model file. It contains a set of 3 | # configuration directives to install a simple Lustre filesystem. 4 | # 5 | 6 | ### Section 1 - Required directives 7 | 8 | # fs_name 9 | # The Lustre filesystem name (8 characters max). 10 | fs_name: example 11 | 12 | # nid_map 13 | # Hosts to Lnet NIDs mapping. 14 | # 15 | # Use multiple lines with the same nodes if you have several nids for 16 | # the same machines. 17 | nid_map: nodes=nova[0-39] nids=nova[0-39]@tcp0 18 | 19 | # mount_path 20 | # Default clients mount point path. 21 | # Some variables could be used in 'mount_path', see -Path Variables- below. 22 | mount_path: /example 23 | 24 | 25 | ### Section 2 - Target definitions 26 | 27 | # Defines your Lustre filesystem targets. 28 | # 29 | # mgt|mdt|ost: [ tag= ] [ node= ] [ dev= ] 30 | # [ index= ] [ jdev= ] [ ha_node= ] 31 | # [ group= ] [ mode={external|managed} ] 32 | # [ network= ] [ active={yes|nocreate|no|manual} ] 33 | # 34 | # Here, we don't use any backend (no File nor ClusterDB), so we have to 35 | # fully describe our targets (no RegExp accepted). For this simple 36 | # example, only minimum target information is provided. 37 | 38 | # MGS 39 | # Management Target 40 | mgt: node=nova4 dev=/dev/sde1 41 | 42 | # Example of MGS with HA (device must have same path on both nodes) 43 | #mgt: node=nova4 ha_node=nova3 dev=/dev/mapper/mgt-persistent-name 44 | 45 | # MDT 46 | # MetaData Target 47 | mdt: node=nova4 dev=/dev/sdf 48 | 49 | # OST 50 | # Object Storage Target(s) 51 | ost: node=nova5 dev=/dev/sdd 52 | ost: node=nova5 dev=/dev/sde 53 | ost: node=nova6 dev=/dev/sdd 54 | ost: node=nova6 dev=/dev/sde 55 | 56 | # client 57 | # FS clients definition. Like targets, use multiple lines if you want. 58 | client: node=nova[8-30] 59 | client: node=nova[31-32] 60 | 61 | # Sometimes it is needed for some nodes to mount this FS on a different 62 | # mount point (not the default mount_path). In that case, use the 63 | # optional client parameter mount_path. 64 | # Some variables could be used in 'mount_path', see -Path Variables- below. 65 | # 66 | client: node=nova[35-38] mount_path=/my/specific/example 67 | 68 | # Also, to override default client mount options, add the following 69 | # mount_options inline option: 70 | client: node=nova39 mount_path=/my/specific/example mount_options=noatime 71 | 72 | # To mount only a sub-directory of the Lustre namespace 73 | client: node=nova33 subdir=/projects 74 | 75 | # Lustre routers 76 | # 77 | # Example: nova7 and nova8 are declared as LNET routers. 78 | #router: node=nova[7-8] 79 | 80 | ### Section 3 - Additionnal directives 81 | 82 | # description 83 | # Optional FS description 84 | description: Example Lustre Filesystem 85 | 86 | # stripe_size 87 | # Specify the stripe size in bytes. Default is 1048576 (1M) 88 | stripe_size: 1048576 89 | 90 | # stripe_count 91 | # Specify the number of OSTs each file should be stripped on. 92 | # If not defined, no explicit value is used and Lustre will apply its default behaviour. 93 | #stripe_count: 1 94 | 95 | # 96 | # mgt_format_params: 97 | # mdt_format_params: 98 | # ost_format_params: 99 | # 100 | # Optional argument that will be used by mkfs.lustre for a target. Default is 101 | # no option. 102 | # 103 | # ie: disable failover mode and enable failout instead 104 | # mdt_format_params: failover.mode=failout 105 | # ost_format_params: failover.mode=failout 106 | 107 | # 108 | # mgt_mkfs_options: 109 | # mdt_mkfs_options: 110 | # ost_mkfs_options: 111 | # 112 | # Optional argument for --mkfsoptions, by target type. You can use ext4 format 113 | # options here. Defaults is no option. 114 | # eg: reduce reserved blocks for super-user. 115 | #mgt_mkfs_options: -m 2 116 | #mdt_mkfs_options: -m 2 117 | #ost_mkfs_options: -m 2 118 | 119 | # 120 | # mgt_mount_options: 121 | # mdt_mount_options: 122 | # ost_mount_options: 123 | # 124 | # Optional argument used when starting a target. Default is no options. 125 | 126 | # ie: Enable ACL for MDT 127 | mdt_mount_options: acl 128 | 129 | 130 | # mount_options 131 | # This define the default options to mount the filesystem on clients. 132 | mount_options: user_xattr 133 | 134 | # 135 | # Quota 136 | # 137 | # Enable quota support. 138 | # In lustre 2.4 and above, all quota options described here are ignored 139 | # Possible values are yes or no (default is no). 140 | quota: no 141 | 142 | # Quota configuration options 143 | # Describe options for quota support, if quota enabled. 144 | # 145 | # quota_type: (default is 'ug') 146 | # quota_iunit: 147 | # quota_bunit: 148 | # quota_itune: 149 | # quota_btune: 150 | 151 | # Target Mount Path patterns 152 | # 153 | # -Path Variables- 154 | # Several variables could be used within these paths: 155 | # $fs_name: Filesystem name defined in 'fs_name:' 156 | # $label: Component label (ie: foo-OST0002) 157 | # $type: Component type ('mdt', 'mgt', 'ost', 'router', 'client') 158 | # $index: Target index, in decimal (ie: 1, 2, 36, ...) or in hex (ie. 0x2, 0xa5, 0x00FA) 159 | # $dev: Base name of target device path (ie: sdc) 160 | # $jdev: Base name of target journal device path 161 | mgt_mount_path: /mnt/$fs_name/mgt 162 | mdt_mount_path: /mnt/$fs_name/mdt/$index 163 | ost_mount_path: /mnt/$fs_name/ost/$index 164 | -------------------------------------------------------------------------------- /conf/shine.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Configuration file for shine, a Lustre administration utility. 3 | # See "man shine.conf" for a complete description of this file. 4 | # 5 | 6 | # 7 | # Default LMF files (.lmf) search directory. 8 | # 9 | #lmf_dir=/etc/shine/models 10 | 11 | # 12 | # File describing Lustre tuning parameters 13 | # (to enable, create the file and uncomment the following line) 14 | # 15 | #tuning_file=/etc/shine/tuning.conf 16 | 17 | # 18 | # Directory path where to store generated configuration files. 19 | # 20 | #conf_dir=/var/cache/shine/conf 21 | 22 | 23 | # 24 | # The Lustre version managed by Shine. 25 | # 26 | #ie: lustre_version=1.8.5 27 | 28 | # 29 | # CLI 30 | # 31 | 32 | # Tell if colored output should be used on command line interface. 33 | # 34 | #color=auto 35 | 36 | 37 | # 38 | # BACKEND 39 | # 40 | 41 | # 42 | # Backend system to use. Its primary goal is to centralize devices information, 43 | # but it can also be used to store Lustre file systems states. 44 | # Match internal backend module name: 45 | # 46 | # Shine.Configuration.Backend.* 47 | # 48 | # Possible values are (case-sensitive): 49 | # 50 | # None No backend. Each model file provides required devices 51 | # information. Recommended for simple, small configs (default). 52 | # 53 | # File Built-in File backend: a single file (see storage_file below) 54 | # centralizes Lustre devices information (cluster-wide). 55 | # Highly recommended if you plan to install more than one Lustre 56 | # file system. 57 | # 58 | # ClusterDB Bull ClusterDB (proprietary external database backend). 59 | # 60 | #backend=None 61 | 62 | # 63 | # The location of the target storage information file. 64 | # 65 | #storage_file=/etc/shine/storage.conf 66 | 67 | # 68 | # Directory used for cached status information. 69 | # 70 | #status_dir=/var/cache/shine/status 71 | 72 | 73 | # 74 | # TIMEOUTS and FANOUT 75 | # 76 | 77 | # Timeout in seconds given to ConnectTimeout parameter of ssh 78 | # 79 | #ssh_connect_timeout=30 80 | 81 | # Maximum number of simultaneous local commands and remote connections. 82 | # (default is ClusterShell default fanout). 83 | # 84 | #ssh_fanout=64 85 | 86 | 87 | # 88 | # COMMANDS 89 | # 90 | 91 | # Additional paths to look for Lustre or ldiskfs specific commands. 92 | # 93 | #command_path=/usr/lib/lustre 94 | -------------------------------------------------------------------------------- /conf/storage.conf: -------------------------------------------------------------------------------- 1 | ############################################################################# 2 | # # 3 | # This file stores information about the storage devices available on the # 4 | # cluster, describing which are OST and which are MDT. This file is # 5 | # composed of lines with the following syntax: # 6 | # # 7 | # : tag=<> node=<> [ ha_node_name=<> ] dev=<> [ jdev=<> ] # 8 | # # 9 | # ost/mdt: This device is chosen to be an OST/MDT. # 10 | # # 11 | # name: The name you want to give to the OST or MDT. For instance, # 12 | # /dev/sdd on node ns13 can be called ns13_sdd. # 13 | # # 14 | # node: The hostname of the node where the device is. # 15 | # # 16 | # index: The Lustre target index (default is automatic indexing) # 17 | # # 18 | # dev: The device path (/dev/sdd for instance) # 19 | # # 20 | # ha_node: The hostname of the back up node in case of shared storage # 21 | # # 22 | # jdev: The name of the device where the ext3 journal will be stored, if # 23 | # you want it to be outside the main device. This parameter is # 24 | # optional. Loop devices cannot be used for this purpose. # 25 | # # 26 | ############################################################################# 27 | 28 | # A sample lines of an MDT on a node named ns13 29 | #mdt: tag=ns13_sdd node=ns13 dev=/dev/sdd jdev=/dev/sdc 30 | 31 | # A sample lines loop device on ns4 use as OST 32 | #ost: tag=ns4_loop2 node=ns4 dev=/tmp/loop2 33 | 34 | # A sample lines using an ns5 ramdisk as OST 35 | #ost: tag=ns5_ram1 node=ns5 dev=/dev/ram1 size=16000 36 | 37 | # Example of range support (be careful of indexes) 38 | #ost: tag=ost[0-3] node=fortoy12 index=[0-3] dev=/dev/mapper/ost[0-3] 39 | 40 | # loop device support (create file before) 41 | #ost: tag=ost4 node=fortoy12 dev=/tmp/ost4 42 | -------------------------------------------------------------------------------- /conf/tuning.conf.example: -------------------------------------------------------------------------------- 1 | # 2 | # Shine Example Tuning File 3 | # 4 | 5 | # Alias line 6 | # ---------- 7 | # 8 | # alias = 9 | # 10 | # : Alias name which will be used in tuning declarations. 11 | # 12 | # : Path it refers to, which should be tuned. 13 | # 14 | # 15 | # Tuning line 16 | # ----------- 17 | # 18 | # 19 | # 20 | # : Content to set in the provided alias. Could be quoted with 21 | # double-quotes. 22 | # 23 | # : Alias previously defined where value will be echoed. 24 | # 25 | # : Could be a mix of node type and node names, 26 | # separated by ';'. 27 | # Supported node type: MGS, MDS, OSS, CLIENT, CLT, ROUTER, RTR 28 | # ie: MDS;OSS;foo[1-5] 29 | 30 | # 31 | # Alias declarations 32 | # 33 | alias debug=/proc/sys/lnet/debug 34 | #alias max_rpcs_in_flight=/proc/fs/lustre/osc/*${ost}*/max_rpcs_in_flight 35 | 36 | # 37 | # Tuning parameters 38 | # 39 | "-all" debug MGS;MDS;OSS;ROUTER;CLT 40 | #32 max_rpcs_in_flight CLT 41 | -------------------------------------------------------------------------------- /debian/README.debian: -------------------------------------------------------------------------------- 1 | Dear user, this package provides the vim addon shine, but it is not enabled 2 | per default. 3 | 4 | If you want to enable it for your user account just execute 5 | 6 | vim-addons install shine 7 | 8 | Similarly, to enable it for all users of this system just execute (as root): 9 | 10 | vim-addons -w install shine 11 | 12 | vim-addons is provided by the vim-addon-manager package, have a look at its 13 | manpage for more information. 14 | -------------------------------------------------------------------------------- /debian/changelog: -------------------------------------------------------------------------------- 1 | shine (1.3.1-1) unstable; urgency=medium 2 | 3 | * New upstream release. 4 | 5 | -- Arnaud Guignard Wed, 29 Apr 2015 14:54:37 +0200 6 | 7 | shine (0.911.82) unstable; urgency=low 8 | 9 | * Initial release. (Closes: #632) 10 | 11 | -- Antonio J. Russo Thu, 26 Jul 2012 09:54:21 +0200 12 | -------------------------------------------------------------------------------- /debian/compat: -------------------------------------------------------------------------------- 1 | 9 2 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: shine 2 | Section: admin 3 | Priority: optional 4 | Maintainer: Arnaud Guignard 5 | Build-Depends: debhelper (>= 9),python (>= 2.6.6-3), python-setuptools (>= 0.6) 6 | X-Python-Version: >= 2.4 7 | Standards-Version: 3.9.5 8 | Vcs-Git: http://github.com/cea-hpc/shine.git 9 | Vcs-Browser: https://github.com/cea-hpc/shine/commits/master 10 | 11 | Package: shine 12 | Architecture: all 13 | Depends: ${python:Depends}, ${misc:Depends}, clustershell 14 | Recommends: vim-addon-manager 15 | Description: Lustre file system administration utility 16 | Command line tools designed to setup and manage a Lustre file system on a 17 | cluster. 18 | -------------------------------------------------------------------------------- /debian/copyright: -------------------------------------------------------------------------------- 1 | Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Upstream-Name: shine 3 | Source: https://github.com/cea-hpc/shine 4 | 5 | Files: * 6 | Copyright: 2008-2017 Commissariat à l'Energie Atomique et aux Energies Alternatives (CEA) 7 | Authors: Stephane Thiell 8 | Aurelien Degremont 9 | License: GPL-2+ 10 | This program is free software: you can redistribute it and/or modify 11 | it under the terms of the GNU General Public License as published by 12 | the Free Software Foundation, either version 2 of the License, or 13 | (at your option) any later version. 14 | . 15 | This program is distributed in the hope that it will be useful, 16 | but WITHOUT ANY WARRANTY; without even the implied warranty of 17 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 | GNU General Public License for more details. 19 | . 20 | You should have received a copy of the GNU General Public License 21 | along with this program. If not, see . 22 | . 23 | On Debian systems, the complete text of the GNU General 24 | Public License version 2 can be found in "/usr/share/common-licenses/GPL-2". 25 | 26 | Files: debian/* 27 | Copyright: 2012 - Antonio J. Russo 28 | 2014-2015 - Arnaud Guignard 29 | License: GPL-3 30 | This package is free software; you can redistribute it and/or modify 31 | it under the terms of the GNU General Public License as published by 32 | the Free Software Foundation; either version 3 of the License, or 33 | (at your option) any later version. 34 | . 35 | This package is distributed in the hope that it will be useful, 36 | but WITHOUT ANY WARRANTY; without even the implied warranty of 37 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 38 | GNU General Public License for more details. 39 | . 40 | You should have received a copy of the GNU General Public License 41 | along with this program. If not, see 42 | . 43 | On Debian systems, the complete text of the GNU General 44 | Public License version 3 can be found in "/usr/share/common-licenses/GPL-3". 45 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | 3 | override_dh_auto_install: 4 | dh_auto_install 5 | rm -rf debian/shine/usr/share/vim/vimfiles/ 6 | rm -rf debian/shine/usr/share/shine/ 7 | rm -f debian/shine/var/cache/shine/conf/README 8 | 9 | override_dh_installdocs: 10 | dh_installdocs 11 | mv debian/shine/usr/share/doc/shine/ChangeLog debian/shine/usr/share/doc/shine/changelog 12 | 13 | %: 14 | dh $@ --with python2 --buildsystem=python_distutils 15 | -------------------------------------------------------------------------------- /debian/shine.default: -------------------------------------------------------------------------------- 1 | # Configuration for the Lustre shine mounting script 2 | 3 | # Should shine be enabled? 4 | #ENABLED=1 5 | 6 | # Filesystems to start/stop. 7 | # If empty, all configured filesystems are started/stopped. They can be seen 8 | # with `shine show fs -v`. 9 | #FSLIST=ptmp,testfs 10 | -------------------------------------------------------------------------------- /debian/shine.dirs: -------------------------------------------------------------------------------- 1 | etc/shine/models 2 | var/cache/shine/conf 3 | -------------------------------------------------------------------------------- /debian/shine.docs: -------------------------------------------------------------------------------- 1 | README 2 | ChangeLog 3 | -------------------------------------------------------------------------------- /debian/shine.examples: -------------------------------------------------------------------------------- 1 | conf/*.conf.example 2 | conf/models 3 | -------------------------------------------------------------------------------- /debian/shine.init: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | ### BEGIN INIT INFO 3 | # Provides: shine 4 | # Required-Start: $network $remote_fs $syslog 5 | # Required-Stop: $network $remote_fs $syslog 6 | # Default-Start: 2 3 4 5 7 | # Default-Stop: 0 1 6 8 | # Short-Description: Lustre shine mounting 9 | # Description: This script mounts the Lustre filesystems with shine. 10 | ### END INIT INFO 11 | 12 | # Author: Arnaud Guignard 13 | 14 | # Do NOT "set -e" 15 | 16 | # PATH should only include /usr/* if it runs after the mountnfs.sh script 17 | PATH=/sbin:/usr/sbin:/bin:/usr/bin 18 | NAME=shine 19 | DAEMON=/usr/sbin/$NAME 20 | SCRIPTNAME=/etc/init.d/$NAME 21 | LOCKFILE=/var/lock/subsys/$NAME 22 | 23 | # Exit if the package is not installed 24 | [ -x "$DAEMON" ] || exit 0 25 | 26 | # Read configuration variable file if it is present 27 | ENABLED=0 28 | [ -r /etc/default/$NAME ] && . /etc/default/$NAME 29 | 30 | test "$ENABLED" != "0" || exit 0 31 | 32 | . /lib/lsb/init-functions 33 | 34 | do_start() 35 | { 36 | $DAEMON mount -L -q ${FSLIST:+-f $FSLIST} 37 | RETVAL=$? 38 | [ $RETVAL -eq 0 ] && touch $LOCKFILE 39 | return $RETVAL 40 | } 41 | 42 | do_stop() 43 | { 44 | $DAEMON umount -L -q ${FSLIST:+-f $FSLIST} 45 | RETVAL=$? 46 | [ $RETVAL -eq 0 ] && rm -f $LOCKFILE 47 | return $RETVAL 48 | } 49 | 50 | do_status() 51 | { 52 | $DAEMON status -L ${FSLIST:+-f $FSLIST} 53 | } 54 | 55 | case "$1" in 56 | start) 57 | log_begin_msg "Starting $NAME:" 58 | do_start 59 | log_end_msg $? 60 | ;; 61 | stop) 62 | log_begin_msg "Stopping $NAME:" 63 | do_stop 64 | log_end_msg $? 65 | ;; 66 | status) 67 | do_status 68 | ;; 69 | restart|force-reload) 70 | log_daemon_msg "Restarting $NAME:" 71 | do_stop 72 | case "$?" in 73 | 0) 74 | do_start 75 | case "$?" in 76 | 0) log_end_msg 0 ;; 77 | *) log_end_msg 1 ;; # Failed to start 78 | esac 79 | ;; 80 | *) 81 | # Failed to stop 82 | log_end_msg 1 83 | ;; 84 | esac 85 | ;; 86 | *) 87 | echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 88 | exit 3 89 | ;; 90 | esac 91 | 92 | : 93 | -------------------------------------------------------------------------------- /debian/shine.install: -------------------------------------------------------------------------------- 1 | conf/*.conf etc/shine 2 | debian/shine.yaml /usr/share/vim/registry 3 | doc/extras/vim/ftdetect/shine.vim /usr/share/vim/addons/ftdetect 4 | doc/extras/vim/syntax/shine.vim /usr/share/vim/addons/syntax 5 | doc/extras/vim/syntax/shinefs.vim /usr/share/vim/addons/syntax 6 | -------------------------------------------------------------------------------- /debian/shine.manpages: -------------------------------------------------------------------------------- 1 | ./doc/shine.1 2 | ./doc/shine.conf.5 3 | -------------------------------------------------------------------------------- /debian/shine.postrm: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | case "$1" in 6 | purge) 7 | rm -rf /var/cache/shine 8 | ;; 9 | remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) 10 | ;; 11 | *) 12 | echo "postrm called with unknown argument \`$1'" >&2 13 | exit 1 14 | esac 15 | 16 | #DEBHELPER# 17 | 18 | exit 0 19 | 20 | # vim: set et sw=2 ts=2 sts=2: 21 | -------------------------------------------------------------------------------- /debian/shine.yaml: -------------------------------------------------------------------------------- 1 | addon: shine 2 | description: "Syntax highlighting for shine" 3 | files: 4 | - ftdetect/shine.vim 5 | - syntax/shine.vim 6 | - syntax/shinefs.vim 7 | -------------------------------------------------------------------------------- /debian/source/format: -------------------------------------------------------------------------------- 1 | 3.0 (quilt) 2 | -------------------------------------------------------------------------------- /doc/extras/vim/ftdetect/shine.vim: -------------------------------------------------------------------------------- 1 | " 2 | " Installed As: vim/ftdetect/shine.vim 3 | " 4 | au BufNewFile,BufRead *shine.conf setlocal filetype=shine 5 | au BufNewFile,BufRead /etc/shine/models/* setlocal filetype=shinefs 6 | au BufNewFile,BufRead /var/cache/shine/conf/* setlocal filetype=shinefs 7 | 8 | -------------------------------------------------------------------------------- /doc/extras/vim/syntax/shine.vim: -------------------------------------------------------------------------------- 1 | " Vim syntax file 2 | if version < 600 3 | syntax clear 4 | elseif exists("b:current_syntax") 5 | finish 6 | endif 7 | 8 | if version < 600 9 | source :p:h/conf.vim 10 | else 11 | runtime! syntax/conf.vim 12 | endif 13 | 14 | syn keyword shineConfKey tuning_file 15 | syn keyword shineConfKey conf_dir 16 | syn keyword shineConfKey lmf_dir 17 | syn keyword shineConfKey backend 18 | 19 | syn keyword shineConfKey storage_file 20 | syn keyword shineConfKey status_dir 21 | 22 | syn keyword shineConfKey log_file 23 | syn keyword shineConfKey log_level 24 | 25 | syn keyword shineConfKey color 26 | 27 | syn keyword shineConfKey command_path 28 | 29 | syn keyword shineConfKey ssh_connect_timeout 30 | syn keyword shineConfKey ssh_fanout 31 | syn keyword shineConfKey default_timeout 32 | syn keyword shineConfKey start_timeout 33 | syn keyword shineConfKey stop_timeout 34 | syn keyword shineConfKey status_timeout 35 | syn keyword shineConfKey mount_timeout 36 | syn keyword shineConfKey umount_timeout 37 | syn keyword shineConfKey set_tuning_timeout 38 | 39 | 40 | 41 | " Define the default highlighting. 42 | " For version 5.7 and earlier: only when not done already 43 | " For version 5.8 and later: only when an item doesn't have highlighting yet 44 | if version >= 508 || !exists("did_shine_syntax_inits") 45 | if version < 508 46 | let did_shine_syntax_inits = 1 47 | command -nargs=+ HiLink hi link 48 | else 49 | command -nargs=+ HiLink hi def link 50 | endif 51 | 52 | HiLink shineConfKey Identifier 53 | "hi basicMathsOperator term=bold cterm=bold gui=bold 54 | 55 | delcommand HiLink 56 | endif 57 | 58 | let b:current_syntax = "shine" 59 | 60 | " vim: ts=8 61 | -------------------------------------------------------------------------------- /doc/extras/vim/syntax/shinefs.vim: -------------------------------------------------------------------------------- 1 | " Vim syntax file 2 | if version < 600 3 | syntax clear 4 | elseif exists("b:current_syntax") 5 | finish 6 | endif 7 | 8 | 9 | if version < 600 10 | source :p:h/conf.vim 11 | else 12 | runtime! syntax/conf.vim 13 | endif 14 | 15 | syn keyword shineFSKey description 16 | syn keyword shineFSKey failover 17 | syn keyword shineFSKey fs_name 18 | syn keyword shineFSKey mdt_mkfs_options 19 | syn keyword shineFSKey mdt_mount_options 20 | syn keyword shineFSKey mdt_mount_path 21 | syn keyword shineFSKey mdt_format_params 22 | syn keyword shineFSKey mount_options 23 | syn keyword shineFSKey mount_path 24 | syn keyword shineFSKey subdir 25 | syn keyword shineFSKey mgt_mkfs_options 26 | syn keyword shineFSKey mgt_mount_options 27 | syn keyword shineFSKey mgt_mount_path 28 | syn keyword shineFSKey mgt_format_params 29 | syn keyword shineFSKey nid_map contained 30 | syn keyword shineFSKey ost_mkfs_options 31 | syn keyword shineFSKey ost_mount_options 32 | syn keyword shineFSKey ost_mount_path 33 | syn keyword shineFSKey ost_format_params 34 | syn keyword shineFSKey quota 35 | syn keyword shineFSKey quota_type 36 | syn keyword shineFSKey quota_iunit 37 | syn keyword shineFSKey quota_bunit 38 | syn keyword shineFSKey quota_btune 39 | syn keyword shineFSKey quota_itune 40 | syn keyword shineFSKey stripe_count 41 | syn keyword shineFSKey stripe_size 42 | syn keyword shineFSKey mgt 43 | syn keyword shineFSKey mdt 44 | syn keyword shineFSKey ost 45 | syn keyword shineFSKey client 46 | syn keyword shineFSKey router 47 | 48 | syn match shineVariable /\$\(fs_name\|index\)/ 49 | 50 | syn match shineTargetTagKey /\(tag\|node\|dev\|jdev\|index\|ha_node\|mode\|group\)=/me=e-1 51 | 52 | syn match shineComment " #.*"ms=s+1 contained 53 | 54 | syn match shineHostPattern /=[^ @]\+/ms=s+1 contained 55 | syn match shineNidPattern /=[^ ]\+@[^ ]\+/ms=s+1 contained 56 | syn keyword shineNidKey nodes contained 57 | syn keyword shineNidKey nids contained 58 | syn match shineNidMap /^nid_map: *[^ ]\+ *[^ ]\+@[^ ]\+.*/ contains=shineFSKey,shineNidKey,shineHostPattern,shineNidPattern,shineComment 59 | 60 | 61 | " Define the default highlighting. 62 | " For version 5.7 and earlier: only when not done already 63 | " For version 5.8 and later: only when an item doesn't have highlighting yet 64 | if version >= 508 || !exists("did_shine_syntax_inits") 65 | if version < 508 66 | let did_shine_syntax_inits = 1 67 | command -nargs=+ HiLink hi link 68 | else 69 | command -nargs=+ HiLink hi def link 70 | endif 71 | 72 | HiLink shineComment Comment 73 | HiLink shineFSKey Identifier 74 | HiLink shineNidKey Identifier 75 | HiLink shineTargetTagKey Identifier 76 | HiLink shineTargetNodeKey Identifier 77 | HiLink shineTargetDevKey Identifier 78 | HiLink shineTargetJdevKey Identifier 79 | HiLink shineTargetSizeKey Identifier 80 | HiLink shineTargetJsizeKey Identifier 81 | HiLink shineTargetIndexKey Identifier 82 | 83 | " HiLink shineNidMap Label 84 | HiLink shineHostPattern Type 85 | HiLink shineNidPattern String 86 | HiLink shineVariable PreProc 87 | 88 | delcommand HiLink 89 | endif 90 | 91 | let b:current_syntax = "shinefs" 92 | 93 | " vim: ts=8 94 | -------------------------------------------------------------------------------- /doc/shine.conf.5: -------------------------------------------------------------------------------- 1 | .\" -*- nroff -*- 2 | .\" Copyright (c) 2007-2013 CEA. All rights reserved. 3 | .\" 4 | .\" This file may be copied under the terms of the GNU Public License. 5 | .\" Redistribution and use in source and binary forms, with or without 6 | .\" modification, are permitted provided that the following conditions 7 | .\" are met: 8 | .\" 9 | .\" 1. Redistributions of source code must retain the above copyright 10 | .\" notice, this list of conditions and the following disclaimer. 11 | .\" 12 | .\" 2. Redistributions in binary form must reproduce the above copyright 13 | .\" notice, this list of conditions and the following disclaimer in the 14 | .\" documentation and/or other materials provided with the distribution. 15 | .\" 16 | .\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 | .\" "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 | .\" TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 | .\" PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 20 | .\" CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 21 | .\" EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 22 | .\" PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 23 | .\" OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 | .\" WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 25 | .\" OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE EVEN IF 26 | .\" ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | .\" 28 | .Dd March 10, 2015 \" DATE 29 | .Dt shine.conf 5 \" Program name and manual section number 30 | .Os Linux 31 | .Sh NAME \" Section Header 32 | .Nm shine.conf 33 | .Nd Configuration file for 34 | .Tn shine 35 | .Sh DESCRIPTION \" Section Header 36 | The 37 | .Nm 38 | file controls the behavior of 39 | .Xr shine 1 40 | command. It is located in the 41 | .Pa /etc/shine 42 | directory and is only needed on the management(s) node(s). 43 | .Pp 44 | The configuration is made up of several lines of paramaters such as: 45 | .Pp 46 | .D1 Ar key Ns = Ns Ar value 47 | .Pp 48 | where 49 | .Ar key 50 | represents a parameter name, and 51 | .Ar value 52 | would be the parameter's assigned value. Lines that begin with a # are 53 | treated as comments. 54 | .Pp 55 | 56 | .Ss General 57 | .Bl -tag -width Ds -compact 58 | .It Ic lmf_dir Ns = Ns Ar pathname 59 | is the directory where model files are looked for, in 60 | .Ic install Ns 61 | command. Default is 62 | .Pa /etc/shine/models . 63 | .It Ic lustre_version Ns = Ns Ar x.y.z 64 | The Lustre version of the filesystem Shine is managing. This is used to 65 | handle incompatibilities between several Lustre releases. By example, 66 | set it to 67 | .Pa 1.8 Ns 68 | if you are using any of 1.8 Lustre version. 69 | .El 70 | 71 | .Ss Storage backend 72 | .Bl -tag -width Ds -compact 73 | .It Ic backend Ns = Ns None|File|ClusterDB 74 | specifies selected storage backend type. If this variable is set to 75 | .Ic None Ns , 76 | storage information is retrieved directly from each Lustre model file 77 | (this way, storage info like device names is not centralized). 78 | If this variable is set to 79 | .Ic ClusterDB Ns , 80 | storage, filesystem and mount information is retrieved and stored from 81 | tables lustre_ost, lustre_mdt, lustre_mount and lustre_fs of ClusterDB. 82 | If backend is set to 83 | .Ic File Ns , 84 | storage information is retrieved from 85 | .Ar storage_file 86 | and status information is stored in 87 | .Ar cache_dir 88 | directory on the management node. 89 | Default is 90 | .Ic None Ns . 91 | .El 92 | 93 | .Ss File backend specific settings 94 | .Bl -tag -width Ds -compact 95 | .It Ic status_dir Ns = Ns Ar pathname 96 | is the cache directory used for status information. 97 | Default directory is 98 | .Pa /var/cache/shine/status 99 | .It Ic storage_file Ns = Ns Ar pathname 100 | is the file used to retrieve targets storage information. 101 | Default is 102 | .Pa /etc/shine/storage.conf 103 | .El 104 | 105 | .Ss Command line interface 106 | .Bl -tag -width Ds -compact 107 | .It Ic color Ns = Ns Ar [auto, always, never] 108 | Tell if colored output should be used on command line interface. If 'auto', color is used only if standard output is a TTY. This value could be changed using --color option on command line. 109 | .El 110 | 111 | .Ss Commands options 112 | .Bl -tag -width Ds -compact 113 | .It Ic command_path Ns = Ns Ar path 114 | Additional paths to look for Lustre or ldiskfs specific commands. 115 | .El 116 | 117 | .Ss Cluster-wide applicable settings 118 | .Bl -tag -width Ds -compact 119 | .It Ic conf_dir Ns = Ns Ar pathname 120 | is a directory present on each Lustre nodes to store generated configuration files. 121 | The content of this directory is managed by shine and shouldn't be modified directly. 122 | Default directory is 123 | .Pa /var/cache/shine/conf 124 | .El 125 | .Ss Optional configuration files 126 | .Bl -tag -width Ds -compact 127 | .It Ic tuning_file Ns = Ns Ar pathname 128 | is the file describing Lustre tuning parameters applicable for all installed filesystems. 129 | If not set, no tuning is applied. 130 | .El 131 | .Ss Advanced settings 132 | .Bl -tag -width Ds -compact 133 | .It Ic ssh_fanout Ns = Ns Ar number 134 | is the maximum number of simultaneous local commands and remote connections. 135 | .It Ic ssh_connect_timeout Ns = Ns Ar secs 136 | is the timeout in seconds for ssh connections. 137 | .El 138 | .Sh FILES \" File used or created by the topic of the man page 139 | .Bl -tag -width "/Library/StartupItems/balanced/uninstall.sh" -compact 140 | .It Pa /etc/shine/shine.conf 141 | .It Pa /etc/shine/tuning.conf 142 | .El 143 | .\" .Sh BUGS \" Document known, unremedied bugs 144 | .Sh HISTORY \" Document history if command behaves in a unique manner 145 | .Nm 146 | settings were previously in 147 | .Pa /etc/lustre/lustre_util.conf 148 | and 149 | .Pa /etc/lustre/lustre.cfg 150 | .Sh SEE ALSO 151 | .Xr shine 1 152 | -------------------------------------------------------------------------------- /lib/Shine/CLI/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cea-hpc/shine/11f329dcceae798eb9a4262f9be25f158e63c3f8/lib/Shine/CLI/__init__.py -------------------------------------------------------------------------------- /lib/Shine/Commands/Base/CommandRCDefs.py: -------------------------------------------------------------------------------- 1 | # CommandRCDefs.py -- Shine command return code constants 2 | # Copyright (C) 2009-2016 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | 22 | """ 23 | Shine command return code constants definition. 24 | 25 | Shine makes use of return codes to indicate the status of the completed 26 | command and often adds some state information about the filesystem. 27 | Below are defined the base return code flags used by shine to build its 28 | return codes. Please note that a return code of 0 (zero) indicates that 29 | the selected Lustre targets or clients are currently running with an 30 | healthy state. 31 | """ 32 | 33 | # Command return code masks 34 | 35 | RC_FLAG_CLIENT = 0x02 36 | RC_FLAG_RECOVERING = 0x04 37 | RC_FLAG_OFFLINE = 0x08 38 | RC_FLAG_ERROR = 0x10 39 | RC_FLAG_UNHEALTHY = 0x20 40 | RC_FLAG_USER_ERROR = 0x40 41 | RC_FLAG_RUNTIME_ERROR = 0x80 42 | RC_FLAG_EXTERNAL = 0x100 43 | 44 | 45 | # Shine command return codes 46 | 47 | RC_OK = 0 48 | RC_FAILURE = 1 49 | 50 | # Status 51 | RC_ST_ONLINE = 0 52 | RC_ST_MIGRATED = 2 53 | RC_ST_RECOVERING = 4 54 | RC_ST_OFFLINE = 8 55 | RC_ST_EXTERNAL = 16 56 | 57 | # Errors 58 | RC_TARGET_ERROR = 16 59 | RC_CLIENT_ERROR = 18 60 | RC_UNHEALTHY = 32 61 | RC_USER_ERROR = 64 62 | RC_RUNTIME_ERROR = 128 63 | -------------------------------------------------------------------------------- /lib/Shine/Commands/Base/FSLiveCommand.py: -------------------------------------------------------------------------------- 1 | # FSLiveCommand.py -- Base commands class : live filesystem 2 | # Copyright (C) 2009-2015 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | """ 22 | Base class for live filesystem commands (start, stop, status, etc.). 23 | """ 24 | 25 | from __future__ import print_function 26 | 27 | from Shine.Configuration.Globals import Globals 28 | 29 | from Shine.Commands.Base.Command import RemoteCommand, CommandHelpException 30 | 31 | # Command helper 32 | from Shine.FSUtils import open_lustrefs 33 | 34 | # Error handling 35 | from Shine.Commands.Base.CommandRCDefs import RC_RUNTIME_ERROR 36 | 37 | class FSLiveCommand(RemoteCommand): 38 | """ 39 | shine [-f ] [-n ] [-dqv] 40 | 41 | 'CRITICAL' could be set if command will run action that can 42 | damage filesystems. 43 | """ 44 | 45 | GLOBAL_EH = None 46 | LOCAL_EH = None 47 | 48 | CRITICAL = False 49 | 50 | TARGET_STATUS_RC_MAP = {} 51 | 52 | def fs_status_to_rc(self, status_set): 53 | mapped_status = [self.TARGET_STATUS_RC_MAP.get(st, RC_RUNTIME_ERROR) for st in status_set] 54 | return max(mapped_status) 55 | 56 | def _open_fs(self, fsname, eh): 57 | return open_lustrefs(fsname, 58 | self.options.targets, 59 | nodes=self.options.nodes, 60 | excluded=self.options.excludes, 61 | failover=self.options.failover, 62 | indexes=self.options.indexes, 63 | labels=self.options.labels, 64 | event_handler=eh) 65 | 66 | def execute_fs(self, fs, fs_conf, eh, vlevel): 67 | raise NotImplemented("Derived class must implement.") 68 | 69 | def execute(self): 70 | first = True 71 | 72 | # Option sanity check 73 | self.forbidden(self.options.model, "-m, use -f") 74 | 75 | # Do not allow implicit filesystems format. 76 | if self.CRITICAL and not self.options.fsnames: 77 | msg = "A filesystem is required (use -f)." 78 | raise CommandHelpException(msg, self) 79 | 80 | result = 0 81 | 82 | self.init_execute() 83 | 84 | # Install appropriate event handler. 85 | local_eh = None 86 | global_eh = None 87 | if self.LOCAL_EH: 88 | local_eh = self.LOCAL_EH(self) 89 | if self.GLOBAL_EH: 90 | global_eh = self.GLOBAL_EH(self) 91 | eh = self.install_eventhandler(local_eh, global_eh) 92 | 93 | for fsname in self.iter_fsname(): 94 | 95 | # Open configuration and instantiate a Lustre FS. 96 | fs_conf, fs = self._open_fs(fsname, eh) 97 | 98 | # Define debuggin level 99 | fs.set_debug(self.options.debug) 100 | 101 | # Separate each fsname with a blank line 102 | if not first: 103 | print() 104 | first = False 105 | 106 | # Run the real job 107 | vlevel = self.options.verbose 108 | result = max(result, self.execute_fs(fs, fs_conf, eh, vlevel)) 109 | 110 | return result 111 | -------------------------------------------------------------------------------- /lib/Shine/Commands/Base/RemoteCallEventHandler.py: -------------------------------------------------------------------------------- 1 | # RemoteCallEventHandler.py -- Lustre remote call event handling (for -R) 2 | # Copyright (C) 2009-2015 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | import sys 22 | 23 | from Shine.Lustre.EventHandler import EventHandler 24 | from Shine.Lustre.Actions.Proxy import shine_msg_pack 25 | 26 | class RemoteCallEventHandler(EventHandler): 27 | """ 28 | Special shine EventHandler installed when called with -R (remote 29 | call), which aims to serialize all events instead of printing human 30 | output. 31 | """ 32 | 33 | def event_callback(self, evtype, **kwargs): 34 | """Convert each event it receives into an encoded line on stdout.""" 35 | # For distant message, we do not need to send the node. It will 36 | # be extract from the incoming server name. 37 | if 'node' in kwargs: 38 | del kwargs['node'] 39 | msg = shine_msg_pack(evtype=evtype, **kwargs) 40 | sys.stdout.write(msg) 41 | sys.stdout.flush() 42 | -------------------------------------------------------------------------------- /lib/Shine/Commands/Base/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cea-hpc/shine/11f329dcceae798eb9a4262f9be25f158e63c3f8/lib/Shine/Commands/Base/__init__.py -------------------------------------------------------------------------------- /lib/Shine/Commands/Config.py: -------------------------------------------------------------------------------- 1 | # Config.py -- Display component configuration 2 | # Copyright (C) 2012-2015 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | from __future__ import print_function 22 | 23 | from Shine.CLI.Display import display 24 | 25 | from Shine.Commands.Base.FSLiveCommand import FSLiveCommand 26 | 27 | # 28 | # NOTE: This command is declared as a FSLiveCommand but it does not do any live 29 | # actions! 30 | # This is for convenience only as FSLiveCommand has already most of what this 31 | # needs to run. 32 | # 33 | class Config(FSLiveCommand): 34 | """ 35 | shine config -f foo -O "%fsname" 36 | """ 37 | 38 | NAME = "config" 39 | DESCRIPTION = "Display filesystem component information" 40 | 41 | def execute_fs(self, fs, fs_conf, hdl, vlevel): 42 | print(display(self, fs)) 43 | -------------------------------------------------------------------------------- /lib/Shine/Commands/Execute.py: -------------------------------------------------------------------------------- 1 | # Execute.py -- Execute a custom command for any component. 2 | # Copyright (C) 2012-2015 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | """ 22 | Shine `execute' command classes. 23 | 24 | """ 25 | 26 | from __future__ import print_function 27 | 28 | # Command base class 29 | from Shine.Commands.Base.FSLiveCommand import FSLiveCommand, \ 30 | CommandHelpException 31 | from Shine.Commands.Base.CommandRCDefs import RC_OK, \ 32 | RC_FAILURE, RC_TARGET_ERROR, \ 33 | RC_CLIENT_ERROR, RC_RUNTIME_ERROR 34 | 35 | # Lustre events and errors 36 | from Shine.Commands.Base.FSEventHandler import FSGlobalEventHandler, \ 37 | FSLocalEventHandler 38 | from Shine.Lustre.FileSystem import MOUNTED, RECOVERING, EXTERNAL, OFFLINE, \ 39 | TARGET_ERROR, CLIENT_ERROR, RUNTIME_ERROR, \ 40 | MIGRATED 41 | 42 | 43 | class Execute(FSLiveCommand): 44 | """ 45 | shine execute [-f ] -o "..." 46 | """ 47 | 48 | NAME = "execute" 49 | DESCRIPTION = "Execute a custom command" 50 | 51 | GLOBAL_EH = FSGlobalEventHandler 52 | LOCAL_EH = FSLocalEventHandler 53 | 54 | TARGET_STATUS_RC_MAP = { \ 55 | MOUNTED : RC_OK, 56 | MIGRATED : RC_OK, 57 | RECOVERING : RC_OK, 58 | EXTERNAL : RC_OK, 59 | OFFLINE : RC_OK, 60 | TARGET_ERROR : RC_TARGET_ERROR, 61 | CLIENT_ERROR : RC_CLIENT_ERROR, 62 | RUNTIME_ERROR : RC_RUNTIME_ERROR } 63 | 64 | def execute(self): 65 | # Do not allow implicit filesystems format. 66 | if not self.options.additional: 67 | msg = "A custom command (-o) is required." 68 | raise CommandHelpException(msg, self) 69 | 70 | return FSLiveCommand.execute(self) 71 | 72 | def execute_fs(self, fs, fs_conf, eh, vlevel): 73 | 74 | # Warn if trying to act on wrong nodes 75 | all_nodes = fs.components.managed().servers() 76 | if not self.check_valid_list(fs.fs_name, all_nodes, 'execute'): 77 | return RC_FAILURE 78 | 79 | # Will call the handle_pre() method defined by the event handler. 80 | if hasattr(eh, 'pre'): 81 | eh.pre(fs) 82 | 83 | fs_result = fs.execute(failover=self.options.failover, 84 | addopts=self.options.additional, 85 | fanout=self.options.fanout, 86 | dryrun=self.options.dryrun, 87 | mountdata=self.options.mountdata) 88 | 89 | rc = self.fs_status_to_rc(fs_result) 90 | 91 | if rc == RC_OK: 92 | if vlevel > 0: 93 | print("Execute successful.") 94 | 95 | elif rc == RC_RUNTIME_ERROR: 96 | self.display_proxy_errors(fs) 97 | print() 98 | 99 | # Call a post_format method if defined by the event handler. 100 | if hasattr(eh, 'post'): 101 | eh.post(fs) 102 | 103 | return rc 104 | -------------------------------------------------------------------------------- /lib/Shine/Commands/Format.py: -------------------------------------------------------------------------------- 1 | # Format.py -- Format file system targets 2 | # Copyright (C) 2007-2015 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | """ 22 | Shine `format' classes. 23 | Format or reformat any installed filesystem. 24 | """ 25 | 26 | from __future__ import print_function 27 | 28 | # Command base class 29 | from Shine.Commands.Base.FSLiveCommand import FSLiveCommand 30 | from Shine.Commands.Base.CommandRCDefs import RC_OK, RC_ST_EXTERNAL, \ 31 | RC_FAILURE, RC_TARGET_ERROR, \ 32 | RC_CLIENT_ERROR, RC_RUNTIME_ERROR 33 | # Lustre events 34 | from Shine.Commands.Base.FSEventHandler import FSGlobalEventHandler, \ 35 | FSLocalEventHandler 36 | 37 | from Shine.Lustre.FileSystem import MOUNTED, RECOVERING, EXTERNAL, OFFLINE, \ 38 | TARGET_ERROR, CLIENT_ERROR, RUNTIME_ERROR, \ 39 | MIGRATED 40 | 41 | 42 | class Format(FSLiveCommand): 43 | """ 44 | shine format -f [-t ] [-i ] [-n ] 45 | """ 46 | 47 | NAME = "format" 48 | DESCRIPTION = "Format file system targets." 49 | 50 | CRITICAL = True 51 | 52 | GLOBAL_EH = FSGlobalEventHandler 53 | LOCAL_EH = FSLocalEventHandler 54 | 55 | TARGET_STATUS_RC_MAP = { \ 56 | MOUNTED : RC_FAILURE, 57 | MIGRATED : RC_FAILURE, 58 | EXTERNAL : RC_ST_EXTERNAL, 59 | RECOVERING : RC_FAILURE, 60 | OFFLINE : RC_OK, 61 | TARGET_ERROR : RC_TARGET_ERROR, 62 | CLIENT_ERROR : RC_CLIENT_ERROR, 63 | RUNTIME_ERROR : RC_RUNTIME_ERROR } 64 | 65 | def execute_fs(self, fs, fs_conf, eh, vlevel): 66 | 67 | # Warn if trying to act on wrong nodes 68 | servers = fs.components.managed(supports='format').servers() 69 | if not self.check_valid_list(fs.fs_name, servers, 'format'): 70 | return RC_FAILURE 71 | 72 | if not self.ask_confirm("Format %s on %s: are you sure?" % (fs.fs_name, 73 | servers)): 74 | return RC_FAILURE 75 | 76 | mkfs_options = {} 77 | format_params = {} 78 | for target_type in [ 'mgt', 'mdt', 'ost' ]: 79 | format_params[target_type] = \ 80 | fs_conf.get_target_format_params(target_type) 81 | mkfs_options[target_type] = \ 82 | fs_conf.get_target_mkfs_options(target_type) 83 | 84 | # Call a pre_format method if defined by the event handler. 85 | if hasattr(eh, 'pre'): 86 | eh.pre(fs) 87 | 88 | # Format really. 89 | status = fs.format(stripecount=fs_conf.get_stripecount(), 90 | stripesize=fs_conf.get_stripesize(), 91 | format_params=format_params, 92 | mkfs_options=mkfs_options, 93 | quota=fs_conf.has_quota(), 94 | quota_type=fs_conf.get_quota_type(), 95 | addopts=self.options.additional, 96 | failover=self.options.failover, 97 | fanout=self.options.fanout, 98 | dryrun=self.options.dryrun, 99 | mountdata=self.options.mountdata) 100 | 101 | rc = self.fs_status_to_rc(status) 102 | 103 | if rc == RC_OK: 104 | if vlevel > 0: 105 | print("Format successful.") 106 | else: 107 | if rc == RC_RUNTIME_ERROR: 108 | self.display_proxy_errors(fs) 109 | if vlevel > 0: 110 | print("Format failed") 111 | 112 | # Call a post_format method if defined by the event handler. 113 | if hasattr(eh, 'post'): 114 | eh.post(fs) 115 | 116 | return rc 117 | -------------------------------------------------------------------------------- /lib/Shine/Commands/Fsck.py: -------------------------------------------------------------------------------- 1 | # Fsck.py -- Check backend file system for each target 2 | # Copyright (C) 2010 BULL S.A.S, CEA 3 | # Copyright (C) 2012-2015 CEA 4 | # 5 | # This file is part of shine 6 | # 7 | # This program is free software; you can redistribute it and/or 8 | # modify it under the terms of the GNU General Public License 9 | # as published by the Free Software Foundation; either version 2 10 | # of the License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program; if not, write to the Free Software 19 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 20 | # 21 | 22 | """ 23 | Shine `fsck' command. 24 | Run a low-level filesystem check for filesystem targets. 25 | """ 26 | 27 | from __future__ import print_function 28 | 29 | import sys 30 | 31 | # Command base class 32 | from Shine.Commands.Base.FSLiveCommand import FSLiveCommand 33 | from Shine.Commands.Base.CommandRCDefs import RC_OK, RC_ST_EXTERNAL, \ 34 | RC_FAILURE, RC_TARGET_ERROR, \ 35 | RC_CLIENT_ERROR, RC_RUNTIME_ERROR 36 | # Lustre events 37 | from Shine.Commands.Base.FSEventHandler import FSGlobalEventHandler, \ 38 | FSLocalEventHandler 39 | 40 | from Shine.Lustre.FileSystem import MOUNTED, RECOVERING, EXTERNAL, OFFLINE, \ 41 | TARGET_ERROR, CLIENT_ERROR, RUNTIME_ERROR, \ 42 | MIGRATED 43 | 44 | class GlobalFsckEventHandler(FSGlobalEventHandler): 45 | """Display a global progress status for all components.""" 46 | 47 | def __init__(self, command): 48 | FSGlobalEventHandler.__init__(self, command) 49 | self._comps = {} 50 | self._current = 0 51 | 52 | def action_start(self, node, action, comp): 53 | self._comps[comp] = 0 54 | 55 | def action_progress(self, node, action, comp, result): 56 | self._comps[comp] = result.progress 57 | self._current = sum(self._comps.values()) // len(self._comps) 58 | header = self.command.NAME.capitalize() 59 | sys.stdout.write("%s in progress: %d %%\r" % (header, self._current)) 60 | sys.stdout.flush() 61 | if self._current == 100: 62 | sys.stdout.write("\n") 63 | 64 | 65 | class LocalFsckEventHandler(FSLocalEventHandler): 66 | """Display a global progress status for all components.""" 67 | 68 | def __init__(self, command): 69 | FSLocalEventHandler.__init__(self, command) 70 | self._comps = {} 71 | self._current = 0 72 | 73 | def action_start(self, node, action, comp): 74 | self._comps[comp] = 0 75 | 76 | def action_progress(self, node, action, comp, result): 77 | self._comps[comp] = result.progress 78 | self._current = sum(self._comps.values()) // len(self._comps) 79 | header = self.command.NAME.capitalize() 80 | sys.stdout.write("%s in progress: %d %%\r" % (header, self._current)) 81 | sys.stdout.flush() 82 | if self._current == 100: 83 | sys.stdout.write("\n") 84 | 85 | 86 | class Fsck(FSLiveCommand): 87 | """ 88 | shine fsck -f [-t ] [-i ] [-n ] 89 | """ 90 | 91 | NAME = "fsck" 92 | DESCRIPTION = "Fsck on targets backend file system." 93 | 94 | CRITICAL = True 95 | 96 | GLOBAL_EH = GlobalFsckEventHandler 97 | LOCAL_EH = LocalFsckEventHandler 98 | 99 | TARGET_STATUS_RC_MAP = { \ 100 | MOUNTED : RC_FAILURE, 101 | MIGRATED : RC_FAILURE, 102 | EXTERNAL : RC_ST_EXTERNAL, 103 | RECOVERING : RC_FAILURE, 104 | OFFLINE : RC_OK, 105 | TARGET_ERROR : RC_TARGET_ERROR, 106 | CLIENT_ERROR : RC_CLIENT_ERROR, 107 | RUNTIME_ERROR : RC_RUNTIME_ERROR } 108 | 109 | def execute_fs(self, fs, fs_conf, eh, vlevel): 110 | 111 | # Warn if trying to act on wrong nodes 112 | servers = fs.components.managed(supports='fsck').servers() 113 | if not self.check_valid_list(fs.fs_name, servers, "fsck"): 114 | return RC_FAILURE 115 | 116 | if not self.ask_confirm("Fsck %s on %s: are you sure?" % (fs.fs_name, 117 | servers)): 118 | return RC_FAILURE 119 | 120 | # Call a pre_fsck method if defined by the event handler. 121 | if hasattr(eh, 'pre'): 122 | eh.pre(fs) 123 | 124 | # Fsck really. 125 | status = fs.fsck(addopts=self.options.additional, 126 | failover=self.options.failover, 127 | fanout=self.options.fanout, 128 | dryrun=self.options.dryrun, 129 | mountdata=self.options.mountdata) 130 | 131 | rc = self.fs_status_to_rc(status) 132 | 133 | if rc == RC_OK: 134 | if vlevel > 0: 135 | print("Fsck successful.") 136 | else: 137 | if rc == RC_RUNTIME_ERROR: 138 | self.display_proxy_errors(fs) 139 | if vlevel > 0: 140 | print("Fsck failed") 141 | 142 | # Call a post_fsck method if defined by the event handler. 143 | if hasattr(eh, 'post'): 144 | eh.post(fs) 145 | 146 | return rc 147 | -------------------------------------------------------------------------------- /lib/Shine/Commands/Install.py: -------------------------------------------------------------------------------- 1 | # Install.py -- File system installation commands 2 | # Copyright (C) 2007-2013 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | from __future__ import print_function 22 | 23 | import sys 24 | 25 | from Shine.Configuration.Globals import Globals 26 | 27 | from Shine.FSUtils import create_lustrefs 28 | from Shine.Lustre.FileSystem import FSRemoteError 29 | 30 | from Shine.Commands.Base.Command import Command, CommandHelpException 31 | from Shine.Commands.Base.CommandRCDefs import RC_OK, RC_FAILURE 32 | 33 | # Lustre events 34 | from Shine.Commands.Base.FSEventHandler import FSGlobalEventHandler 35 | 36 | 37 | class Install(Command): 38 | """ 39 | shine install -m /path/to/model.lmf 40 | """ 41 | 42 | NAME = "install" 43 | DESCRIPTION = "Install a new file system." 44 | 45 | def execute(self): 46 | 47 | # Option sanity check 48 | self.forbidden(self.options.fsnames, "-f, see -m") 49 | self.forbidden(self.options.labels, "-l") 50 | self.forbidden(self.options.indexes, "-i") 51 | self.forbidden(self.options.failover, "-F") 52 | 53 | rc = RC_OK 54 | 55 | if not self.options.model: 56 | raise CommandHelpException("Lustre model file path" 57 | "(-m ) argument required.", self) 58 | 59 | eh = FSGlobalEventHandler(self) 60 | 61 | # Use this Shine.FSUtils convenience function. 62 | lmf = self.get_lmf_path() 63 | if lmf: 64 | print("Using Lustre model file %s" % lmf) 65 | else: 66 | raise CommandHelpException("Lustre model file for ``%s'' not found:" 67 | " please use filename or full LMF path.\n" 68 | "Your default model files directory (lmf_dir) is: %s" % 69 | (self.options.model, Globals().get_lmf_dir()), self) 70 | 71 | install_nodes = self.options.nodes 72 | excluded_nodes = self.options.excludes 73 | 74 | fs_conf, fs = create_lustrefs(self.get_lmf_path(), 75 | event_handler=eh, nodes=install_nodes, 76 | excluded=excluded_nodes) 77 | 78 | # Register the filesystem in backend 79 | print("Registering FS %s to backend..." % fs.fs_name) 80 | if self.options.dryrun: 81 | rc = 0 82 | else: 83 | rc = self.register_fs(fs_conf) 84 | if rc: 85 | msg = "Error: failed to register FS to backend (rc=%d)" % rc 86 | print(msg, file=sys.stderr) 87 | else: 88 | print("Filesystem %s registered." % fs.fs_name) 89 | 90 | # Helper message. 91 | # If user specified nodes which were not used, warn him about it. 92 | actual_nodes = fs.components.managed().servers() 93 | if not self.check_valid_list(fs_conf.get_fs_name(), \ 94 | actual_nodes, "install"): 95 | return RC_FAILURE 96 | 97 | # Install file system configuration files; normally, this should 98 | # not be done by the Shine.Lustre.FileSystem object itself, but as 99 | # all proxy methods are currently handled by it, it is more 100 | # convenient this way... 101 | try: 102 | fs.install(fs_conf.get_cfg_filename(), 103 | dryrun=self.options.dryrun) 104 | 105 | tuning_conf = Globals().get_tuning_file() 106 | if tuning_conf: 107 | fs.install(tuning_conf, dryrun=self.options.dryrun) 108 | 109 | except FSRemoteError as error: 110 | print("WARNING: Due to error, installation skipped on %s" 111 | % error.nodes) 112 | rc = RC_FAILURE 113 | 114 | if not install_nodes and not excluded_nodes: 115 | # Give pointer to next user step. 116 | print("Use `shine format -f %s' to initialize the file system." % 117 | fs_conf.get_fs_name()) 118 | 119 | return rc 120 | 121 | def register_fs(self, fs_conf): 122 | # register file system configuration to the backend 123 | fs_conf.register_fs() 124 | 125 | fs_conf.register_targets() 126 | -------------------------------------------------------------------------------- /lib/Shine/Commands/List.py: -------------------------------------------------------------------------------- 1 | # List.py -- List installed filename names. 2 | # Copyright (C) 2012 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | """ 22 | Shine `list' command classes. 23 | 24 | List installed filename names. 25 | """ 26 | 27 | from __future__ import print_function 28 | 29 | from Shine.Commands.Base.Command import Command 30 | 31 | class List(Command): 32 | """ 33 | shine list 34 | """ 35 | 36 | NAME = "list" 37 | DESCRIPTION = "List installed filename names." 38 | 39 | def execute(self): 40 | 41 | # Option sanity check 42 | self.forbidden(self.options.fsnames, "-f") 43 | self.forbidden(self.options.model, "-m") 44 | self.forbidden(self.options.labels, "-f") 45 | self.forbidden(self.options.labels, "-l") 46 | self.forbidden(self.options.indexes, "-i") 47 | self.forbidden(self.options.failover, "-F") 48 | 49 | print("\n".join(self.iter_fsname())) 50 | -------------------------------------------------------------------------------- /lib/Shine/Commands/Mount.py: -------------------------------------------------------------------------------- 1 | # Mount.py -- Mount file system on clients 2 | # Copyright (C) 2007-2015 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | """ 22 | Shine `mount' command classes. 23 | 24 | The mount command aims to start Lustre filesystem clients. 25 | """ 26 | 27 | from __future__ import print_function 28 | 29 | # Command helper 30 | from Shine.Commands.Tune import Tune 31 | 32 | # Command base class 33 | from Shine.Commands.Base.FSLiveCommand import FSLiveCommand 34 | from Shine.Commands.Base.CommandRCDefs import RC_OK, \ 35 | RC_FAILURE, RC_TARGET_ERROR, \ 36 | RC_CLIENT_ERROR, RC_RUNTIME_ERROR 37 | # Lustre events 38 | from Shine.Commands.Base.FSEventHandler import FSGlobalEventHandler, \ 39 | FSLocalEventHandler 40 | 41 | from Shine.Lustre.FileSystem import MOUNTED, RECOVERING, OFFLINE, \ 42 | TARGET_ERROR, CLIENT_ERROR, RUNTIME_ERROR 43 | 44 | class Mount(FSLiveCommand): 45 | """ 46 | shine mount 47 | """ 48 | 49 | NAME = "mount" 50 | DESCRIPTION = "Mount file system clients." 51 | 52 | GLOBAL_EH = FSGlobalEventHandler 53 | LOCAL_EH = FSLocalEventHandler 54 | 55 | TARGET_STATUS_RC_MAP = { \ 56 | MOUNTED : RC_OK, 57 | RECOVERING : RC_FAILURE, 58 | OFFLINE : RC_FAILURE, 59 | TARGET_ERROR : RC_TARGET_ERROR, 60 | CLIENT_ERROR : RC_CLIENT_ERROR, 61 | RUNTIME_ERROR : RC_RUNTIME_ERROR } 62 | 63 | def execute_fs(self, fs, fs_conf, eh, vlevel): 64 | 65 | # Warn if trying to act on wrong nodes 66 | comps = fs.components.managed(supports='mount') 67 | if not self.check_valid_list(fs.fs_name, comps.servers(), "mount"): 68 | return RC_FAILURE 69 | 70 | # Will call the handle_pre() method defined by the event handler. 71 | if hasattr(eh, 'pre'): 72 | eh.pre(fs) 73 | 74 | status = fs.mount(addopts=self.options.additional, 75 | fanout=self.options.fanout, 76 | dryrun=self.options.dryrun, 77 | tunings=Tune.get_tuning(fs_conf, fs.components)) 78 | 79 | rc = self.fs_status_to_rc(status) 80 | 81 | if not self.options.remote: 82 | if rc == RC_OK: 83 | if vlevel > 0: 84 | key = lambda c: c.state == MOUNTED 85 | print("%s was successfully mounted on %s" % 86 | (fs.fs_name, comps.filter(key=key).servers())) 87 | elif rc == RC_RUNTIME_ERROR: 88 | self.display_proxy_errors(fs) 89 | 90 | if hasattr(eh, 'post'): 91 | eh.post(fs) 92 | 93 | return rc 94 | -------------------------------------------------------------------------------- /lib/Shine/Commands/Remove.py: -------------------------------------------------------------------------------- 1 | # Remove.py -- File system removing commands 2 | # Copyright (C) 2007, 2008 BULL S.A.S 3 | # Copyright (C) 2009-2015 CEA 4 | # 5 | # This file is part of shine 6 | # 7 | # This program is free software; you can redistribute it and/or 8 | # modify it under the terms of the GNU General Public License 9 | # as published by the Free Software Foundation; either version 2 10 | # of the License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program; if not, write to the Free Software 19 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 20 | # 21 | 22 | """ 23 | Shine 'remove' command classes. 24 | 25 | The remove command aims to uninstall a Lustre filesystem setup with Shine. 26 | This will interact with the backend and will remove local cached files. 27 | """ 28 | 29 | from __future__ import print_function 30 | 31 | import sys 32 | 33 | from Shine.Configuration.FileSystem import ModelFileIOError 34 | 35 | from Shine.Commands.Base.FSLiveCommand import FSLiveCommand 36 | from Shine.Commands.Base.CommandRCDefs import RC_OK, RC_FAILURE 37 | 38 | from Shine.Lustre.FileSystem import MOUNTED, RECOVERING, RUNTIME_ERROR 39 | 40 | from Shine.Commands.Base.FSEventHandler import FSGlobalEventHandler 41 | 42 | 43 | class Remove(FSLiveCommand): 44 | """ 45 | This Remove Command object is used to completly remove the 46 | File System description from the Shine environment. 47 | All datas are lost after the Remove command completion. 48 | """ 49 | 50 | NAME = "remove" 51 | DESCRIPTION = "Remove a previously installed file system" 52 | 53 | CRITICAL = True 54 | 55 | GLOBAL_EH = FSGlobalEventHandler 56 | 57 | def execute(self): 58 | 59 | # Option sanity check 60 | self.forbidden(self.options.model, "-m, use -f") 61 | 62 | try: 63 | return FSLiveCommand.execute(self) 64 | except ModelFileIOError: 65 | if self.has_local_flag(): 66 | return 0 67 | raise 68 | 69 | def execute_fs(self, fs, fs_conf, eh, vlevel): 70 | 71 | rc = RC_OK 72 | 73 | # Warn if trying to act on wrong nodes 74 | servers = fs.components.managed().servers() 75 | if not self.check_valid_list(fs.fs_name, servers, 'uninstall'): 76 | return RC_FAILURE 77 | 78 | # Admin mode 79 | if not self.has_local_flag(): 80 | 81 | # Get first the status of any FS components and display some 82 | # warnings if filesystem is not OK. 83 | fs.status() 84 | for state, targets in \ 85 | fs.components.managed().groupby(attr='state'): 86 | 87 | # Mounted filesystem! 88 | if state in [MOUNTED, RECOVERING]: 89 | labels = targets.labels() 90 | print("WARNING: Some targets are started: %s" % labels) 91 | # Error, won't be able to remove on these nodes 92 | elif state == RUNTIME_ERROR: 93 | self.display_proxy_errors(fs) 94 | print("WARNING: Removing %s might failed on some nodes " 95 | "(see above)!" % fs.fs_name) 96 | 97 | # Confirmation 98 | if not self.ask_confirm("Please confirm the removal of filesystem" \ 99 | " ``%s''" % fs.fs_name): 100 | return RC_FAILURE 101 | 102 | # Do the job now! 103 | print("Removing filesystem %s..." % fs.fs_name) 104 | if fs.remove(dryrun=self.options.dryrun): 105 | print("WARNING: failed to remove all filesystem %s " 106 | "configuration files" % fs.fs_name) 107 | 108 | # XXX: This is not really nice. Need to find a better way. 109 | if not self.options.nodes \ 110 | and not self.options.excludes \ 111 | and not self.options.targets \ 112 | and not self.options.labels \ 113 | and not self.options.failover \ 114 | and not self.options.indexes: 115 | 116 | print("Unregistering FS %s from backend..." % fs.fs_name) 117 | if self.options.dryrun: 118 | retcode = 0 119 | else: 120 | retcode = self.unregister_fs(fs_conf) 121 | if retcode: 122 | msg = "Error: failed to unregister FS from backend " \ 123 | "(rc = %d)" % retcode 124 | print(msg, file=sys.stderr) 125 | return RC_FAILURE 126 | 127 | print("Filesystem %s removed." % fs.fs_name) 128 | 129 | # Local mode (either -R or -L) 130 | else: 131 | if fs.remove(dryrun=self.options.dryrun): 132 | if self.options.local: 133 | msg = "Error: failed to remove filesystem ```%s'' " \ 134 | "configuration files" % fs.fs_name 135 | print(msg, file=sys.stderr) 136 | return RC_FAILURE 137 | 138 | elif self.options.local: 139 | print("Filesystem %s removed." % fs.fs_name) 140 | 141 | return rc 142 | 143 | def unregister_fs(self, fs_conf): 144 | """ 145 | Unregister all client nodes and the filesystem from the backend. 146 | """ 147 | fs_conf.unregister_targets() 148 | 149 | # Unregister file system configuration from the backend 150 | fs_conf.unregister_fs() 151 | -------------------------------------------------------------------------------- /lib/Shine/Commands/Start.py: -------------------------------------------------------------------------------- 1 | # Start.py -- Start file system 2 | # Copyright (C) 2007-2016 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | """ 22 | Shine `start' command classes. 23 | 24 | The start command aims to start Lustre filesystem servers or just some 25 | of the filesystem targets on local or remote servers. It is available 26 | for any filesystems previously installed and formatted. 27 | """ 28 | 29 | from __future__ import print_function 30 | 31 | from Shine.Commands.Tune import Tune 32 | 33 | # Command base class 34 | from Shine.Commands.Base.FSLiveCommand import FSLiveCommand 35 | from Shine.Commands.Base.CommandRCDefs import RC_OK, RC_ST_EXTERNAL, \ 36 | RC_FAILURE, RC_TARGET_ERROR, \ 37 | RC_CLIENT_ERROR, RC_RUNTIME_ERROR 38 | # Lustre events 39 | from Shine.Commands.Base.FSEventHandler import FSGlobalEventHandler, \ 40 | FSLocalEventHandler 41 | 42 | from Shine.Lustre.FileSystem import MOUNTED, RECOVERING, EXTERNAL, OFFLINE, \ 43 | TARGET_ERROR, CLIENT_ERROR, RUNTIME_ERROR, \ 44 | MIGRATED 45 | 46 | 47 | class Start(FSLiveCommand): 48 | """ 49 | shine start [-f ] [-t ] [-i ] [-n ] [-qv] 50 | """ 51 | 52 | NAME = "start" 53 | DESCRIPTION = "Start file system servers." 54 | 55 | GLOBAL_EH = FSGlobalEventHandler 56 | LOCAL_EH = FSLocalEventHandler 57 | 58 | # return RC_OK for MIGRATED and RECOVERING as the *start* action 59 | # worked as expected. If in the future start becomes omniscient 60 | # and decides to start targets on failover nodes we might want to 61 | # reconsider this 62 | TARGET_STATUS_RC_MAP = { \ 63 | MOUNTED : RC_OK, 64 | MIGRATED : RC_OK, 65 | RECOVERING : RC_OK, 66 | EXTERNAL : RC_ST_EXTERNAL, 67 | OFFLINE : RC_FAILURE, 68 | TARGET_ERROR : RC_TARGET_ERROR, 69 | CLIENT_ERROR : RC_CLIENT_ERROR, 70 | RUNTIME_ERROR : RC_RUNTIME_ERROR } 71 | 72 | def execute_fs(self, fs, fs_conf, eh, vlevel): 73 | 74 | # Prepare options... 75 | mount_options = {} 76 | mount_paths = {} 77 | for tgt_type in [ 'mgt', 'mdt', 'ost' ]: 78 | mount_options[tgt_type] = fs_conf.get_target_mount_options(tgt_type) 79 | mount_paths[tgt_type] = fs_conf.get_target_mount_path(tgt_type) 80 | 81 | # Warn if trying to act on wrong nodes 82 | comps = fs.components.managed(supports='start') 83 | if not self.check_valid_list(fs.fs_name, comps.servers(), 'start'): 84 | return RC_FAILURE 85 | 86 | # Will call the handle_pre() method defined by the event handler. 87 | if hasattr(eh, 'pre'): 88 | eh.pre(fs) 89 | 90 | status = fs.start(mount_options=mount_options, 91 | mount_paths=mount_paths, 92 | addopts=self.options.additional, 93 | failover=self.options.failover, 94 | fanout=self.options.fanout, 95 | dryrun=self.options.dryrun, 96 | mountdata=self.options.mountdata, 97 | tunings=Tune.get_tuning(fs_conf, fs.components)) 98 | 99 | rc = self.fs_status_to_rc(status) 100 | 101 | if rc == RC_OK: 102 | if vlevel > 0: 103 | print("Start successful.") 104 | elif rc == RC_RUNTIME_ERROR: 105 | self.display_proxy_errors(fs) 106 | 107 | if hasattr(eh, 'post'): 108 | eh.post(fs) 109 | 110 | return rc 111 | -------------------------------------------------------------------------------- /lib/Shine/Commands/Status.py: -------------------------------------------------------------------------------- 1 | # Status.py -- Check remote filesystem servers and targets status 2 | # Copyright (C) 2009-2016 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | """ 22 | Shine `status' command classes. 23 | 24 | The status command aims to return the real state of a Lustre filesystem 25 | and its components, depending of the requested "view". Status views let 26 | the Lustre administrator to either stand back and get a global status 27 | of the filesystem, or if needed, to enquire about filesystem components 28 | detailed states. 29 | """ 30 | 31 | from __future__ import print_function 32 | 33 | # Command base class 34 | from Shine.Commands.Base.FSLiveCommand import FSLiveCommand 35 | from Shine.Commands.Base.CommandRCDefs import RC_ST_OFFLINE, RC_ST_EXTERNAL, \ 36 | RC_ST_ONLINE, RC_ST_RECOVERING, \ 37 | RC_ST_MIGRATED, \ 38 | RC_FAILURE, RC_TARGET_ERROR, \ 39 | RC_CLIENT_ERROR, RC_RUNTIME_ERROR 40 | 41 | # Lustre events and errors 42 | from Shine.Commands.Base.FSEventHandler import FSGlobalEventHandler, \ 43 | FSLocalEventHandler 44 | from Shine.Lustre.FileSystem import MOUNTED, RECOVERING, EXTERNAL, OFFLINE, \ 45 | TARGET_ERROR, CLIENT_ERROR, RUNTIME_ERROR, \ 46 | MIGRATED 47 | 48 | from Shine.FSUtils import open_lustrefs 49 | 50 | class Status(FSLiveCommand): 51 | """ 52 | shine status [-f ] [-t ] [-i ] [-n ] [-qv] 53 | """ 54 | 55 | NAME = "status" 56 | DESCRIPTION = "Check for file system target status." 57 | 58 | GLOBAL_EH = FSGlobalEventHandler 59 | LOCAL_EH = FSLocalEventHandler 60 | 61 | TARGET_STATUS_RC_MAP = { \ 62 | MOUNTED : RC_ST_ONLINE, 63 | MIGRATED : RC_ST_MIGRATED, 64 | RECOVERING : RC_ST_RECOVERING, 65 | EXTERNAL : RC_ST_EXTERNAL, 66 | OFFLINE : RC_ST_OFFLINE, 67 | TARGET_ERROR : RC_TARGET_ERROR, 68 | CLIENT_ERROR : RC_CLIENT_ERROR, 69 | RUNTIME_ERROR : RC_RUNTIME_ERROR } 70 | 71 | def execute_fs(self, fs, fs_conf, eh, vlevel): 72 | 73 | # Warn if trying to act on wrong nodes 74 | all_nodes = fs.components.managed().allservers() 75 | if not self.check_valid_list(fs.fs_name, all_nodes, "check"): 76 | return RC_FAILURE 77 | 78 | # Apply 'status' only to required components 79 | comps = fs.components 80 | if self.options.view.startswith("target"): 81 | comps = comps.filter(supports='index') 82 | if self.options.view.startswith("disk"): 83 | comps = comps.filter(supports='dev') 84 | 85 | # Will call the handle_pre() method defined by the event handler. 86 | if hasattr(eh, 'pre'): 87 | eh.pre(fs) 88 | 89 | fs_result = fs.status(comps, 90 | failover=self.options.failover, 91 | dryrun=self.options.dryrun, 92 | fanout=self.options.fanout, 93 | mountdata=self.options.mountdata) 94 | 95 | # Display error messages for each node that failed. 96 | if len(fs.proxy_errors) > 0: 97 | self.display_proxy_errors(fs) 98 | print() 99 | 100 | result = self.fs_status_to_rc(fs_result) 101 | 102 | # Call a handle_post() method if defined by the event handler. 103 | if hasattr(eh, 'post'): 104 | eh.post(fs) 105 | 106 | return result 107 | 108 | def _open_fs(self, fsname, eh): 109 | # Status command needs to open the filesystem in extended mode. 110 | # See FSUtils.instantiate_lustrefs() for the use of this argument. 111 | fs_conf, fs = open_lustrefs(fsname, 112 | self.options.targets, 113 | nodes=self.options.nodes, 114 | excluded=self.options.excludes, 115 | failover=self.options.failover, 116 | indexes=self.options.indexes, 117 | labels=self.options.labels, 118 | event_handler=eh, 119 | extended=True) 120 | return fs_conf, fs 121 | -------------------------------------------------------------------------------- /lib/Shine/Commands/Stop.py: -------------------------------------------------------------------------------- 1 | # Stop.py -- Stop file system 2 | # Copyright (C) 2007-2015 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | """ 22 | Shine `stop' command classes. 23 | 24 | The stop command aims to stop Lustre filesystem servers or just some 25 | of the filesystem targets on local or remote servers. It is available 26 | for any filesystems previously installed and formatted. 27 | """ 28 | 29 | from __future__ import print_function 30 | 31 | # Command base class 32 | from Shine.Commands.Base.FSLiveCommand import FSLiveCommand 33 | from Shine.Commands.Base.CommandRCDefs import RC_OK, RC_ST_EXTERNAL, \ 34 | RC_FAILURE, RC_TARGET_ERROR, \ 35 | RC_CLIENT_ERROR, RC_RUNTIME_ERROR 36 | # Lustre events 37 | from Shine.Commands.Base.FSEventHandler import FSGlobalEventHandler, \ 38 | FSLocalEventHandler 39 | 40 | from Shine.Lustre.FileSystem import MOUNTED, RECOVERING, EXTERNAL, OFFLINE, \ 41 | TARGET_ERROR, CLIENT_ERROR, RUNTIME_ERROR, \ 42 | MIGRATED 43 | 44 | 45 | class Stop(FSLiveCommand): 46 | """ 47 | shine stop [-f ] [-t ] [-i ] [-n ] [-qv] 48 | """ 49 | 50 | NAME = "stop" 51 | DESCRIPTION = "Stop file system servers." 52 | 53 | GLOBAL_EH = FSGlobalEventHandler 54 | LOCAL_EH = FSLocalEventHandler 55 | 56 | TARGET_STATUS_RC_MAP = { \ 57 | MOUNTED : RC_FAILURE, 58 | MIGRATED : RC_FAILURE, 59 | RECOVERING : RC_FAILURE, 60 | EXTERNAL : RC_ST_EXTERNAL, 61 | OFFLINE : RC_OK, 62 | TARGET_ERROR : RC_TARGET_ERROR, 63 | CLIENT_ERROR : RC_CLIENT_ERROR, 64 | RUNTIME_ERROR : RC_RUNTIME_ERROR } 65 | 66 | def execute_fs(self, fs, fs_conf, eh, vlevel): 67 | 68 | # Prepare options... 69 | mount_options = {} 70 | mount_paths = {} 71 | for tgt_type in [ 'mgt', 'mdt', 'ost' ]: 72 | mount_options[tgt_type] = fs_conf.get_target_mount_options(tgt_type) 73 | mount_paths[tgt_type] = fs_conf.get_target_mount_path(tgt_type) 74 | 75 | # Warn if trying to act on wrong nodes 76 | servers = fs.components.managed(supports='stop').servers() 77 | if not self.check_valid_list(fs.fs_name, servers, "stop"): 78 | return RC_FAILURE 79 | 80 | # Will call the handle_pre() method defined by the event handler. 81 | if hasattr(eh, 'pre'): 82 | eh.pre(fs) 83 | 84 | status = fs.stop(addopts=self.options.additional, 85 | failover=self.options.failover, 86 | fanout=self.options.fanout, 87 | dryrun=self.options.dryrun, 88 | mountdata=self.options.mountdata) 89 | 90 | rc = self.fs_status_to_rc(status) 91 | 92 | if rc == RC_OK: 93 | if vlevel > 0: 94 | print("Stop successful.") 95 | elif rc == RC_RUNTIME_ERROR: 96 | self.display_proxy_errors(fs) 97 | 98 | if hasattr(eh, 'post'): 99 | eh.post(fs) 100 | 101 | return rc 102 | -------------------------------------------------------------------------------- /lib/Shine/Commands/Tunefs.py: -------------------------------------------------------------------------------- 1 | # Tunefs.py -- Tune file system targets 2 | # Copyright (C) 2011-2013 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | """ 22 | Shine `tunefs' command classes. 23 | 24 | The tunefs command aims to modify target on-disk parameter without reformating 25 | it. 26 | """ 27 | 28 | from __future__ import print_function 29 | 30 | # Command base class 31 | from Shine.Commands.Base.FSLiveCommand import FSLiveCommand 32 | from Shine.Commands.Base.CommandRCDefs import RC_OK, RC_ST_EXTERNAL, \ 33 | RC_FAILURE, RC_TARGET_ERROR, \ 34 | RC_CLIENT_ERROR, RC_RUNTIME_ERROR 35 | # Lustre events 36 | from Shine.Commands.Base.FSEventHandler import FSGlobalEventHandler, \ 37 | FSLocalEventHandler 38 | 39 | from Shine.Lustre.FileSystem import MOUNTED, RECOVERING, EXTERNAL, OFFLINE, \ 40 | TARGET_ERROR, CLIENT_ERROR, RUNTIME_ERROR, \ 41 | MIGRATED 42 | 43 | 44 | class Tunefs(FSLiveCommand): 45 | """ 46 | shine tunefs -f [...] 47 | """ 48 | 49 | NAME = "tunefs" 50 | DESCRIPTION = "Tune file system targets." 51 | 52 | CRITICAL = True 53 | 54 | GLOBAL_EH = FSGlobalEventHandler 55 | LOCAL_EH = FSLocalEventHandler 56 | 57 | TARGET_STATUS_RC_MAP = { \ 58 | MOUNTED : RC_FAILURE, 59 | MIGRATED : RC_FAILURE, 60 | EXTERNAL : RC_ST_EXTERNAL, 61 | RECOVERING : RC_FAILURE, 62 | OFFLINE : RC_OK, 63 | TARGET_ERROR : RC_TARGET_ERROR, 64 | CLIENT_ERROR : RC_CLIENT_ERROR, 65 | RUNTIME_ERROR : RC_RUNTIME_ERROR } 66 | 67 | def execute_fs(self, fs, fs_conf, eh, vlevel): 68 | 69 | # Warn if trying to act on wrong nodes 70 | servers = fs.components.managed(supports='tunefs').servers() 71 | if not self.check_valid_list(fs.fs_name, servers, "tunefs"): 72 | return RC_FAILURE 73 | 74 | if not self.ask_confirm("Tunefs %s on %s: are you sure?" % (fs.fs_name, 75 | servers)): 76 | return RC_FAILURE 77 | 78 | format_params = {} 79 | for target_type in [ 'mgt', 'mdt', 'ost' ]: 80 | format_params[target_type] = \ 81 | fs_conf.get_target_format_params(target_type) 82 | 83 | # Call a pre_format method if defined by the event handler. 84 | if hasattr(eh, 'pre'): 85 | eh.pre(fs) 86 | 87 | # Format really. 88 | status = fs.tunefs(stripecount=fs_conf.get_stripecount(), 89 | stripesize=fs_conf.get_stripesize(), 90 | format_params=format_params, 91 | quota=fs_conf.has_quota(), 92 | quota_type=fs_conf.get_quota_type(), 93 | addopts=self.options.additional, 94 | fanout=self.options.fanout, 95 | dryrun=self.options.dryrun, 96 | failover=self.options.failover, 97 | writeconf=True, 98 | mountdata=self.options.mountdata) 99 | 100 | rc = self.fs_status_to_rc(status) 101 | 102 | if rc == RC_OK: 103 | if vlevel > 0: 104 | print("Tunefs successful.") 105 | else: 106 | if rc == RC_RUNTIME_ERROR: 107 | self.display_proxy_errors(fs) 108 | if vlevel > 0: 109 | print("Tunefs failed") 110 | 111 | # Call a post_tunefs method if defined by the event handler. 112 | if hasattr(eh, 'post'): 113 | eh.post(fs) 114 | 115 | return rc 116 | -------------------------------------------------------------------------------- /lib/Shine/Commands/Umount.py: -------------------------------------------------------------------------------- 1 | # Umount.py -- Unmount file system on clients 2 | # Copyright (C) 2007-2015 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | """ 22 | Shine `umount' command classes. 23 | 24 | The umount command aims to stop Lustre filesystem clients. 25 | """ 26 | 27 | from __future__ import print_function 28 | 29 | # Command base class 30 | from Shine.Commands.Base.FSLiveCommand import FSLiveCommand 31 | from Shine.Commands.Base.CommandRCDefs import RC_OK, \ 32 | RC_FAILURE, RC_TARGET_ERROR, \ 33 | RC_CLIENT_ERROR, RC_RUNTIME_ERROR 34 | # Lustre events 35 | from Shine.Commands.Base.FSEventHandler import FSGlobalEventHandler, \ 36 | FSLocalEventHandler 37 | 38 | from Shine.Lustre.FileSystem import MOUNTED, RECOVERING, OFFLINE, \ 39 | TARGET_ERROR, CLIENT_ERROR, RUNTIME_ERROR 40 | 41 | 42 | class Umount(FSLiveCommand): 43 | """ 44 | shine umount 45 | """ 46 | 47 | NAME = "umount" 48 | DESCRIPTION = "Unmount file system clients." 49 | 50 | GLOBAL_EH = FSGlobalEventHandler 51 | LOCAL_EH = FSLocalEventHandler 52 | 53 | TARGET_STATUS_RC_MAP = { \ 54 | MOUNTED : RC_FAILURE, 55 | RECOVERING : RC_FAILURE, 56 | OFFLINE : RC_OK, 57 | TARGET_ERROR : RC_TARGET_ERROR, 58 | CLIENT_ERROR : RC_CLIENT_ERROR, 59 | RUNTIME_ERROR : RC_RUNTIME_ERROR } 60 | 61 | def execute_fs(self, fs, fs_conf, eh, vlevel): 62 | 63 | # Warn if trying to act on wrong nodes 64 | comps = fs.components.managed(supports='umount') 65 | if not self.check_valid_list(fs.fs_name, comps.servers(), "unmount"): 66 | return RC_FAILURE 67 | 68 | # Will call the handle_pre() method defined by the event handler. 69 | if hasattr(eh, 'pre'): 70 | eh.pre(fs) 71 | 72 | status = fs.umount(addopts=self.options.additional, 73 | dryrun=self.options.dryrun, 74 | fanout=self.options.fanout) 75 | 76 | rc = self.fs_status_to_rc(status) 77 | 78 | if not self.options.remote: 79 | if rc == RC_OK: 80 | if vlevel > 0: 81 | key = lambda c: c.state == OFFLINE 82 | print("%s was successfully unmounted on %s" % 83 | (fs.fs_name, comps.filter(key=key).servers())) 84 | elif rc == RC_RUNTIME_ERROR: 85 | self.display_proxy_errors(fs) 86 | 87 | if hasattr(eh, 'post'): 88 | eh.post(fs) 89 | 90 | return rc 91 | -------------------------------------------------------------------------------- /lib/Shine/Commands/__init__.py: -------------------------------------------------------------------------------- 1 | # Commands/__init__.py -- Commands module initialization 2 | # Copyright (C) 2007-2012 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | 22 | # ---------------------------------------------------------------------- 23 | # List of enabled commands classes. 24 | # ---------------------------------------------------------------------- 25 | 26 | COMMAND_LIST = {} 27 | 28 | for cmd in [ "Show", 29 | "Config", 30 | "List", 31 | "Install", 32 | "Update", 33 | "Remove", 34 | "Format", 35 | "Status", 36 | "Start", 37 | "Stop", 38 | "Fsck", 39 | "Mount", 40 | "Umount", 41 | "Tune", 42 | "Tunefs", 43 | "Execute"]: 44 | 45 | # Import command class file 46 | mod = __import__("Shine.Commands." + cmd, globals(), locals(), [cmd]) 47 | 48 | # Add class to global command list 49 | cls = getattr(mod, cmd) 50 | COMMAND_LIST[cls.NAME] = cls 51 | -------------------------------------------------------------------------------- /lib/Shine/Configuration/Backend/Backend.py: -------------------------------------------------------------------------------- 1 | # Backend.py -- File system config backend point of view 2 | # Copyright (C) 2007-2013 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | 20 | 21 | NIEXC="Derived classes must implement." 22 | 23 | class Backend: 24 | """ 25 | An interface representing config backend storage resources for a file system. 26 | """ 27 | 28 | MOUNT_COMPLETE = 1 29 | MOUNT_FAILED = 2 30 | MOUNT_WARNING = 3 31 | UMOUNT_COMPLETE = 4 32 | UMOUNT_FAILED = 5 33 | UMOUNT_WARNING = 6 34 | 35 | # Integers which represents the different target status 36 | TARGET_UNKNOWN=1 37 | TARGET_KO=2 38 | TARGET_AVAILABLE=3 39 | TARGET_FORMATING=4 40 | TARGET_FORMAT_FAILED=5 41 | TARGET_FORMATED=6 42 | TARGET_OFFLINE=7 43 | TARGET_STARTING=8 44 | TARGET_ONLINE=9 45 | TARGET_CRITICAL=10 46 | TARGET_STOPPING=11 47 | TARGET_UNREACHABLE=12 48 | TARGET_CHECKING=13 49 | 50 | def __init__(self): 51 | "Initializer." 52 | pass 53 | 54 | # Public accessors. 55 | 56 | def get_name(self): 57 | raise NotImplementedError(NIEXC) 58 | 59 | def get_desc(self): 60 | raise NotImplementedError(NIEXC) 61 | 62 | # Public methods. 63 | 64 | def start(self): 65 | """ 66 | The config backend storage system has been selected. 67 | """ 68 | raise NotImplementedError(NIEXC) 69 | 70 | def stop(self): 71 | """ 72 | Stop operations 73 | """ 74 | raise NotImplementedError(NIEXC) 75 | 76 | def get_target_devices(self, target, fs_name=None, update_mode=False): 77 | """ 78 | Get the targets configuration, as a TargetDevice list (for mgt, mdt, ost). 79 | """ 80 | raise NotImplementedError(NIEXC) 81 | 82 | def register_fs(self, fs): 83 | """ 84 | This function is used to register a a filesystem configuration to the backend 85 | """ 86 | raise NotImplementedError(NIEXC) 87 | 88 | def unregister_fs(self, fs): 89 | """ 90 | This function is used to remove a filesystem configuration to the backend 91 | """ 92 | raise NotImplementedError(NIEXC) 93 | 94 | def register_target(self, fs, target): 95 | """ 96 | Set the specified `target', used by `fs', as 'in use' in the backend. 97 | 98 | This target could not be use anymore for other filesystems. 99 | """ 100 | raise NotImplementedError(NIEXC) 101 | 102 | def unregister_target(self, fs, target): 103 | """ 104 | Set the specified `target', used by `fs', as available in the backend. 105 | 106 | This target could be now reuse, for other targets of the same 107 | filesystem or any other one. 108 | """ 109 | raise NotImplementedError(NIEXC) 110 | -------------------------------------------------------------------------------- /lib/Shine/Configuration/Backend/BackendRegistry.py: -------------------------------------------------------------------------------- 1 | # BackendRegistry.py -- Registry for config backends 2 | # Copyright (C) 2007,2013 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | """ 22 | Load and maintain the list of available configuration backends. 23 | """ 24 | 25 | from Shine.Configuration.Globals import Globals 26 | 27 | class BackendRegistry: 28 | """Container object to deal with available storage systems.""" 29 | 30 | def __init__(self): 31 | self.backends = {} 32 | 33 | def __len__(self): 34 | """Return the number of backend storages.""" 35 | return len(self.backends) 36 | 37 | def __iter__(self): 38 | """Iterate over available backend storages.""" 39 | for backend in self.backends.values(): 40 | yield backend 41 | 42 | def get(self, name): 43 | """Load an return a instance of backend with the specified name.""" 44 | 45 | if name == "None": 46 | return None 47 | 48 | # Import Backend if not already done 49 | if name not in self.backends: 50 | mod = __import__(name, globals(), locals(), []) 51 | cls = getattr(mod, mod.BACKEND_MODNAME) 52 | self.register(cls()) 53 | 54 | return self.backends[name] 55 | 56 | def selected(self): 57 | """Return the Backend specified in global configuration.""" 58 | return self.get(Globals().get_backend()) 59 | 60 | def register(self, obj): 61 | """Register a new config backend storage system.""" 62 | self.backends[obj.get_name()] = obj 63 | 64 | -------------------------------------------------------------------------------- /lib/Shine/Configuration/Backend/File.py: -------------------------------------------------------------------------------- 1 | # File.py -- File backend module 2 | # Copyright (C) 2007-2013 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | import os 22 | import shelve 23 | 24 | from Shine.Configuration.Backend.Backend import Backend 25 | from Shine.Configuration.Globals import Globals 26 | from Shine.Configuration.ModelFile import ModelFile 27 | from Shine.Configuration.TargetDevice import TargetDevice 28 | 29 | BACKEND_MODNAME = "File" 30 | 31 | 32 | class Storage(ModelFile): 33 | """Storage file for backend File.""" 34 | 35 | def __init__(self, sep=":", linesep="\n"): 36 | ModelFile.__init__(self, sep, linesep) 37 | self.add_custom('mgt', FileDevice(), multiple=True) 38 | self.add_custom('mdt', FileDevice(), multiple=True) 39 | self.add_custom('ost', FileDevice(), multiple=True) 40 | 41 | def get_target_devices(self, tgt_type): 42 | return [TargetDevice(tgt_type, tgt) 43 | for tgt in self.elements(tgt_type).as_dict()] 44 | 45 | 46 | class FileDevice(ModelFile): 47 | """ModelFile sub element representing a Target.""" 48 | 49 | def __init__(self, sep='=', linesep=' '): 50 | ModelFile.__init__(self, sep, linesep) 51 | self.add_element('tag', check='string') 52 | self.add_element('node', check='string') 53 | self.add_element('ha_node', check='string', multiple=True) 54 | self.add_element('dev', check='path') 55 | self.add_element('size', check='digit') 56 | self.add_element('jdev', check='path') 57 | self.add_element('jsize', check='digit') 58 | 59 | 60 | class File(Backend): 61 | 62 | def __init__(self): 63 | Backend.__init__(self) 64 | self.storage_file = None 65 | self.status_clients = {} 66 | 67 | def get_name(self): 68 | return "File" 69 | 70 | def get_desc(self): 71 | return "File Backend System." 72 | 73 | def start(self): 74 | pass 75 | 76 | def stop(self): 77 | for cli in self.status_clients.values(): 78 | cli.close() 79 | self.status_clients = {} 80 | 81 | def _start_storage(self): 82 | storage = Storage() 83 | storage.load(Globals().get_storage_file()) 84 | self.storage_file = storage 85 | 86 | def _start_status_client(self, fs_name): 87 | 88 | status_dir = Globals().get_status_dir() 89 | if not os.path.exists(status_dir): 90 | os.mkdir(status_dir) 91 | 92 | status_file = os.path.join(status_dir, fs_name) 93 | 94 | self.status_clients[fs_name] = shelve.open(status_file) 95 | 96 | def get_target_devices(self, target, fs_name=None, update_mode=None): 97 | """ 98 | Get target storage devices. 99 | """ 100 | if not self.storage_file: 101 | self._start_storage() 102 | return self.storage_file.get_target_devices(target) 103 | 104 | def register_fs(self, fs): 105 | """ 106 | This function is used to register a a filesystem configuration to the backend 107 | """ 108 | pass 109 | 110 | def unregister_fs(self, fs): 111 | """ 112 | This function is used to remove a filesystem configuration to the backend 113 | """ 114 | pass 115 | 116 | def register_target(self, fs, target): 117 | """ 118 | Set the specified `target', used by `fs', as 'in use' in the backend. 119 | 120 | This target could not be use anymore for other filesystems. 121 | """ 122 | pass 123 | 124 | def unregister_target(self, fs, target): 125 | """ 126 | Set the specified `target', used by `fs', as available in the backend. 127 | 128 | This target could be now reuse, for other targets of the same 129 | filesystem or any other one. 130 | """ 131 | pass 132 | -------------------------------------------------------------------------------- /lib/Shine/Configuration/Backend/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cea-hpc/shine/11f329dcceae798eb9a4262f9be25f158e63c3f8/lib/Shine/Configuration/Backend/__init__.py -------------------------------------------------------------------------------- /lib/Shine/Configuration/Exceptions.py: -------------------------------------------------------------------------------- 1 | # Exceptions.py -- Configuration exception classes 2 | # Copyright (C) 2007-2011 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | class ConfigException(Exception): 22 | """Generic exception for Shine.Configuration.""" 23 | 24 | class ConfigDeviceNotFoundError(ConfigException): 25 | """A target, described in a model file, cannot be found.""" 26 | def __init__(self, model_dev): 27 | msg = "No matching device found for \"%s\"" % model_dev 28 | ConfigException.__init__(self, msg) 29 | self.model_dev = model_dev 30 | 31 | class ConfigInvalidFileSystem(ConfigException): 32 | """Error indicating the filesystem configuration is not correct.""" 33 | def __init__(self, fs, message): 34 | ConfigException.__init__(self, message) 35 | self._fs = fs 36 | -------------------------------------------------------------------------------- /lib/Shine/Configuration/Globals.py: -------------------------------------------------------------------------------- 1 | # Globals.py -- Configuration of global parameters 2 | # Copyright (C) 2007-2015 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | """ 22 | Classes used to manipulate Shine global configuration file. 23 | 24 | This is mostly done using the Globals singleton. 25 | """ 26 | 27 | import os 28 | 29 | from Shine.Configuration.ModelFile import ModelFile, SimpleElement 30 | 31 | 32 | class Globals(object): 33 | """ 34 | Global paramaters configuration class. 35 | Design Pattern: Singleton 36 | """ 37 | __instance = None 38 | 39 | DEFAULT_CONF_FILE = "/etc/shine/shine.conf" 40 | 41 | def __new__(cls): 42 | if not Globals.__instance: 43 | Globals.__instance = Globals._Globals() 44 | # Load config file 45 | if os.path.exists(cls.DEFAULT_CONF_FILE): 46 | Globals.__instance.load(cls.DEFAULT_CONF_FILE) 47 | return Globals.__instance 48 | 49 | def __getattr__(self, attr): 50 | return getattr(self.__instance, attr) 51 | 52 | def __setattr__(self, attr, val): 53 | return setattr(self.__instance, attr, val) 54 | 55 | 56 | class _Globals(ModelFile): 57 | 58 | def __init__(self, sep="=", linesep="\n"): 59 | ModelFile.__init__(self, sep, linesep) 60 | 61 | # Backend stuff 62 | self.add_element('backend', check='enum', 63 | default='None', values=['ClusterDB', 'File', 'None']) 64 | self.add_element('storage_file', check='path', 65 | default='/etc/shine/storage.conf') 66 | self.add_element('status_dir', check='path', 67 | default='/var/cache/shine/status') 68 | 69 | # Config dirs 70 | self.add_element('conf_dir', check='path', 71 | default='/var/cache/shine/conf') 72 | self.add_element('lmf_dir', check='path', 73 | default='/etc/shine/models') 74 | self.add_element('tuning_file', check='path') 75 | 76 | # Timeouts 77 | self.add_element('ssh_connect_timeout', check='digit', 78 | default=30) 79 | self.add_element('ssh_fanout', check='digit', 80 | default=0) 81 | self.add_element('default_timeout', check='digit', 82 | default=30) 83 | 84 | # Commands 85 | self.add_element('command_path', check='path') 86 | 87 | # Lustre version 88 | self.add_element('lustre_version', check='string') 89 | 90 | # CLI 91 | self.add_element('color', check='enum', 92 | default='auto', values=['never', 'always', 'auto']) 93 | 94 | # TO BE IMPLEMENTED 95 | self.add_element('start_timeout', check='digit') 96 | self.add_element('mount_timeout', check='digit') 97 | self.add_element('stop_timeout', check='digit') 98 | self.add_element('status_timeout', check='digit') 99 | self.add_element('log_file', check='path') 100 | self.add_element('log_level', check='string') 101 | 102 | def add_element(self, name, multiple=False, fold=False, **kwargs): 103 | """ 104 | For this class, all elements are replaced by the local 105 | DefaultElement class. 106 | 107 | Only for convenience. 108 | """ 109 | self.add_custom(name, DefaultElement(**kwargs), multiple, fold) 110 | 111 | def lustre_version_is_smaller(self, version): 112 | """ 113 | Return true if the Lustre version defined in configuration 114 | is smaller than the one provided. 115 | If no version is speficied in configuration, it always returns 116 | False. 117 | """ 118 | if 'lustre_version' in self: 119 | return self['lustre_version'].split('.') < version.split('.') 120 | else: 121 | return False 122 | 123 | def get_backend(self): 124 | return self.get('backend') 125 | 126 | def get_storage_file(self): 127 | return self.get('storage_file') 128 | 129 | def get_status_dir(self): 130 | return self.get('status_dir') 131 | 132 | def get_conf_dir(self): 133 | return self.get('conf_dir') 134 | 135 | def get_lmf_dir(self): 136 | return self.get('lmf_dir') 137 | 138 | def get_tuning_file(self): 139 | return self.get('tuning_file') 140 | 141 | def get_ssh_connect_timeout(self): 142 | return self.get('ssh_connect_timeout') 143 | 144 | def get_ssh_fanout(self): 145 | return self.get('ssh_fanout') 146 | 147 | 148 | class DefaultElement(SimpleElement): 149 | """ 150 | SimpleElement with a special handling of default value. 151 | 152 | Currently, SimpleElement does return the default value only when get() is 153 | called. As I'm not sure it is a good idea to modify the global behaviour, 154 | this class is here to fix this only for Globals(). 155 | """ 156 | 157 | def __iter__(self): 158 | yield self.get() 159 | 160 | def __str__(self): 161 | return str(self.get()) 162 | 163 | def __len__(self): 164 | return int(self.get() is not None) 165 | -------------------------------------------------------------------------------- /lib/Shine/Configuration/TargetDevice.py: -------------------------------------------------------------------------------- 1 | # TargetDevice.py -- Representation of a Target Device 2 | # Copyright (C) 2007 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | 22 | import copy 23 | 24 | # 25 | # ost: tag=ost1_cors115 node=cors115 dev=/dev/cciss/c0d3 index=3 26 | # 27 | 28 | class TargetDevice: 29 | """ Objet representation of a target (mgt, mdt, ost) device (/dev/stuff) 30 | """ 31 | def __init__(self, target, dic): 32 | self.target = target 33 | self.params = copy.copy(dic) 34 | 35 | def __contains__(self, key): 36 | return key in self.params 37 | 38 | def get(self, key): 39 | return self.params.get(key) 40 | 41 | def getline(self): 42 | line = "" 43 | for k, v in self.params.items(): 44 | if isinstance(v, list): 45 | for lv in v: 46 | line += "%s=%s " % (k, lv) 47 | else: 48 | line += "%s=%s " % (k, v) 49 | return line.strip() 50 | 51 | def has_index(self): 52 | return 'index' in self.params 53 | 54 | def add_index(self, index): 55 | self.params['index'] = index 56 | 57 | def index(self): 58 | return int(self.params['index']) 59 | 60 | def add_active(self, active): 61 | self.params['active'] = active 62 | 63 | def __str__(self): 64 | node = self.params.get('node', '') 65 | ha_node = self.params.get('ha_node') 66 | if not ha_node: 67 | ha_node = "" 68 | else: 69 | ha_node = ",%s" % ",".join(ha_node) 70 | index = self.params.get('index', 'AUTO') 71 | tag = self.params.get('tag', '') 72 | dev = self.params.get('dev', '') 73 | jdev = self.params.get('jdev', '') 74 | if jdev: 75 | jdev = "jdev=%s, " % jdev 76 | group = self.params.get('group', '') 77 | if group: 78 | group = ", group=%s" % group 79 | return "%s on %s%s tag=\"%s\" (%sindex=%s%s)" % \ 80 | (dev, node, ha_node, tag, jdev, index, group) 81 | 82 | -------------------------------------------------------------------------------- /lib/Shine/Configuration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cea-hpc/shine/11f329dcceae798eb9a4262f9be25f158e63c3f8/lib/Shine/Configuration/__init__.py -------------------------------------------------------------------------------- /lib/Shine/Lustre/Actions/Execute.py: -------------------------------------------------------------------------------- 1 | # Execute.py -- Generic command execution 2 | # Copyright (C) 2012 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | """ 22 | This module contains the specific version of FSAction class to implement 23 | Execute Action. 24 | """ 25 | 26 | from Shine.Lustre.Actions.Action import FSAction 27 | 28 | class Execute(FSAction): 29 | """Generic command execution for any component.""" 30 | 31 | NAME = 'execute' 32 | 33 | CHECK_MOUNTDATA = False 34 | 35 | def _prepare_cmd(self): 36 | """ 37 | There is no real preparation for Execute, as the command syntax is 38 | exactly the content of additional options (-o). 39 | """ 40 | return [ self.addopts ] 41 | -------------------------------------------------------------------------------- /lib/Shine/Lustre/Actions/Fsck.py: -------------------------------------------------------------------------------- 1 | # Fsck.py -- Lustre action class: fsck 2 | # Copyright (C) 2010-2013 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | """ 22 | Action class to check (fsck) target filesystem coherency. 23 | """ 24 | 25 | import time 26 | import logging 27 | 28 | from Shine.Lustre.Actions.Action import Action, FSAction, Result, ErrorResult, \ 29 | ACT_OK, ACT_ERROR 30 | 31 | 32 | class FsckProgress(Result): 33 | """ 34 | Result message sent with 'progress' event, to provide fsck command 35 | progression. 36 | """ 37 | 38 | # Number of passes in a e2fsck run. 39 | _NB_PASSES = 5 40 | 41 | def __init__(self, passid, current, total): 42 | Result.__init__(self) 43 | self.pass_id = int(passid) 44 | self.pass_progress = float(current) / float(total) 45 | 46 | @property 47 | def progress(self): 48 | """Current fsck command progression value, between 1 and 100.""" 49 | return ((self.pass_id - 1 + self.pass_progress) // self._NB_PASSES) * 100 50 | 51 | class Fsck(FSAction): 52 | """ 53 | File system check using 'e2fsck'. 54 | """ 55 | 56 | NAME = 'fsck' 57 | 58 | # No mountdata check for fsck has it could be corrupted 59 | CHECK_MOUNTDATA = False 60 | 61 | def __init__(self, target, **kwargs): 62 | FSAction.__init__(self, target, **kwargs) 63 | 64 | # e2fsck send its progression on stderr 65 | self.stderr = True 66 | # As stderr msgtree is disabled, we have to track output ourselves. 67 | self._output = [] 68 | 69 | # Logging 70 | self.logger = logging.getLogger(__name__) 71 | 72 | # To track message rate 73 | self._last_progress = 0 74 | 75 | def _already_done(self): 76 | """Raise an exception if the target is mounted.""" 77 | self.comp.raise_if_started("Cannot fsck") 78 | return None 79 | 80 | def _prepare_cmd(self): 81 | """ 82 | Create the command line to run 'e2fsck'. 83 | """ 84 | command = ["e2fsck", '-f -C2', self.comp.dev] 85 | 86 | # Process additional options 87 | command.append(self.addopts or '-y') 88 | 89 | return command 90 | 91 | def _shell(self): 92 | """Call superclass _shell() method and add logging.""" 93 | FSAction._shell(self) 94 | self.logger.info("%-16s %s" % (self.comp.label, "Starting fsck")) 95 | 96 | def ev_read(self, worker): 97 | self.logger.info("%-16s %s" % (self.comp.label, worker.current_msg)) 98 | self._output.append(worker.current_msg) 99 | 100 | def ev_error(self, worker): 101 | try: 102 | line = worker.current_errmsg 103 | passid, current, total, dummy = line.split(' ', 3) 104 | 105 | result = FsckProgress(passid, current, total) 106 | # Limit message rate to one message per second max. 107 | if result.progress == 100 or self._last_progress + 1 < time.time(): 108 | self._last_progress = time.time() 109 | self.comp.action_event(self, 'progress', result=result) 110 | 111 | except ValueError: 112 | # Other error messages could be important 113 | self._output.append(worker.current_errmsg) 114 | 115 | def ev_close(self, worker): 116 | """ 117 | Check process termination status and generate appropriate events. 118 | 119 | Note that if fsck has correctly fixed some errors, actions will be 120 | considered as successful. 121 | """ 122 | 123 | if worker.did_timeout(): 124 | return FSAction.ev_close(self, worker) 125 | 126 | # We want to skip FSAction.ev_close(), just call the upper layer. 127 | Action.ev_close(self, worker) 128 | 129 | self.comp.lustre_check() 130 | 131 | # fsck returns 0=NOERROR, 1=OK_BUT_CORRECTION, 2=OK_BUT_REBOOT. 132 | # see man fsck. 133 | if worker.retcode() in (0, 1, 2, 4): 134 | # action succeeded 135 | result = Result(duration=self.duration, retcode=worker.retcode()) 136 | if worker.retcode() in (1, 2): 137 | result.message = "Errors corrected" 138 | if worker.retcode() == 4: # -n 139 | result.message = "Errors found but NOT corrected" 140 | self.comp.action_event(self, 'done', result) 141 | self.set_status(ACT_OK) 142 | else: 143 | # action failed 144 | msg = "\n".join(self._output) 145 | result = ErrorResult(msg, self.duration, worker.retcode()) 146 | self.comp.action_event(self, 'failed', result) 147 | self.set_status(ACT_ERROR) 148 | -------------------------------------------------------------------------------- /lib/Shine/Lustre/Actions/Install.py: -------------------------------------------------------------------------------- 1 | # Install.py -- Install Lustre FS configuration 2 | # Copyright (C) 2007-2015 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | import os.path 22 | 23 | from ClusterShell.NodeSet import NodeSet 24 | 25 | from Shine.Lustre.Actions.Action import Action, CommonAction, ACT_OK, ACT_ERROR 26 | 27 | class Install(CommonAction): 28 | """ 29 | Action class: install file configuration requirements on remote nodes. 30 | """ 31 | 32 | NAME = 'install' 33 | 34 | def __init__(self, nodes, fs, config_file, comps=None, **kwargs): 35 | CommonAction.__init__(self) 36 | assert config_file is not None 37 | self.nodes = nodes 38 | self.fs = fs 39 | self.config_file = config_file 40 | self._comps = comps 41 | self.dryrun = kwargs.get('dryrun', False) 42 | 43 | def _launch(self): 44 | """Copy local configuration file to remote nodes.""" 45 | msg = '[COPY] %s on %s' % (self.config_file, self.nodes) 46 | self.fs.hdlr.log('detail', msg=msg) 47 | if self.dryrun: 48 | self.set_status(ACT_OK) 49 | else: 50 | self.task.copy(self.config_file, self.config_file, 51 | nodes=self.nodes, handler=self) 52 | 53 | def ev_start(self, worker): 54 | CommonAction.ev_start(self, worker) 55 | name = os.path.basename(self.config_file) 56 | if len(self.nodes) > 8: 57 | msg = "Updating configuration file `%s' on %d servers" % \ 58 | (name, len(self.nodes)) 59 | else: 60 | msg = "Updating configuration file `%s' on %s" % (name, self.nodes) 61 | self.fs.hdlr.log('verbose', msg) 62 | 63 | def ev_close(self, worker): 64 | """ 65 | Check process termination status and generate appropriate events. 66 | """ 67 | Action.ev_close(self, worker) 68 | 69 | # Action timed out 70 | if worker.did_timeout(): 71 | nodes = NodeSet.fromlist(worker.iter_keys_timeout()) 72 | self.fs._handle_shine_proxy_error(nodes, "Nodes timed out") 73 | self.set_status(ACT_ERROR) 74 | 75 | # Action succeeded 76 | elif max(rc for rc, _ in worker.iter_retcodes()) == 0: 77 | self.set_status(ACT_OK) 78 | 79 | # Action failed 80 | else: 81 | for rc, nodes in worker.iter_retcodes(): 82 | if rc == 0: 83 | continue 84 | 85 | # Avoid warnings, flag this component in error state 86 | for comp in self._comps or []: 87 | comp.sanitize_state(nodes=worker.nodes) 88 | 89 | for output, nodes in worker.iter_buffers(match_keys=nodes): 90 | nodes = NodeSet.fromlist(nodes) 91 | msg = "Copy failed: %s" % output 92 | self.fs._handle_shine_proxy_error(nodes, msg) 93 | self.set_status(ACT_ERROR) 94 | -------------------------------------------------------------------------------- /lib/Shine/Lustre/Actions/Modules.py: -------------------------------------------------------------------------------- 1 | # Modules.py -- Load/Unload Lustre modules 2 | # Copyright (C) 2013-2015 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | import os 22 | 23 | """ 24 | Action classes for Lustre module managements. 25 | """ 26 | 27 | from Shine.Configuration.Globals import Globals 28 | 29 | from Shine.Lustre import ServerError 30 | from Shine.Lustre.Actions.Action import CommonAction, ACT_OK, ACT_ERROR, \ 31 | Result, ErrorResult, Action, ActionInfo 32 | 33 | class ServerAction(CommonAction): 34 | """ 35 | Base class for any server-specific Action. 36 | 37 | At minimum, _shell() method should be overloaded. 38 | """ 39 | 40 | def __init__(self, srv, **kwargs): 41 | CommonAction.__init__(self) 42 | self.dryrun = kwargs.get('dryrun', False) 43 | self.server = srv 44 | 45 | def info(self): 46 | """Return a ActionInfo describing this action.""" 47 | return ActionInfo(self, self.server) 48 | 49 | def _already_done(self): 50 | """ 51 | Verify if the action work is already done. 52 | 53 | Return a Result object if done, None otherwise. 54 | """ 55 | return False 56 | 57 | def _launch(self): 58 | """ 59 | Run the command to process the action. 60 | 61 | It checks the command could be really be run before running it. 62 | """ 63 | self.server.action_event(self, 'start') 64 | try: 65 | self.server.lustre_check() 66 | 67 | result = self._already_done() 68 | if not result: 69 | self._shell() 70 | else: 71 | self.server.action_event(self, 'done', result) 72 | self.set_status(ACT_OK) 73 | 74 | except ServerError as error: 75 | self.server.action_event(self, 'failed', Result(str(error))) 76 | self.set_status(ACT_ERROR) 77 | 78 | def _prepare_cmd(self): 79 | """ 80 | Return an array of command and arguments to be run by launch() method. 81 | """ 82 | raise NotImplementedError 83 | 84 | def _shell(self): 85 | """Create a command line and schedule it to be run by self.task""" 86 | # Call specific method to prepare command line 87 | command = self._prepare_cmd() 88 | 89 | # Extent path if defined 90 | path = Globals().get('command_path') 91 | if path: 92 | command.insert(0, "export PATH=%s:${PATH};" % path) 93 | 94 | # Add the command to be scheduled 95 | cmdline = ' '.join(command) 96 | 97 | self.server.hdlr.log('detail', msg='[RUN] %s' % cmdline) 98 | 99 | if self.dryrun: 100 | self.server.action_event(self, 'done') 101 | self.set_status(ACT_OK) 102 | else: 103 | self.task.shell(cmdline, handler=self) 104 | 105 | def ev_close(self, worker): 106 | """ 107 | Check process termination status and set action status. 108 | """ 109 | Action.ev_close(self, worker) 110 | 111 | self.server.lustre_check() 112 | 113 | # Action timed out 114 | if worker.did_timeout(): 115 | self.server.action_event(self, 'timeout') 116 | self.set_status(ACT_ERROR) 117 | 118 | # Action succeeded 119 | elif worker.retcode() == 0: 120 | result = Result(duration=self.duration, retcode=worker.retcode()) 121 | self.server.action_event(self, 'done', result) 122 | self.set_status(ACT_OK) 123 | 124 | # Action failed 125 | else: 126 | result = ErrorResult(worker.read(), self.duration, worker.retcode()) 127 | self.server.action_event(self, 'failed', result) 128 | self.set_status(ACT_ERROR) 129 | 130 | class LoadModules(ServerAction): 131 | """ 132 | Load some lustre modules using modprobe. 133 | 134 | By default, it is 'lustre', use `modname' if you want to load another one. 135 | """ 136 | 137 | NAME = 'load modules' 138 | 139 | def __init__(self, srv, modname='lustre', options=None, **kwargs): 140 | ServerAction.__init__(self, srv, **kwargs) 141 | self._modname = modname 142 | self._options = options 143 | 144 | def info(self): 145 | """Return a ActionInfo describing this action.""" 146 | return ActionInfo(self, self.server, 147 | "load module '%s'" % self._modname) 148 | 149 | def _already_done(self): 150 | if self._modname in self.server.modules: 151 | return Result("'%s' is already loaded" % self._modname) 152 | 153 | def _prepare_cmd(self): 154 | command = ['modprobe %s' % self._modname] 155 | if self._options is not None: 156 | command.append(' "%s"' % self._options) 157 | return command 158 | 159 | 160 | class UnloadModules(ServerAction): 161 | """ 162 | Unload all lustre modules using 'lustre_rmmod' 163 | """ 164 | 165 | NAME = 'unload modules' 166 | 167 | def _device_count(self): 168 | """Return the number of loaded Lustre devices.""" 169 | count = 0 170 | try: 171 | devicesfile = '/sys/kernel/debug/lustre/devices' 172 | # Compat code for lustre versions prior to 2.10 (2.9 and below) 173 | if os.access('/proc/fs/lustre/devices', os.F_OK): 174 | devicesfile = '/proc/fs/lustre/devices' 175 | 176 | devices = open(devicesfile) 177 | 178 | for line in devices.readlines(): 179 | 180 | # Workaround for racy Lustre behaviour. 181 | # When unmounting, some lustre devs could be a little bit slow 182 | # to be cleared, ignore it as module unloading will probably be 183 | # ok. 184 | _, state, _, _, _, refcnt = line.split() 185 | if state == 'ST' and refcnt == '0': 186 | continue 187 | 188 | count += 1 189 | 190 | devices.close() 191 | except IOError: 192 | pass 193 | 194 | return count 195 | 196 | def _already_done(self): 197 | if len(self.server.modules) == 0: 198 | return Result("modules already unloaded") 199 | 200 | # If some devices are still loaded, do not try to unload 201 | # and do not consider this as an error. 202 | count = self._device_count() 203 | if count > 0: 204 | return Result('ignoring, still %d in-use lustre device(s)' % count) 205 | 206 | # Check still in use? 207 | self.server.raise_if_mod_in_use() 208 | 209 | def _prepare_cmd(self): 210 | return ['lustre_rmmod'] 211 | -------------------------------------------------------------------------------- /lib/Shine/Lustre/Actions/StartClient.py: -------------------------------------------------------------------------------- 1 | # StartClient.py -- Mount client 2 | # Copyright (C) 2009-2013 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | """ 22 | This module contains the FSAction class for mounting a Lustre filesystem, from 23 | a client side. 24 | """ 25 | 26 | from Shine.Lustre.Actions.Action import FSAction, Result 27 | 28 | class StartClient(FSAction): 29 | """ 30 | File system client start (ie. mount) action class. 31 | """ 32 | 33 | NAME = 'mount' 34 | NEEDED_MODULES = ['lustre'] 35 | 36 | def _already_done(self): 37 | """Return a Result object if the client is already mounted.""" 38 | if self.comp.is_started(): 39 | return Result("%s is already mounted on %s" % 40 | (self.comp.fs.fs_name, self.comp.mtpt)) 41 | else: 42 | return None 43 | 44 | def _prepare_cmd(self): 45 | """ 46 | Prepare client file system mount command line. 47 | """ 48 | 49 | mount_path = self._vars_substitute(self.comp.mount_path) 50 | 51 | command = ["mkdir -p \"%s\"" % mount_path] 52 | command += ["&& /bin/mount -t lustre"] 53 | 54 | options = [] 55 | 56 | # Mount options from configuration 57 | if self.comp.mount_options: 58 | options += [ self.comp.mount_options ] 59 | 60 | # Mount options from command line 61 | if self.addopts: 62 | options += [ self.addopts ] 63 | 64 | if len(options): 65 | command.append("-o " + ','.join(options)) 66 | 67 | # MGS NIDs 68 | # List of node nids ['foo1@tcp0,foo1@tcp1', 'foo2@tcp0,foo2@tcp1'] 69 | nodenids = [','.join(nids) for nids in self.comp.fs.get_mgs_nids()] 70 | mgsfullnid = ':'.join(nodenids) 71 | command.append("%s:/%s" % (mgsfullnid, self.comp.fspath)) 72 | 73 | command.append(mount_path) 74 | 75 | return command 76 | -------------------------------------------------------------------------------- /lib/Shine/Lustre/Actions/StartRouter.py: -------------------------------------------------------------------------------- 1 | # StartRouter.py -- Start router 2 | # Copyright (C) 2010-2013 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | """Action class to handle router start command and event handling.""" 22 | 23 | from Shine.Lustre.Actions.Action import FSAction, Result 24 | 25 | class StartRouter(FSAction): 26 | """ 27 | File system router (ie: start lnet) start class 28 | """ 29 | 30 | NAME = 'start' 31 | 32 | def _already_done(self): 33 | """Return a Result object is the router is already enabled.""" 34 | if self.comp.is_started(): 35 | return Result('router is already enabled') 36 | else: 37 | return None 38 | 39 | def _prepare_cmd(self): 40 | """Start LNET which will start router if properly configured.""" 41 | return [ "/sbin/modprobe ptlrpc" ] 42 | -------------------------------------------------------------------------------- /lib/Shine/Lustre/Actions/StartTarget.py: -------------------------------------------------------------------------------- 1 | # StartTarget.py -- Lustre action class : start (mount) target 2 | # Copyright (C) 2009-2013 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | """ 22 | This module contains a FSAction class to start a Lustre target. 23 | """ 24 | 25 | import os 26 | 27 | from ClusterShell.Task import task_self 28 | 29 | from Shine.Configuration.Globals import Globals 30 | 31 | from Shine.Lustre.Actions.Action import FSAction, Result 32 | 33 | class StartTarget(FSAction): 34 | """ 35 | File system target start action class. 36 | 37 | Lustre, since 1.6, starts a target simply by mounting it. 38 | """ 39 | 40 | NAME = 'start' 41 | 42 | def __init__(self, target, **kwargs): 43 | FSAction.__init__(self, target, **kwargs) 44 | self.mount_options = kwargs.get('mount_options') 45 | self.mount_paths = kwargs.get('mount_paths') 46 | 47 | def _already_done(self): 48 | """Return a Result object is the target is already mounted.""" 49 | 50 | # Already done? 51 | if self.comp.is_started(): 52 | return Result("%s is already started" % self.comp.label) 53 | 54 | # LBUG #18624 55 | if not self.comp.dev_isblk: 56 | task_self().set_info("fanout", 1) 57 | 58 | return None 59 | 60 | def _prepare_cmd(self): 61 | """Mount file system target.""" 62 | 63 | # If there is a user-defined path 64 | if self.mount_paths and self.comp.TYPE in self.mount_paths: 65 | mount_path = self.mount_paths[self.comp.TYPE] 66 | else: 67 | # Default mount path 68 | mount_path = "/mnt/$fs_name/$type/$index" 69 | 70 | # Replace variables 71 | var_map = {'index': str(self.comp.index), 72 | 'dev' : os.path.basename(self.comp.dev)} 73 | if self.comp.journal: 74 | var_map['jdev'] = os.path.basename(self.comp.journal.dev) 75 | 76 | mount_path = self._vars_substitute(mount_path, var_map) 77 | 78 | # 79 | # Build command 80 | # 81 | command = ["mkdir -p \"%s\"" % mount_path] 82 | command += ["&& /bin/mount -t lustre"] 83 | 84 | # Loop devices handling 85 | if not self.comp.dev_isblk: 86 | command.append("-o loop") 87 | 88 | options = [] 89 | # Mount options from configuration 90 | if self.mount_options and self.mount_options.get(self.comp.TYPE): 91 | options += [self.mount_options.get(self.comp.TYPE)] 92 | # Mount options from command line 93 | if self.addopts: 94 | options += [self.addopts] 95 | 96 | # When device detection order is variable, jdev could have a different 97 | # major/minor than the one it has on previous mount. 98 | # In this case, we must be sure we use the current one to avoid error. 99 | # 100 | # (Note: We can use `blkid' instead of jdev and extract the current 101 | # journal UUID if we have issue using directly jdev path.) 102 | if self.comp.journal: 103 | majorminor = os.stat(self.comp.journal.dev).st_rdev 104 | options += ["journal_dev=%#x" % majorminor] 105 | 106 | if len(options): 107 | command.append('-o ' + ','.join(options)) 108 | 109 | command.append(self.comp.dev) 110 | command.append(mount_path) 111 | 112 | return command 113 | 114 | def needed_modules(self): 115 | if Globals().lustre_version_is_smaller('2.4') or \ 116 | not Globals().lustre_version_is_smaller('2.5'): 117 | return ['lustre', 'ldiskfs'] 118 | else: 119 | # lustre 2.4 needs fsfilt_ldiskfs 120 | return ['lustre', 'fsfilt_ldiskfs'] 121 | -------------------------------------------------------------------------------- /lib/Shine/Lustre/Actions/Status.py: -------------------------------------------------------------------------------- 1 | # Status.py -- Check a component status 2 | # Copyright (C) 2012-2015 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | """ 22 | This module contains the version of FSAction class to implement component 23 | status checking. 24 | """ 25 | 26 | from Shine.Lustre.Actions.Action import FSAction, ACT_OK 27 | 28 | class Status(FSAction): 29 | """ 30 | Status action triggers component status checking. 31 | 32 | It does not run an external command. 33 | """ 34 | 35 | NAME = 'status' 36 | 37 | def _shell(self): 38 | """ 39 | No-op method. Status command does not need to run an external command. 40 | """ 41 | self.set_status(ACT_OK) 42 | self.comp.action_event(self, 'done') 43 | -------------------------------------------------------------------------------- /lib/Shine/Lustre/Actions/StopClient.py: -------------------------------------------------------------------------------- 1 | # StopClient.py -- Umount client 2 | # Copyright (C) 2009-2012 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | """ 22 | Action class to stop Lustre client. 23 | """ 24 | 25 | from Shine.Lustre.Actions.Action import FSAction, Result 26 | 27 | class StopClient(FSAction): 28 | """ 29 | File system client stop (ie. umount) action class. 30 | """ 31 | 32 | NAME = 'umount' 33 | 34 | def _already_done(self): 35 | """Return a Result object if the filesystem is not mounted already.""" 36 | if self.comp.is_stopped(): 37 | return Result("%s is not mounted" % self.comp.fs.fs_name) 38 | else: 39 | return None 40 | 41 | def _prepare_cmd(self): 42 | """ 43 | Unmount file system client. 44 | """ 45 | command = ["umount"] 46 | 47 | # Process additional option for umount command 48 | if self.addopts: 49 | command.append(self.addopts) 50 | 51 | mount_path = self._vars_substitute(self.comp.mount_path) 52 | command.append(mount_path) 53 | 54 | return command 55 | -------------------------------------------------------------------------------- /lib/Shine/Lustre/Actions/StopRouter.py: -------------------------------------------------------------------------------- 1 | # StopRouter.py -- Stop router 2 | # Copyright (C) 2010-2013 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | """Action class to handle router stop command and event handling.""" 22 | 23 | from Shine.Lustre.Actions.Action import FSAction, Result 24 | 25 | class StopRouter(FSAction): 26 | """ 27 | File system router (ie: stop lnet) stop class. 28 | """ 29 | 30 | NAME = 'stop' 31 | 32 | def _already_done(self): 33 | """Return a Result object if the router is already stopped.""" 34 | if self.comp.is_stopped(): 35 | return Result('router is already disabled') 36 | else: 37 | return None 38 | 39 | def _prepare_cmd(self): 40 | """Stop LNET.""" 41 | # Depending on what was done before on the node (only router) or if some 42 | # targets or clients were also started, the only safe and simple way to 43 | # stop a router is to unload all modules. 44 | return [ "lustre_rmmod" ] 45 | -------------------------------------------------------------------------------- /lib/Shine/Lustre/Actions/StopTarget.py: -------------------------------------------------------------------------------- 1 | # StopTarget.py -- Lustre action class: stop (umount) target 2 | # Copyright (C) 2009-2012 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | """ 22 | Action class to stop Lustre target. 23 | """ 24 | 25 | from ClusterShell.Task import task_self 26 | 27 | from Shine.Lustre.Actions.Action import FSAction, Result 28 | 29 | class StopTarget(FSAction): 30 | """ 31 | File system target start action class. 32 | """ 33 | 34 | NAME = 'stop' 35 | 36 | def _already_done(self): 37 | """Return a Result object is the target is already unmounted.""" 38 | if self.comp.is_stopped(): 39 | return Result(message="%s is already stopped" % self.comp.label) 40 | 41 | # LBUG #18624 42 | if not self.comp.dev_isblk: 43 | task_self().set_info("fanout", 1) 44 | 45 | return None 46 | 47 | def _prepare_cmd(self): 48 | """ 49 | Unmount file system target. 50 | """ 51 | 52 | command = ["umount"] 53 | 54 | # Also free the loop device if needed 55 | if not self.comp.dev_isblk: 56 | command.append("-d") 57 | 58 | # Process additional umount options 59 | if self.addopts: 60 | command.append(self.addopts) 61 | 62 | command.append(self.comp.mntdev) 63 | 64 | return command 65 | -------------------------------------------------------------------------------- /lib/Shine/Lustre/Actions/Tune.py: -------------------------------------------------------------------------------- 1 | # Tune.py -- Tune Lustre /proc files. 2 | # Copyright (C) 2013-2015 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | """ 22 | Classes used the apply tuning on local node, either from tuning.conf or 23 | dynamically created. 24 | """ 25 | 26 | from Shine.Lustre.Actions.Action import CommonAction, ActionGroup, \ 27 | ActionInfo, \ 28 | ACT_OK, ACT_ERROR, ErrorResult 29 | 30 | _SRVTYPE_MAP = { 31 | 'mgt': 'mgs', 32 | 'mdt': 'mds', 33 | 'ost': 'oss', 34 | 'client': 'client', 35 | 'router': 'router' 36 | } 37 | 38 | class _TuningAction(CommonAction): 39 | """Action handling the command to modify only 1 tuning.""" 40 | 41 | NAME = "tuning" 42 | 43 | def __init__(self, action, command): 44 | CommonAction.__init__(self) 45 | self._tune = action 46 | self._command = command 47 | self.dryrun = action.dryrun 48 | 49 | def _launch(self): 50 | self._tune._server.hdlr.log('detail', 51 | msg='[RUN] %s' % self._command) 52 | if self.dryrun: 53 | self.set_status(ACT_OK) 54 | else: 55 | self.task.shell(self._command, handler=self) 56 | 57 | 58 | class Tune(ActionGroup): 59 | """Action to apply all tunings for the local node.""" 60 | 61 | NAME = "tune" 62 | 63 | def __init__(self, srv, tuning_conf, comps, fsname, **kwargs): 64 | ActionGroup.__init__(self) 65 | self._server = srv 66 | self._comps = comps 67 | self._conf = tuning_conf 68 | self._fsname = fsname 69 | self._init = False 70 | self.dryrun = kwargs.get('dryrun', False) 71 | 72 | def info(self): 73 | """Return a ActionInfo describing this action.""" 74 | return ActionInfo(self, self._server, 'apply tunings') 75 | 76 | def _add_actions(self): 77 | """ 78 | Create all individual tunings Action. 79 | 80 | To be run before this fake group action is really launched. 81 | """ 82 | srvtypes = set([_SRVTYPE_MAP.get(comp.TYPE) for comp in self._comps]) 83 | srvname = str(self._server.hostname) 84 | 85 | tunings = self._conf.get_params_for_name(srvname, srvtypes) 86 | for tuning in tunings: 87 | cmds = tuning.build_tuning_command(self._fsname) 88 | for command in cmds: 89 | self.add(_TuningAction(self, command)) 90 | 91 | def set_status(self, status): 92 | """ 93 | Update action status. 94 | 95 | If this is a final state, raise corresponding events. 96 | """ 97 | # Do not raise events if start event was not raised. 98 | # Start event is raised when init is set. 99 | if self._init: 100 | if status == ACT_OK: 101 | self._server.action_event(self, 'done') 102 | elif status == ACT_ERROR: 103 | # Build an error string 104 | errors = [] 105 | for act in self: 106 | if act.status() == ACT_ERROR: 107 | errors.append("'%s' failed" % act._command) 108 | result = ErrorResult("\n".join(errors)) 109 | self._server.action_event(self, 'failed', result) 110 | 111 | ActionGroup.set_status(self, status) 112 | 113 | def _launch(self): 114 | # First time launch is called, we need to create all the sub actions. 115 | # As the graph is calling launch a couple of times, we need to do this 116 | # only once. 117 | if not self._init: 118 | self._server.action_event(self, 'start') 119 | self._add_actions() 120 | self._init = True 121 | # Then call the real _launch() 122 | ActionGroup._launch(self) 123 | -------------------------------------------------------------------------------- /lib/Shine/Lustre/Actions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cea-hpc/shine/11f329dcceae798eb9a4262f9be25f158e63c3f8/lib/Shine/Lustre/Actions/__init__.py -------------------------------------------------------------------------------- /lib/Shine/Lustre/Disk.py: -------------------------------------------------------------------------------- 1 | # Disk.py -- Pythonized Lustre Disk 2 | # Copyright (C) 2009-2013 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | 22 | """ 23 | Lustre Disk abstraction module. 24 | """ 25 | 26 | import copy 27 | import os 28 | import stat 29 | import subprocess 30 | 31 | from Shine.Configuration.Globals import Globals 32 | 33 | ### From lustre/include/lustre_disk.h: 34 | 35 | # persistent mount data 36 | LDD_F_NEED_INDEX = 0x0010 # need an index assignment 37 | LDD_F_VIRGIN = 0x0020 # never registered 38 | LDD_F_UPDATE = 0x0040 # update all related config logs 39 | LDD_F_REWRITE_LDD = 0x0080 # rewrite the LDD 40 | LDD_F_WRITECONF = 0x0100 # regenerate all logs for this fs 41 | LDD_F_UPGRADE14 = 0x0200 # COMPAT_14 42 | LDD_F_PARAM = 0x0400 # process as lctl conf_param 43 | 44 | 45 | class DiskDeviceError(Exception): 46 | """ 47 | Associated device error. 48 | """ 49 | def __init__(self, disk, message=None): 50 | Exception.__init__(self, message) 51 | self._disk = disk 52 | 53 | 54 | class Disk: 55 | """ 56 | Represents a low-level Lustre Disk as defined in lustre/include/ 57 | lustre_disk.h. Base class for Lustre Target (see Target.py). 58 | """ 59 | 60 | def __init__(self, dev): 61 | self.dev = dev 62 | 63 | # filled by _device_check 64 | self.dev_isblk = False 65 | self.dev_size = 0 66 | 67 | # filled by _mountdata_check (use provided accessors if needed) 68 | self.ldd_svname = None 69 | self._ldd_flags = 0 70 | 71 | def update(self, other): 72 | """ 73 | Update my serializable fields from other/distant object. 74 | """ 75 | self.dev_isblk = other.dev_isblk 76 | self.dev_size = other.dev_size 77 | self.ldd_svname = copy.copy(other.ldd_svname) 78 | self._ldd_flags = other._ldd_flags 79 | 80 | def _device_check(self): 81 | """ 82 | Device sanity checking based on the stat() syscall. 83 | """ 84 | try: 85 | info = os.stat(self.dev) 86 | except OSError as error: 87 | raise DiskDeviceError(self, str(error)) 88 | 89 | mode = info[stat.ST_MODE] 90 | 91 | if stat.S_ISBLK(mode): 92 | # block device 93 | self.dev_isblk = True 94 | # get dev size 95 | partitions = open("/proc/partitions", 'r') 96 | try: 97 | dev = os.path.basename(os.path.realpath(self.dev)) 98 | for line in partitions: 99 | d_info = line.rstrip('\n').split(' ') 100 | if len(d_info) > 1 and d_info[-1] == dev: 101 | self.dev_size = int(d_info[-2]) * 1024 102 | break 103 | finally: 104 | partitions.close() 105 | 106 | elif stat.S_ISREG(mode): 107 | # regular file 108 | self.dev_isblk = False 109 | self.dev_size = int(info[stat.ST_SIZE]) 110 | else: 111 | # unsupported 112 | raise DiskDeviceError(self, "unsupported device type") 113 | 114 | def _mountdata_check(self, label_check=None): 115 | """Read device flags using 'tunefs.lustre'""" 116 | 117 | cmd = "tunefs.lustre --dryrun %s" % self.dev 118 | path = Globals().get('command_path') 119 | if path: 120 | cmd = "export PATH=%s:${PATH}; %s" % (path, cmd) 121 | 122 | process = subprocess.Popen([cmd], stdout=subprocess.PIPE, 123 | stderr=subprocess.PIPE, shell=True) 124 | output = process.communicate()[0] 125 | if process.returncode > 0: 126 | raise DiskDeviceError(self, "Failed to run 'tunefs.lustre' to " + 127 | "read flags (rc=%d)" % process.returncode) 128 | 129 | for line in output.splitlines(): 130 | line = line.strip() 131 | if line.startswith('Flags:'): 132 | self._ldd_flags = int(line.split(':')[1], 16) 133 | elif line.startswith('Target:'): 134 | self.ldd_svname = line.split(':', 1)[1].strip() 135 | elif line.startswith('Permanent disk data:'): 136 | break 137 | 138 | if label_check: 139 | # Lustre 2.3 changed the label patterns. 140 | # fsname and svname could be separated by '-', ':' and '=' 141 | # For compatibility reasons, we ignore ':' and '='. 142 | if len(self.ldd_svname) > 8 and self.ldd_svname[-8] in (':', '='): 143 | self.ldd_svname = "%s-%s" % (self.ldd_svname[:-8], 144 | self.ldd_svname[-7:]) 145 | 146 | if self.ldd_svname != label_check: 147 | raise DiskDeviceError(self, 148 | "Found service %s != %s on %s" % 149 | (self.ldd_svname, label_check, self.dev)) 150 | 151 | def flags(self): 152 | """Return a list of text flags set on this disk.""" 153 | lst = [] 154 | if self.has_need_index_flag(): 155 | lst.append("need_index") 156 | if self.has_first_time_flag(): 157 | lst.append("first_time") 158 | if self.has_update_flag(): 159 | lst.append("update") 160 | if self.has_rewrite_ldd_flag(): 161 | lst.append("rewrite_ldd") 162 | if self.has_writeconf_flag(): 163 | lst.append("writeconf") 164 | if self.has_upgrade14_flag(): 165 | lst.append("upgrade14") 166 | if self.has_param_flag(): 167 | lst.append("conf_param") 168 | return lst 169 | 170 | def has_need_index_flag(self): 171 | """LDD flag: need an index assignment""" 172 | return self._ldd_flags & LDD_F_NEED_INDEX 173 | 174 | def has_first_time_flag(self): 175 | """LDD flag: never registered""" 176 | return self._ldd_flags & LDD_F_VIRGIN 177 | 178 | def has_update_flag(self): 179 | """LDD flag: update all related config logs""" 180 | return self._ldd_flags & LDD_F_UPDATE 181 | 182 | def has_rewrite_ldd_flag(self): 183 | """LDD flag: rewrite the LDD""" 184 | return self._ldd_flags & LDD_F_REWRITE_LDD 185 | 186 | def has_writeconf_flag(self): 187 | """LDD flag: regenerate all logs for this fs""" 188 | return self._ldd_flags & LDD_F_WRITECONF 189 | 190 | def has_upgrade14_flag(self): 191 | """LDD flag: COMPAT 14""" 192 | return self._ldd_flags & LDD_F_UPGRADE14 193 | 194 | def has_param_flag(self): 195 | """LDD flag: process as lctl conf_param""" 196 | return self._ldd_flags & LDD_F_PARAM 197 | 198 | -------------------------------------------------------------------------------- /lib/Shine/Lustre/EventHandler.py: -------------------------------------------------------------------------------- 1 | # EventHandler.py -- Lustre event handling 2 | # Copyright (C) 2009-2015 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | __all__ = ['EventHandler'] 22 | 23 | # 24 | # Duplicate this method here to avoid cyclic import loop with 25 | # Shine.Lustre.Server 26 | # 27 | import socket 28 | _CACHE_HOSTNAME_SHORT = None 29 | def hostname_short(): 30 | """Return cached short host name. 31 | 32 | If not already cached, resolve and cache it. 33 | """ 34 | global _CACHE_HOSTNAME_SHORT 35 | if _CACHE_HOSTNAME_SHORT is None: 36 | _CACHE_HOSTNAME_SHORT = socket.getfqdn().split('.', 1)[0] 37 | return _CACHE_HOSTNAME_SHORT 38 | 39 | class EventHandler(object): 40 | """ 41 | Base class EventHandler. Event-based applications using the Shine library 42 | should override this class and handle events of their choice. 43 | """ 44 | 45 | def log(self, level, msg, **kwargs): 46 | """Send a log message through event handling""" 47 | self.local_event('log', level=level, msg=msg, **kwargs) 48 | 49 | def local_event(self, evtype, **kwargs): 50 | """Raise an event, automatically providing local node information.""" 51 | self.event_callback(evtype, node=hostname_short(), **kwargs) 52 | 53 | def event_callback(self, evtype, **kwargs): 54 | """ 55 | Base event handler. It is called for each event received. 56 | 57 | This event handler could be overload to implement your own event 58 | management. 59 | """ 60 | pass 61 | -------------------------------------------------------------------------------- /lib/Shine/Lustre/Router.py: -------------------------------------------------------------------------------- 1 | # Router.py -- Shine Lustre Router 2 | # Copyright (C) 2010-2013 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | """ 22 | Classes for Shine framework to manage Lustre LNET routers. 23 | """ 24 | 25 | import os 26 | 27 | from Shine.Lustre.Component import Component, ComponentError, \ 28 | MOUNTED, OFFLINE, TARGET_ERROR, RUNTIME_ERROR 29 | 30 | from Shine.Lustre.Actions.StartRouter import StartRouter 31 | from Shine.Lustre.Actions.StopRouter import StopRouter 32 | 33 | class Router(Component): 34 | """ 35 | Manages a LNET router in Shine framework. 36 | """ 37 | 38 | TYPE = 'router' 39 | DISPLAY_ORDER = 1 40 | START_ORDER = 1 41 | 42 | # 43 | # Text form for different router states. 44 | # 45 | # Could be nearly merged with Target state_text_map if MOUNTED value 46 | # becomes the same. 47 | STATE_TEXT_MAP = { 48 | None: "unknown", 49 | OFFLINE: "offline", 50 | TARGET_ERROR: "ERROR", 51 | MOUNTED: "online", 52 | RUNTIME_ERROR: "CHECK FAILURE" 53 | } 54 | 55 | def longtext(self): 56 | """ 57 | Return the routeur server name. 58 | """ 59 | return "router on %s" % self.server 60 | 61 | def lustre_check(self): 62 | """ 63 | Check Router health at Lustre level. 64 | 65 | Check LNET routing capabilities and change object state 66 | based on the results. 67 | """ 68 | 69 | # LNET is not loaded 70 | # Lustre 2.11+ moved lnet to sysfs, try both paths 71 | routesfile = '/sys/kernel/debug/lnet/routes' 72 | if not os.path.isfile(routesfile): 73 | routesfile='/proc/sys/lnet/routes' 74 | if not os.path.isfile(routesfile): 75 | self.state = OFFLINE 76 | return 77 | 78 | # Read routing information 79 | try: 80 | routes = open(routesfile) 81 | # read only first line 82 | state = routes.readline().strip().lower() 83 | except: 84 | self.state = RUNTIME_ERROR 85 | raise ComponentError(self, "Could not read routing information") 86 | 87 | # routing info tells this is ok? 88 | if state == "routing enabled": 89 | self.state = MOUNTED 90 | elif state == "routing disabled": 91 | self.state = TARGET_ERROR 92 | raise ComponentError(self, "Misconfigured router") 93 | else: 94 | self.state = RUNTIME_ERROR 95 | raise ComponentError(self, "Bad routing status") 96 | 97 | # 98 | # Client actions 99 | # 100 | 101 | def start(self, **kwargs): 102 | """Start a Lustre router.""" 103 | return StartRouter(self, **kwargs) 104 | 105 | def stop(self, **kwargs): 106 | """Stop a Lustre router.""" 107 | return StopRouter(self, **kwargs) 108 | -------------------------------------------------------------------------------- /lib/Shine/Lustre/Server.py: -------------------------------------------------------------------------------- 1 | # Server.py -- Lustre Server base class 2 | # Copyright (C) 2007-2015 CEA 3 | # 4 | # This file is part of shine 5 | # 6 | # This program is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU General Public License 8 | # as published by the Free Software Foundation; either version 2 9 | # of the License, or (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | # 20 | 21 | """ 22 | Lustre server management. 23 | """ 24 | 25 | import socket 26 | 27 | from ClusterShell.Task import NodeSet 28 | 29 | from Shine.Lustre import ServerError 30 | from Shine.Lustre.EventHandler import EventHandler 31 | from Shine.Lustre.Actions.Modules import LoadModules, UnloadModules 32 | from Shine.Lustre.Actions.Tune import Tune 33 | 34 | class ServerGroup(object): 35 | """ 36 | List of Server instance, with helpers to filter of display them. 37 | """ 38 | 39 | def __init__(self, iterable=None): 40 | self._list = [] 41 | if iterable is not None: 42 | self._list = list(iterable) 43 | 44 | def __len__(self): 45 | return len(self._list) 46 | 47 | def __getitem__(self, index): 48 | return self._list[index] 49 | 50 | def __iter__(self): 51 | return iter(self._list) 52 | 53 | def append(self, server): 54 | """Append the provided server at the end of group.""" 55 | self._list.append(server) 56 | 57 | def select(self, nodeset): 58 | """ 59 | Return a ServerGroup containing only server which hostname are 60 | present in provided nodeset. 61 | """ 62 | return ServerGroup((srv for srv in self if srv.hostname in nodeset)) 63 | 64 | def distant(self): 65 | """Return a new ServerGroup with only distant servers.""" 66 | return ServerGroup((srv for srv in self if not srv.is_local())) 67 | 68 | def nodeset(self): 69 | """Return a NodeSet from server hostnames.""" 70 | return NodeSet.fromlist((srv.hostname for srv in self)) 71 | 72 | 73 | class Server(object): 74 | """ 75 | Represents a node in the cluster, by its hostname and NIDs. 76 | 77 | Currently, it is link to no specific filesystem nor components. 78 | """ 79 | 80 | _CACHE_HOSTNAME_SHORT = None 81 | _CACHE_HOSTNAME_LONG = None 82 | 83 | def __init__(self, hostname, nids, hdlr=None): 84 | assert isinstance(nids, list) 85 | self.nids = nids 86 | self.hostname = NodeSet(hostname) 87 | self.modules = dict() 88 | self.action_enabled = True 89 | 90 | self.hdlr = hdlr or EventHandler() 91 | self._running_actions = [] 92 | 93 | def __str__(self): 94 | return "%s (%s)" % (self.hostname, ','.join(self.nids)) 95 | 96 | def __lt__(self, other): 97 | # Cast hostname into str until NodeSet is sortable 98 | return (str(self.hostname), self.nids) < (str(other.hostname), self.nids) 99 | 100 | @classmethod 101 | def hostname_long(cls): 102 | """ 103 | Return cached long host name. If not already cached, resolve and cache 104 | it. 105 | """ 106 | if not cls._CACHE_HOSTNAME_LONG: 107 | cls._CACHE_HOSTNAME_LONG = socket.getfqdn() 108 | return cls._CACHE_HOSTNAME_LONG 109 | 110 | @classmethod 111 | def hostname_short(cls): 112 | """ 113 | Return cached short host name. If not already cached, resolve and cache 114 | it. 115 | """ 116 | if not cls._CACHE_HOSTNAME_SHORT: 117 | cls._CACHE_HOSTNAME_SHORT = cls.hostname_long().split('.', 1)[0] 118 | return cls._CACHE_HOSTNAME_SHORT 119 | 120 | @classmethod 121 | def distant_servers(cls, servers): 122 | """ 123 | Filter the local host from the provided server list. 124 | """ 125 | if cls.hostname_long() in servers: 126 | return servers.difference(cls.hostname_long()) 127 | elif cls.hostname_short() in servers: 128 | return servers.difference(cls.hostname_short()) 129 | else: 130 | return servers 131 | 132 | def is_local(self): 133 | """ 134 | Return true if the node where this code is running matches the server 135 | node_name. 136 | This means node_name should either match the machine fully qualified 137 | domain name or machine short-name. 138 | """ 139 | srvname = str(self.hostname) 140 | return srvname in (self.hostname_long(), self.hostname_short()) 141 | 142 | def raise_if_mod_in_use(self): 143 | """Raise a ServerError if Lustre modules are currently in use.""" 144 | if self.modules.get('lustre', 0) > 0: 145 | raise ServerError(self, "Lustre modules are busy") 146 | 147 | def lustre_check(self): 148 | """ 149 | Verify server Lustre sanity. 150 | 151 | It analyzes which Lustre module is loaded and keeps it in self.modules 152 | """ 153 | self.modules.clear() 154 | try: 155 | modlist = open('/proc/modules') 156 | for line in modlist: 157 | modname, _, count, _ = line.split(' ', 3) 158 | if modname in ('libcfs', 'lustre', 'ldiskfs', 'fsfilt_ldiskfs'): 159 | self.modules[modname] = int(count) 160 | finally: 161 | modlist.close() 162 | # 163 | # Inprogress action methods 164 | # 165 | def _add_action(self, act): 166 | """Add the named action to the running action list.""" 167 | assert act not in self._running_actions 168 | self._running_actions.append(act) 169 | 170 | def _del_action(self, act): 171 | """Remove the named action from the running action list.""" 172 | self._running_actions.remove(act) 173 | 174 | def _list_action(self): 175 | """Return the running action list.""" 176 | return self._running_actions 177 | 178 | # 179 | # Event raising methods 180 | # 181 | def local_event(self, **kwargs): 182 | self.hdlr.local_event('server', **kwargs) 183 | 184 | def action_event(self, act, status, result=None): 185 | """Send an event.""" 186 | if status == 'start': 187 | self._add_action(act.NAME) 188 | elif status in ('done', 'timeout', 'failed'): 189 | self._del_action(act.NAME) 190 | self.local_event(info=act.info(), status=status, result=result) 191 | 192 | # 193 | # Actions 194 | # 195 | 196 | def tune(self, tuning_model, types, fs_name, **kwargs): 197 | """Tune server parameters.""" 198 | return Tune(self, tuning_model, types, fs_name, **kwargs) 199 | 200 | def load_modules(self, **kwargs): 201 | """Load lustre kernel modules.""" 202 | return LoadModules(self, **kwargs) 203 | 204 | def unload_modules(self, **kwargs): 205 | """Unload all lustre kernel modules.""" 206 | return UnloadModules(self, **kwargs) 207 | -------------------------------------------------------------------------------- /lib/Shine/Lustre/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | class ServerError(Exception): 3 | """Any error related a specific Server.""" 4 | def __init__(self, srv, message): 5 | Exception.__init__(self, message) 6 | self.server = srv 7 | 8 | class ComponentError(Exception): 9 | """Generic exception for any components.""" 10 | def __init__(self, comp, message): 11 | Exception.__init__(self, message) 12 | self.comp = comp 13 | -------------------------------------------------------------------------------- /lib/Shine/__init__.py: -------------------------------------------------------------------------------- 1 | public_version = "1.5" 2 | __version__ = "1.5" 3 | -------------------------------------------------------------------------------- /mkrelease.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Script to release shine RPMS 3 | 4 | VERSIONFILE="lib/Shine/__init__.py" 5 | 6 | # check usage 7 | if [ -z $1 ]; then 8 | echo "$0 " 9 | exit 1 10 | fi 11 | 12 | version=$1 13 | 14 | # check version 15 | if [ `grep -c "^public_version = \"$version\"" $VERSIONFILE` -eq 0 ]; then 16 | echo "Version doesn't match $VERSIONFILE:" 17 | echo 18 | cat $VERSIONFILE 19 | exit 1 20 | fi 21 | 22 | # build a source distribution 23 | python setup.py sdist || exit 1 24 | 25 | # build RPMs 26 | rpmbuild -ba \ 27 | --define "_rpmdir $PWD/dist" \ 28 | --define "_srcrpmdir $PWD/dist" \ 29 | --define "_sourcedir $PWD/dist" \ 30 | --clean shine.spec 31 | -------------------------------------------------------------------------------- /scripts/shine.init.redhat: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # chkconfig: 345 61 22 4 | # description: Lustre shine mounting script 5 | # config: /etc/sysconfig/shine 6 | # 7 | # This program is free software; you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation; either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program; if not, write to the Free Software 19 | # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 20 | 21 | # See also the shine homepage at: 22 | # http://shine.sourceforge.net/ 23 | 24 | 25 | # Config file, example of content: 26 | # FSLIST=ptmp,testfs 27 | # 28 | # If empty, all configured FS are started/stopped. You can see them 29 | # with ``shine show fs -v''. 30 | # 31 | CONFIG=/etc/sysconfig/shine 32 | 33 | # Load config file 34 | if [ -f "$CONFIG" ]; then 35 | . "$CONFIG" 36 | fi 37 | 38 | SHINE=/usr/sbin/shine 39 | 40 | # Source function library. 41 | . /etc/init.d/functions 42 | 43 | RETVAL=0 44 | prog="shine" 45 | 46 | start() { 47 | action $"Starting $prog: " $SHINE mount -L -q ${FSLIST:+-f $FSLIST} 48 | RETVAL=$? 49 | [ $RETVAL -eq 0 ] && touch /var/lock/subsys/shine 50 | } 51 | 52 | stop() { 53 | action $"Stopping $prog: " $SHINE umount -L -q ${FSLIST:+-f $FSLIST} 54 | RETVAL=$? 55 | [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/shine 56 | } 57 | 58 | dostatus() { 59 | $SHINE status -L ${FSLIST:+-f $FSLIST} 60 | RETVAL=$? 61 | } 62 | 63 | restart() { 64 | stop 65 | start 66 | RETVAL=$? 67 | } 68 | 69 | condrestart() { 70 | [ -e /var/lock/subsys/shine ] && restart || : 71 | } 72 | 73 | # See how we were called. 74 | case "$1" in 75 | start) 76 | start 77 | ;; 78 | stop) 79 | stop 80 | ;; 81 | status) 82 | dostatus 83 | ;; 84 | restart|reload) 85 | restart 86 | ;; 87 | condrestart) 88 | condrestart 89 | ;; 90 | *) 91 | echo $"Usage: $0 {start|stop|status|restart|condrestart}" 92 | exit 1 93 | esac 94 | 95 | exit $RETVAL 96 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [install] 2 | optimize = 1 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright (C) 2007-2017 CEA 4 | # 5 | # This file is part of shine 6 | # 7 | # This program is free software; you can redistribute it and/or 8 | # modify it under the terms of the GNU General Public License 9 | # as published by the Free Software Foundation; either version 2 10 | # of the License, or (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program; if not, write to the Free Software 19 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 20 | # 21 | 22 | from setuptools import setup, find_packages 23 | 24 | VERSION='1.5' 25 | 26 | setup(name='shine', 27 | version=VERSION, 28 | license='GPLv2+', 29 | description='Lustre administration utility', 30 | author='Aurelien Degremont', 31 | author_email='aurelien.degremont@cea.fr', 32 | url='https://github.com/cea-hpc/shine', 33 | download_url='https://github.com/cea-hpc/shine/releases/v%s' % VERSION, 34 | package_dir={'': 'lib'}, 35 | packages=find_packages('lib'), 36 | data_files=[('/etc/shine', 37 | ['conf/shine.conf', 38 | 'conf/storage.conf', 39 | 'conf/tuning.conf.example']), 40 | ('/etc/shine/models', ['conf/models/example.lmf']), 41 | ('/var/cache/shine/conf', ['conf/cache/README']), 42 | ('/usr/share/vim/vimfiles/syntax', 43 | ['doc/extras/vim/syntax/shine.vim', 44 | 'doc/extras/vim/syntax/shinefs.vim']), 45 | ('/usr/share/vim/vimfiles/ftdetect', 46 | ['doc/extras/vim/ftdetect/shine.vim']), 47 | ('/usr/share/shine', ['scripts/shine.init.redhat'])], 48 | entry_points={'console_scripts': ['shine=Shine.Controller:run']}, 49 | classifiers=[ 50 | "Development Status :: 5 - Production/Stable", 51 | "Environment :: Console", 52 | "Intended Audience :: System Administrators", 53 | "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)", 54 | "Operating System :: POSIX :: Linux", 55 | "Programming Language :: Python", 56 | "Programming Language :: Python :: 2.4", 57 | "Programming Language :: Python :: 2.5", 58 | "Programming Language :: Python :: 2.6", 59 | "Programming Language :: Python :: 2.7", 60 | "Topic :: System :: Clustering", 61 | "Topic :: System :: Distributed Computing" 62 | ], 63 | ) 64 | -------------------------------------------------------------------------------- /shine.spec: -------------------------------------------------------------------------------- 1 | %define vimdatadir %{_datadir}/vim/vimfiles 2 | 3 | Name: shine 4 | Version: 1.5 5 | Release: 1%{?dist} 6 | Vendor: CEA 7 | License: GPLv2+ 8 | Summary: Lustre administration utility 9 | Url: https://github.com/cea-hpc/shine 10 | Source0: https://github.com/cea-hpc/shine/archive/v%{version}.tar.gz#/%{name}-%{version}.tar.gz 11 | Group: Development/Libraries 12 | BuildArch: noarch 13 | Requires: clustershell >= 1.5.1 14 | 15 | %description 16 | Python-based Lustre utility to easily control Lustre filesystem 17 | devices, mount points and routers. 18 | 19 | %prep 20 | %setup -q 21 | 22 | %build 23 | %{__python} setup.py build 24 | 25 | %install 26 | %{__python} setup.py install -O1 --skip-build --root %{buildroot} 27 | 28 | # move 'shine' into /usr/sbin 29 | mv %{buildroot}/usr/bin %{buildroot}/usr/sbin 30 | # man pages 31 | install -d %{buildroot}/%{_mandir}/{man1,man5} 32 | install -p -m 0644 doc/shine.1 %{buildroot}/%{_mandir}/man1/ 33 | install -p -m 0644 doc/shine.conf.5 %{buildroot}/%{_mandir}/man5/ 34 | 35 | %files 36 | %defattr(-,root,root) 37 | %{_sysconfdir}/shine/*.conf.example 38 | %config(noreplace) %{_sysconfdir}/shine/*.conf 39 | %config(noreplace) %{_sysconfdir}/shine/models/*.lmf 40 | %{_sbindir}/shine 41 | %{python_sitelib}/Shine/ 42 | %{python_sitelib}/shine-*-py?.?.egg-info 43 | %{vimdatadir}/ftdetect/*.vim 44 | %{vimdatadir}/syntax/*.vim 45 | %doc LICENSE README.md ChangeLog 46 | %{_mandir}/man1/shine.1* 47 | %{_mandir}/man5/shine.conf.5* 48 | %{_usr}/share/shine/shine.init.redhat 49 | %dir %{_localstatedir}/cache/shine/conf 50 | %{_localstatedir}/cache/shine/conf/README 51 | 52 | %changelog 53 | * Wed May 24 2017 - 1.5-1 54 | - Update to shine 1.5 55 | 56 | * Wed Apr 29 2015 - 1.4-1 57 | - Update to shine 1.4 58 | 59 | * Tue Mar 11 2014 - 1.3.1-1 60 | - Update to shine 1.3.1 61 | 62 | * Thu Oct 10 2013 - 1.3-1 63 | - Update to shine 1.3 64 | -------------------------------------------------------------------------------- /tests/CLI/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cea-hpc/shine/11f329dcceae798eb9a4262f9be25f158e63c3f8/tests/CLI/__init__.py -------------------------------------------------------------------------------- /tests/Configuration/BackendFileTest.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Shine.Configuration.Backend.File test suite 3 | # Written by A. Degremont 2010-11-20 4 | 5 | 6 | """Unit test for Backend File""" 7 | 8 | import unittest 9 | 10 | from Utils import makeTempFile, setup_tempdirs, clean_tempdirs 11 | from Shine.Configuration.Globals import Globals 12 | from Shine.Configuration.Configuration import Configuration 13 | 14 | class BackendFileTest(unittest.TestCase): 15 | 16 | def setUp(self): 17 | self._conf = None 18 | setup_tempdirs() 19 | Globals().replace('backend', 'File') 20 | 21 | def tearDown(self): 22 | Globals().replace('storage_file', '/etc/shine/storage.conf') 23 | if self._conf is not None: 24 | self._conf.unregister_fs() 25 | clean_tempdirs() 26 | Globals().replace('backend', 'None') 27 | 28 | def make_temp_conf(self, txt): 29 | self._storagefile = makeTempFile(txt) 30 | Globals().replace('storage_file', self._storagefile.name) 31 | 32 | def make_temp_fs(self, txt): 33 | self._fsfile = makeTempFile(txt) 34 | self._conf = Configuration.create_from_model(self._fsfile.name) 35 | self._model = self._conf._fs.model 36 | 37 | def test_simple_mgs(self): 38 | self.make_temp_conf("mgt: node=foo1 dev=/dev/sda") 39 | self.make_temp_fs("""fs_name: example 40 | nid_map: nodes=foo[1-10] nids=foo[1-10]@tcp0 41 | mgt: node=foo1""") 42 | self.assertEqual(self._model.get('mgt')[0].get('dev'), '/dev/sda') 43 | 44 | def test_ha_node(self): 45 | """target with ha_node""" 46 | self.make_temp_conf("mgt: node=foo1 dev=/dev/sda ha_node=foo1") 47 | self.make_temp_fs("""fs_name: example 48 | nid_map: nodes=foo[1-10] nids=foo[1-10]@tcp0 49 | mgt: node=foo1""") 50 | self.assertEqual(self._model.get('mgt')[0].get('ha_node'), ['foo1']) 51 | 52 | def test_ha_nodes(self): 53 | """target with several ha_nodes""" 54 | self.make_temp_conf("mgt: node=foo1 dev=/dev/sda ha_node=foo1 ha_node=foo2") 55 | self.make_temp_fs("""fs_name: example 56 | nid_map: nodes=foo[1-10] nids=foo[1-10]@tcp0 57 | mgt: node=foo1 ha_node=foo1""") 58 | self.assertEqual(self._model.get('mgt')[0].get('ha_node'), ['foo1', 'foo2']) 59 | 60 | def test_multiple_matches(self): 61 | self.make_temp_conf( 62 | """mgt: node=foo1 dev=/dev/sda 63 | mdt: node=foo1 dev=/dev/sdd 64 | ost: node=foo1 dev=/dev/sdb 65 | ost: node=foo1 dev=/dev/sdc 66 | ost: node=foo2 dev=/dev/sdb""") 67 | self.make_temp_fs("""fs_name: example 68 | nid_map: nodes=foo[1-10] nids=foo[1-10]@tcp0 69 | mgt: node=foo1 70 | mdt: node=foo1 71 | ost: node=foo1""") 72 | self.assertEqual(self._model.get('mgt')[0].get('dev'), '/dev/sda') 73 | self.assertEqual(self._model.get('mdt')[0].get('dev'), '/dev/sdd') 74 | self.assertEqual(len(self._model.elements('ost')), 2) 75 | self.assertEqual(self._model.get('ost')[0].get('dev'), '/dev/sdb') 76 | self.assertEqual(self._model.get('ost')[1].get('dev'), '/dev/sdc') 77 | 78 | def test_index_external(self): 79 | self.make_temp_conf( 80 | """mgt: node=foo1 dev=/dev/sda 81 | mdt: node=foo1 dev=/dev/sdd 82 | ost: node=foo1 dev=/dev/sdb 83 | ost: node=foo1 dev=/dev/sdc 84 | ost: node=foo2 dev=/dev/sdb""") 85 | self.make_temp_fs("""fs_name: example 86 | nid_map: nodes=foo[1-10] nids=foo[1-10]@tcp0 87 | mgt: node=foo1 88 | mdt: node=foo1 mode=external 89 | ost: node=foo2 90 | ost: node=foo1 dev=/dev/sdb index=0 91 | ost: node=foo1 dev=/dev/sdc""") 92 | self.assertEqual(self._model.get('mgt')[0].get('dev'), '/dev/sda') 93 | self.assertEqual(self._model.get('mdt')[0].get('mode'), 'external') 94 | self.assertEqual(len(self._model.elements('ost')), 3) 95 | self.assertEqual(self._model.get('ost')[0].get('dev'), '/dev/sdb') 96 | self.assertEqual(self._model.get('ost')[0].get('index'), 1) 97 | self.assertEqual(self._model.get('ost')[1].get('dev'), '/dev/sdb') 98 | self.assertEqual(self._model.get('ost')[1].get('index'), 0) 99 | self.assertEqual(self._model.get('ost')[2].get('dev'), '/dev/sdc') 100 | self.assertEqual(self._model.get('ost')[2].get('index'), 2) 101 | -------------------------------------------------------------------------------- /tests/Configuration/GlobalsTest.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Shine.Configuration.Globals test suite 3 | # Written by A. Degremont 2010-11-07 4 | 5 | 6 | """Unit test for Globals""" 7 | 8 | import unittest 9 | 10 | from Shine.Configuration.Globals import Globals 11 | 12 | 13 | class GlobalsTest(unittest.TestCase): 14 | 15 | def testNoFileIsOk(self): 16 | """test that Globals load if not file is found""" 17 | self.assertEqual(Globals().get('backend'), 'None') 18 | 19 | def testLoadExample(self): 20 | backup = Globals.DEFAULT_CONF_FILE 21 | Globals.DEFAULT_CONF_FILE = "../conf/shine.conf" 22 | self.assertEqual(Globals().get('backend'), 'None') 23 | self.assertEqual(Globals().get('storage_file'), 24 | '/etc/shine/storage.conf') 25 | Globals.DEFAULT_CONF_FILE = backup 26 | 27 | def test_lustre_version(self): 28 | conf = Globals() 29 | self.assertFalse(conf.lustre_version_is_smaller('1.6.5')) 30 | 31 | conf.add('lustre_version', '1.8.5') 32 | self.assertFalse(conf.lustre_version_is_smaller('1.6.7')) 33 | self.assertTrue(conf.lustre_version_is_smaller('2.0.0.1')) 34 | -------------------------------------------------------------------------------- /tests/Configuration/TuningModelTest.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Shine.Configuration.TuningModel test suite 3 | # Written by A. Degremont 2010-01-19 4 | 5 | 6 | """Unit test for Model""" 7 | 8 | import unittest 9 | 10 | from Utils import makeTempFile 11 | from ClusterShell.NodeSet import NodeSet 12 | from Shine.Configuration.TuningModel import TuningModel, TuningParameter, TuningError 13 | 14 | class TuningModelTest(unittest.TestCase): 15 | 16 | def makeTempTuningModel(self, text): 17 | """ 18 | Create a temporary file instance and returns a TuningModel with it. 19 | """ 20 | self.f = makeTempFile(text) 21 | model = TuningModel(filename=self.f.name) 22 | return model 23 | 24 | def test_tuning_param_str(self): 25 | """test TuningParameter.__str__()""" 26 | param = TuningParameter("foo", 0, ('mds', 'clt'), ["toto15"]) 27 | self.assertEqual(str(param), "foo=0 types=mds,client nodes=toto15") 28 | 29 | def testExampleFile(self): 30 | """test example tuning""" 31 | m = TuningModel(filename="../conf/tuning.conf.example") 32 | m.parse() 33 | 34 | def testEmptyFile(self): 35 | """test empty tuning""" 36 | m = self.makeTempTuningModel("") 37 | m.parse() 38 | self.assertEqual(len(m._parameter_dict.keys()), 0) 39 | 40 | def testComments(self): 41 | """test comments and blank lines""" 42 | m = self.makeTempTuningModel(""" 43 | # This is a comment 44 | 45 | #### Another one ## 46 | 47 | """) 48 | m.parse() 49 | self.assertEqual(len(m._parameter_dict.keys()), 0) 50 | 51 | def testSimpleFile(self): 52 | """test simple tuning""" 53 | m = self.makeTempTuningModel(""" 54 | alias panic_on_lbug=/proc/sys/lnet/panic_on_lbug 55 | 1 panic_on_lbug MDS;OSS;CLT""") 56 | m.parse() 57 | 58 | # We have one tuning for each 59 | tuning = TuningParameter("/proc/sys/lnet/panic_on_lbug", "1", \ 60 | ["mds","oss","clt"], None) 61 | for t in ["oss","mds","client"]: 62 | tunings = m.get_params_for_name(None, [t]) 63 | self.assertEqual(len(tunings), 1) 64 | self.assertTrue(tunings[0], tuning) 65 | 66 | def testAliasMissing(self): 67 | """test tuning with missing alias""" 68 | m = self.makeTempTuningModel("bar foo CLT") 69 | self.assertRaises(TuningError, TuningModel.parse, m) 70 | 71 | def testWrongSyntax(self): 72 | """test tuning with wrong syntax""" 73 | m = self.makeTempTuningModel(""" 74 | alias foo=/foo 75 | bar foo CLT ; MDS""") 76 | self.assertRaises(TuningError, TuningModel.parse, m) 77 | 78 | def testFileValue(self): 79 | """test tuning with path as value""" 80 | m = self.makeTempTuningModel(""" 81 | alias daemon_file = /proc/sys/lnet/daemon_file 82 | /tmp/toto.log daemon_file MDS;OSS;CLT""") 83 | m.parse() 84 | 85 | # We have one tuning for each 86 | tuning = TuningParameter("/proc/sys/lnet/daemon_file", \ 87 | "/tmp/toto.log", \ 88 | ["mds","oss","clt"], None) 89 | for t in ["oss","mds","client"]: 90 | tunings = m.get_params_for_name(None, [t]) 91 | self.assertEqual(len(tunings), 1) 92 | self.assertTrue(tunings[0], tuning) 93 | 94 | def testQuotedValue(self): 95 | """test tuning with a quoted value""" 96 | m = self.makeTempTuningModel(""" 97 | alias daemon_file=/proc/sys/lnet/daemon_file 98 | "/tmp/toto space.log" daemon_file MDS;OSS;CLT""") 99 | m.parse() 100 | 101 | # We have one tuning for each 102 | tuning = TuningParameter("/proc/sys/lnet/daemon_file", \ 103 | '"/tmp/toto space.log"', \ 104 | ["mds","oss","clt"], None) 105 | for t in ["oss","mds","client"]: 106 | tunings = m.get_params_for_name(None, [t]) 107 | self.assertEqual(len(tunings), 1) 108 | self.assertTrue(tunings[0] == tuning) 109 | 110 | def testOnNodes(self): 111 | """test tuning with nodes""" 112 | m = self.makeTempTuningModel(""" 113 | alias panic_on_lbug=/proc/sys/lnet/panic_on_lbug 114 | 1 panic_on_lbug MDS;CLT;foo[1-5]""") 115 | m.parse() 116 | 117 | # We have one tuning for each 118 | tuning = TuningParameter("/proc/sys/lnet/panic_on_lbug", "1", \ 119 | ["mds","clt"], NodeSet("foo[1-5]")) 120 | # Check node types 121 | for t in ["mds","client"]: 122 | tunings = m.get_params_for_name(None, [t]) 123 | self.assertEqual(len(tunings), 1) 124 | self.assertTrue(tunings[0] == tuning) 125 | # Check node name 126 | tunings = m.get_params_for_name(NodeSet("foo[1-2]"), []) 127 | self.assertEqual(len(tunings), 1) 128 | self.assertTrue(tunings[0] == tuning) 129 | 130 | def test_router_is_supported(self): 131 | """test ROUTER is a supported node type""" 132 | model = self.makeTempTuningModel(""" 133 | alias panic_on_lbug=/proc/sys/lnet/panic_on_lbug 134 | 1 panic_on_lbug ROUTER""") 135 | model.parse() 136 | tuning = TuningParameter("/proc/sys/lnet/panic_on_lbug", "1", 137 | ['router']) 138 | tunings = model.get_params_for_name(None, ['router']) 139 | self.assertEqual(len(tunings), 1) 140 | self.assertEqual(tunings[0], tuning) 141 | 142 | def test_double_param_declaration(self): 143 | """test that a parameter declared twice raises an error""" 144 | model = self.makeTempTuningModel(""" 145 | alias panic_on_lbug=/proc/sys/lnet/panic_on_lbug 146 | 1 panic_on_lbug ROUTER 147 | 0 panic_on_lbug ROUTER""") 148 | self.assertRaises(TuningError, model.parse) 149 | -------------------------------------------------------------------------------- /tests/Configuration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cea-hpc/shine/11f329dcceae798eb9a4262f9be25f158e63c3f8/tests/Configuration/__init__.py -------------------------------------------------------------------------------- /tests/Lustre/Actions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cea-hpc/shine/11f329dcceae798eb9a4262f9be25f158e63c3f8/tests/Lustre/Actions/__init__.py -------------------------------------------------------------------------------- /tests/Lustre/ClientTest.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Shine.Lustre.Client test suite 3 | # Written by A. Degremont 2010-12-20 4 | 5 | """Unit test for Client""" 6 | 7 | import unittest 8 | 9 | from Shine.Lustre.FileSystem import FileSystem 10 | from Shine.Lustre.Server import Server 11 | from Shine.Lustre.Client import Client 12 | 13 | class ClientTest(unittest.TestCase): 14 | 15 | def test_allservers(self): 16 | """test client.allservers()""" 17 | fs = FileSystem('foo') 18 | srv = Server('foo1', ['foo1@tcp']) 19 | client = fs.new_client(srv, '/foo') 20 | self.assertEqual(str(client.allservers().nodeset()), 'foo1') 21 | 22 | def test_unique_id(self): 23 | """test client.uniqueid()""" 24 | fs1 = FileSystem('uniqueid') 25 | srv1 = Server('foo1', ['foo1@tcp']) 26 | client1 = fs1.new_client(srv1, '/foo') 27 | 28 | fs2 = FileSystem('uniqueid') 29 | srv2 = Server('foo1', ['foo1@tcp']) 30 | client2 = fs2.new_client(srv2, '/foo') 31 | 32 | self.assertEqual(client1.uniqueid(), client2.uniqueid()) 33 | 34 | def test_uniqueid_diff_mountpath(self): 35 | """test client.uniqueid() (diff mount_path)""" 36 | fs1 = FileSystem('uniqueid') 37 | srv1 = Server('foo1', ['foo1@tcp']) 38 | client1 = fs1.new_client(srv1, '/foo1') 39 | 40 | fs2 = FileSystem('uniqueid') 41 | srv2 = Server('foo1', ['foo1@tcp']) 42 | client2 = fs2.new_client(srv2, '/foo2') 43 | 44 | self.assertNotEqual(client1.uniqueid(), client2.uniqueid()) 45 | -------------------------------------------------------------------------------- /tests/Lustre/DiskTest.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Shine.Lustre.Disk test suite 3 | # Written by A. Degremont 2009-08-03 4 | 5 | 6 | """Unit test for Shine.Lustre.Disk""" 7 | 8 | import unittest 9 | from subprocess import Popen, PIPE, STDOUT 10 | 11 | import Utils 12 | from Shine.Lustre.Disk import Disk, DiskDeviceError 13 | 14 | class DiskLoopbackTest(unittest.TestCase): 15 | 16 | def format(self, *opts): 17 | cmd = [] 18 | cmd += ["/usr/sbin/mkfs.lustre", "--device-size", str(self.size / 1024)] 19 | cmd += opts 20 | cmd.append(self.dev) 21 | process = Popen(cmd, stdout=PIPE, stderr=STDOUT) 22 | process.wait() 23 | self.assertEqual(process.returncode, 0) 24 | 25 | def setUp(self): 26 | self.size = 450 * 1024 * 1024 27 | self.diskfile = Utils.make_disk(450) 28 | self.dev = self.diskfile.name 29 | self.disk = Disk(dev=self.dev) 30 | 31 | def testLoopbackDevCheck(self): 32 | """test device check with a loopback device""" 33 | self.disk._device_check() 34 | self.assertEqual(self.disk.dev_isblk, False) 35 | self.assertEqual(self.disk.dev_size, self.size) 36 | 37 | @Utils.rootonly 38 | def testMountDataFlags(self): 39 | """test mountdata flag check when freshly formatted""" 40 | 41 | self.format("--fsname=nonreg", "--ost", "--mgsnode=127.0.0.1@lo") 42 | 43 | self.disk._mountdata_check() 44 | 45 | # first_time TRUE 46 | self.assertTrue(self.disk.has_first_time_flag()) 47 | # need_index TRUE 48 | self.assertTrue(self.disk.has_need_index_flag()) 49 | # update TRUE 50 | self.assertTrue(self.disk.has_update_flag()) 51 | # rewrite_ldd FALSE 52 | self.assertFalse(self.disk.has_rewrite_ldd_flag()) 53 | # writeconf FALSE 54 | self.assertFalse(self.disk.has_writeconf_flag()) 55 | # upgrade14 FALSE 56 | self.assertFalse(self.disk.has_upgrade14_flag()) 57 | # param FALSE 58 | self.assertFalse(self.disk.has_param_flag()) 59 | 60 | self.assertEqual(self.disk.flags(), 61 | ['need_index', 'first_time', 'update']) 62 | 63 | @Utils.rootonly 64 | def test_mountdata_label_mgs(self): 65 | """test mountdata check with fsname and MGS as label""" 66 | 67 | self.format("--fsname=nonreg", "--mgs", "--param", "sys.timeout=40") 68 | 69 | self.disk._mountdata_check(label_check="MGS") 70 | 71 | # update TRUE 72 | self.assertTrue(self.disk.has_update_flag()) 73 | # rewrite_ldd FALSE 74 | self.assertFalse(self.disk.has_rewrite_ldd_flag()) 75 | # writeconf FALSE 76 | self.assertFalse(self.disk.has_writeconf_flag()) 77 | # upgrade14 FALSE 78 | self.assertFalse(self.disk.has_upgrade14_flag()) 79 | # param FALSE 80 | self.assertFalse(self.disk.has_param_flag()) 81 | 82 | # Starting from Lustre 2.4, 'need_index' and 'first_time' flags are no 83 | # more set 84 | flags = self.disk.flags() 85 | if 'need_index' in flags: 86 | flags.remove('need_index') 87 | if 'first_time' in flags: 88 | flags.remove('first_time') 89 | self.assertEqual(flags, ['update']) 90 | 91 | @Utils.rootonly 92 | def test_mountdata_label_ost(self): 93 | """test mountdata check with fsname and label for an OST""" 94 | 95 | self.format("--fsname=nonreg", "--ost", "--mgsnode=127.0.0.1@lo", 96 | "--param", "sys.timeout=40") 97 | 98 | self.disk._mountdata_check(label_check="nonreg-OSTffff") 99 | 100 | # first_time TRUE 101 | self.assertTrue(self.disk.has_first_time_flag()) 102 | # need_index TRUE 103 | self.assertTrue(self.disk.has_need_index_flag()) 104 | # update TRUE 105 | self.assertTrue(self.disk.has_update_flag()) 106 | # rewrite_ldd FALSE 107 | self.assertFalse(self.disk.has_rewrite_ldd_flag()) 108 | # writeconf FALSE 109 | self.assertFalse(self.disk.has_writeconf_flag()) 110 | # upgrade14 FALSE 111 | self.assertFalse(self.disk.has_upgrade14_flag()) 112 | # param FALSE 113 | self.assertFalse(self.disk.has_param_flag()) 114 | 115 | self.assertEqual(self.disk.flags(), 116 | ['need_index', 'first_time', 'update']) 117 | 118 | @Utils.rootonly 119 | def testMountDataBadLabel(self): 120 | """test mountdata check with wrong fsname and label""" 121 | self.format("--fsname=nonreg","--mgs") 122 | self.assertRaises(DiskDeviceError, self.disk._mountdata_check, 123 | label_check="MDT") 124 | self.assertRaises(DiskDeviceError, self.disk._mountdata_check, 125 | label_check="bad_label") 126 | 127 | def testMountDataNoFormat(self): 128 | """test mountdata on a not formatted device""" 129 | self.assertRaises(DiskDeviceError, self.disk._mountdata_check) 130 | 131 | 132 | class DiskOtherTest(unittest.TestCase): 133 | 134 | def testErrorDevCheck(self): 135 | """test device check with a wrong device""" 136 | disk = Disk(dev='wrong path') 137 | self.assertRaises(DiskDeviceError, disk._device_check) 138 | 139 | def testBadDevCheck(self): 140 | """test device check with a bad file""" 141 | disk = Disk(dev='/dev/tty0') 142 | self.assertRaises(DiskDeviceError, disk._device_check) 143 | 144 | ### XXX: This does not check the device size 145 | def testBlockDevCheck(self): 146 | """test device check with a block device""" 147 | disk = Disk(Utils.get_block_dev()) 148 | disk._device_check() 149 | self.assertEqual(disk.dev_isblk, True) 150 | self.assertNotEqual(disk.dev_size, 0) 151 | 152 | def testUpdate(self): 153 | """test update a Disk instance with another one""" 154 | disk1 = Disk(dev='a_dev') 155 | disk1.dev_size = 1234 156 | disk2 = Disk(dev='b_dev') 157 | disk2.dev_size = 5678 158 | disk1.update(disk2) 159 | 160 | self.assertEqual(disk1.dev_size, disk2.dev_size) 161 | 162 | def testException(self): 163 | """test Disk exception classes""" 164 | disk = Disk(dev="foo") 165 | exp = DiskDeviceError(disk=disk, message="Something") 166 | self.assertEqual(str(exp), "Something") 167 | -------------------------------------------------------------------------------- /tests/Lustre/ServerTest.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Shine.Lustre.Server test suite 3 | # Written by A. Degremont 2009-08-03 4 | 5 | 6 | """Unit test for Server""" 7 | 8 | import unittest 9 | import socket 10 | 11 | from Shine.Lustre.Server import Server, ServerGroup 12 | # No direct dependancies to NodeSet. This should be fixed. 13 | from ClusterShell.NodeSet import NodeSet 14 | 15 | class ServerTest(unittest.TestCase): 16 | 17 | def testStringRepr(self): 18 | """test string representation""" 19 | srv = Server('localhost', ['localhost@tcp']) 20 | self.assertEqual(str(srv), 'localhost (localhost@tcp)') 21 | 22 | def testHostname(self): 23 | """test hostname resolution""" 24 | # Check hostname methods returns something 25 | self.assertTrue(Server.hostname_short()) 26 | self.assertTrue(Server.hostname_long()) 27 | 28 | def testIsLocal(self): 29 | """test is_local()""" 30 | fqdn = socket.getfqdn() 31 | shortname = socket.gethostname().split('.', 1)[0] 32 | 33 | # Test shortname is local() 34 | srv = Server(shortname, ['%s@tcp' % shortname]) 35 | self.assertTrue(srv.is_local()) 36 | 37 | # Test fqdn is local() 38 | srv = Server(fqdn, ['%s@tcp' % fqdn]) 39 | self.assertTrue(srv.is_local()) 40 | 41 | # Test a dummy shortname should not be local 42 | shortname = shortname + "-shine-false-suffix" 43 | srv = Server(shortname, ['%s@tcp' % shortname]) 44 | self.assertFalse(srv.is_local()) 45 | 46 | # Test a false domain name with a good hostname 47 | othername = shortname + ".shine-false-tld" 48 | srv = Server(othername, ['%s@tcp' % othername]) 49 | self.assertFalse(srv.is_local()) 50 | 51 | # Test something else should not be local 52 | othername = fqdn + ".shine-false-tld" 53 | srv = Server(othername, ['%s@tcp' % othername]) 54 | self.assertFalse(srv.is_local()) 55 | 56 | # Check hostname methods are rightly seen as local 57 | self.assertTrue(Server(Server.hostname_short(), ['foo']).is_local()) 58 | self.assertTrue(Server(Server.hostname_long(), ['foo']).is_local()) 59 | 60 | def testDistantServers(self): 61 | """test distant_servers()""" 62 | nodes = NodeSet("foo,bar") 63 | self.assertEqual(Server.distant_servers(nodes), nodes) 64 | 65 | nodes_short = nodes | NodeSet(Server.hostname_short()) 66 | self.assertEqual(Server.distant_servers(nodes_short), nodes) 67 | 68 | nodes_long = nodes | NodeSet(Server.hostname_long()) 69 | self.assertEqual(Server.distant_servers(nodes_long), nodes) 70 | 71 | 72 | class ServerGroupTest(unittest.TestCase): 73 | 74 | def testSimple(self): 75 | """test ServerGroup simple tests""" 76 | grp = ServerGroup() 77 | self.assertEqual(len(grp), 0) 78 | 79 | srv = Server('foo', ['foo@tcp']) 80 | grp.append(srv) 81 | self.assertEqual(len(grp), 1) 82 | self.assertEqual(grp[0], srv) 83 | 84 | def testIter(self): 85 | """test ServerGroup.__iter__()""" 86 | srv1 = Server('foo1', ['foo1@tcp']) 87 | srv2 = Server('foo2', ['foo2@tcp']) 88 | grp = ServerGroup([srv1, srv2]) 89 | self.assertEqual(list(iter(grp)), [srv1, srv2]) 90 | 91 | def testSelect(self): 92 | """test ServerGroup.select()""" 93 | srv1 = Server('foo1', ['foo1@tcp']) 94 | srv2 = Server('foo2', ['foo2@tcp']) 95 | srv3 = Server('foo3', ['foo3@tcp']) 96 | grp = ServerGroup([srv1, srv2, srv3]) 97 | subgrp = grp.select(NodeSet("foo[1,3]")) 98 | self.assertEqual(list(iter(subgrp)), [srv1, srv3]) 99 | 100 | def testNodeSet(self): 101 | """test ServerGroup.nodeset()""" 102 | srv1 = Server('foo1', ['foo1@tcp']) 103 | srv2 = Server('foo2', ['foo2@tcp']) 104 | grp = ServerGroup([srv1, srv2]) 105 | self.assertEqual(grp.nodeset(), NodeSet('foo[1-2]')) 106 | 107 | def testDistant(self): 108 | """test ServerGroup.nodeset()""" 109 | fqdn = socket.getfqdn() 110 | shortname = socket.gethostname().split('.', 1)[0] 111 | 112 | srv1 = Server(shortname, ['%s@tcp' % shortname]) 113 | srv2 = Server('foo', ['foo@tcp']) 114 | grp = ServerGroup([srv1, srv2]) 115 | subgrp = grp.distant() 116 | self.assertEqual(list(iter(subgrp)), [ srv2 ]) 117 | -------------------------------------------------------------------------------- /tests/Lustre/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cea-hpc/shine/11f329dcceae798eb9a4262f9be25f158e63c3f8/tests/Lustre/__init__.py -------------------------------------------------------------------------------- /tests/Utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Utilities function for Shine unit tests. 3 | # Written by A. Degremont 4 | 5 | from __future__ import print_function 6 | 7 | import os 8 | import socket 9 | import tempfile 10 | 11 | from Shine.Configuration.Globals import Globals 12 | 13 | 14 | # Used when a real hostname is required 15 | HOSTNAME = socket.gethostname().split('.')[0] 16 | 17 | # 18 | # Test decorators 19 | # 20 | 21 | def rootonly(method): 22 | def root_tested(self): 23 | if os.getuid() == 0: 24 | method(self) 25 | else: 26 | print("SKIP. Root permission required.") 27 | root_tested.__name__ = method.__name__ 28 | root_tested.__doc__ = method.__doc__ 29 | return root_tested 30 | 31 | # 32 | # Temp fake disk 33 | # 34 | def make_disk(size=200): 35 | real_size = size * 1024 * 1024 36 | disk = tempfile.NamedTemporaryFile(prefix='shine-test-disk-') 37 | disk.truncate(real_size) 38 | disk.flush() 39 | return disk 40 | 41 | # 42 | # Some tests need a block device to stat. 43 | # Index is used to get a different one if possible 44 | # 45 | def get_block_dev(i=0): 46 | with open("/proc/partitions", 'r') as partitions: 47 | # skip first two lines (header) 48 | lines = partitions.readlines()[2:] 49 | i %= len(lines) 50 | d_info = lines[i].rstrip('\n').split(' ') 51 | return '/dev/' + d_info[-1] 52 | 53 | # 54 | # Temp files 55 | # 56 | def makeTempFilename(): 57 | """Return a temporary name for a file.""" 58 | return (tempfile.mkstemp(prefix='shine-test-'))[1] 59 | 60 | def makeTempFile(text): 61 | """ Create a temporary file with the provided text.""" 62 | tmp = tempfile.NamedTemporaryFile(prefix='shine-test-', mode='w') 63 | tmp.write(text) 64 | tmp.flush() 65 | return tmp 66 | 67 | # 68 | # Temp directories 69 | # 70 | def make_tempdir(): 71 | """Create and return a temporary directory for Shine tests.""" 72 | return tempfile.mkdtemp(prefix='shine-test-') 73 | 74 | def clean_tempdir(path): 75 | """Delete a directory created by make_tempdir()""" 76 | os.rmdir(path) 77 | 78 | def setup_tempdirs(): 79 | Globals.DEFAULT_CONF_FILE = "../conf/shine.conf" 80 | Globals().replace('conf_dir', make_tempdir()) 81 | 82 | def clean_tempdirs(): 83 | clean_tempdir(Globals().get('conf_dir')) 84 | --------------------------------------------------------------------------------