├── .gitignore ├── .gitmodules ├── .pylintrc ├── CONTRIBUTING.md ├── Dockerfile ├── Jenkinsfile ├── LICENSE ├── README ├── README.md ├── banner.sh ├── emailer.py ├── fixterm_src ├── .gitignore ├── compile.sh ├── fixterm.c └── resize.sh ├── package_map.py ├── release.sh ├── requirements.txt ├── setup.py ├── shutit.py ├── shutit_assets.py ├── shutit_background.py ├── shutit_class.py ├── shutit_exam.py ├── shutit_global.py ├── shutit_login_stack.py ├── shutit_module.py ├── shutit_patterns ├── __init__.py ├── bash.py ├── docker.py ├── docker_tutorial.py ├── shutitfile.py └── vagrant.py ├── shutit_pexpect_session.py ├── shutit_pexpect_session_environment.py ├── shutit_sendspec.py ├── shutit_session_setup ├── __init__.py ├── vagrant.py └── virtualization.py ├── shutit_setup.py ├── shutit_skeleton.py ├── shutit_threads.py └── shutit_util.py /.gitignore: -------------------------------------------------------------------------------- 1 | #The MIT License (MIT) 2 | # 3 | #Copyright (C) 2014 OpenBet Limited 4 | # 5 | #Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | #this software and associated documentation files (the "Software"), to deal in 7 | #the Software without restriction, including without limitation the rights to 8 | #use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 9 | #of the Software, and to permit persons to whom the Software is furnished to do 10 | #so, subject to the following conditions: 11 | # 12 | #The above copyright notice and this permission notice shall be included in all 13 | #copies or substantial portions of the Software. 14 | # 15 | #THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | #IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | #ITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 | #THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | #LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | #OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | #SOFTWARE. 22 | 23 | # Python 24 | *.pyc 25 | # Configs 26 | **.cnf 27 | !**build.cnf 28 | !**push.cnf 29 | !**defaults.cnf 30 | # Logs 31 | **.log 32 | # Nohups and outs 33 | nohup.out 34 | out 35 | # Lock files 36 | **.lck 37 | # Docs 38 | docs/_build 39 | # Artifacts 40 | artifacts/* 41 | # Resources 42 | *resources/* 43 | !*resources/README.md 44 | !*shutit_resources/README.md 45 | # Keys 46 | examples/ianmiellaws/context/pems/*.pem 47 | pubring.gpg~ 48 | secring.gpg 49 | examples/digital_ocean/context/access_token.dat 50 | # Show Config 51 | show_config/* 52 | build/* 53 | dist/* 54 | shutit.egg* 55 | .eric6* 56 | shutit.e4p 57 | #bak 58 | *bak 59 | bugs 60 | TODO 61 | secret 62 | shutitskel* 63 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "shutit-test"] 2 | path = shutit-test 3 | url = https://github.com/ianmiell/shutit-test 4 | branch = master 5 | [submodule "shutit-docs"] 6 | path = shutit-docs 7 | url = https://github.com/ianmiell/shutit-docs 8 | branch = master 9 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | #The MIT License (MIT) 2 | # 3 | #Copyright (C) 2014 OpenBet Limited 4 | # 5 | #Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | #this software and associated documentation files (the "Software"), to deal in 7 | #the Software without restriction, including without limitation the rights to 8 | #use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 9 | #of the Software, and to permit persons to whom the Software is furnished to do 10 | #so, subject to the following conditions: 11 | # 12 | #The above copyright notice and this permission notice shall be included in all 13 | #copies or substantial portions of the Software. 14 | # 15 | #THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | #IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | #ITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 | #THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | #LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | #OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | #SOFTWARE. 22 | 23 | [MASTER] 24 | 25 | # Specify a configuration file. 26 | #rcfile= 27 | 28 | # Python code to execute, usually for sys.path manipulation such as 29 | # pygtk.require(). 30 | #init-hook= 31 | 32 | # Profiled execution. 33 | profile=no 34 | 35 | # Add files or directories to the blacklist. They should be base names, not 36 | # paths. 37 | ignore=CVS 38 | 39 | # Pickle collected data for later comparisons. 40 | persistent=yes 41 | 42 | # List of plugins (as comma separated values of python modules names) to load, 43 | # usually to register additional checkers. 44 | load-plugins= 45 | 46 | 47 | [MESSAGES CONTROL] 48 | 49 | # Enable the message, report, category or checker with the given id(s). You can 50 | # either give multiple identifier separated by comma (,) or put this option 51 | # multiple time. See also the "--disable" option for examples. 52 | #enable= 53 | 54 | # Disable the message, report, category or checker with the given id(s). You 55 | # can either give multiple identifiers separated by comma (,) or put this 56 | # option multiple times (only on the command line, not in the configuration 57 | # file where it should appear only once).You can also use "--disable=all" to 58 | # disable everything first and then reenable specific checks. For example, if 59 | # you want to run only the similarities checker, you can use "--disable=all 60 | # --enable=similarities". If you want to run only the classes checker, but have 61 | # no Warning level messages displayed, use"--disable=all --enable=classes 62 | # --disable=W" 63 | #[C0326] Exactly one space required after comma 64 | #[C0301] Line too long 65 | #[W0621] Redefining name '?' from outer scope 66 | #[C0111] Missing function docstring 67 | #[C0103] Invalid variable name #(single letter variables...masks other issues?) 68 | #[C0321] More than one statement on a single line 69 | #[W0511] TODO 70 | #[R0913] Too many arguments 71 | #[R0914] Too many local variables 72 | #[R0912] Too many branches 73 | #bad-continuation Some truly bizarre opinions about python line wrapping 74 | disable=C0326,C0301,W0621,C0111,C0103,C0321,W0511,R0913,R0914,R0912,bad-continuation 75 | 76 | 77 | [REPORTS] 78 | 79 | # Set the output format. Available formats are text, parseable, colorized, msvs 80 | # (visual studio) and html. You can also give a reporter class, eg 81 | # mypackage.mymodule.MyReporterClass. 82 | output-format=text 83 | 84 | # Put messages in a separate file for each module / package specified on the 85 | # command line instead of printing them on stdout. Reports (if any) will be 86 | # written in a file name "pylint_global.[txt|html]". 87 | files-output=no 88 | 89 | # Tells whether to display a full report or only the messages 90 | reports=yes 91 | 92 | # Python expression which should return a note less than 10 (10 is the highest 93 | # note). You have access to the variables errors warning, statement which 94 | # respectively contain the number of errors / warnings messages and the total 95 | # number of statements analyzed. This is used by the global evaluation report 96 | # (RP0004). 97 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 98 | 99 | # Add a comment according to your evaluation note. This is used by the global 100 | # evaluation report (RP0004). 101 | comment=no 102 | 103 | # Template used to display messages. This is a python new-style format string 104 | # used to format the message information. See doc for all details 105 | #msg-template= 106 | 107 | 108 | [MISCELLANEOUS] 109 | 110 | # List of note tags to take in consideration, separated by a comma. 111 | notes=FIXME,XXX,TODO 112 | 113 | 114 | [VARIABLES] 115 | 116 | # Tells whether we should check for unused import in __init__ files. 117 | init-import=no 118 | 119 | # A regular expression matching the beginning of the name of dummy variables 120 | # (i.e. not used). 121 | dummy-variables-rgx=_$|dummy 122 | 123 | # List of additional names supposed to be defined in builtins. Remember that 124 | # you should avoid to define new builtins when possible. 125 | additional-builtins= 126 | 127 | 128 | [TYPECHECK] 129 | 130 | # Tells whether missing members accessed in mixin class should be ignored. A 131 | # mixin class is detected if its name ends with "mixin" (case insensitive). 132 | ignore-mixin-members=yes 133 | 134 | # List of classes names for which member attributes should not be checked 135 | # (useful for classes with attributes dynamically set). 136 | ignored-classes=SQLObject 137 | 138 | # When zope mode is activated, add a predefined set of Zope acquired attributes 139 | # to generated-members. 140 | zope=no 141 | 142 | # List of members which are set dynamically and missed by pylint inference 143 | # system, and so shouldn't trigger E0201 when accessed. Python regular 144 | # expressions are accepted. 145 | generated-members=REQUEST,acl_users,aq_parent 146 | 147 | 148 | [BASIC] 149 | 150 | # Required attributes for module, separated by a comma 151 | required-attributes= 152 | 153 | # List of builtins function names that should not be used, separated by a comma 154 | bad-functions=map,filter,apply,input 155 | 156 | # Regular expression which should only match correct module names 157 | module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 158 | 159 | # Regular expression which should only match correct module level names 160 | const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 161 | 162 | # Regular expression which should only match correct class names 163 | class-rgx=[A-Z_][a-zA-Z0-9]+$ 164 | 165 | # Regular expression which should only match correct function names 166 | function-rgx=[a-z_][a-z0-9_]{2,30}$ 167 | 168 | # Regular expression which should only match correct method names 169 | method-rgx=[a-z_][a-z0-9_]{2,30}$ 170 | 171 | # Regular expression which should only match correct instance attribute names 172 | attr-rgx=[a-z_][a-z0-9_]{2,30}$ 173 | 174 | # Regular expression which should only match correct argument names 175 | argument-rgx=[a-z_][a-z0-9_]{2,30}$ 176 | 177 | # Regular expression which should only match correct variable names 178 | variable-rgx=[a-z_][a-z0-9_]{2,30}$ 179 | 180 | # Regular expression which should only match correct attribute names in class 181 | # bodies 182 | class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 183 | 184 | # Regular expression which should only match correct list comprehension / 185 | # generator expression variable names 186 | inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ 187 | 188 | # Good variable names which should always be accepted, separated by a comma 189 | good-names=i,j,k,ex,Run,_ 190 | 191 | # Bad variable names which should always be refused, separated by a comma 192 | bad-names=foo,bar,baz,toto,tutu,tata 193 | 194 | # Regular expression which should only match function or class names that do 195 | # not require a docstring. 196 | no-docstring-rgx=__.*__ 197 | 198 | # Minimum line length for functions/classes that require docstrings, shorter 199 | # ones are exempt. 200 | docstring-min-length=-1 201 | 202 | 203 | [SIMILARITIES] 204 | 205 | # Minimum lines number of a similarity. 206 | min-similarity-lines=4 207 | 208 | # Ignore comments when computing similarities. 209 | ignore-comments=yes 210 | 211 | # Ignore docstrings when computing similarities. 212 | ignore-docstrings=yes 213 | 214 | # Ignore imports when computing similarities. 215 | ignore-imports=no 216 | 217 | 218 | [FORMAT] 219 | 220 | # Maximum number of characters on a single line. 221 | max-line-length=80 222 | 223 | # Regexp for a line that is allowed to be longer than the limit. 224 | ignore-long-lines=^\s*(# )??$ 225 | 226 | # Allow the body of an if to be on the same line as the test if there is no 227 | # else. 228 | single-line-if-stmt=no 229 | 230 | # List of optional constructs for which whitespace checking is disabled 231 | no-space-check=trailing-comma,dict-separator 232 | 233 | # Maximum number of lines in a module 234 | max-module-lines=1000 235 | 236 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 237 | # tab). 238 | indent-string='\t' 239 | 240 | 241 | [IMPORTS] 242 | 243 | # Deprecated modules which should not be used, separated by a comma 244 | deprecated-modules=regsub,TERMIOS,Bastion,rexec 245 | 246 | # Create a graph of every (i.e. internal and external) dependencies in the 247 | # given file (report RP0402 must not be disabled) 248 | import-graph= 249 | 250 | # Create a graph of external dependencies in the given file (report RP0402 must 251 | # not be disabled) 252 | ext-import-graph= 253 | 254 | # Create a graph of internal dependencies in the given file (report RP0402 must 255 | # not be disabled) 256 | int-import-graph= 257 | 258 | 259 | [DESIGN] 260 | 261 | # Maximum number of arguments for function / method 262 | max-args=5 263 | 264 | # Argument names that match this expression will be ignored. Default to name 265 | # with leading underscore 266 | ignored-argument-names=_.* 267 | 268 | # Maximum number of locals for function / method body 269 | max-locals=15 270 | 271 | # Maximum number of return / yield for function / method body 272 | max-returns=6 273 | 274 | # Maximum number of branch for function / method body 275 | max-branches=12 276 | 277 | # Maximum number of statements in function / method body 278 | max-statements=50 279 | 280 | # Maximum number of parents for a class (see R0901). 281 | max-parents=7 282 | 283 | # Maximum number of attributes for a class (see R0902). 284 | max-attributes=7 285 | 286 | # Minimum number of public methods for a class (see R0903). 287 | min-public-methods=2 288 | 289 | # Maximum number of public methods for a class (see R0904). 290 | max-public-methods=20 291 | 292 | 293 | [CLASSES] 294 | 295 | # List of interface methods to ignore, separated by a comma. This is used for 296 | # instance to not check methods defines in Zope's Interface base class. 297 | ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by 298 | 299 | # List of method names used to declare (i.e. assign) instance attributes. 300 | defining-attr-methods=__init__,__new__,setUp 301 | 302 | # List of valid names for the first argument in a class method. 303 | valid-classmethod-first-arg=cls 304 | 305 | # List of valid names for the first argument in a metaclass class method. 306 | valid-metaclass-classmethod-first-arg=mcs 307 | 308 | 309 | [EXCEPTIONS] 310 | 311 | # Exceptions that will emit a warning when being caught. Defaults to 312 | # "Exception" 313 | overgeneral-exceptions=Exception 314 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributions welcome. 2 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine 2 | 3 | # ShutIt in a container. 4 | 5 | RUN apk add --update py-pip 6 | RUN pip install shutit 7 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | #!groovy 2 | 3 | def builddir='shutit-' + env.BUILD_NUMBER 4 | def branch=env.BRANCH_NAME 5 | 6 | try { 7 | lock('shutit_tests') { 8 | stage('setupenv') { 9 | node() { 10 | sh 'mkdir -p ' + builddir 11 | dir(builddir) { 12 | checkout([$class: 'GitSCM', branches: [[name: '*/' + branch]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption', disableSubmodules: false, parentCredentials: false, recursiveSubmodules: true, reference: '', trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[url: 'https://github.com/ianmiell/shutit']]]) 13 | } 14 | } 15 | } 16 | stage('shutit_tests') { 17 | node() { 18 | dir(builddir + '/shutit-test') { 19 | sh('PATH=$(pwd)/..:${PATH} ./run.sh -s tk.shutit.shutit_test shutit_branch ' + branch + ' -l info 2>&1') 20 | } 21 | } 22 | } 23 | } 24 | mail bcc: '', body: '''See: http://jenkins.meirionconsulting.tk/job/shutit''', cc: '', from: 'shutit-jenkins@jenkins.meirionconsulting.tk', replyTo: '', subject: 'Build OK', to: 'ian.miell@gmail.com' 25 | } catch(err) { 26 | mail bcc: '', body: '''See: http://jenkins.meirionconsulting.tk/job/shutit 27 | 28 | ''' + err, cc: '', from: 'shutit-jenkins@jenkins.meirionconsulting.tk', replyTo: '', subject: 'Build failure', to: 'ian.miell@gmail.com' 29 | throw(err) 30 | } 31 | 32 | 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (C) 2014 OpenBet Limited 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | 11 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | README.md -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [ShutIt](http://shutit.tk) 2 | ========================== 3 | 4 | 5 | [![Join the chat at https://gitter.im/ianmiell/shutit](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/ianmiell/shutit?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 6 | 7 | A versatile automation framework. 8 | 9 | ShutIt is an automation tool that models a user's actions on a terminal. 10 | 11 | It can automate any process that can be run by a human on the command line with little effort. 12 | 13 | It was originally written to manage complex Docker builds, but is a now general-purpose automation tool that supports bash, Docker, Vagrant, ssh and arbitrary build contexts. 14 | 15 | ShutIt can also be used as an educational tool, as it can produce videos of demos, capture reproducible steps required to set environments up, and even challenge you to get the right output (see [grep-scales](https://github.com/ianmiell/grep-scales)). 16 | 17 | If you want to know more about Docker, see the [official site](https://www.docker.com/) or take a look at the book by the creators of ShutIt - [Docker in Practice](http://docker-in-practice.github.io/). 18 | 19 | Really Quick Overview 20 | ===================== 21 | Some use cases: 22 | 23 | - You like bash, want to automate tasks, have structure and support, but don't want to learn a configuration management framework that takes you away from the command line you know and love. 24 | 25 | - Want to create [complex Vagrant environments](https://medium.com/@zwischenzugs/a-complete-openshift-cluster-on-vagrant-step-by-step-7465e9816d98) to model clusters of machines. 26 | 27 | - Want to create instructive [walkthroughs](https://asciinema.org/a/30598?t=70): 28 | 29 | - Are interested in "phoenix deployment". 30 | 31 | - Want to take your scripts and turn them into stateless containers quickly, without needing to maintain (or learn) a configuration management solution designed for moving-target systems. 32 | 33 | - You're programmer who wants highly configurable stateless containers development, testing, and production. 34 | 35 | - Want to [build everything from source](https://github.com/ianmiell/shutit-distro/blob/master/README.md) in a way that's comprehensible and auditable. 36 | 37 | 38 | What Does it Do (bash Builds)? 39 | ============================== 40 | 41 | ShutIt acts as a modular and easy to use wrapper around [pexpect](https://github.com/pexpect/pexpect). 42 | 43 | Here is a simple example of a script that creates a file and a directory if they are not there already: 44 | 45 | [![Simple Example](https://asciinema.org/a/47076.png)](https://asciinema.org/a/47076) 46 | 47 | What Does it Do (Tutorials)? 48 | ============================ 49 | 50 | This builds on the docker features (see below), but allows you to interrupt the run at points of your choosing with 'challenges' for the user to overcome. 51 | 52 | Two types of 'challenge' exist in ShutIt: 53 | 54 | - scales 55 | - free form 56 | 57 | Scales tell you to run a specific command before continuing. This is useful when you want to get certain commands or flags 'under your fingers', which does not happen without dedicated and direct practice. 58 | 59 | [![grep Scales](https://asciinema.org/a/41308.png)](https://asciinema.org/a/41308) 60 | 61 | Free form exercises give you a task to perform, and free access to the shell. This is to give the user a realistic environment in which to hone their skills. You can check man pages, look around the directories, search for useful utils (even install new ones!). When you are finished, a pre-specified command is run to check the system is in an appropriate state. Here's an example for the [basics of git](github.com/ianmiell/git-101-tutorial/blob/master/git_101_tutorial.py): 62 | 63 | [![git 101 Tutorial](https://asciinema.org/a/44937.png)](https://asciinema.org/a/44937) 64 | 65 | If you use a Docker-based tutorial and you mess the environment up, the state can be restored to a known one by hitting CTRL-G. 66 | 67 | 68 | What Does it Do (Vagrant)? 69 | ========================== 70 | Uses a bash build to set up n vagrant machines, and uses Landrush to give them useful hostnames accessible from the hosts and in the guest VMs. 71 | 72 | It supports both Virtualbox and Libvirt providers. 73 | 74 | This allows another kind of contained environment for more infrastructural projects than Docker allows for. 75 | 76 | This example demonstrates a reproducible build that sets up Docker on an Ubuntu VM (on a Linux host), then runs a CentOS image within Docker within the Ubuntu VM. 77 | 78 | It deposits the user into a shell mid-build to interrogate the environment, after which the user re-runs the build to add a directive to ensure ps is installed in the image. 79 | 80 | [![Docker on Ubuntu VM running a CentOS image](https://asciinema.org/a/47078.png)](https://asciinema.org/a/47078) 81 | 82 | 83 | 84 | Auto-Generate Modules 85 | ===================== 86 | 87 | ShutIt provides a means for auto-generation of modules (either bare ones, or from existing Dockerfiles) with its skeleton command. See [here](http://ianmiell.github.io/shutit/) for an example. 88 | 89 | [Really Quick Start](http://ianmiell.github.io/shutit) 90 | ==================== 91 | 92 | [Full User Guide](http://github.com/ianmiell/shutit-docs/blob/master/USER_GUIDE.md) 93 | ============== 94 | 95 | [API](http://github.com/ianmiell/shutit-docs/blob/master/API.md) 96 | ====== 97 | 98 | [Installation](http://github.com/ianmiell/shutit-docs/blob/master/INSTALL.md) 99 | ============== 100 | 101 | Known Issues 102 | ============= 103 | Since a core technology used in this application is pexpect - and a typical usage pattern is to expect the prompt to return. 104 | Unusual shell prompts and escape sequences have been known to cause problems. Use the shutit.setup_prompt() function to help manage this by setting up a more sane prompt. 105 | Use of COMMAND_PROMPT with echo -ne has been seen to cause problems with overwriting of shells and pexpect patterns. 106 | 107 | [![ScreenShot](https://raw.github.com/GabLeRoux/WebMole/master/ressources/WebMole_Youtube_Video.png)](https://www.youtube.com/watch?v=gsEtaX207a4) 108 | 109 | Licence 110 | ------------ 111 | The MIT License (MIT) 112 | 113 | Copyright (C) 2014 OpenBet Limited 114 | 115 | Permission is hereby granted, free of charge, to any person obtaining a copy of 116 | this software and associated documentation files (the "Software"), to deal in 117 | the Software without restriction, including without limitation the rights to 118 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 119 | of the Software, and to permit persons to whom the Software is furnished to do 120 | so, subject to the following conditions: 121 | 122 | The above copyright notice and this permission notice shall be included in all 123 | copies or substantial portions of the Software. 124 | 125 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 126 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 127 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 128 | THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 129 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 130 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 131 | SOFTWARE. 132 | 133 | -------------------------------------------------------------------------------- /banner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #http://patorjk.com/software/taag 3 | DATA[0]=" SSSS hh tt III tt "; 4 | DATA[1]=" SSS hh uu uu tttt III tttt"; 5 | DATA[2]=" SSS hhhh uu uu tt I tt"; 6 | DATA[3]=" SSS hh hh uuuuu tt III tt"; 7 | DATA[4]="SSSS hh hh uu uu tt III tt"; 8 | REALoOFFSEToX=0; REALoOFFSEToY=0; 9 | drawochar() { VoCOORDoX=$1; VoCOORDoY=$2; tput cup $((REALoOFFSEToY + VoCOORDoY)) $((REALoOFFSEToX + VoCOORDoX)); printf %c ${DATA[VoCOORDoY]:VoCOORDoX:1}; }; 10 | trap 'exit 1' INT TERM; trap 'tput setaf 9; tput cvvis; clear' EXIT; tput civis; clear; 11 | for ((b=0; b<5; b++)); 12 | do for c in 0 1 2 3 4 5 0; 13 | do tput setaf $c; 14 | for ((x=0; x<${#DATA[0]}; x++)); 15 | do for ((y=0; y<=4; y++)); 16 | do drawochar $x $y; 17 | done; 18 | done; 19 | done; 20 | done 21 | 22 | # self.send_file('/tmp/asd','''#!/bin/bash 23 | #DATA[0]=" SSSS hh tt III tt "; 24 | #DATA[1]=" SSS hh uu uu tttt III tttt"; 25 | #DATA[2]=" SSS hhhh uu uu tt I tt"; 26 | #DATA[3]=" SSS hh hh uuuuu tt III tt"; 27 | #DATA[4]="SSSS hh hh uu uu tt III tt"; 28 | #REALoOFFSEToX=0; REALoOFFSEToY=0; 29 | #drawochar() { VoCOORDoX=$1; VoCOORDoY=$2; tput cup $((REALoOFFSEToY + VoCOORDoY)) $((REALoOFFSEToX + VoCOORDoX)); printf %c ${DATA[VoCOORDoY]:VoCOORDoX:1}; }; 30 | #trap 'exit 1' INT TERM; trap 'tput setaf 9; tput cvvis; clear' EXIT; tput civis; clear; 31 | #for ((b=0; b<5; b++)); 32 | # do for c in 0 1 2 3 4 5 0; 33 | # do tput setaf $c; 34 | # for ((x=0; x<${#DATA[0]}; x++)); 35 | # do for ((y=0; y<=4; y++)); 36 | # do drawochar $x $y; 37 | # done; 38 | # done; 39 | # done; 40 | #done''') 41 | # self.send('chmod +x /tmp/asd') 42 | # self.send('/tmp/asd',echo=True) 43 | # self.send('rm -f /tmp/asd',echo=True) 44 | -------------------------------------------------------------------------------- /emailer.py: -------------------------------------------------------------------------------- 1 | """Utility object for sending emails reports via shutit 2 | 3 | Example code:: 4 | 5 | e = shutit.get_emailer('shutit.tk.mysql.mysql',shutit) 6 | for line in ['your message line 1', 'your message line 2']: 7 | e.add_line(line) 8 | for attach in ['/tmp/filetoattach1','/tmp/filetoattach2']: 9 | e.attach(attach) 10 | e.send() 11 | 12 | Example cfg:: 13 | 14 | [shutit.tk.mysql.mysql] 15 | shutit.core.alerting.emailer.mailto:recipient@example.com 16 | shutit.core.alerting.emailer.mailfrom:sender@example.com 17 | shutit.core.alerting.emailer.smtp_server:localhost 18 | shutit.core.alerting.emailer.subject:Shutit Report 19 | shutit.core.alerting.emailer.signature:--Angry Shutit 20 | shutit.core.alerting.emailer.compress:yes 21 | shutit.core.alerting.emailer.username: 22 | shutit.core.alerting.emailer.password: 23 | shutit.core.alerting.emailer.safe_mode: True 24 | shutit.core.alerting.emailer.mailto_maintainer: True 25 | 26 | """ 27 | 28 | #The MIT License (MIT) 29 | # 30 | #Copyright (C) 2014 OpenBet Limited 31 | # 32 | #Permission is hereby granted, free of charge, to any person obtaining a copy of 33 | #this software and associated documentation files (the "Software"), to deal in 34 | #the Software without restriction, including without limitation the rights to 35 | #use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 36 | #of the Software, and to permit persons to whom the Software is furnished to do 37 | #so, subject to the following conditions: 38 | # 39 | #The above copyright notice and this permission notice shall be included in all 40 | #copies or substantial portions of the Software. 41 | # 42 | #THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 43 | #IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 44 | #ITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 45 | #THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 46 | #LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 47 | #OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 48 | #SOFTWARE. 49 | 50 | from __future__ import print_function 51 | from email.mime.text import MIMEText 52 | from email.mime.application import MIMEApplication 53 | from email.mime.multipart import MIMEMultipart 54 | from smtplib import SMTP, SMTP_SSL, SMTPSenderRefused 55 | import os 56 | import gzip 57 | import shutit_global 58 | import logging 59 | 60 | class Emailer(): 61 | """ Emailer class definition 62 | """ 63 | 64 | def __init__( self, cfg_section, shutit): 65 | """Initialise the emailer object 66 | cfg_section - section in shutit config to look for email configuration items, allowing easier config according to shutit_module. 67 | e.g. 'com.my_module','shutit.core.alerting.emailer.subject': My Module Build Failed! 68 | Config Items: 69 | shutit.core.alerting.emailer.mailto - address to send the mail to (no default) 70 | shutit.core.alerting.emailer.mailfrom - address to send the mail from (angry@shutit.tk) 71 | shutit.core.alerting.emailer.smtp_server - server to send the mail (localhost) 72 | shutit.core.alerting.emailer.smtp_port - port to contact the smtp server on (587) 73 | shutit.core.alerting.emailer.use_tls - should we use tls to connect (True) 74 | shutit.core.alerting.emailer.subject - subject of the email (Shutit Report) 75 | shutit.core.alerting.emailer.signature - --Angry Shutit 76 | shutit.core.alerting.emailer.compress - gzip attachments? (True) 77 | shutit.core.alerting.emailer.username - mail username 78 | shutit.core.alerting.emailer.password - mail password 79 | shutit.core.alerting.emailer.safe_mode - don't fail the build if we get an exception 80 | shutit.core.alerting.emailer.mailto_maintainer - email the maintainer of the module as well as the shutit.core.alerting.emailer.mailto address 81 | """ 82 | self.shutit = shutit 83 | self.config = {} 84 | self.__set_config(cfg_section) 85 | self.lines = [] 86 | self.attaches = [] 87 | 88 | def __set_config(self, cfg_section): 89 | """Set a local config array up according to 90 | defaults and main shutit configuration 91 | 92 | cfg_section - see __init__ 93 | """ 94 | defaults = [ 95 | 'shutit.core.alerting.emailer.mailto', None, 96 | 'shutit.core.alerting.emailer.mailfrom', 'angry@shutit.tk', 97 | 'shutit.core.alerting.emailer.smtp_server', 'localhost', 98 | 'shutit.core.alerting.emailer.smtp_port', 25, 99 | 'shutit.core.alerting.emailer.use_tls', True, 100 | 'shutit.core.alerting.emailer.send_mail', True, 101 | 'shutit.core.alerting.emailer.subject', 'Shutit Report', 102 | 'shutit.core.alerting.emailer.signature', '--Angry Shutit', 103 | 'shutit.core.alerting.emailer.compress', True, 104 | 'shutit.core.alerting.emailer.username', '', 105 | 'shutit.core.alerting.emailer.password', '', 106 | 'shutit.core.alerting.emailer.safe_mode', True, 107 | 'shutit.core.alerting.emailer.maintainer','', 108 | 'shutit.core.alerting.emailer.mailto_maintainer', True 109 | ] 110 | 111 | for cfg_name, cfg_default in zip(defaults[0::2], defaults[1::2]): 112 | try: 113 | self.config[cfg_name] = self.shutit.cfg[cfg_section][cfg_name] 114 | except KeyError: 115 | if cfg_default is None: 116 | raise Exception(cfg_section + ' ' + cfg_name + ' must be set') 117 | else: 118 | self.config[cfg_name] = cfg_default 119 | 120 | # only send a mail to the module's maintainer if configured correctly 121 | if self.config['shutit.core.alerting.emailer.mailto_maintainer'] and \ 122 | (self.config['shutit.core.alerting.emailer.maintainer'] == "" or \ 123 | self.config['shutit.core.alerting.emailer.maintainer'] == self.config['shutit.core.alerting.emailer.mailto']): 124 | self.config['shutit.core.alerting.emailer.mailto_maintainer'] = False 125 | self.config['shutit.core.alerting.emailer.maintainer'] = "" 126 | 127 | @staticmethod 128 | def __gzip(filename): 129 | """ Compress a file returning the new filename (.gz) 130 | """ 131 | zipname = filename + '.gz' 132 | file_pointer = open(filename,'rb') 133 | zip_pointer = gzip.open(zipname,'wb') 134 | zip_pointer.writelines(file_pointer) 135 | file_pointer.close() 136 | zip_pointer.close() 137 | return zipname 138 | 139 | def __get_smtp(self): 140 | """ Return the appropraite smtplib depending on wherther we're using TLS 141 | """ 142 | use_tls = self.config['shutit.core.alerting.emailer.use_tls'] 143 | if use_tls: 144 | smtp = SMTP(self.config['shutit.core.alerting.emailer.smtp_server'], self.config['shutit.core.alerting.emailer.smtp_port']) 145 | smtp.starttls() 146 | else: 147 | smtp = SMTP_SSL(self.config['shutit.core.alerting.emailer.smtp_server'], self.config['shutit.core.alerting.emailer.smtp_port']) 148 | return smtp 149 | 150 | def add_line(self, line): 151 | """Add a single line to the email body 152 | """ 153 | self.lines.append(line) 154 | 155 | def add_body(self, msg): 156 | """Add an entire email body as a string, will be split on newlines 157 | and overwrite anything currently in the body (e.g added by add_lines) 158 | """ 159 | self.lines = msg.rsplit('\n') 160 | 161 | def attach(self, filename, filetype="txt"): 162 | """Attach a file - currently needs to be entered as root (shutit) 163 | 164 | Filename - absolute path, relative to the target host! 165 | filetype - MIMEApplication._subtype 166 | """ 167 | shutit = self.shutit 168 | host_path = '/tmp' 169 | host_fn = shutit.get_file(filename, host_path) 170 | if self.config['shutit.core.alerting.emailer.compress']: 171 | filetype = 'x-gzip-compressed' 172 | filename = self.__gzip(host_fn) 173 | host_fn = os.path.join(host_path, os.path.basename(filename)) 174 | file_pointer = open(host_fn, 'rb') 175 | attach = MIMEApplication(file_pointer.read(), _subtype=filetype) 176 | file_pointer.close() 177 | attach.add_header('Content-Disposition', 'attachment', filename=os.path.basename(filename)) 178 | self.attaches.append(attach) 179 | 180 | def __compose(self): 181 | """ Compose the message, pulling together body, attachments etc 182 | """ 183 | msg = MIMEMultipart() 184 | msg['Subject'] = self.config['shutit.core.alerting.emailer.subject'] 185 | msg['To'] = self.config['shutit.core.alerting.emailer.mailto'] 186 | msg['From'] = self.config['shutit.core.alerting.emailer.mailfrom'] 187 | # add the module's maintainer as a CC if configured 188 | if self.config['shutit.core.alerting.emailer.mailto_maintainer']: 189 | msg['Cc'] = self.config['shutit.core.alerting.emailer.maintainer'] 190 | if self.config['shutit.core.alerting.emailer.signature'] != '': 191 | signature = '\n\n' + self.config['shutit.core.alerting.emailer.signature'] 192 | else: 193 | signature = self.config['shutit.core.alerting.emailer.signature'] 194 | body = MIMEText('\n'.join(self.lines) + signature) 195 | msg.attach(body) 196 | for attach in self.attaches: 197 | msg.attach(attach) 198 | return msg 199 | 200 | def send(self, attachment_failure=False): 201 | """Send the email according to the configured setup 202 | 203 | attachment_failure - used to indicate a recursive call after the 204 | smtp server has refused based on file size. 205 | Should not be used externally 206 | """ 207 | if not self.config['shutit.core.alerting.emailer.send_mail']: 208 | self.shutit.log('emailer.send: Not configured to send mail!',level=logging.INFO) 209 | return True 210 | msg = self.__compose() 211 | mailto = [self.config['shutit.core.alerting.emailer.mailto']] 212 | smtp = self.__get_smtp() 213 | if self.config['shutit.core.alerting.emailer.username'] != '': 214 | smtp.login(self.config['shutit.core.alerting.emailer.username'], self.config['shutit.core.alerting.emailer.password']) 215 | if self.config['shutit.core.alerting.emailer.mailto_maintainer']: 216 | mailto.append(self.config['shutit.core.alerting.emailer.maintainer']) 217 | try: 218 | self.shutit.log('Attempting to send email',level=logging.INFO) 219 | smtp.sendmail(self.config['shutit.core.alerting.emailer.mailfrom'], mailto, msg.as_string()) 220 | except SMTPSenderRefused as refused: 221 | code = refused.args[0] 222 | if code == 552 and not attachment_failure: 223 | self.shutit.log("Mailserver rejected message due to " + "oversize attachments, attempting to resend without",level=logging.INFO) 224 | self.attaches = [] 225 | self.lines.append("Oversized attachments not sent") 226 | self.send(attachment_failure=True) 227 | else: 228 | self.shutit.log("Unhandled SMTP error:" + str(refused),level=logging.INFO) 229 | if not self.config['shutit.core.alerting.emailer.safe_mode']: 230 | raise refused 231 | except Exception as error: 232 | self.shutit.log('Unhandled exception: ' + str(error),level=logging.INFO) 233 | if not self.config['shutit.core.alerting.emailer.safe_mode']: 234 | raise error 235 | finally: 236 | smtp.quit() 237 | -------------------------------------------------------------------------------- /fixterm_src/.gitignore: -------------------------------------------------------------------------------- 1 | fixterm 2 | fixterm_base64 3 | -------------------------------------------------------------------------------- /fixterm_src/compile.sh: -------------------------------------------------------------------------------- 1 | gcc -o fixterm fixterm.c 2 | base64 fixterm > fixterm_base64 3 | echo now add fixterm_base64 to shutit_assets 4 | rm fixterm 5 | -------------------------------------------------------------------------------- /fixterm_src/fixterm.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #define BUFFER_SIZE (32) 10 | #define NUM_PARAMS (5) 11 | 12 | struct termio old_term; 13 | 14 | void set_raw() { 15 | struct termio new_term; 16 | if (ioctl(1, TCGETA, &old_term) == -1) { 17 | exit(10); 18 | } 19 | memcpy(&new_term, &old_term, sizeof(struct termio)); 20 | new_term.c_lflag &= ~(ICANON|ECHO|ECHOE|ECHOK|ECHONL); 21 | new_term.c_cc[VMIN] = 0; 22 | new_term.c_cc[VTIME] = 0; 23 | if (ioctl(1, TCSETA, &new_term) == -1) { 24 | exit(11); 25 | } 26 | } 27 | 28 | void set_cooked() { 29 | if (ioctl(1, TCSETA, &old_term) == -1) { 30 | exit(12); 31 | } 32 | } 33 | 34 | void set_window_size(int w, int h) { 35 | struct winsize ws; 36 | if (ioctl(1, TIOCGWINSZ, &ws) == -1) { 37 | set_cooked(); 38 | exit(13); 39 | } 40 | ws.ws_col = w; 41 | ws.ws_row = h; 42 | if (ioctl(1, TIOCSWINSZ, &ws) == -1) { 43 | set_cooked(); 44 | exit(14); 45 | } 46 | } 47 | void handle_alarm(int x) { 48 | set_cooked(); 49 | exit(2); 50 | } 51 | void handle_abort(int x) { 52 | set_cooked(); 53 | exit(3); 54 | } 55 | int main(int argc, char **argv) { 56 | int state = 0; 57 | int eof_count = 0; 58 | char buffer[BUFFER_SIZE]; 59 | char *bp = buffer; 60 | char *param_start = buffer; 61 | int params[NUM_PARAMS]; 62 | int param_index = 0; 63 | int i; 64 | for (i=0; i 20) { 78 | goto parse_error; 79 | } else { 80 | usleep(50000); 81 | } 82 | continue; 83 | } 84 | *bp = c; 85 | bp++; 86 | switch (state) { 87 | case 0: 88 | if (c == 27) { 89 | state = 1; 90 | continue; 91 | } else { 92 | goto parse_error; 93 | } 94 | break; 95 | case 1: 96 | if (c == '[') { 97 | state = 2; 98 | param_start = bp; 99 | continue; 100 | } else { 101 | goto parse_error; 102 | } 103 | break; 104 | case 2: 105 | if (c == 't') { 106 | state = 3; 107 | } 108 | if (bp >= (buffer + BUFFER_SIZE - 1)) { 109 | goto parse_error; 110 | } 111 | 112 | if (c == ';') { 113 | if (param_index >= NUM_PARAMS) { 114 | goto parse_error; 115 | } 116 | params[param_index] = atoi(param_start); 117 | param_start=bp; 118 | param_index++; 119 | } 120 | break; 121 | } 122 | if (state == 3) { break; } 123 | } 124 | *bp = '\0'; 125 | if (param_index >= NUM_PARAMS) { 126 | goto parse_error; 127 | } 128 | params[param_index] = atoi(param_start); 129 | param_start=bp; 130 | param_index++; 131 | int width, height; 132 | if (param_index == 2) { 133 | height = params[0]; 134 | width = params[1]; 135 | } else if ((param_index == 3) && (params[0] == 8)) { 136 | height = params[1]; 137 | width = params[2]; 138 | } else { 139 | set_cooked(); 140 | exit(13); 141 | } 142 | set_window_size(width, height); 143 | set_cooked(); 144 | exit(0); 145 | parse_error: 146 | set_cooked(); 147 | exit(1); 148 | } 149 | -------------------------------------------------------------------------------- /fixterm_src/resize.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # http://cafbit.com/entry/terminal_window_size_detection_over 3 | echo -en "\e[18t" # returns \e[8;??;??t 4 | IFS='[;' 5 | read -d t -s esc params 6 | set -- $params 7 | [ $# = 3 -a "$1" = 8 ] && shift 8 | [ $# != 2 ] && echo error >&2 && exit 1 9 | echo $2 10 | echo $1 11 | echo setting terminal to "$2x$1" >&2 12 | #stty rows "$1" cols "$2" 13 | -------------------------------------------------------------------------------- /package_map.py: -------------------------------------------------------------------------------- 1 | """Stores known package maps for different distributions. 2 | """ 3 | 4 | from __future__ import print_function 5 | import logging 6 | import shutit_pexpect_session 7 | 8 | from shutit_sendspec import ShutItSendSpec 9 | 10 | #The MIT License (MIT) 11 | # 12 | #Copyright (C) 2014 OpenBet Limited 13 | # 14 | #Permission is hereby granted, free of charge, to any person obtaining a copy of 15 | #this software and associated documentation files (the "Software"), to deal in 16 | #the Software without restriction, including without limitation the rights to 17 | #use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 18 | #of the Software, and to permit persons to whom the Software is furnished to do 19 | #so, subject to the following conditions: 20 | # 21 | #The above copyright notice and this permission notice shall be included in all 22 | #copies or substantial portions of the Software. 23 | # 24 | #THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 25 | #IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 26 | #ITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 27 | #THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 28 | #LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 29 | #OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 | #SOFTWARE. 31 | 32 | # TODO: figure out how to install pip in yum, and/or return a function that 33 | # does that rather than a string (see https://github.com/ianmiell/shutit/issues/313) 34 | 35 | # Structured by package, then another dict with 36 | # install_type -> mapped package inside that. 37 | # The keys are then the canonical package names. 38 | 39 | def yum_install_pip(shutit_pexpect_session): 40 | # https://www.liquidweb.com/kb/how-to-install-pip-on-centos-7/ 41 | shutit_pexpect_session.send(ShutItSendSpec(shutit_pexpect_session, 42 | send='yum install -y epel-release', 43 | loglevel=logging.INFO)) 44 | shutit_pexpect_session.send(ShutItSendSpec(shutit_pexpect_session, 45 | send='yum -y update', 46 | loglevel=logging.INFO)) 47 | shutit_pexpect_session.send(ShutItSendSpec(shutit_pexpect_session, 48 | send='yum -y install python-pip', 49 | loglevel=logging.INFO)) 50 | 51 | PACKAGE_MAP = { 52 | 'apache2': { 'yum':'httpd'}, 53 | 'httpd': {'apt':'apache2'}, 54 | 'adduser': {'apt':'adduser', 'yum':''}, 55 | 'php5': { 'yum':'php'}, 56 | 'ruby-dev': { 'yum':'ruby-devel', 'brew':'ruby-build'}, 57 | 'git': { 'emerge':'dev-vcs/git'}, 58 | 'vagrant': { 'brew':'Caskroom/cask/vagrant'}, 59 | 'virtualbox': { 'brew':'Caskroom/cask/virtualbox'}, 60 | 'build-essential': { 'yum':'gcc make gcc-c++', 'brew':'gcc'}, 61 | 'sudo': { 'brew':''}, 62 | 'netcat': { 'yum':'nc'}, 63 | 'nc': {'apt':'netcat'}, 64 | 'python-dev': { 'yum':'python-devel'}, 65 | 'python-devel': {'apt':'python-dev'}, 66 | 'mysql-devel': {'apt':'libmysqlclient-dev'}, 67 | 'libmysqlclient-dev': { 'yum':'mysql-devel'}, 68 | 'libkrb5-dev': { 'yum':'krb5-devel'}, 69 | 'libffi-dev': { 'yum':'libffi-devel'}, 70 | 'libffi-devel': {'apt':'libffi-dev'}, 71 | 'libsasl2-dev': { 'yum':''}, 72 | 'libssl-dev': { 'yum':'openssl-devel'}, 73 | 'kvm': {'apt':'qemu-kvm'}, 74 | 'libvirt': {'apt':'libvirt-bin'}, 75 | 'libvirt-dev': { 'yum':'libvirt-devel'}, 76 | 'libvirt-devel': {'apt':'libvirt-dev'}, 77 | 'docker': {'apt':'docker.io'}, 78 | 'asciinema': { 'yum':'epel-release asciinema'}, 79 | 'run-one': { 'yum':''}, 80 | 'python-pip': { 'yum': yum_install_pip}, 81 | 'piptest': {'apt':'python-pip', 'yum': yum_install_pip}, 82 | 'lsb-release': { 'yum': 'redhat-lsb-core'}, 83 | } 84 | 85 | 86 | # A list of OS Family members 87 | # Suse = SLES, SLED, OpenSuSE, Suse 88 | # Archlinux = Archlinux 89 | # Mandrake = Mandriva, Mandrake 90 | # Solaris = Solaris, Nexenta, OmniOS, OpenIndiana, SmartOS 91 | # AIX = AIX 92 | # FreeBSD = FreeBSD 93 | # HP-UK = HPUX 94 | # OSDIST_DICT = {'/etc/vmware-release':'VMwareESX','/etc/openwrt_release':'OpenWrt','/etc/system-release':'OtherLinux','/etc/release':'Solaris','/etc/arch-release':'Archlinux','/etc/SuSE-release':'SuSE','/etc/gentoo-release':'Gentoo'} 95 | # # A list of dicts. If there is a platform with more than one package manager, put the preferred one last. If there is an ansible module, use that as the value for the 'name' key. 96 | #PKG_MGRS = [{'path':'/usr/bin/zypper','name':'zypper'},{'path':'/usr/sbin/urpmi','name':'urpmi'},{'path':'/usr/bin/pacman','name':'pacman'},{'path':'/bin/opkg','name':'opkg'},{'path':'/opt/local/bin/pkgin','name':'pkgin'},{'path':'/opt/local/bin/port','name':'macports'},{'path':'/usr/sbin/pkg','name':'pkgng'},{'path':'/usr/sbin/swlist','name':'SD-UX'},{'path':'/usr/sbin/pkgadd','name':'svr4pkg'},{'path':'/usr/bin/pkg','name':'pkg'}, 97 | # ] 98 | # Map install types based on /etc/issue contents 99 | INSTALL_TYPE_MAP = {'ubuntu':'apt', 100 | 'debian':'apt', 101 | 'raspbian':'apt', 102 | 'linuxmint':'apt', 103 | 'steamos':'apt', 104 | 'red hat':'yum', 105 | 'redhatenterpriseserver':'yum', 106 | 'oracleserver':'yum', 107 | 'amazon':'yum', 108 | 'amazonami':'yum', 109 | 'centos':'yum', 110 | 'fedora':'yum', 111 | 'fedora_dnf':'dnf', 112 | 'alpine':'apk', 113 | 'shutit':'src', 114 | 'coreos':'docker', 115 | 'gentoo':'emerge', 116 | 'osx':'brew', 117 | 'arch':'pacman', 118 | 'manjarolinux':'pacman', 119 | 'minishift':'none', 120 | 'minikube':'none', 121 | 'cygwin':'apt-cyg'} 122 | 123 | 124 | 125 | def map_packages(shutit_pexpect_session, package_str, install_type): 126 | res = '' 127 | for package in package_str.split(): 128 | map_package_res = map_package(shutit_pexpect_session, package,install_type) 129 | if map_package_res == '': 130 | return res 131 | res += ' ' + map_package_res 132 | return res 133 | 134 | 135 | def map_package(shutit_pexpect_session, package, install_type): 136 | """If package mapping exists, then return it, else return package. 137 | """ 138 | if package in PACKAGE_MAP.keys(): 139 | for itype in PACKAGE_MAP[package].keys(): 140 | if itype == install_type: 141 | ret = PACKAGE_MAP[package][install_type] 142 | if isinstance(ret,str): 143 | return ret 144 | if callable(ret): 145 | ret(shutit_pexpect_session) 146 | return '' 147 | # Otherwise, simply return package 148 | return package 149 | -------------------------------------------------------------------------------- /release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # pip release scripts that auto-updates version number and keeps trying until successful 3 | set -x 4 | set -u 5 | i=0 6 | #https://towardsdatascience.com/how-to-upload-your-python-package-to-pypi-de1b363a1b3 7 | pip install twine 8 | while true 9 | do 10 | rm -rf build 11 | i=$((i+1)) 12 | output=$(grep version= setup.py | awk -F'=' '{print $2}' | sed "s/'\([0-9][0-9]*\)\.\([0-9][0-9]*\)\.\([0-9][0-9]*\)',/\1 \2 \3/") 13 | major=$(echo "${output}" | awk '{print $1}') 14 | minor=$(echo "${output}" | awk '{print $2}') 15 | point=$(echo "${output}" | awk '{print $3}') 16 | newpoint=$((point+1)) 17 | sed -i "s/\([ \s]\)*version=\(.\)$major.$minor.$point\(.\).*/\1version=\2$major.$minor.$newpoint\3,/" setup.py 18 | sed -i "s/^shutit_version=\(.\)${major}.${minor}.[0-9][0-9]*\(.\).*/shutit_version=\1${major}.${minor}.${newpoint}\2/" shutit.py 19 | #if python setup.py sdist bdist_wheel upload 20 | if python setup.py sdist 21 | then 22 | twine upload dist/* 23 | break 24 | fi 25 | # wait a minute 26 | sleep 60 27 | done 28 | git commit -am "release: ${major}.${minor}.${newpoint}" 29 | echo Success after ${i} attempts 30 | git push 31 | git tag -f "${major}.${minor}.${newpoint}" 32 | git push --tags 33 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pexpect>=4.0 2 | texttable>=0.1 3 | jinja2>=0.1 4 | six>=1.10 5 | future>=0.15 6 | cjkwrap 7 | curtsies>=0.3.0 8 | pygithub 9 | jinja2>=3.0.0 10 | MarkupSafe>=2.0 11 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # To use a consistent encoding 2 | from __future__ import print_function 3 | from codecs import open 4 | # Always prefer setuptools over distutils 5 | from setuptools import setup, find_packages 6 | 7 | setup( 8 | name='shutit', 9 | 10 | # Versions should comply with PEP440. For a discussion on single-sourcing 11 | # the version across setup.py and the project code, see 12 | # https://packaging.python.org/en/latest/single_source_version.html 13 | 14 | version='1.0.173', 15 | description='A programmable automation tool designed for complex builds', 16 | long_description='A programmable shell-based (pexpect) automation tool designed for complex builds. See: http://ianmiell.github.io/shutit', 17 | 18 | # The project's main homepage. 19 | url='http://ianmiell.github.io/shutit/', 20 | 21 | # Author details 22 | author='Ian Miell', 23 | author_email='ian.miell@gmail.com', 24 | 25 | # Choose your license 26 | license='MIT', 27 | 28 | # See https://pypi.python.org/pypi?%3Aaction=list_classifiers 29 | classifiers=[ 30 | # How mature is this project? Common values are 31 | # 3 - Alpha 32 | # 4 - Beta 33 | # 5 - Production/Stable 34 | 'Development Status :: 4 - Beta', 35 | 36 | # Indicate who your project is intended for 37 | 'Intended Audience :: Developers', 38 | 'Topic :: Software Development :: Build Tools', 39 | 40 | # Pick your license as you wish (should match "license" above) 41 | 'License :: OSI Approved :: MIT License', 42 | 43 | # Specify the Python versions you support here. In particular, ensure 44 | # that you indicate whether you support Python 2, Python 3 or both. 45 | 'Programming Language :: Python :: 2.7', 46 | ], 47 | 48 | # What does your project relate to? 49 | keywords='Docker pexpect expect automation build', 50 | 51 | # You can just specify the packages manually here if your project is 52 | # simple. Or you can use find_packages(). 53 | packages=['.'] + find_packages(), 54 | 55 | # List run-time dependencies here. These will be installed by pip when 56 | # your project is installed. For an analysis of "install_requires" vs pip's 57 | # requirements files see: 58 | # https://packaging.python.org/en/latest/requirements.html 59 | install_requires=['pexpect>=4.0','jinja2>=0.1','texttable>=0.1','six>=1.10','future>=0.15','curtsies>=0.3.0', 'cjkwrap'], 60 | 61 | 62 | # List additional groups of dependencies here (e.g. development 63 | # dependencies). You can install these using the following syntax, 64 | # for example: 65 | # $ pip install -e .[dev,test] 66 | extras_require={ 67 | 'dev': [], 68 | 'test': [], 69 | }, 70 | 71 | # If there are data files included in your packages that need to be 72 | # installed, specify them here. If using Python 2.6 or less, then these 73 | # have to be included in MANIFEST.in as well. 74 | package_data={}, 75 | 76 | # Although 'package_data' is the preferred approach, in some case you may 77 | # need to place data files outside of your packages. See: 78 | # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files # noqa 79 | # In this case, 'data_file' will be installed into '/my_data' 80 | data_files=[], 81 | 82 | # To provide executable scripts, use entry points in preference to the 83 | # "scripts" keyword. Entry points provide cross-platform support and allow 84 | # pip to create the appropriate form of executable for the target platform. 85 | entry_points={ 86 | 'console_scripts': [ 87 | 'shutit=shutit:main', 88 | ], 89 | }, 90 | ) 91 | -------------------------------------------------------------------------------- /shutit.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # The MIT License (MIT) 3 | # 4 | # Copyright (C) 2014 OpenBet Limited 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy of 7 | # this software and associated documentation files (the "Software"), to deal in 8 | # the Software without restriction, including without limitation the rights to 9 | # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 10 | # of the Software, and to permit persons to whom the Software is furnished to do 11 | # so, subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # ITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | 26 | """ShutIt is a means of building stateless target hosts in a flexible and predictable way. 27 | """ 28 | import sys 29 | import shutit_global 30 | import shutit_util 31 | 32 | 33 | def create_session(docker_image=None, 34 | docker_rm=None, 35 | echo=False, 36 | loglevel='', 37 | nocolor=False, 38 | session_type='bash', 39 | vagrant_session_name=None, 40 | vagrant_image='ubuntu/xenial64', 41 | vagrant_gui=False, 42 | vagrant_memory='1024', 43 | vagrant_num_machines='1', 44 | vagrant_provider='virtualbox', 45 | vagrant_root_folder=None, 46 | vagrant_swapsize='2G', 47 | vagrant_version='1.8.6', 48 | vagrant_virt_method='virtualbox', 49 | vagrant_cpu='1', 50 | video=-1, 51 | walkthrough=False): 52 | """Creates a distinct ShutIt session. Sessions can be of type: 53 | 54 | bash - a bash shell is spawned and 55 | vagrant - a Vagrantfile is created and 'vagrant up'ped 56 | """ 57 | assert session_type in ('bash','docker','vagrant'), shutit_util.print_debug() 58 | shutit_global_object = shutit_global.shutit_global_object 59 | if video != -1 and video > 0: 60 | walkthrough = True 61 | if session_type in ('bash','docker'): 62 | return shutit_global_object.create_session(session_type, 63 | docker_image=docker_image, 64 | rm=docker_rm, 65 | echo=echo, 66 | walkthrough=walkthrough, 67 | walkthrough_wait=video, 68 | nocolor=nocolor, 69 | loglevel=loglevel) 70 | elif session_type == 'vagrant': 71 | if vagrant_session_name is None: 72 | vagrant_session_name = 'shutit' + shutit_util.random_id() 73 | if isinstance(vagrant_num_machines, int): 74 | vagrant_num_machines = str(vagrant_num_machines) 75 | assert isinstance(vagrant_num_machines, str) 76 | assert isinstance(int(vagrant_num_machines), int) 77 | if vagrant_root_folder is None: 78 | vagrant_root_folder = shutit_global.shutit_global_object.owd 79 | return create_session_vagrant(vagrant_session_name, 80 | vagrant_num_machines, 81 | vagrant_image, 82 | vagrant_provider, 83 | vagrant_gui, 84 | vagrant_memory, 85 | vagrant_swapsize, 86 | echo, 87 | walkthrough, 88 | nocolor, 89 | video, 90 | vagrant_version, 91 | vagrant_virt_method, 92 | vagrant_root_folder, 93 | vagrant_cpu, 94 | loglevel) 95 | 96 | 97 | def create_session_vagrant(session_name, 98 | num_machines, 99 | vagrant_image, 100 | vagrant_provider, 101 | gui, 102 | memory, 103 | swapsize, 104 | echo, 105 | walkthrough, 106 | nocolor, 107 | video, 108 | vagrant_version, 109 | virt_method, 110 | root_folder, 111 | cpu, 112 | loglevel): 113 | if video != -1 and video > 0: 114 | walkthrough = True 115 | assert isinstance(memory, str) 116 | assert isinstance(swapsize, str) 117 | return shutit_global.shutit_global_object.create_session_vagrant(session_name, 118 | num_machines, 119 | vagrant_image, 120 | vagrant_provider, 121 | gui, 122 | memory, 123 | swapsize, 124 | echo, 125 | walkthrough, 126 | video, 127 | nocolor, 128 | vagrant_version, 129 | virt_method, 130 | root_folder, 131 | loglevel) 132 | 133 | def main(): 134 | """Main ShutIt function. 135 | 136 | Handles the configured actions: 137 | 138 | - skeleton - create skeleton module 139 | - list_configs - output computed configuration 140 | - depgraph - output digraph of module dependencies 141 | """ 142 | # Create base shutit object. 143 | shutit = shutit_global.shutit_global_object.shutit_objects[0] 144 | if sys.version_info[0] == 2: 145 | if sys.version_info[1] < 7: 146 | shutit.fail('Python version must be 2.7+') # pragma: no cover 147 | try: 148 | shutit.setup_shutit_obj() 149 | except KeyboardInterrupt: 150 | shutit_util.print_debug(sys.exc_info()) 151 | shutit_global.shutit_global_object.shutit_print('Keyboard interrupt caught, exiting with status 1') 152 | sys.exit(1) 153 | 154 | 155 | shutit_version='1.0.173' 156 | 157 | if __name__ == '__main__': 158 | shutit_global.setup_signals() 159 | main() 160 | -------------------------------------------------------------------------------- /shutit_background.py: -------------------------------------------------------------------------------- 1 | r"""Represents a ShutIt background command. 2 | 3 | - options to: 4 | - cancel command 5 | - get status (running, suspended etc) 6 | - check_timeout 7 | """ 8 | 9 | from __future__ import print_function 10 | import sys 11 | import time 12 | import logging 13 | import traceback 14 | import shutit_global 15 | import shutit_util 16 | 17 | 18 | class ShutItBackgroundCommand(object): 19 | """Background command in ShutIt 20 | """ 21 | def __init__(self, 22 | sendspec, 23 | shutit_obj): 24 | # Stub this with a simple command for now 25 | self.sendspec = sendspec 26 | self.block_other_commands = sendspec.block_other_commands 27 | self.retry = sendspec.retry 28 | self.tries = 0 29 | self.pid = None 30 | self.return_value = None 31 | self.start_time = None 32 | self.run_state = 'N' # State as per ps man page, but 'C' == Complete, 'N' == not started, 'F' == failed, 'S' == sleeping/running, 'T' == timed out by ShutIt 33 | self.cwd = self.sendspec.shutit_pexpect_child.send_and_get_output(' command pwd', ignore_background=True) 34 | self.id = shutit_util.random_id() 35 | self.output_file = '/tmp/shutit_background_' + self.id + '_output.log' 36 | self.exit_code_file = '/tmp/shutit_background_' + self.id + '_exit_code_file.log' 37 | self.command_file = '/tmp/shutit_background_' + self.id + '_command.log' 38 | if self.sendspec.run_in_background: 39 | # TODO: consider separating out into a simple send for the part that creates the command file, the cd and the output file. Perhaps send file first and run that in the background? 40 | self.sendspec.send = ' set +m && { : $(echo "' + self.sendspec.original_send + '" >' + self.command_file + ' && command cd "' + self.cwd + '">' + self.output_file + ' && ' + self.sendspec.send + ' >>' + self.output_file + ' 2>&1; echo $? >' + self.exit_code_file + ') & } 2>/dev/null' 41 | self.shutit_obj = shutit_obj 42 | 43 | 44 | def __str__(self): 45 | string = str(self.sendspec) 46 | string += '\n---- Background object BEGIN ----' 47 | string += '\n block_other_commands: ' + str(self.block_other_commands) 48 | string += '| cwd: ' + str(self.cwd) 49 | string += '| pid: ' + str(self.pid) 50 | string += '| retry: ' + str(self.retry) 51 | string += '| return_value: ' + str(self.return_value) 52 | string += '| run_state: ' + str(self.run_state) 53 | string += '| start_time: ' + str(self.start_time) 54 | string += '| tries: ' + str(self.tries) 55 | string += '|---- Background object END ----' 56 | return string 57 | 58 | 59 | def run_background_command(self): 60 | # reset object 61 | self.pid = None 62 | self.return_value = None 63 | self.run_state = 'N' 64 | self.start_time = time.time() # record start time 65 | 66 | # run command 67 | self.tries += 1 68 | if self.sendspec.run_in_background: 69 | ## Required to reset terminal before a background send. (TODO: why?) 70 | #self.sendspec.shutit_pexpect_child.reset_terminal() 71 | # Run in the background 72 | self.sendspec.shutit_pexpect_child.quick_send(self.sendspec.send,loglevel=logging.DEBUG) 73 | # Put into an 'S' state as that means 'running' 74 | self.run_state = 'S' 75 | # Required to reset terminal after a background send. (TODO: why?) 76 | self.sendspec.shutit_pexpect_child.reset_terminal() 77 | # record pid 78 | self.pid = self.sendspec.shutit_pexpect_child.send_and_get_output(" echo ${!}",ignore_background=True) 79 | else: 80 | # Run synchronously and mark complete 81 | # We need to set this to ignore background before we run it, so that 82 | # it does not block itself and end up in an infinite loop. 83 | self.sendspec.ignore_background = True 84 | self.sendspec.shutit_pexpect_child.send(self.sendspec) 85 | self.run_state = 'C' 86 | 87 | self.sendspec.started = True 88 | 89 | assert self.run_state in ('C','S','F'), shutit_util.print_debug() 90 | return True 91 | 92 | 93 | def check_background_command_state(self): 94 | self.shutit_obj.log('CHECKING background task: ' + self.sendspec.send + ', id: ' + self.id,level=logging.DEBUG) 95 | assert self.start_time is not None, shutit_util.print_debug() 96 | # Check the command has been started 97 | if not self.sendspec.started: 98 | assert self.run_state == 'N', shutit_util.print_debug() 99 | return self.run_state 100 | if self.run_state in ('C','F'): 101 | assert self.sendspec.started, shutit_util.print_debug() 102 | return self.run_state 103 | try: 104 | assert self.run_state in ('S',), shutit_util.print_debug(msg='State should be in S, is in fact: ' + self.run_state) 105 | except AssertionError: 106 | _, _, tb = sys.exc_info() 107 | traceback.print_tb(tb) # Fixed format 108 | tb_info = traceback.extract_tb(tb) 109 | _, line, _, text = tb_info[-1] 110 | shutit_global.shutit_global_object.shutit_print('An error occurred on line {} in statement {}'.format(line, text)) 111 | # Update the run state. 112 | updated_run_state = self.sendspec.shutit_pexpect_child.send_and_get_output(""" command ps -o stat """ + self.pid + """ | command sed '1d' """, ignore_background=True) 113 | # Ensure we get the first character only, if one exists. 114 | if len(updated_run_state) > 0: 115 | # Task is unfinished. 116 | self.run_state = updated_run_state[0] 117 | updated_run_state = None 118 | # state The state is given by a sequence of characters, for example, ``RWNA''. The first character indicates the run state of the process: 119 | # I Marks a process that is idle (sleeping for longer than about 20 seconds). 120 | # R Marks a runnable process. 121 | # S Marks a process that is sleeping for less than about 20 seconds. 122 | # T Marks a stopped process. 123 | # U Marks a process in uninterruptible wait. 124 | # Z Marks a dead process (a ``zombie''). 125 | if self.run_state in ('I','R','T','U','Z'): 126 | self.shutit_obj.log('background task run state: ' + self.run_state, level=logging.DEBUG) 127 | self.run_state = 'S' 128 | try: 129 | assert self.run_state in ('S',), shutit_util.print_debug(msg='State should be in S having gleaned from ps, is in fact: ' + self.run_state) 130 | except AssertionError: 131 | _, _, tb = sys.exc_info() 132 | traceback.print_tb(tb) # Fixed format 133 | tb_info = traceback.extract_tb(tb) 134 | _, line, _, text = tb_info[-1] 135 | shutit_global.shutit_global_object.shutit_print('An error occurred on line {} in statement {}'.format(line, text)) 136 | shutit_global.shutit_global_object.shutit_print(self) 137 | # honour sendspec.timeout 138 | if self.sendspec.timeout is not None: 139 | current_time = time.time() 140 | time_taken = current_time - self.start_time 141 | if time_taken > self.sendspec.timeout: 142 | # We've timed out, kill with prejudice. 143 | self.sendspec.shutit_pexpect_child.quick_send(' kill -9 ' + self.pid,loglevel=logging.DEBUG) 144 | self.run_state = 'T' 145 | assert self.run_state in ('S','T'), shutit_util.print_debug() 146 | return self.run_state 147 | else: 148 | # Task is finished. 149 | self.run_state = 'C' 150 | self.shutit_obj.log('background task: ' + self.sendspec.send + ', id: ' + self.id + ' complete',level=logging.DEBUG) 151 | # Stop this from blocking other commands from here. 152 | assert self.return_value is None, shutit_util.print_debug(msg='check_background_command_state called with self.return_value already set?' + str(self)) 153 | self.sendspec.shutit_pexpect_child.quick_send(' wait ' + self.pid,loglevel=logging.DEBUG) 154 | # Get the exit code. 155 | self.return_value = self.sendspec.shutit_pexpect_child.send_and_get_output(' cat ' + self.exit_code_file, ignore_background=True) 156 | # If the return value is deemed a failure: 157 | if self.return_value not in self.sendspec.exit_values: 158 | self.shutit_obj.log('background task: ' + self.sendspec.send + ' failed with exit code: ' + self.return_value, level=logging.DEBUG) 159 | self.shutit_obj.log('background task: ' + self.sendspec.send + ' failed with output: ' + self.sendspec.shutit_pexpect_child.send_and_get_output(' cat ' + self.output_file, ignore_background=True), level=logging.DEBUG) 160 | if self.retry > 0: 161 | self.shutit_obj.log('background task: ' + self.sendspec.send + ' retrying',level=logging.DEBUG) 162 | self.retry -= 1 163 | self.run_background_command() 164 | # recurse 165 | return self.check_background_command_state() 166 | else: 167 | self.shutit_obj.log('background task final failure: ' + self.sendspec.send + ' failed with exit code: ' + self.return_value, level=logging.DEBUG) 168 | self.run_state = 'F' 169 | assert self.run_state in ('C','F'), shutit_util.print_debug() 170 | return self.run_state 171 | else: 172 | # Task succeeded. 173 | self.shutit_obj.log('background task: ' + self.sendspec.send + ' succeeded with exit code: ' + self.return_value, level=logging.DEBUG) 174 | assert self.run_state in ('C',), shutit_util.print_debug() 175 | return self.run_state 176 | # Should never get here. 177 | assert False, shutit_util.print_debug() 178 | -------------------------------------------------------------------------------- /shutit_exam.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import sys 3 | import time 4 | import signal 5 | 6 | class ShutItExamSessionStage(object): 7 | 8 | # difficulty - a proportion of the default difficulty=1 9 | # reduction_per_minute - the degree to which success value decreases every minute 10 | # reduction_per_reset - the degree to which a reset affects the final score 11 | # reduction_per_hint - the degree to which a hint affects the final score 12 | # grace_period - the time under which a success scores full points 13 | # 14 | # A difficulty of zero means the challenge has no significance to the overall score. 15 | def __init__(self, 16 | shutit, 17 | difficulty=1.0, 18 | reduction_per_minute=0.2, 19 | reduction_per_reset=0, 20 | reduction_per_hint=0.5, 21 | grace_period=30): 22 | self.shutit = shutit 23 | self.difficulty = difficulty 24 | self.reduction_per_minute = reduction_per_minute 25 | self.reduction_per_reset = reduction_per_reset 26 | self.reduction_per_hint = reduction_per_hint 27 | self.grace_period = grace_period 28 | self.result = '' 29 | self.num_resets = 0 30 | self.num_hints = 0 31 | self.start_time = None 32 | self.end_time = None 33 | self.total_time = None 34 | self.score = -1 35 | 36 | 37 | def __str__(self): 38 | string = '| num_resets = ' + str(self.num_resets) 39 | string += '| num_hints = ' + str(self.num_hints) 40 | string += '| result = ' + str(self.result) 41 | string += '| start_time = ' + str(self.start_time) 42 | string += '| end_time = ' + str(self.end_time) 43 | string += '| score = ' + str(self.score) 44 | return string 45 | 46 | 47 | def start_timer(self): 48 | if self.start_time != None: 49 | self.shutit.fail('start_timer called with start_time already set') # pragma: no cover 50 | self.start_time = time.time() 51 | 52 | 53 | def end_timer(self): 54 | if self.start_time is None: 55 | self.shutit.fail('end_timer called with no start_time set') # pragma: no cover 56 | if self.end_time is not None: 57 | self.shutit.fail('end_time already set') # pragma: no cover 58 | self.end_time = time.time() 59 | self.total_time = self.end_time - self.start_time 60 | 61 | 62 | def is_complete(self): 63 | if self.result == '': 64 | return False 65 | else: 66 | return True 67 | 68 | 69 | 70 | class ShutItExamSession(object): 71 | 72 | def __init__(self, shutit, num_stages=0): 73 | self.shutit = shutit 74 | self.stages = [] 75 | self.num_stages = num_stages 76 | self.final_score = 0.0 77 | self.curr_stage = len(self.stages)+1 78 | # Switch off CTRL-C etc 79 | signal.signal(signal.SIGINT, signal.SIG_IGN) 80 | signal.signal(signal.SIGQUIT, signal.SIG_IGN) 81 | signal.signal(signal.SIGPIPE, signal.SIG_IGN) 82 | signal.signal(signal.SIGTSTP, signal.SIG_IGN) 83 | 84 | def __str__(self): 85 | string = '' 86 | n=0 87 | for stage in self.stages: 88 | n+=1 89 | stage_desc = 'Stage ' + str(n) + ' of ' + str(len(self.stages)) 90 | string += '\n' + stage_desc 91 | string += '\n' + str(stage) 92 | string += '\n\nFinal score: ' + str(self.final_score) + '%\n' 93 | return string 94 | 95 | def new_stage(self, 96 | difficulty, 97 | reduction_per_minute=0.2, 98 | reduction_per_reset=0, 99 | reduction_per_hint=0.5, 100 | grace_period=30): 101 | difficulty = float(difficulty) 102 | stage = ShutItExamSessionStage(self.shutit, difficulty,reduction_per_minute,reduction_per_reset,reduction_per_hint,grace_period) 103 | self.stages.append(stage) 104 | self.curr_stage = len(self.stages)+1 105 | return stage 106 | 107 | def add_reset(self): 108 | if self.stages == []: 109 | self.shutit.fail('add_reset: no stages to reset') # pragma: no cover 110 | stage = self.stages[-1] 111 | stage.num_resets += 1 112 | 113 | def add_skip(self): 114 | if self.stages == []: 115 | self.shutit.fail('add_skip: no stages to skip') # pragma: no cover 116 | stage = self.stages[-1] 117 | if stage.result != '': 118 | self.shutit.fail('add_skip: result already determined') # pragma: no cover 119 | else: 120 | stage.result = 'SKIP' 121 | 122 | def add_fail(self): 123 | if self.stages == []: 124 | self.shutit.fail('add_fail: no stages to fail') # pragma: no cover 125 | stage = self.stages[-1] 126 | if stage.result != '': 127 | self.shutit.fail('add_fail: result already determined') # pragma: no cover 128 | else: 129 | stage.result = 'FAIL' 130 | 131 | def add_ok(self): 132 | if self.stages == []: 133 | self.shutit.fail('add_ok: no stages to ok') # pragma: no cover 134 | stage = self.stages[-1] 135 | if stage.result != '': 136 | self.shutit.fail('add_ok: result already determined') # pragma: no cover 137 | else: 138 | stage.result = 'OK' 139 | 140 | def add_hint(self): 141 | if self.stages == []: 142 | self.shutit.fail('add_hint: no stages to add hint for') # pragma: no cover 143 | stage = self.stages[-1] 144 | stage.num_hints += 1 145 | 146 | def start_timer(self): 147 | if self.stages == []: 148 | self.shutit.fail('start_timer: no stages to time') # pragma: no cover 149 | stage = self.stages[-1] 150 | stage.start_timer() 151 | 152 | def end_timer(self): 153 | if self.stages == []: 154 | self.shutit.fail('end_timer: no stages to time') # pragma: no cover 155 | stage = self.stages[-1] 156 | stage.end_timer() 157 | 158 | def calculate_score(self): 159 | max_score = 0.0 160 | total_score = 0.0 161 | for stage in self.stages: 162 | max_score += stage.difficulty 163 | # If they succeeded, start with the diffulty score (100%) 164 | if stage.result == 'OK': 165 | stage.score = stage.difficulty 166 | for item in range(0,stage.num_resets): 167 | item = item # pylint 168 | stage.score = stage.score - (stage.score * stage.reduction_per_reset) 169 | for item in range(0,stage.num_hints): 170 | stage.score = stage.score - (stage.score * stage.reduction_per_hint) 171 | total_time = stage.end_time - stage.start_time 172 | total_time -= stage.grace_period 173 | if total_time > 0: 174 | num_minutes = total_time / 60 175 | num_seconds = total_time % 60 176 | num_minutes = num_minutes + (num_seconds / 60) 177 | while num_minutes > 1: 178 | num_minutes -= 1 179 | stage.score = stage.score - (stage.score * stage.reduction_per_minute) 180 | if num_minutes > 0: 181 | stage.score = stage.score - (stage.score * stage.reduction_per_minute * num_minutes) 182 | total_score = total_score + stage.score 183 | else: 184 | stage.score = 0 185 | self.final_score = total_score / max_score * 100.00 186 | return self.final_score 187 | -------------------------------------------------------------------------------- /shutit_login_stack.py: -------------------------------------------------------------------------------- 1 | """Represents a ShutItLoginStack object. 2 | 3 | Every time ShutItPexpectSession.login() is run a new item is added, and every 4 | time it a corresponding logout() function is called, it is popped. 5 | 6 | It also holds within it information about ShutItBackgroundCommand objects 7 | belonging to this login. 8 | 9 | """ 10 | 11 | from __future__ import print_function 12 | import logging 13 | import shutit_global 14 | from shutit_background import ShutItBackgroundCommand 15 | 16 | class ShutItLoginStack(object): 17 | 18 | 19 | def __init__(self, shutit_obj): 20 | """ 21 | """ 22 | self.stack = [] 23 | self.shutit_obj = shutit_obj 24 | 25 | 26 | def append(self, login_id): 27 | self.stack.append(ShutItLoginStackItem(login_id, self.shutit_obj)) 28 | return True 29 | 30 | 31 | def pop(self): 32 | return self.stack.pop() 33 | 34 | 35 | def length(self): 36 | return len(self.stack) 37 | 38 | 39 | def get_current_login_id(self): 40 | if self.stack: 41 | return self.stack[-1].login_id 42 | return None 43 | 44 | 45 | def get_current_login_item(self): 46 | if self.stack: 47 | return self.stack[-1] 48 | return None 49 | 50 | 51 | def find_sendspec(self): 52 | for stack_item in self.stack: 53 | if stack_item.find_sendspec(): 54 | return True 55 | return False 56 | 57 | 58 | def __str__(self): 59 | string = '\n---- Login stack object BEGIN ----' 60 | string += '\nstack: ' + str(self.stack) 61 | for item in self.stack: 62 | string += '\nlogin_item: ' + str(item) 63 | string = '\n---- Login stack object END ----' 64 | return string 65 | 66 | 67 | class ShutItLoginStackItem(object): 68 | 69 | 70 | def __init__(self, login_id, shutit_obj): 71 | """ 72 | """ 73 | self.login_id = login_id 74 | self.background_objects = [] 75 | self.background_objects_completed = [] 76 | self.shutit_obj = shutit_obj 77 | 78 | 79 | def append_background_send(self,sendspec): 80 | shutit_background_command_object = ShutItBackgroundCommand(sendspec, self.shutit_obj) 81 | self.background_objects.append(shutit_background_command_object) 82 | return shutit_background_command_object 83 | 84 | 85 | def has_blocking_background_send(self): 86 | """Check whether any blocking background commands are waiting to run. 87 | If any are, return True. If none are, return False. 88 | """ 89 | for background_object in self.background_objects: 90 | # If it's running, or not started yet, it should block other tasks. 91 | if background_object.block_other_commands and background_object.run_state in ('S','N'): 92 | self.shutit_obj.log('All objects are: ' + str(self),level=logging.DEBUG) 93 | self.shutit_obj.log('The current blocking send object is: ' + str(background_object),level=logging.DEBUG) 94 | return True 95 | elif background_object.block_other_commands and background_object.run_state in ('F','C','T'): 96 | assert False, shutit_util.print_debug(msg='Blocking command should have been removed, in run_state: ' + background_object.run_state) 97 | else: 98 | assert background_object.block_other_commands is False, shutit_util.print_debug() 99 | return False 100 | 101 | 102 | def check_background_commands_complete(self): 103 | """Check whether any background commands are running or to be run. 104 | If none are, return True. If any are, return False. 105 | """ 106 | unstarted_command_exists = False 107 | self.shutit_obj.log('In check_background_commands_complete: all background objects: ' + str(self.background_objects),level=logging.DEBUG) 108 | self.shutit_obj.log('Login id: ' + str(self.login_id),level=logging.DEBUG) 109 | for background_object in self.background_objects: 110 | self.shutit_obj.log('Background object send: ' + str(background_object.sendspec.send),level=logging.DEBUG) 111 | background_objects_to_remove = [] 112 | def remove_background_objects(a_background_objects_to_remove): 113 | for background_object in a_background_objects_to_remove: 114 | self.background_objects.remove(background_object) 115 | for background_object in self.background_objects: 116 | self.shutit_obj.log('Checking background object: ' + str(background_object),level=logging.DEBUG) 117 | state = background_object.check_background_command_state() 118 | self.shutit_obj.log('State is: ' + state,level=logging.DEBUG) 119 | if state in ('C','F','T'): 120 | background_objects_to_remove.append(background_object) 121 | self.background_objects_completed.append(background_object) 122 | elif state == 'S': 123 | # Running command exists 124 | self.shutit_obj.log('check_background_command_state returning False (S) for ' + str(background_object),level=logging.DEBUG) 125 | remove_background_objects(background_objects_to_remove) 126 | return False, 'S', background_object 127 | elif state == 'N': 128 | self.shutit_obj.log('UNSTARTED COMMAND! ' + str(background_object.sendspec.send),level=logging.DEBUG) 129 | unstarted_command_exists = True 130 | else: 131 | remove_background_objects(background_objects_to_remove) 132 | assert False, shutit_util.print_debug(msg='Un-handled: ' + state) 133 | if state == 'F': 134 | self.shutit_obj.log('check_background_command_state returning False (F) for ' + str(background_object),level=logging.DEBUG) 135 | remove_background_objects(background_objects_to_remove) 136 | return False, 'F', background_object 137 | remove_background_objects(background_objects_to_remove) 138 | self.shutit_obj.log('Checking background objects done.',level=logging.DEBUG) 139 | if unstarted_command_exists: 140 | # Start up an unstarted one (in order), and return False 141 | for background_object in self.background_objects: 142 | state = background_object.check_background_command_state() 143 | if state == 'N': 144 | background_object.run_background_command() 145 | self.shutit_obj.log('check_background_command_state returning False (N) for ' + str(background_object),level=logging.DEBUG) 146 | return False, 'N', background_object 147 | # Nothing left to do - return True. 148 | self.shutit_obj.log('check_background_command_state returning True (OK)',level=logging.DEBUG) 149 | return True, 'OK', None 150 | 151 | 152 | def find_sendspec(self,sendspec): 153 | for background_object in self.background_objects: 154 | if background_object == sendspec: 155 | return True 156 | return False 157 | 158 | 159 | def __str__(self): 160 | string = '\n---- Login stack item object ----' 161 | string += '\nlogin_id: ' + str(self.login_id) 162 | for background_object in self.background_objects: 163 | string += '\nbackground_objects: ' + str(background_object) 164 | string += '\n---- ----' 165 | return string 166 | -------------------------------------------------------------------------------- /shutit_module.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # 3 | # Copyright (C) 2014 OpenBet Limited 4 | # 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | # this software and associated documentation files (the "Software"), to deal in 7 | # the Software without restriction, including without limitation the rights to 8 | # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 9 | # of the Software, and to permit persons to whom the Software is furnished to do 10 | # so, subject to the following conditions: 11 | # 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # ITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | """Abstract class that defines how a ShutIt module should be written. 23 | """ 24 | 25 | from __future__ import print_function 26 | import decimal 27 | import inspect 28 | import shutit_util 29 | from abc import ABCMeta, abstractmethod 30 | from six import with_metaclass, iteritems 31 | 32 | 33 | # TODO: these don't belong here, but this module is 'top level' and doesn't depend on any other shutit files. 34 | class ShutItException(Exception): 35 | """Placeholder exception. Implementation TODO. 36 | """ 37 | pass 38 | 39 | 40 | class ShutItModuleError(ShutItException): 41 | """Placeholder exception. Implementation TODO. 42 | """ 43 | pass 44 | 45 | 46 | class ShutItFailException(ShutItException): 47 | """Placeholder exception. Implementation TODO. 48 | """ 49 | pass 50 | 51 | 52 | 53 | def shutit_method_scope(func): 54 | """Notifies the ShutIt object whenever we call a shutit module method. 55 | This allows setting values for the 'scope' of a function. 56 | """ 57 | def wrapper(self, shutit): 58 | """Wrapper to call a shutit module method, notifying the ShutIt object. 59 | """ 60 | ret = func(self, shutit) 61 | return ret 62 | return wrapper 63 | 64 | 65 | class ShutItMeta(ABCMeta): 66 | """Abstract class that defines what a ShutIt module must implement 67 | to be registered. 68 | """ 69 | ShutItModule = None 70 | def __new__(mcs, name, bases, local): 71 | """Checks this is a ShutItModule, and wraps any ShutItModule methods 72 | that have been overridden in the subclass. 73 | """ 74 | 75 | # Don't wrap methods of the ShutItModule class, only subclasses 76 | if name != 'ShutItModule': 77 | 78 | sim = mcs.ShutItModule 79 | assert sim is not None, shutit_util.print_debug() 80 | 81 | # Wrap any of the ShutItModule (self, shutit) methods that have been 82 | # overridden in a subclass 83 | for fname, method in iteritems(local): 84 | if not hasattr(sim, fname): 85 | continue 86 | if not callable(method): 87 | continue 88 | sim_method = getattr(sim, fname) 89 | if sim_method is method: # pragma: no cover 90 | continue 91 | args = inspect.getargspec(sim_method)[0] 92 | if args != ['self', 'shutit']: 93 | continue 94 | local[fname] = shutit_method_scope(method) 95 | 96 | cls = super(ShutItMeta, mcs).__new__(mcs, name, bases, local) 97 | if name == 'ShutItModule': 98 | mcs.ShutItModule = cls 99 | return cls 100 | 101 | 102 | class ShutItModule(with_metaclass(ShutItMeta)): 103 | """Class that takes a ShutIt object and defines what a ShutIt module must 104 | implement to be registered. 105 | 106 | Build order: 107 | 108 | - Gather core config (build, remove, tag) 109 | - Gather module-specific config 110 | - FOR MODULE 0 111 | - Build module 0 112 | - FOR ALL MODULES: 113 | - Determine dependency requirements are met 114 | - Determine conflict requirements are met. 115 | - Remove any modules that are configured for removal. 116 | - Build if not installed 117 | - Do repo work if not installed (commit, tag, push) 118 | - Test all modules (in reverse) 119 | - Finalize all modules 120 | - FOR MODULE 0 121 | - Do repo work on build 122 | """ 123 | 124 | def __init__(self, module_id, run_order, description='', maintainer='', depends=None, conflicts=None, delivery_methods=None): 125 | """Constructor. 126 | Sets up module_id, run_order, deps and conflicts. 127 | Also checks types for safety. 128 | """ 129 | # Module id for the module (a string). 130 | # Following the Java standard is recommended, eg 'com.bigcorp.project.alpha.mymodule' 131 | # Duplicate module ids are rejected if within the configured 132 | # shutit_module_path. 133 | self.module_id = module_id 134 | if not isinstance(module_id, str): # pragma: no cover 135 | err = str(module_id) + '\'s module_id is not a string' 136 | shutit_global.shutit_global_object.shutit_print(err) 137 | raise ShutItModuleError(err) 138 | # run_order for the module (a float). 139 | # It should be a float and not duplicated within the shutit_module path. 140 | # Module 0 is special. It is expected to: 141 | # - Set up a target (see shutit_setup.py) 142 | # - Set up pexpect children with relevant keys and populate 143 | # shutit_pexpect_children. 144 | if isinstance(run_order, (float, int, str)): 145 | run_order = decimal.Decimal(run_order) 146 | # Check that run_order is a float - this will throw an error as a 147 | # side effect if float doesn't work. 148 | if not isinstance(run_order, decimal.Decimal): # pragma: no cover 149 | err = module_id + '\'s run order is not a decimal' 150 | shutit_global.shutit_global_object.shutit_print(err) 151 | raise ShutItModuleError(err) 152 | self.run_order = run_order 153 | # module ids depended on 154 | self.depends_on = [] 155 | if depends is not None: 156 | self.depends_on = [dep for dep in depends] 157 | # module ids this is known to conflict with. 158 | self.conflicts_with = [] 159 | if conflicts is not None: 160 | self.conflicts_with = [conflict for conflict in conflicts] 161 | self.description = description 162 | self.maintainer = maintainer 163 | if not delivery_methods: 164 | # default to all 165 | delivery_methods = ['ssh','dockerfile','bash','docker'] 166 | if isinstance(delivery_methods, str): 167 | delivery_methods = [delivery_methods] 168 | self.ok_delivery_methods = delivery_methods 169 | 170 | 171 | ######################################################################## 172 | # Abstract methods 173 | ######################################################################## 174 | def get_config(self, shutit): 175 | """Gets all config items necessary for this module to be built 176 | """ 177 | return True 178 | 179 | def check_ready(self, shutit): 180 | """Checks whether we are ready to build this module. 181 | 182 | This is called before the build, to ensure modules have 183 | their requirements in place before we commence the build. 184 | Checking whether the build will happen at all (and 185 | therefore whether the check should take place) will be 186 | determined by the framework. 187 | 188 | Should return True if it's ready to run, else False. 189 | """ 190 | return True 191 | 192 | def remove(self, shutit): 193 | """Remove the module, which should ensure the module has been deleted 194 | from the system. 195 | 196 | Returns True if all removed without any errors, else False. 197 | """ 198 | return False 199 | 200 | def start(self, shutit): 201 | """Run when module should be installed (is_installed() or configured 202 | to build is true) 203 | Run after repository work. 204 | Returns True if all started ok. 205 | """ 206 | return True 207 | 208 | def stop(self, shutit): 209 | """Runs when module should be stopped. 210 | Runs before repo work, and before finalize is called. 211 | Returns True if all stopped ok. 212 | """ 213 | return True 214 | 215 | def is_installed(self, shutit): 216 | """Determines whether the module has been built in this target host 217 | already. 218 | 219 | Returns True if it is certain it's there, else False. 220 | 221 | Required. 222 | """ 223 | return shutit.is_shutit_installed(self.module_id) 224 | 225 | @abstractmethod 226 | def build(self, shutit): 227 | """Runs the build part of the module, which should ensure the module has been set up. If is_installed determines that the module is already there, this is not run. 228 | 229 | Returns True if it has succeeded in building, else False. 230 | 231 | Required. 232 | """ 233 | pass 234 | 235 | def test(self, shutit): 236 | """Tests the module is OK. 237 | Returns True if all is OK, else False. 238 | This is run regardless of whether the module is installed or not. 239 | """ 240 | return True 241 | 242 | def finalize(self, shutit): 243 | """Finalize the module, ie do things that need doing after final module 244 | has been run and before we exit, eg updatedb. 245 | """ 246 | return True 247 | -------------------------------------------------------------------------------- /shutit_patterns/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ianmiell/shutit/fde11a8d76136884865ca0553246df4598f8bcf6/shutit_patterns/__init__.py -------------------------------------------------------------------------------- /shutit_patterns/bash.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import os 3 | from . import shutitfile 4 | 5 | def setup_bash_pattern(shutit, 6 | skel_path, 7 | skel_delivery, 8 | skel_domain, 9 | skel_module_name, 10 | skel_shutitfiles, 11 | skel_domain_hash, 12 | skel_depends): 13 | 14 | runsh_filename = skel_path + '/run.sh' 15 | runsh_file = open(runsh_filename,'w+') 16 | runsh_file.write('''#!/bin/bash 17 | [[ -z "$SHUTIT" ]] && SHUTIT="$1/shutit" 18 | [[ ! -a "$SHUTIT" ]] || [[ -z "$SHUTIT" ]] && SHUTIT="$(which shutit)" 19 | if [[ ! -a "$SHUTIT" ]] 20 | then 21 | echo "Must have shutit on path, eg export PATH=$PATH:/path/to/shutit_dir" 22 | exit 1 23 | fi 24 | $SHUTIT build -d ''' + skel_delivery + ''' "$@" 25 | if [[ $? != 0 ]] 26 | then 27 | exit 1 28 | fi''') 29 | runsh_file.close() 30 | os.chmod(runsh_filename,0o755) 31 | 32 | # build.cnf file 33 | os.system('mkdir -p ' + skel_path + '/configs') 34 | 35 | # User message 36 | shutit.log('''# Run: 37 | cd ''' + skel_path + ''' && ./run.sh 38 | # to run.''',transient=True) 39 | 40 | if skel_shutitfiles: 41 | _count = 0 42 | _total = len(skel_shutitfiles) 43 | for skel_shutitfile in skel_shutitfiles: 44 | _count += 1 45 | module_modifier = '_' + str(_count) 46 | new_module_filename = skel_path + '/' + os.path.join(skel_module_name + module_modifier + '.py') 47 | shutit.cfg['skeleton']['module_modifier'] = module_modifier 48 | (sections, skel_module_id, skel_module_name, _, _) = shutitfile.shutitfile_to_shutit_module(shutit, skel_shutitfile,skel_path,skel_domain,skel_module_name,skel_domain_hash,skel_delivery,skel_depends,_count,_total,module_modifier) 49 | shutit.cfg['skeleton']['header_section'] = sections['header_section'] 50 | shutit.cfg['skeleton']['config_section'] = sections['config_section'] 51 | shutit.cfg['skeleton']['build_section'] = sections['build_section'] 52 | shutit.cfg['skeleton']['finalize_section'] = sections['finalize_section'] 53 | shutit.cfg['skeleton']['test_section'] = sections['test_section'] 54 | shutit.cfg['skeleton']['isinstalled_section'] = sections['isinstalled_section'] 55 | shutit.cfg['skeleton']['start_section'] = sections['start_section'] 56 | shutit.cfg['skeleton']['stop_section'] = sections['stop_section'] 57 | shutit.cfg['skeleton']['final_section'] = sections['final_section'] 58 | module_file = open(new_module_filename,'w+') 59 | module_file.write(shutit.cfg['skeleton']['header_section'] + ''' 60 | 61 | def build(self, shutit): 62 | ''' + shutit.cfg['skeleton']['build_section'] + ''' 63 | return True 64 | 65 | def get_config(self, shutit): 66 | ''' + shutit.cfg['skeleton']['config_section'] + ''' 67 | return True 68 | 69 | def test(self, shutit): 70 | ''' + shutit.cfg['skeleton']['test_section'] + ''' 71 | return True 72 | 73 | def finalize(self, shutit): 74 | ''' + shutit.cfg['skeleton']['finalize_section'] + ''' 75 | return True 76 | 77 | def is_installed(self, shutit): 78 | ''' + shutit.cfg['skeleton']['isinstalled_section'] + ''' 79 | return False 80 | 81 | def start(self, shutit): 82 | ''' + shutit.cfg['skeleton']['start_section'] + ''' 83 | return True 84 | 85 | def stop(self, shutit): 86 | ''' + shutit.cfg['skeleton']['stop_section'] + ''' 87 | return True 88 | 89 | ''' + shutit.cfg['skeleton']['final_section']) 90 | module_file.close() 91 | # Set up build.cnf 92 | build_cnf_filename = skel_path + '/configs/build.cnf' 93 | if _count == 1: 94 | build_cnf_file = open(build_cnf_filename,'w+') 95 | build_cnf_file.write('''############################################################################### 96 | # PLEASE NOTE: This file should be changed only by the maintainer. 97 | # PLEASE NOTE: This file is only sourced if the "shutit build" command is run 98 | # and this file is in the relative path: configs/build.cnf 99 | # This is to ensure it is only sourced if _this_ module is the 100 | # target. 101 | ############################################################################### 102 | # When this module is the one being built, which modules should be built along with it by default? 103 | # This feeds into automated testing of each module. 104 | [''' + skel_module_id + '''] 105 | shutit.core.module.build:yes''') 106 | build_cnf_file.close() 107 | else: 108 | build_cnf_file = open(build_cnf_filename,'a') 109 | build_cnf_file.write(''' 110 | [''' + skel_domain + '''.''' + skel_module_name + module_modifier + '''] 111 | shutit.core.module.build:yes''') 112 | build_cnf_file.close() 113 | os.chmod(build_cnf_filename,0o400) 114 | else: 115 | shutit.cfg['skeleton']['header_section'] = 'from shutit_module import ShutItModule\n\nclass ' + skel_module_name + '(ShutItModule):\n' 116 | shutit.cfg['skeleton']['config_section'] = '' 117 | shutit.cfg['skeleton']['build_section'] = '' 118 | shutit.cfg['skeleton']['finalize_section'] = '' 119 | shutit.cfg['skeleton']['test_section'] = '' 120 | shutit.cfg['skeleton']['isinstalled_section'] = '' 121 | shutit.cfg['skeleton']['start_section'] = '' 122 | shutit.cfg['skeleton']['stop_section'] = '' 123 | shutit.cfg['skeleton']['final_section'] = """def module(): 124 | return """ + skel_module_name + """( 125 | '""" + skel_domain + '''.''' + skel_module_name + """', """ + skel_domain_hash + """.0001, 126 | description='', 127 | maintainer='', 128 | delivery_methods=['""" + skel_delivery + """'], 129 | depends=['""" + skel_depends + """'] 130 | )""" 131 | new_module_filename = skel_path + '/' + os.path.join(skel_module_name) + '.py' 132 | module_file = open(new_module_filename,'w+') 133 | module_file.write(shutit.cfg['skeleton']['header_section'] + ''' 134 | 135 | def build(self, shutit): 136 | ''' + shutit.cfg['skeleton']['build_section'] + ''' 137 | return True 138 | 139 | def get_config(self, shutit): 140 | ''' + shutit.cfg['skeleton']['config_section'] + ''' 141 | return True 142 | 143 | def test(self, shutit): 144 | ''' + shutit.cfg['skeleton']['test_section'] + ''' 145 | return True 146 | 147 | def finalize(self, shutit): 148 | ''' + shutit.cfg['skeleton']['finalize_section'] + ''' 149 | return True 150 | 151 | def is_installed(self, shutit): 152 | ''' + shutit.cfg['skeleton']['isinstalled_section'] + ''' 153 | return False 154 | 155 | def start(self, shutit): 156 | ''' + shutit.cfg['skeleton']['start_section'] + ''' 157 | return True 158 | 159 | def stop(self, shutit): 160 | ''' + shutit.cfg['skeleton']['stop_section'] + ''' 161 | return True 162 | 163 | ''' + shutit.cfg['skeleton']['final_section']) 164 | module_file.close() 165 | 166 | build_cnf_filename = skel_path + '/configs/build.cnf' 167 | build_cnf_file = open(build_cnf_filename,'w+') 168 | build_cnf_file.write('''############################################################################### 169 | # PLEASE NOTE: This file should be changed only by the maintainer. 170 | # PLEASE NOTE: This file is only sourced if the "shutit build" command is run 171 | # and this file is in the relative path: configs/build.cnf 172 | # This is to ensure it is only sourced if _this_ module is the 173 | # target. 174 | ############################################################################### 175 | # When this module is the one being built, which modules should be built along with it by default? 176 | # This feeds into automated testing of each module. 177 | ['''+skel_domain+'''.'''+skel_module_name+'''] 178 | shutit.core.module.build:yes''') 179 | build_cnf_file.close() 180 | os.chmod(build_cnf_filename,0o400) 181 | -------------------------------------------------------------------------------- /shutit_patterns/docker.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import os 3 | import logging 4 | from . import shutitfile 5 | 6 | def setup_docker_pattern(shutit, 7 | skel_path, 8 | skel_delivery, 9 | skel_domain, 10 | skel_module_name, 11 | skel_shutitfiles, 12 | skel_domain_hash, 13 | skel_depends): 14 | 15 | # Set up shutitfile cfg 16 | shutit.shutitfile['base_image'] = shutit.cfg['skeleton']['base_image'] 17 | shutit.shutitfile['cmd'] = """/bin/sh -c 'sleep infinity'""" 18 | shutit.shutitfile['expose'] = [] 19 | shutit.shutitfile['env'] = [] 20 | shutit.shutitfile['volume'] = [] 21 | shutit.shutitfile['onbuild'] = [] 22 | shutit.shutitfile['script'] = [] 23 | 24 | # arguments 25 | shutit.cfg['skeleton']['volumes_arg'] = '' 26 | for varg in shutit.shutitfile['volume']: 27 | shutit.cfg['skeleton']['volumes_arg'] += ' -v ' + varg + ':' + varg 28 | shutit.cfg['skeleton']['ports_arg'] = '' 29 | if isinstance(shutit.shutitfile['expose'], str): 30 | for parg in shutit.shutitfile['expose']: 31 | shutit.cfg['skeleton']['ports_arg'] += ' -p ' + parg + ':' + parg 32 | else: 33 | for parg in shutit.shutitfile['expose']: 34 | for port in parg.split(): 35 | shutit.cfg['skeleton']['ports_arg'] += ' -p ' + port + ':' + port 36 | shutit.cfg['skeleton']['env_arg'] = '' 37 | for earg in shutit.shutitfile['env']: 38 | shutit.cfg['skeleton']['env_arg'] += ' -e ' + earg.split()[0] + ':' + earg.split()[1] 39 | 40 | os.system('mkdir -p ' + skel_path) 41 | build_bin_filename = skel_path + '/build.sh' 42 | build_bin_file = open(build_bin_filename,'w+') 43 | build_bin_file.write('''#!/bin/bash 44 | [[ -z "$SHUTIT" ]] && SHUTIT="$1/shutit" 45 | [[ ! -a "$SHUTIT" ]] || [[ -z "$SHUTIT" ]] && SHUTIT="$(which shutit)" 46 | if [[ ! -a "$SHUTIT" ]] 47 | then 48 | echo "Must have shutit on path, eg export PATH=$PATH:/path/to/shutit_dir" 49 | exit 1 50 | fi 51 | $SHUTIT build -d ''' + skel_delivery + ''' "$@" 52 | if [[ $? != 0 ]] 53 | then 54 | exit 1 55 | fi''') 56 | build_bin_file.close() 57 | os.chmod(build_bin_filename,0o755) 58 | run_bin_filename = skel_path + '/run.sh' 59 | run_bin_file = open(run_bin_filename,'w+') 60 | # TODO: sort out entrypoint properly 61 | entrypoint = '' 62 | run_bin_file.write('''#!/bin/bash 63 | # Example for running 64 | DOCKER=${DOCKER:-docker} 65 | IMAGE_NAME=%s 66 | CONTAINER_NAME=$IMAGE_NAME 67 | DOCKER_ARGS='' 68 | while getopts "i:c:a:" opt 69 | do 70 | case "$opt" in 71 | i) 72 | IMAGE_NAME=$OPTARG 73 | ;; 74 | c) 75 | CONTAINER_NAME=$OPTARG 76 | ;; 77 | a) 78 | DOCKER_ARGS=$OPTARG 79 | ;; 80 | esac 81 | done 82 | ${DOCKER} run -d --name ${CONTAINER_NAME} ''' + skel_module_name + ''' ''' + shutit.cfg['skeleton']['ports_arg'] + ''' ''' + shutit.cfg['skeleton']['ports_arg'] + ''' ''' + shutit.cfg['skeleton']['env_arg'] + ''' ${DOCKER_ARGS} ${IMAGE_NAME} ''' + entrypoint + ''' ''' + shutit.shutitfile['cmd']) 83 | run_bin_file.close() 84 | os.chmod(run_bin_filename,0o755) 85 | test_bin_filename = skel_path + '/test.sh' 86 | test_bin_file = open(test_bin_filename,'w+') 87 | test_bin_file.write('''#!/bin/bash 88 | # Test the building of this module 89 | if [ $0 != test.sh ] && [ $0 != ./test.sh ] 90 | then 91 | echo 92 | echo "Called as: $0" 93 | echo "Must be run as test.sh or ./test.sh" 94 | exit 95 | fi 96 | ./build.sh "$@"''') 97 | test_bin_file.close() 98 | os.chmod(test_bin_filename,0o755) 99 | os.system('mkdir -p ' + skel_path + '/configs') 100 | 101 | push_cnf_filename = skel_path + '/configs/push.cnf' 102 | push_cnf_file = open(push_cnf_filename,'w+') 103 | push_cnf_file.write('''############################################################################### 104 | # PLEASE NOTE: This file should be changed only by the maintainer. 105 | # PLEASE NOTE: IF YOU WANT TO CHANGE THE CONFIG, PASS IN 106 | # --config configfilename 107 | # OR ADD DETAILS TO YOUR 108 | # ~/.shutit/config 109 | # FILE 110 | ############################################################################### 111 | [target] 112 | rm:false 113 | 114 | [repository] 115 | # COPY THESE TO YOUR ~/.shutit/config FILE AND FILL OUT ITEMS IN CAPS 116 | #user:YOUR_USERNAME 117 | ## Fill these out in server- and username-specific config (also in this directory) 118 | #password:YOUR_REGISTRY_PASSWORD_OR_BLANK 119 | ## Fill these out in server- and username-specific config (also in this directory) 120 | #email:YOUR_REGISTRY_EMAIL_OR_BLANK 121 | #tag:no 122 | #push:yes 123 | #save:no 124 | #export:no 125 | ##server:REMOVE_ME_FOR_DOCKER_INDEX 126 | ## tag suffix, defaults to "latest", eg registry/username/repository:latest. 127 | ## empty is also "latest" 128 | #tag_name:latest 129 | #suffix_date:no 130 | #suffix_format:%s''') 131 | push_cnf_file.close() 132 | os.chmod(push_cnf_filename,0o400) 133 | dockerfile_filename = skel_path + '/Dockerfile' 134 | dockerfile_file = open(dockerfile_filename,'w+') 135 | dockerfile_file.write('''FROM ''' + shutit.shutitfile['base_image'] + ''' 136 | 137 | RUN apt-get -qq update 138 | RUN apt-get -qq install -y git python-pip python-dev 139 | RUN pip install shutit 140 | 141 | WORKDIR /opt 142 | # Change the next two lines to build your ShutIt module. 143 | RUN git clone https://github.com/yourname/yourshutitproject.git 144 | WORKDIR /opt/yourshutitproject 145 | RUN shutit build --delivery dockerfile 146 | CMD ["/bin/bash"]''') 147 | dockerfile_file.close() 148 | 149 | # User message 150 | shutit.log('''# Run: 151 | cd ''' + skel_path + ''' && ./build.sh 152 | # to build. 153 | # And then: 154 | ./run.sh 155 | # to run.''',transient=True) 156 | 157 | if skel_shutitfiles: 158 | shutit.log('Processing ShutItFiles: ' + str(skel_shutitfiles),level=logging.DEBUG) 159 | _total = len(skel_shutitfiles) 160 | _count = 0 161 | for skel_shutitfile in skel_shutitfiles: 162 | _count += 1 163 | shutit.log('Processing ShutItFile: ' + str(skel_shutitfile),level=logging.INFO) 164 | module_modifier = '_' + str(_count) 165 | new_module_filename = skel_path + '/' + os.path.join(skel_module_name + module_modifier + '.py') 166 | shutit.cfg['skeleton']['module_modifier'] = module_modifier 167 | (sections, skel_module_id, skel_module_name, _, _) = shutitfile.shutitfile_to_shutit_module(shutit, skel_shutitfile,skel_path,skel_domain,skel_module_name,skel_domain_hash,skel_delivery,skel_depends,_count,_total,module_modifier) 168 | shutit.cfg['skeleton']['header_section'] = sections['header_section'] 169 | shutit.cfg['skeleton']['config_section'] = sections['config_section'] 170 | shutit.cfg['skeleton']['build_section'] = sections['build_section'] 171 | shutit.cfg['skeleton']['finalize_section'] = sections['finalize_section'] 172 | shutit.cfg['skeleton']['test_section'] = sections['test_section'] 173 | shutit.cfg['skeleton']['isinstalled_section'] = sections['isinstalled_section'] 174 | shutit.cfg['skeleton']['start_section'] = sections['start_section'] 175 | shutit.cfg['skeleton']['stop_section'] = sections['stop_section'] 176 | shutit.cfg['skeleton']['final_section'] = sections['final_section'] 177 | module_file = open(new_module_filename,'w+') 178 | module_file.write(shutit.cfg['skeleton']['header_section'] + ''' 179 | 180 | def build(self, shutit): 181 | ''' + shutit.cfg['skeleton']['build_section'] + ''' 182 | return True 183 | 184 | def get_config(self, shutit): 185 | ''' + shutit.cfg['skeleton']['config_section'] + ''' 186 | return True 187 | 188 | def test(self, shutit): 189 | ''' + shutit.cfg['skeleton']['test_section'] + ''' 190 | return True 191 | 192 | def finalize(self, shutit): 193 | ''' + shutit.cfg['skeleton']['finalize_section'] + ''' 194 | return True 195 | 196 | def is_installed(self, shutit): 197 | ''' + shutit.cfg['skeleton']['isinstalled_section'] + ''' 198 | return False 199 | 200 | def start(self, shutit): 201 | ''' + shutit.cfg['skeleton']['start_section'] + ''' 202 | return True 203 | 204 | def stop(self, shutit): 205 | ''' + shutit.cfg['skeleton']['stop_section'] + ''' 206 | return True 207 | 208 | ''' + shutit.cfg['skeleton']['final_section']) 209 | module_file.close() 210 | # Set up build.cnf 211 | build_cnf_filename = skel_path + '/configs/build.cnf' 212 | if _count == 1: 213 | build_cnf_file = open(build_cnf_filename,'w+') 214 | build_cnf_file.write('''############################################################################### 215 | # PLEASE NOTE: This file should be changed only by the maintainer. 216 | # PLEASE NOTE: This file is only sourced if the "shutit build" command is run 217 | # and this file is in the relative path: configs/build.cnf 218 | # This is to ensure it is only sourced if _this_ module is the 219 | # target. 220 | ############################################################################### 221 | # When this module is the one being built, which modules should be built along with it by default? 222 | # This feeds into automated testing of each module. 223 | [''' + skel_module_id + '''] 224 | shutit.core.module.build:yes 225 | # Allowed images as a regexp, eg ["ubuntu:12.*"], or [".*"], or ["centos"]. 226 | # It's recommended this is locked down as far as possible. 227 | shutit.core.module.allowed_images:["''' + shutit.shutitfile['base_image'] + '''"] 228 | 229 | # Aspects of build process 230 | [build] 231 | base_image:''' + shutit.shutitfile['base_image'] + ''' 232 | 233 | # Volume arguments wanted as part of the build 234 | [target] 235 | volumes: 236 | 237 | [repository] 238 | name:''' + skel_module_name) 239 | build_cnf_file.close() 240 | else: 241 | build_cnf_file = open(build_cnf_filename,'a') 242 | build_cnf_file.write(''' 243 | [''' + skel_domain + '''.''' + skel_module_name + module_modifier + '''] 244 | shutit.core.module.build:yes''') 245 | build_cnf_file.close() 246 | os.chmod(build_cnf_filename,0o400) 247 | else: 248 | shutit.cfg['skeleton']['header_section'] = 'from shutit_module import ShutItModule\n\nclass ' + skel_module_name + '(ShutItModule):\n' 249 | shutit.cfg['skeleton']['config_section'] = '' 250 | shutit.cfg['skeleton']['build_section'] = '' 251 | shutit.cfg['skeleton']['finalize_section'] = '' 252 | shutit.cfg['skeleton']['test_section'] = '' 253 | shutit.cfg['skeleton']['isinstalled_section'] = '' 254 | shutit.cfg['skeleton']['start_section'] = '' 255 | shutit.cfg['skeleton']['stop_section'] = '' 256 | shutit.cfg['skeleton']['final_section'] = """def module(): 257 | return """ + skel_module_name + """( 258 | '""" + skel_domain + '''.''' + skel_module_name + """', """ + skel_domain_hash + """.0001, 259 | description='', 260 | maintainer='', 261 | delivery_methods=['""" + skel_delivery + """'], 262 | depends=['""" + skel_depends + """'] 263 | )""" 264 | new_module_filename = skel_path + '/' + os.path.join(skel_module_name) + '.py' 265 | module_file = open(new_module_filename,'w+') 266 | module_file.write(shutit.cfg['skeleton']['header_section'] + ''' 267 | 268 | def build(self, shutit): 269 | ''' + shutit.cfg['skeleton']['build_section'] + ''' 270 | return True 271 | 272 | def get_config(self, shutit): 273 | ''' + shutit.cfg['skeleton']['config_section'] + ''' 274 | return True 275 | 276 | def test(self, shutit): 277 | ''' + shutit.cfg['skeleton']['test_section'] + ''' 278 | return True 279 | 280 | def finalize(self, shutit): 281 | ''' + shutit.cfg['skeleton']['finalize_section'] + ''' 282 | return True 283 | 284 | def is_installed(self, shutit): 285 | ''' + shutit.cfg['skeleton']['isinstalled_section'] + ''' 286 | return False 287 | 288 | def start(self, shutit): 289 | ''' + shutit.cfg['skeleton']['start_section'] + ''' 290 | return True 291 | 292 | def stop(self, shutit): 293 | ''' + shutit.cfg['skeleton']['stop_section'] + ''' 294 | return True 295 | 296 | ''' + shutit.cfg['skeleton']['final_section']) 297 | module_file.close() 298 | 299 | build_cnf_filename = skel_path + '/configs/build.cnf' 300 | build_cnf_file = open(build_cnf_filename,'w+') 301 | build_cnf_file.write('''############################################################################### 302 | # PLEASE NOTE: This file should be changed only by the maintainer. 303 | # PLEASE NOTE: This file is only sourced if the "shutit build" command is run 304 | # and this file is in the relative path: configs/build.cnf 305 | # This is to ensure it is only sourced if _this_ module is the 306 | # target. 307 | ############################################################################### 308 | # When this module is the one being built, which modules should be built along with it by default? 309 | # This feeds into automated testing of each module. 310 | [''' + skel_domain + '''.''' + skel_module_name + '''] 311 | shutit.core.module.build:yes 312 | # Allowed images as a regexp, eg ["ubuntu:12.*"], or [".*"], or ["centos"]. 313 | # It's recommended this is locked down as far as possible. 314 | shutit.core.module.allowed_images:["''' + shutit.shutitfile['base_image'] + '''"] 315 | 316 | # Aspects of build process 317 | [build] 318 | base_image:''' + shutit.shutitfile['base_image'] + ''' 319 | 320 | # Volume arguments wanted as part of the build 321 | [target] 322 | volumes: 323 | 324 | [repository] 325 | name:''' + skel_module_name) 326 | build_cnf_file.close() 327 | os.chmod(build_cnf_filename,0o400) 328 | -------------------------------------------------------------------------------- /shutit_patterns/docker_tutorial.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import os 3 | import logging 4 | from . import shutitfile 5 | 6 | configs_build_cnf_file = """############################################################################### 7 | # PLEASE NOTE: This file should be changed only by the maintainer. 8 | # PLEASE NOTE: This file is only sourced if the "shutit build" command is run 9 | # and this file is in the relative path: configs/build.cnf 10 | # This is to ensure it is only sourced if _this_ module is the 11 | # target. 12 | ############################################################################### 13 | # When this module is the one being built, which modules should be built along with it by default? 14 | # This feeds into automated testing of each module. 15 | [tk.shutit.git_101_tutorial] 16 | shutit.core.module.build:yes 17 | # Allowed images as a regexp, eg ["ubuntu:12.*"], or [".*"], or ["centos"]. 18 | # It's recommended this is locked down as far as possible. 19 | shutit.core.module.allowed_images:["imiell/git-101-tutorial.*"] 20 | 21 | # Aspects of build process 22 | [build] 23 | base_image:imiell/git-101-tutorial:step_4 24 | 25 | # Volume arguments wanted as part of the build 26 | [target] 27 | volumes: 28 | 29 | [repository] 30 | name:git_101_tutorial""" 31 | 32 | dockerfile_file = """ 33 | FROM debian 34 | # Step 2 done 35 | RUN apt-get -qq update && apt-get -qq install -y git lsb-release vim bsdmainutils man-db manpages && mkdir -p myproject && touch /root/.bash_history 36 | # Step 3 done 37 | WORKDIR /myproject 38 | # Step 4 done 39 | CMD /bin/bash 40 | # Step 12 done 41 | """ 42 | 43 | tutorial_py_file = """ 44 | from shutit_module import ShutItModule 45 | 46 | class git_101_tutorial(ShutItModule): 47 | 48 | def build(self, shutit): 49 | shutit.send('cd /myproject') 50 | shutit.challenge( 51 | '''In this tutorial you will be asked to set up git on your machine, 52 | create a repository, and add and commit some code to it. 53 | 54 | You have a full bash shell, so can use vi, less, man etc.. 55 | 56 | If any tools are missing or there are bugs raise a github request or contact 57 | @ianmiell on twitter. 58 | 59 | CTRL-] (right angle bracket) to continue. 60 | ''', 61 | '1', 62 | challenge_type='golf', 63 | expect_type='exact', 64 | hints=['Hit CTRL-]'], 65 | congratulations='OK!', 66 | follow_on_context={ 67 | 'check_command':'echo 1', 68 | 'context':'docker', 69 | 'reset_container_name':'imiell/git-101-tutorial:step_4', 70 | 'ok_container_name':'imiell/git-101-tutorial:step_4' 71 | } 72 | ) 73 | shutit.pause_point('Tutorial complete! Feel free to mess around at the back :)') 74 | return True 75 | 76 | def module(): 77 | return git_101_tutorial( 78 | 'tk.shutit.git_101_tutorial', 1845506479.0001, 79 | description='', 80 | maintainer='', 81 | delivery_methods=['docker'], 82 | depends=['shutit.tk.setup'] 83 | ) 84 | """ 85 | 86 | eg_images_sh_file = """ 87 | #!/bin/bash 88 | set -x 89 | set -u 90 | set -e 91 | IMAGE_NAME="imiell/git-101-tutorial" 92 | #docker images | tac | grep "$IMAGE_NAME" | awk '{print $3}' | xargs docker rmi 93 | docker build -t $IMAGE_NAME . 94 | x=1 95 | docker history -q "${IMAGE_NAME}:latest" | tac 96 | for id in $(docker history -q "${IMAGE_NAME}:latest" | tac) 97 | do 98 | docker tag -f "${id}" "${IMAGE_NAME}:step_$x" 99 | ((x++)) 100 | done 101 | docker push "${IMAGE_NAME}" 102 | """ 103 | 104 | 105 | def setup_docker_tutorial_pattern(shutit, 106 | skel_path, 107 | skel_delivery, 108 | skel_pattern, 109 | skel_domain, 110 | skel_module_name, 111 | skel_shutitfiles, 112 | skel_domain_hash, 113 | skel_depends): 114 | 115 | # Set up shutitfile cfg 116 | shutit.shutitfile['base_image'] = shutit.cfg['skeleton']['base_image'] 117 | shutit.shutitfile['cmd'] = """/bin/sh -c 'sleep infinity'""" 118 | shutit.shutitfile['expose'] = [] 119 | shutit.shutitfile['env'] = [] 120 | shutit.shutitfile['volume'] = [] 121 | shutit.shutitfile['onbuild'] = [] 122 | shutit.shutitfile['script'] = [] 123 | 124 | 125 | os.system('mkdir -p ' + skel_path) 126 | build_bin_filename = skel_path + '/build.sh' 127 | build_bin_file = open(build_bin_filename,'w+') 128 | build_bin_file.write('''#!/bin/bash 129 | [[ -z "$SHUTIT" ]] && SHUTIT="$1/shutit" 130 | [[ ! -a "$SHUTIT" ]] || [[ -z "$SHUTIT" ]] && SHUTIT="$(which shutit)" 131 | if [[ ! -a "$SHUTIT" ]] 132 | then 133 | echo "Must have shutit on path, eg export PATH=$PATH:/path/to/shutit_dir" 134 | exit 1 135 | fi 136 | $SHUTIT build -d ''' + skel_delivery + ''' "$@" 137 | if [[ $? != 0 ]] 138 | then 139 | exit 1 140 | fi''') 141 | build_bin_file.close() 142 | os.chmod(build_bin_filename,0o755) 143 | run_bin_filename = skel_path + '/run.sh' 144 | run_bin_file = open(run_bin_filename,'w+') 145 | os.system('mkdir -p ' + skel_path + '/configs') 146 | 147 | push_cnf_filename = skel_path + '/configs/push.cnf' 148 | push_cnf_file = open(push_cnf_filename,'w+') 149 | push_cnf_file.write('''############################################################################### 150 | # PLEASE NOTE: This file should be changed only by the maintainer. 151 | # PLEASE NOTE: IF YOU WANT TO CHANGE THE CONFIG, PASS IN 152 | # --config configfilename 153 | # OR ADD DETAILS TO YOUR 154 | # ~/.shutit/config 155 | # FILE 156 | ############################################################################### 157 | [target] 158 | rm:false 159 | 160 | [repository] 161 | # COPY THESE TO YOUR ~/.shutit/config FILE AND FILL OUT ITEMS IN CAPS 162 | #user:YOUR_USERNAME 163 | ## Fill these out in server- and username-specific config (also in this directory) 164 | #password:YOUR_REGISTRY_PASSWORD_OR_BLANK 165 | ## Fill these out in server- and username-specific config (also in this directory) 166 | #email:YOUR_REGISTRY_EMAIL_OR_BLANK 167 | #tag:no 168 | #push:yes 169 | #save:no 170 | #export:no 171 | ##server:REMOVE_ME_FOR_DOCKER_INDEX 172 | ## tag suffix, defaults to "latest", eg registry/username/repository:latest. 173 | ## empty is also "latest" 174 | #tag_name:latest 175 | #suffix_date:no 176 | #suffix_format:%s''') 177 | push_cnf_file.close() 178 | os.chmod(push_cnf_filename,0o400) 179 | dockerfile_filename = skel_path + '/Dockerfile' 180 | dockerfile_file = open(dockerfile_filename,'w+') 181 | dockerfile_file.write('''FROM ''' + shutit.shutitfile['base_image'] + ''' 182 | 183 | RUN apt-get -qq update 184 | RUN apt-get -qq install -y git python-pip python-dev 185 | RUN pip install shutit 186 | 187 | WORKDIR /opt 188 | # Change the next two lines to build your ShutIt module. 189 | RUN git clone https://github.com/yourname/yourshutitproject.git 190 | WORKDIR /opt/yourshutitproject 191 | RUN shutit build --delivery dockerfile 192 | CMD ["/bin/bash"]''') 193 | dockerfile_file.close() 194 | 195 | # User message 196 | shutit.log('''# Run: 197 | cd ''' + skel_path + ''' && ./build.sh 198 | # to build. 199 | # And then: 200 | ./run.sh 201 | # to run.''',transient=True) 202 | 203 | if skel_shutitfiles: 204 | shutit.log('Processing ShutItFiles: ' + str(skel_shutitfiles),level=logging.DEBUG) 205 | _total = len(skel_shutitfiles) 206 | _count = 0 207 | for skel_shutitfile in skel_shutitfiles: 208 | _count += 1 209 | shutit.log('Processing ShutItFile: ' + str(skel_shutitfile),level=logging.INFO) 210 | module_modifier = '_' + str(_count) 211 | new_module_filename = skel_path + '/' + os.path.join(skel_module_name + module_modifier + '.py') 212 | shutit.cfg['skeleton']['module_modifier'] = module_modifier 213 | (sections, skel_module_id, skel_module_name, default_include, ok) = shutitfile.shutitfile_to_shutit_module(shutit, skel_shutitfile,skel_path,skel_domain,skel_module_name,skel_domain_hash,skel_delivery,skel_depends,_count,_total,module_modifier) 214 | shutit.cfg['skeleton']['header_section'] = sections['header_section'] 215 | shutit.cfg['skeleton']['config_section'] = sections['config_section'] 216 | shutit.cfg['skeleton']['build_section'] = sections['build_section'] 217 | shutit.cfg['skeleton']['finalize_section'] = sections['finalize_section'] 218 | shutit.cfg['skeleton']['test_section'] = sections['test_section'] 219 | shutit.cfg['skeleton']['isinstalled_section'] = sections['isinstalled_section'] 220 | shutit.cfg['skeleton']['start_section'] = sections['start_section'] 221 | shutit.cfg['skeleton']['stop_section'] = sections['stop_section'] 222 | shutit.cfg['skeleton']['final_section'] = sections['final_section'] 223 | module_file = open(new_module_filename,'w+') 224 | module_file.write(shutit.cfg['skeleton']['header_section'] + ''' 225 | 226 | def build(self, shutit): 227 | ''' + shutit.cfg['skeleton']['build_section'] + ''' 228 | return True 229 | 230 | def get_config(self, shutit): 231 | ''' + shutit.cfg['skeleton']['config_section'] + ''' 232 | return True 233 | 234 | def test(self, shutit): 235 | ''' + shutit.cfg['skeleton']['test_section'] + ''' 236 | return True 237 | 238 | def finalize(self, shutit): 239 | ''' + shutit.cfg['skeleton']['finalize_section'] + ''' 240 | return True 241 | 242 | def is_installed(self, shutit): 243 | ''' + shutit.cfg['skeleton']['isinstalled_section'] + ''' 244 | return False 245 | 246 | def start(self, shutit): 247 | ''' + shutit.cfg['skeleton']['start_section'] + ''' 248 | return True 249 | 250 | def stop(self, shutit): 251 | ''' + shutit.cfg['skeleton']['stop_section'] + ''' 252 | return True 253 | 254 | ''' + shutit.cfg['skeleton']['final_section']) 255 | module_file.close() 256 | # Set up build.cnf 257 | build_cnf_filename = skel_path + '/configs/build.cnf' 258 | if _count == 1: 259 | build_cnf_file = open(build_cnf_filename,'w+') 260 | build_cnf_file.write('''############################################################################### 261 | # PLEASE NOTE: This file should be changed only by the maintainer. 262 | # PLEASE NOTE: This file is only sourced if the "shutit build" command is run 263 | # and this file is in the relative path: configs/build.cnf 264 | # This is to ensure it is only sourced if _this_ module is the 265 | # target. 266 | ############################################################################### 267 | # When this module is the one being built, which modules should be built along with it by default? 268 | # This feeds into automated testing of each module. 269 | [''' + skel_module_id + '''] 270 | shutit.core.module.build:yes 271 | # Allowed images as a regexp, eg ["ubuntu:12.*"], or [".*"], or ["centos"]. 272 | # It's recommended this is locked down as far as possible. 273 | shutit.core.module.allowed_images:["''' + shutit.shutitfile['base_image'] + '''"] 274 | 275 | # Aspects of build process 276 | [build] 277 | base_image:''' + shutit.shutitfile['base_image'] + ''' 278 | 279 | # Volume arguments wanted as part of the build 280 | [target] 281 | volumes: 282 | 283 | [repository] 284 | name:''' + skel_module_name) 285 | build_cnf_file.close() 286 | else: 287 | build_cnf_file = open(build_cnf_filename,'a') 288 | build_cnf_file.write(''' 289 | [''' + skel_domain + '''.''' + skel_module_name + module_modifier + '''] 290 | shutit.core.module.build:yes''') 291 | build_cnf_file.close() 292 | os.chmod(build_cnf_filename,0o400) 293 | else: 294 | shutit.cfg['skeleton']['header_section'] = 'from shutit_module import ShutItModule\n\nclass ' + skel_module_name + '(ShutItModule):\n' 295 | shutit.cfg['skeleton']['config_section'] = '' 296 | shutit.cfg['skeleton']['build_section'] = '' 297 | shutit.cfg['skeleton']['finalize_section'] = '' 298 | shutit.cfg['skeleton']['test_section'] = '' 299 | shutit.cfg['skeleton']['isinstalled_section'] = '' 300 | shutit.cfg['skeleton']['start_section'] = '' 301 | shutit.cfg['skeleton']['stop_section'] = '' 302 | shutit.cfg['skeleton']['final_section'] = """def module(): 303 | return """ + skel_module_name + """( 304 | '""" + skel_domain + '''.''' + skel_module_name + """', """ + skel_domain_hash + """.0001, 305 | description='', 306 | maintainer='', 307 | delivery_methods=['""" + skel_delivery + """'], 308 | depends=['""" + skel_depends + """'] 309 | )""" 310 | new_module_filename = skel_path + '/' + os.path.join(skel_module_name) + '.py' 311 | module_file = open(new_module_filename,'w+') 312 | module_file.write(shutit.cfg['skeleton']['header_section'] + ''' 313 | 314 | def build(self, shutit): 315 | ''' + shutit.cfg['skeleton']['build_section'] + ''' 316 | return True 317 | 318 | def get_config(self, shutit): 319 | ''' + shutit.cfg['skeleton']['config_section'] + ''' 320 | return True 321 | 322 | def test(self, shutit): 323 | ''' + shutit.cfg['skeleton']['test_section'] + ''' 324 | return True 325 | 326 | def finalize(self, shutit): 327 | ''' + shutit.cfg['skeleton']['finalize_section'] + ''' 328 | return True 329 | 330 | def is_installed(self, shutit): 331 | ''' + shutit.cfg['skeleton']['isinstalled_section'] + ''' 332 | return False 333 | 334 | def start(self, shutit): 335 | ''' + shutit.cfg['skeleton']['start_section'] + ''' 336 | return True 337 | 338 | def stop(self, shutit): 339 | ''' + shutit.cfg['skeleton']['stop_section'] + ''' 340 | return True 341 | 342 | ''' + shutit.cfg['skeleton']['final_section']) 343 | module_file.close() 344 | 345 | build_cnf_filename = skel_path + '/configs/build.cnf' 346 | build_cnf_file = open(build_cnf_filename,'w+') 347 | build_cnf_file.write('''############################################################################### 348 | # PLEASE NOTE: This file should be changed only by the maintainer. 349 | # PLEASE NOTE: This file is only sourced if the "shutit build" command is run 350 | # and this file is in the relative path: configs/build.cnf 351 | # This is to ensure it is only sourced if _this_ module is the 352 | # target. 353 | ############################################################################### 354 | # When this module is the one being built, which modules should be built along with it by default? 355 | # This feeds into automated testing of each module. 356 | [''' + skel_domain + '''.''' + skel_module_name + '''] 357 | shutit.core.module.build:yes 358 | # Allowed images as a regexp, eg ["ubuntu:12.*"], or [".*"], or ["centos"]. 359 | # It's recommended this is locked down as far as possible. 360 | shutit.core.module.allowed_images:["''' + shutit.shutitfile['base_image'] + '''"] 361 | 362 | # Aspects of build process 363 | [build] 364 | base_image:''' + shutit.shutitfile['base_image'] + ''' 365 | 366 | # Volume arguments wanted as part of the build 367 | [target] 368 | volumes: 369 | 370 | [repository] 371 | name:''' + skel_module_name) 372 | build_cnf_file.close() 373 | os.chmod(build_cnf_filename,0o400) 374 | -------------------------------------------------------------------------------- /shutit_pexpect_session_environment.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import shutit_util 3 | 4 | class ShutItPexpectSessionEnvironment(object): 5 | 6 | def __init__(self, 7 | prefix): 8 | """Represents a new 'environment' in ShutIt, which corresponds to a host or any 9 | machine-like location (eg docker container, ssh'd to host, or even a chroot jail 10 | with a /tmp folder that has not been touched by shutit. 11 | """ 12 | if prefix == 'ORIGIN_ENV': 13 | self.environment_id = prefix 14 | else: 15 | self.environment_id = shutit_util.random_id() 16 | self.module_root_dir = '/' 17 | self.modules_installed = [] # has been installed in this build 18 | self.modules_not_installed = [] # modules _known_ not to be installed 19 | self.modules_ready = [] # has been checked for readiness and is ready (in this build) 20 | self.modules_recorded = [] 21 | self.modules_recorded_cache_valid = False 22 | self.install_type = '' 23 | self.distro = '' 24 | self.distro_version = '' 25 | self.users = dict() 26 | self.build = {} 27 | self.build['apt_update_done'] = False 28 | self.build['emerge_update_done'] = False 29 | self.build['apk_update_done'] = False 30 | 31 | def __str__(self): 32 | string = '\n======= SHUTIT PEXPECT SESSION ENVIRONMENT BEGIN ========' 33 | string += '\n| distro = ' + str(self.distro) 34 | string += '| module_root_dir = ' + str(self.module_root_dir) 35 | string += '| modules_installed = ' + str(self.modules_installed) 36 | string += '| modules_not_installed = ' + str(self.modules_not_installed) 37 | string += '| modules_ready = ' + str(self.modules_ready) 38 | string += '| modules_recorded_cache_valid = ' + str(self.modules_recorded) 39 | string += '| install_type = ' + str(self.install_type) 40 | string += '| distro = ' + str(self.distro) 41 | string += '| distro_version = ' + str(self.distro_version) 42 | string += '| users = ' + str(self.users) 43 | string += '| self.build = ' + str(self.build) 44 | string += '\n======= SHUTIT PEXPECT SESSION ENVIRONMENT END ========' 45 | return string 46 | -------------------------------------------------------------------------------- /shutit_sendspec.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import logging 3 | 4 | class ShutItSendSpec(object): 5 | """Specification for arguments to send to shutit functions. 6 | """ 7 | def __init__(self, 8 | shutit_pexpect_child, 9 | send=None, 10 | send_dict=None, 11 | expect=None, 12 | timeout=None, 13 | check_exit=None, 14 | fail_on_empty_before=True, 15 | record_command=True, 16 | exit_values=None, 17 | echo=None, 18 | escape=False, 19 | check_sudo=True, 20 | retry=3, 21 | note=None, 22 | assume_gnu=True, 23 | follow_on_commands=None, 24 | searchwindowsize=None, 25 | maxread=None, 26 | delaybeforesend=None, 27 | secret=False, 28 | nonewline=False, 29 | user=None, 30 | password=None, 31 | is_ssh=None, 32 | go_home=True, 33 | prompt_prefix=None, 34 | remove_on_match=None, 35 | fail_on_fail=True, 36 | ignore_background=False, 37 | run_in_background=False, 38 | block_other_commands=True, 39 | wait_cadence=2, 40 | loglevel=logging.INFO): 41 | """Specification for arguments to send to shutit functions. 42 | 43 | @param send: String to send, ie the command being issued. If set to 44 | None, we consume up to the expect string, which is useful if we 45 | just matched output that came before a standard command that 46 | returns to the prompt. 47 | @param send_dict: dict of sends and expects, eg: 48 | {'interim prompt:',['some input',False],'input password:':['mypassword',True]} 49 | Note that the boolean indicates whether the match results in the removal of the send dict expects from the interaction and assumes a prompt follows. 50 | @param expect: String that we expect to see in the output. Usually a 51 | prompt. Defaults to currently-set expect string (see 52 | set_default_shutit_pexpect_session_expect) 53 | @param shutit_pexpect_child: pexpect child to issue command to. 54 | @param timeout: Timeout on response 55 | @param check_exit: Whether to check the shell exit code of the passed-in 56 | command. If the exit value was non-zero an error is thrown. 57 | (default=None, which takes the currently-configured check_exit 58 | value) See also fail_on_empty_before. 59 | @param fail_on_empty_before: If debug is set, fail on empty match output 60 | string (default=True) If this is set to False, then we don't 61 | check the exit value of the command. 62 | @param record_command: Whether to record the command for output at end. 63 | As a safety measure, if the command matches any 'password's then 64 | we don't record it. 65 | @param exit_values: Array of acceptable exit values as strings 66 | @param echo: Whether to suppress any logging output from pexpect to the 67 | terminal or not. We don't record the command if this is set to 68 | False unless record_command is explicitly passed in as True. 69 | @param escape: Whether to escape the characters in a bash-friendly way, eg $'\\Uxxxxxx' 70 | @param check_sudo: Check whether we have sudo available and if we already have sudo rights 71 | cached. 72 | @param retry: Number of times to retry the command if the first attempt 73 | doesn't work. Useful if going to the network 74 | @param note: If a note is passed in, and we are in walkthrough mode, 75 | pause with the note printed 76 | @param assume_gnu: Assume the gnu version of commands, which are not in 77 | @param follow_on_commands: A dictionary of the form: {match_string: command, match_string2: command2} 78 | which runs commands based on whether the output matched. 79 | Follow-on commands are always foregrounded and always ignore backgrounded processes. 80 | @param searchwindowsize: Passed into pexpect session 81 | @param maxread: Passed into pexpect session 82 | @param delaybeforesend: Passed into pexpect session 83 | @param secret: Whether what is being sent is a secret 84 | @param nonewline: Whether to omit the newline from the send 85 | @param user: If logging in, user to use. Default is 'root'. 86 | @param password: If logging in, password to use. Default is 'root'. 87 | @param is_ssh: Indicates whether the login is an ssh one if it is not an ssh command 88 | @param go_home: On logging in, whether to go to the home dir. Default is True. 89 | @param prompt_prefix: Override of random prompt prefix created by prompt setup. 90 | @param remove_on_match: If the item matches, remove the send_dict from future expects (eg if 91 | it's a password). This makes the 'am I logged in yet?' checking more robust. 92 | @param ignore_background: Whether to block if there are background tasks 93 | running in this session that are blocking, or ignore ALL 94 | background tasks and run anyway. Default is False. 95 | @param run_in_background: Whether to run in the background 96 | @param block_other_commands: Whether to block other commands from running 97 | (unless ignore_background is set on those other commands). 98 | Default is True. 99 | @param wait_cadence: If blocked and waiting on a background tasks, wait this 100 | number of seconds before re-checking. Default is 2. 101 | @param loglevel: Log level at which to operate. 102 | 103 | Background Commands 104 | =================== 105 | 106 | +------------------+-------------------+----------------------+------------------------------------------+ 107 | |run_in_background | ignore_background | block_other_commands | Outcome | 108 | +------------------+-------------------+----------------------+------------------------------------------+ 109 | |T | T | T | 'Just run in background and queue others'| 110 | | | | | Runs the command in the background, | 111 | | | | | ignoring all blocking background tasks | 112 | | | | | even if they are blocking, and blocking | 113 | | | | | new background tasks (if they don't | 114 | | | | | ignore blocking background tasks). | 115 | +------------------+-------------------+----------------------+------------------------------------------+ 116 | |T | F | T | 'Run in background if not blocked, and | 117 | | | | | queue others' | 118 | | | | | Runs the command in the background, | 119 | | | | | but will block if there are blocking | 120 | | | | | background tasks running. It will block | 121 | | | | | new background tasks (if they don't | 122 | | | | | ignore blocking background tasks). | 123 | +------------------+-------------------+----------------------+------------------------------------------+ 124 | |T | F | F | 'Run in background if not blocked, and | 125 | | | | | let others run' | 126 | +------------------+-------------------+----------------------+------------------------------------------+ 127 | |F | T | N/A | 'Run in foreground, ignoring any | 128 | | | | | background commands and block any new | 129 | | | | | background commands.' | 130 | +------------------+-------------------+----------------------+------------------------------------------+ 131 | |F | F | N/A | 'Run in foreground, blocking if there are| 132 | | | | | any background tasks running, and | 133 | | | | | blocking any new background commands.' | 134 | +------------------+-------------------+----------------------+------------------------------------------+ 135 | 136 | Example 137 | ======= 138 | 139 | Scenario is that we want to: 140 | 141 | update the file database with 'updatedb' 142 | 143 | then find a file that we expect to be in that database with 'locate file_to_find' 144 | 145 | and then add a line to that file with 'echo line >> file_to_find' 146 | 147 | Statement: I want to run this command in the background in this ShutIt session. 148 | I want to stop other background commands from running. 149 | I don't care if other background commands are running which block this. 150 | Example send: updatedb 151 | Args: run_in_background=True, ignore_background=True, block_other_commands=True 152 | 153 | Statement: I want to run this command in the background in this ShutIt session. 154 | I want to stop other background commands from running. 155 | I don't want to run if other blocking background commands are running. 156 | Example send: locate file_to_find 157 | Args: run_in_background=True, ignore_background=False, block_other_commands=True 158 | 159 | Statement: I just want to run this command in the background in the ShutIt session and forget about it. 160 | I don't care if there are other background tasks running which block this. 161 | I don't want to block other commands, nothing will depend on this completing. 162 | Example send: echo 'Add line to file' >> /path/to/file_to_find 163 | Args: run_in_background=True, ignore_background=True, block_other_commands=False 164 | """ 165 | self.send = send 166 | self.original_send = send 167 | self.send_dict = send_dict 168 | self.expect = expect 169 | self.shutit_pexpect_child = shutit_pexpect_child 170 | self.timeout = timeout 171 | self.check_exit = check_exit 172 | self.fail_on_empty_before = fail_on_empty_before 173 | self.record_command = record_command 174 | self.exit_values = exit_values 175 | self.echo = echo 176 | self.escape = escape 177 | self.check_sudo = check_sudo 178 | self.retry = retry 179 | self.note = note 180 | self.assume_gnu = assume_gnu 181 | self.follow_on_commands = follow_on_commands 182 | self.searchwindowsize = searchwindowsize 183 | self.maxread = maxread 184 | self.delaybeforesend = delaybeforesend 185 | self.secret = secret 186 | self.nonewline = nonewline 187 | self.loglevel = loglevel 188 | self.user = user 189 | self.password = password 190 | self.is_ssh = is_ssh 191 | self.go_home = go_home 192 | self.prompt_prefix = prompt_prefix 193 | self.remove_on_match = remove_on_match 194 | self.fail_on_fail = fail_on_fail 195 | self.ignore_background = ignore_background 196 | self.run_in_background = run_in_background 197 | self.block_other_commands = block_other_commands 198 | self.wait_cadence = wait_cadence 199 | 200 | # BEGIN Setup/checking 201 | self.started = False 202 | if self.check_exit and self.run_in_background: 203 | self.check_exit = False 204 | #if send_dict and run_in_background: 205 | #assert False, shutit_util.print_debug() 206 | # END Setup/checking 207 | 208 | # send_dict can come in with items that are: val:string, or val:[string,boolean] 209 | # ensure they end up as the latter, defaulting to false. 210 | if self.send_dict is not None: 211 | assert isinstance(self.send_dict, dict), shutit_util.print_debug() 212 | for key in self.send_dict: 213 | val = self.send_dict[key] 214 | assert isinstance(val,(str,list)), shutit_util.print_debug() 215 | if isinstance(val,str): 216 | self.send_dict.update({key:[val,False]}) 217 | elif isinstance(val,list): 218 | assert len(val) == 2, shutit_util.print_debug() 219 | else: 220 | assert False, shutit_util.print_debug(msg='send_dict check should not get here') 221 | 222 | if self.exit_values is None: 223 | self.exit_values = ['0',] 224 | 225 | 226 | def __str__(self): 227 | string = '\n---- Sendspec object BEGIN ----' 228 | string += '| assume_gnu = ' + str(self.assume_gnu) 229 | string += '| block_other_commands = ' + str(self.block_other_commands) 230 | string += '| check_exit = ' + str(self.check_exit) 231 | string += '| check_sudo = ' + str(self.check_sudo) 232 | string += '| delaybeforesend = ' + str(self.delaybeforesend) 233 | string += '| echo = ' + str(self.echo) 234 | string += '| escape = ' + str(self.escape) 235 | string += '| exit_values = ' + str(self.exit_values) 236 | string += '| expect = ' + str(self.expect) 237 | string += '| fail_on_empty_before = ' + str(self.fail_on_empty_before) 238 | string += '| fail_on_fail = ' + str(self.fail_on_fail) 239 | string += '| follow_on_commands = ' + str(self.follow_on_commands) 240 | string += '| go_home = ' + str(self.go_home) 241 | string += '| ignore_background = ' + str(self.ignore_background) 242 | string += '| is_ssh = ' + str(self.is_ssh) 243 | string += '| loglevel = ' + str(self.loglevel) 244 | string += '| maxread = ' + str(self.maxread) 245 | string += '| nonewline = ' + str(self.nonewline) 246 | string += '| note = ' + str(self.note) 247 | string += '| original_send = ' + str(self.original_send) 248 | string += '| password = ' + str(self.password) 249 | string += '| prompt_prefix = ' + str(self.prompt_prefix) 250 | string += '| record_command = ' + str(self.record_command) 251 | string += '| remove_on_match = ' + str(self.remove_on_match) 252 | string += '| retry = ' + str(self.retry) 253 | string += '| run_in_background = ' + str(self.run_in_background) 254 | string += '| searchwindowsize = ' + str(self.searchwindowsize) 255 | string += '| secret = ' + str(self.secret) 256 | string += '| send = ' + str(self.send) 257 | string += '| send_dict = ' + str(self.send_dict) 258 | string += '| started = ' + str(self.started) 259 | string += '| timeout = ' + str(self.timeout) 260 | string += '| user = ' + str(self.user) 261 | string += '| wait_cadence = ' + str(self.wait_cadence) 262 | string += '|---- Sendspec object ENDS ----' 263 | return string 264 | -------------------------------------------------------------------------------- /shutit_session_setup/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ianmiell/shutit/fde11a8d76136884865ca0553246df4598f8bcf6/shutit_session_setup/__init__.py -------------------------------------------------------------------------------- /shutit_session_setup/vagrant.py: -------------------------------------------------------------------------------- 1 | import random 2 | import logging 3 | import string 4 | import time 5 | from shutit_session_setup import virtualization 6 | 7 | 8 | def pre_build(shutit, 9 | vagrant_version='1.8.6', 10 | virt_method='virtualbox'): 11 | if not virtualization.pre_build(shutit, virt_method=virt_method): 12 | return False 13 | processor = shutit.send_and_get_output('uname -p') 14 | if not shutit.command_available('wget'): 15 | shutit.install('wget') 16 | if not shutit.command_available('vagrant'): 17 | if shutit.get_current_shutit_pexpect_session_environment().install_type == 'apt': 18 | pw = shutit.get_env_pass('Input your sudo password to install vagrant') 19 | shutit.send('wget -qO- https://releases.hashicorp.com/vagrant/' + vagrant_version + '/vagrant_' + vagrant_version + '_' + processor + '.deb > /tmp/vagrant.deb',note='Downloading vagrant and installing') 20 | shutit.multisend('sudo dpkg -i /tmp/vagrant.deb',{'assword':pw}) 21 | shutit.send('rm -f /tmp/vagrant.deb') 22 | elif shutit.get_current_shutit_pexpect_session_environment().install_type == 'yum': 23 | pw = shutit.get_env_pass('Input your sudo password to install vagrant') 24 | shutit.send('wget -qO- https://releases.hashicorp.com/vagrant/' + vagrant_version + '/vagrant_' + vagrant_version + '_' + processor + '.rpm > /tmp/vagrant.rpm',note='Downloading vagrant and installing') 25 | shutit.multisend('sudo rpm -i /tmp/vagrant.rpm',{'assword':pw}) 26 | shutit.send('rm -f /tmp/vagrant.rpm') 27 | else: 28 | shutit.install('vagrant') 29 | # do not move this! 30 | if virt_method == 'libvirt' and shutit.send_and_get_output('vagrant plugin list | grep vagrant-libvirt') == '': 31 | if shutit.get_current_shutit_pexpect_session_environment().install_type == 'yum': 32 | shutit.install('gcc-c++') 33 | shutit.install('gcc') 34 | shutit.install('libvirt') 35 | shutit.install('libvirt-devel') 36 | shutit.install('qemu-kvm') 37 | pw = shutit.get_env_pass() 38 | shutit.multisend('sudo /opt/vagrant/embedded/bin/gem source -r https://rubygems.org/',{'assword':pw}) 39 | shutit.multisend('sudo /opt/vagrant/embedded/bin/gem source -a http://rubygems.org/', {'Do you want to add this insecure source?':'y','assword':pw}) 40 | shutit.multisend('sudo /opt/vagrant/embedded/bin/gem update --system --no-doc',{'assword':pw}) 41 | shutit.multisend('sudo /opt/vagrant/embedded/bin/gem source -r http://rubygems.org/',{'assword':pw}) 42 | shutit.multisend('sudo /opt/vagrant/embedded/bin/gem source -a https://rubygems.org/',{'assword':pw}) 43 | shutit.multisend('sudo vagrant plugin install vagrant-libvirt',{'assword':pw}) 44 | if virt_method == 'libvirt': 45 | pw = shutit.get_env_pass() 46 | shutit.multisend('sudo systemctl start libvirtd',{'assword':pw}) 47 | else: 48 | if shutit.send_and_get_output("""vagrant version | head -1 | awk '{print $3}'""") < '1.8.6': 49 | shutit.log('Vagrant version may be too low!') 50 | shutit.send('echo VAGRANT VERSION MAY BE TOO LOW SEE https://github.com/ianmiell/shutit-library/issues/1 && sleep 10') 51 | return True 52 | 53 | 54 | def setup_machines(shutit, 55 | vagrant_image, 56 | virt_method, 57 | gui, 58 | memory, 59 | sourcepath, 60 | module_base_name, 61 | swapsize, 62 | num_machines, 63 | cpu): 64 | assert isinstance(num_machines, str) 65 | assert isinstance(gui, bool) 66 | num_machines = int(num_machines) 67 | vagrant_run_dir = sourcepath + '/vagrant_run' 68 | module_base_name = module_base_name.replace('-','').replace('_','') 69 | module_name = module_base_name + ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(6)).replace('-','').replace('_','') 70 | this_vagrant_run_dir = vagrant_run_dir + '/' + module_name 71 | shutit.send(' command rm -rf ' + this_vagrant_run_dir + ' && command mkdir -p ' + this_vagrant_run_dir + ' && command cd ' + this_vagrant_run_dir, echo=False) 72 | # check whether vagrant box is already up 73 | if shutit.send_and_get_output('''vagrant global-status | sed -n '3,$p' | sed '/^The above/,$d' | awk '{print $2}' | grep ''' + module_name + ''' | wc -l''') != '0': 74 | # TODO: ask first 75 | lines = shutit.send_and_get_output('''vagrant global-status | sed -n '3,$p' | sed '/^The above/,$d' | grep ''' + module_name) 76 | shutit.send('''vagrant global-status | sed -n '3,$p' | sed '/^The above/,$d' | awk '{print $1, $2}' | grep ''' + module_name + ''' | awk {print $1}' | xargs -n1 vagrant-destroy''') 77 | 78 | if shutit.send_and_get_output('vagrant plugin list | grep landrush', echo=False) == '': 79 | shutit.send('vagrant plugin install landrush', echo=False) 80 | shutit.send('vagrant init ' + vagrant_image, echo=False) 81 | 82 | # Data structures. 83 | shutit_sessions = {} 84 | machines = {} 85 | 86 | vagrantfile_contents = '''Vagrant.configure("2") do |config| 87 | config.landrush.enabled = true 88 | config.vm.provider "virtualbox" do |vb| 89 | vb.gui = ''' + str(gui).lower() + ''' 90 | vb.memory = "''' + memory + '''" 91 | vb.cpu = "''' + cpu + '''" 92 | end''' 93 | # TODO: check no hyphens or underscores in module_name as that can confuse things 94 | for m in range(1, num_machines+1): 95 | machine_name = module_base_name + str(m) 96 | vagrantfile_contents += ''' 97 | 98 | config.vm.define "''' + machine_name + '''" do |''' + machine_name + '''| 99 | ''' + machine_name + '''.vm.box = ''' + '"' + vagrant_image + '"' + ''' 100 | ''' + machine_name + '''.vm.hostname = "''' + machine_name + '''.vagrant.test" 101 | ''' + machine_name + '''.vm.provider :virtualbox do |vb| 102 | vb.name = "''' + machine_name + '''" 103 | end 104 | end''' 105 | vagrantfile_contents += ''' 106 | end''' 107 | # machines is a dict of dicts containing information about each machine for you to use. 108 | machine_fqdn = machine_name + '.vagrant.test' 109 | machines.update({machine_name:{'fqdn':machine_fqdn}}) 110 | shutit.send_file(this_vagrant_run_dir+ '/Vagrantfile',vagrantfile_contents) 111 | module_base_name = None 112 | 113 | try: 114 | pw = open('secret').read().strip() 115 | except IOError: 116 | pw = '' 117 | if pw == '': 118 | shutit.log("""You can get round this manual step by creating a 'secret' with your password: 'touch secret && chmod 700 secret'""",level=logging.CRITICAL) 119 | pw = shutit.get_env_pass() 120 | time.sleep(10) 121 | 122 | # Set up the sessions. 123 | shutit_host_session = shutit.create_session(session_type='bash', loglevel='DEBUG') 124 | for machine in sorted(machines.keys()): 125 | shutit_sessions.update({machine:shutit.create_session(session_type='bash', loglevel='DEBUG', walkthrough=False)}) 126 | # Set up and validate landrush. 127 | for machine in sorted(machines.keys()): 128 | shutit_session = shutit_sessions[machine] 129 | machine_name = machines 130 | shutit_session.send('cd ' + this_vagrant_run_dir, echo=False) 131 | shutit_host_session.send('cd ' + this_vagrant_run_dir, echo=False) 132 | # Remove any existing landrush entry. 133 | shutit_host_session.send('vagrant landrush rm ' + machines[machine]['fqdn'], echo=False) 134 | # Needs to be done serially for stability reasons. 135 | try: 136 | shutit_host_session.multisend('vagrant up --provider ' + virt_method + ' ' + machine,{'assword for':pw,'assword:':pw}, echo=False) 137 | except NameError: 138 | shutit_host_session.multisend('vagrant up ' + machine,{'assword for':pw,'assword:':pw},timeout=99999, echo=False) 139 | if shutit_host_session.send_and_get_output("vagrant status 2> /dev/null | grep -w ^" + machine + " | awk '{print $2}'", echo=False) != 'running': 140 | shutit_host_session.pause_point("machine: " + machine + " appears not to have come up cleanly") 141 | ip = shutit_host_session.send_and_get_output('''vagrant landrush ls 2> /dev/null | grep -w ^''' + machines[machine]['fqdn'] + ''' | awk '{print $2}' ''', echo=False) 142 | machines.get(machine).update({'ip':ip}) 143 | shutit_session.login(command='vagrant ssh ' + machine, echo=False) 144 | shutit_session.login(command='sudo su - ', echo=False) 145 | # Correct /etc/hosts 146 | shutit_session.send(r'''cat <(echo $(ip -4 -o addr show scope global | grep -v 10.0.2.15 | head -1 | awk '{print $4}' | sed 's/\(.*\)\/.*/\1/') $(hostname)) <(cat /etc/hosts | grep -v $(hostname -s)) > /tmp/hosts && mv -f /tmp/hosts /etc/hosts''', echo=False) 147 | # Correct any broken ip addresses. 148 | if shutit_host_session.send_and_get_output('''vagrant landrush ls | grep ''' + machine + ''' | grep 10.0.2.15 | wc -l''', echo=False) != '0': 149 | shutit_session.log('A 10.0.2.15 landrush ip was detected for machine: ' + machine + ', correcting.',level=logging.WARNING) 150 | # This beaut gets all the eth0 addresses from the machine and picks the first one that it not 10.0.2.15. 151 | while True: 152 | ipaddr = shutit_session.send_and_get_output(r'''ip -4 -o addr show scope global | grep -v 10.0.2.15 | head -1 | awk '{print $4}' | sed 's/\(.*\)\/.*/\1/' ''', echo=False) 153 | if ipaddr[0] not in ('1','2','3','4','5','6','7','8','9'): 154 | time.sleep(10) 155 | else: 156 | break 157 | # Send this on the host (shutit, not shutit_session) 158 | shutit_host_session.send('vagrant landrush set ' + machines[machine]['fqdn'] + ' ' + ipaddr) 159 | # Check that the landrush entry is there. 160 | shutit_host_session.send('vagrant landrush ls | grep -w ' + machines[machine]['fqdn']) 161 | # All done, so gather landrush info 162 | for machine in sorted(machines.keys()): 163 | ip = shutit_host_session.send_and_get_output('''vagrant landrush ls 2> /dev/null | grep -w ^''' + machines[machine]['fqdn'] + ''' | awk '{print $2}' ''', echo=False) 164 | machines.get(machine).update({'ip':ip}) 165 | 166 | for machine in sorted(machines.keys()): 167 | shutit_session = shutit_sessions[machine] 168 | shutit_session.run_script(r'''#!/bin/sh 169 | # See https://raw.githubusercontent.com/ianmiell/vagrant-swapfile/master/vagrant-swapfile.sh 170 | fallocate -l ''' + swapsize + r''' /swapfile 171 | ls -lh /swapfile 172 | chown root:root /swapfile 173 | chmod 0600 /swapfile 174 | ls -lh /swapfile 175 | mkswap /swapfile 176 | swapon /swapfile 177 | swapon -s 178 | grep -i --color swap /proc/meminfo 179 | echo " 180 | /swapfile none swap sw 0 0" >> /etc/fstab''', echo=False) 181 | shutit_session.multisend('adduser person', 182 | {'Enter new UNIX password':'person', 183 | 'Retype new UNIX password:':'person', 184 | 'Full Name':'', 185 | 'Phone':'', 186 | 'Room':'', 187 | 'Other':'', 188 | 'Is the information correct':'Y'}, echo=False) 189 | 190 | 191 | # TODO: copy ssh keys code 192 | # TODO: docker code 193 | -------------------------------------------------------------------------------- /shutit_session_setup/virtualization.py: -------------------------------------------------------------------------------- 1 | def pre_build(shutit, virt_method='virtualbox'): 2 | if virt_method == 'virtualbox': 3 | if not shutit.command_available('VBoxManage'): 4 | if shutit.get_current_shutit_pexpect_session_environment().install_type == 'apt': 5 | shutit.send('echo "deb http://download.virtualbox.org/virtualbox/debian $(lsb_release -s -c) contrib" >> /etc/apt/sources.list ') 6 | shutit.send('wget -qO- https://www.virtualbox.org/download/oracle_vbox.asc | sudo apt-key add -') 7 | shutit.send('apt-get -qq update') 8 | shutit.install('virtualbox-5.0') 9 | else: 10 | shutit.install('virtualbox') 11 | elif virt_method == 'libvirt': 12 | # Is this a good enough test of whether virsh exists? 13 | if not shutit.command_available('virsh'): 14 | shutit.install('kvm') 15 | shutit.install('libvirt') 16 | shutit.install('libvirt-devel') 17 | shutit.install('qemu-kvm') 18 | shutit.send('systemctl start libvirtd') 19 | else: 20 | return False 21 | return True 22 | -------------------------------------------------------------------------------- /shutit_setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | shutit.tk.setup (core ShutIt setup module) 3 | 4 | Nomenclature: 5 | - Host machine: Machine on which this script is run. 6 | - Target: Environment to which we deploy (docker container or bash shell) 7 | - Container: Docker container created to run the modules on. 8 | 9 | - target_child pexpect-spawned child created to build on target 10 | - host_child pexpect spawned child living on the host machine 11 | """ 12 | 13 | # The MIT License (MIT) 14 | # 15 | # Copyright (C) 2014 OpenBet Limited 16 | # 17 | # Permission is hereby granted, free of charge, to any person obtaining a copy of 18 | # this software and associated documentation files (the "Software"), to deal in 19 | # the Software without restriction, including without limitation the rights to 20 | # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 21 | # of the Software, and to permit persons to whom the Software is furnished to do 22 | # so, subject to the following conditions: 23 | # 24 | # The above copyright notice and this permission notice shall be included in all 25 | # copies or substantial portions of the Software. 26 | # 27 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 28 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 29 | # ITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 30 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 31 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 32 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 33 | # SOFTWARE. 34 | 35 | from __future__ import print_function 36 | import shutit_global 37 | import shutit_util 38 | from shutit_module import ShutItModule 39 | from shutit_sendspec import ShutItSendSpec 40 | from shutit_pexpect_session import ShutItPexpectSession 41 | 42 | 43 | class ShutItConnModule(ShutItModule): 44 | 45 | 46 | def __init__(self, *args, **kwargs): 47 | super(ShutItConnModule, self).__init__(*args, **kwargs) 48 | 49 | 50 | def setup_host_child(self, shutit): 51 | shutit.setup_host_child_environment() 52 | 53 | 54 | def setup_target_child(self, shutit, target_child, target_child_id='target_child',prefix='root'): 55 | shutit.setup_target_child_environment(target_child, target_child_id=target_child_id,prefix=prefix) 56 | 57 | 58 | def build(self, shutit): 59 | return True 60 | 61 | 62 | class ConnDocker(ShutItConnModule): 63 | """Connects ShutIt to docker daemon and starts the container. 64 | """ 65 | 66 | def is_installed(self, shutit): 67 | """Always considered false for ShutIt setup. 68 | """ 69 | return False 70 | 71 | 72 | def destroy_container(self, shutit, host_shutit_session_name, container_shutit_session_name, container_id): 73 | host_child = shutit.get_shutit_pexpect_session_from_id(host_shutit_session_name).pexpect_child 74 | shutit.conn_docker_destroy_container(host_shutit_session_name, container_shutit_session_name, container_id) 75 | shutit.send(' command docker rm -f ' + container_id + ' && rm -f ' + shutit.build['cidfile'],shutit_pexpect_child=host_child,expect=shutit.expect_prompts['ORIGIN_ENV']) 76 | 77 | 78 | def start_container(self, shutit, shutit_session_name): 79 | return shutit.conn_docker_start_container(shutit_session_name) 80 | 81 | 82 | def build(self, shutit): 83 | """Sets up the target ready for building. 84 | """ 85 | target_child = self.start_container(shutit, 'target_child') 86 | self.setup_host_child(shutit) 87 | # TODO: on the host child, check that the image running has bash as its cmd/entrypoint. 88 | self.setup_target_child(shutit, target_child) 89 | shutit.send('chmod -R 777 ' + shutit_global.shutit_global_object.shutit_state_dir + ' && mkdir -p ' + shutit_global.shutit_global_object.shutit_state_dir_build_db_dir + '/' + shutit_global.shutit_global_object.build_id, shutit_pexpect_child=target_child, echo=False) 90 | return True 91 | 92 | 93 | def finalize(self, shutit): 94 | """Finalizes the target, exiting for us back to the original shell 95 | and performing any repository work required. 96 | """ 97 | # Finish with the target 98 | target_child_pexpect_session = shutit.get_shutit_pexpect_session_from_id('target_child') 99 | assert not target_child_pexpect_session.sendline(ShutItSendSpec(target_child_pexpect_session,'exit',ignore_background=True)), shutit_util.print_debug() 100 | host_child_pexpect_session = shutit.get_shutit_pexpect_session_from_id('host_child') 101 | host_child = host_child_pexpect_session.pexpect_child 102 | shutit.set_default_shutit_pexpect_session(host_child_pexpect_session) 103 | shutit.set_default_shutit_pexpect_session_expect(shutit.expect_prompts['ORIGIN_ENV']) 104 | shutit.do_repository_work(shutit.repository['name'], docker_executable=shutit.host['docker_executable'], password=shutit.host['password']) 105 | # Final exits 106 | host_child.sendline('rm -f ' + shutit.build['cidfile']) # Ignore response, just send. 107 | host_child.sendline('exit') # Exit raw bash. Ignore response, just send. 108 | return True 109 | 110 | 111 | def get_config(self, shutit): 112 | return True 113 | 114 | 115 | class ConnBash(ShutItConnModule): 116 | """Connects ShutIt to a machine via bash. 117 | Assumes no docker daemon available for tagging and pushing. 118 | """ 119 | 120 | 121 | def is_installed(self, shutit): 122 | """Always considered false for ShutIt setup. 123 | """ 124 | return False 125 | 126 | 127 | def get_config(self, shutit): 128 | return True 129 | 130 | 131 | def build(self, shutit): 132 | """Sets up the machine ready for building. 133 | """ 134 | shutit_pexpect_session = ShutItPexpectSession(shutit, 'target_child','/bin/bash') 135 | target_child = shutit_pexpect_session.pexpect_child 136 | shutit_pexpect_session.expect(shutit_global.shutit_global_object.base_prompt.strip(), timeout=10) 137 | self.setup_host_child(shutit) 138 | self.setup_target_child(shutit, target_child) 139 | return True 140 | 141 | 142 | def finalize(self, shutit): 143 | """Finalizes the target, exiting for us back to the original shell 144 | and performing any repository work required. 145 | """ 146 | # Finish with the target 147 | target_child_pexpect_session = shutit.get_shutit_pexpect_session_from_id('target_child') 148 | assert not target_child_pexpect_session.sendline(ShutItSendSpec(target_child_pexpect_session,'exit',ignore_background=True)), shutit_util.print_debug() 149 | return True 150 | 151 | 152 | def conn_module(): 153 | """Connects ShutIt to something 154 | """ 155 | return [ 156 | ConnDocker('shutit.tk.conn_docker', -0.1, description='Connect ShutIt to docker'), 157 | ConnBash ('shutit.tk.conn_bash', -0.1, description='Connect ShutIt to a host via bash'), 158 | ] 159 | 160 | 161 | class setup(ShutItModule): 162 | 163 | 164 | def is_installed(self, shutit): 165 | """Always considered false for ShutIt setup. 166 | """ 167 | return False 168 | 169 | 170 | def build(self, shutit): 171 | """Initializes target ready for build and updating package management if in container. 172 | """ 173 | if shutit.build['delivery'] in ('docker','dockerfile'): 174 | if shutit.get_current_shutit_pexpect_session_environment().install_type == 'apt': 175 | shutit.add_to_bashrc('export DEBIAN_FRONTEND=noninteractive') 176 | if not shutit.command_available('lsb_release'): 177 | shutit.install('lsb-release') 178 | shutit.lsb_release() 179 | elif shutit.get_current_shutit_pexpect_session_environment().install_type == 'yum': 180 | # yum updates are so often "bad" that we let exit codes of 1 through. 181 | # TODO: make this more sophisticated 182 | shutit.send('yum update -y', timeout=9999, exit_values=['0', '1']) 183 | shutit.pause_point('Anything you want to do to the target host ' + 'before the build starts?', level=2) 184 | return True 185 | 186 | 187 | def remove(self, shutit): 188 | """Removes anything performed as part of build. 189 | """ 190 | return True 191 | 192 | 193 | def get_config(self, shutit): 194 | """Gets the configured core pacakges, and whether to perform the package 195 | management update. 196 | """ 197 | return True 198 | 199 | 200 | def module(): 201 | return setup('shutit.tk.setup', 0.0, description='Core ShutIt setup') 202 | -------------------------------------------------------------------------------- /shutit_skeleton.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env pythen 2 | 3 | """ShutIt skeleton functions 4 | """ 5 | 6 | # The MIT License (MIT) 7 | # 8 | # Copyright (C) 2014 OpenBet Limited 9 | # 10 | # Permission is hereby granted, free of charge, to any person obtaining a copy of 11 | # this software and associated documentation files (the "Software"), to deal in 12 | # the Software without restriction, including without limitation the rights to 13 | # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 14 | # of the Software, and to permit persons to whom the Software is furnished to do 15 | # so, subject to the following conditions: 16 | # 17 | # The above copyright notice and this permission notice shall be included in all 18 | # copies or substantial portions of the Software. 19 | # 20 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22 | # ITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 23 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 24 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 25 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 26 | # SOFTWARE. 27 | 28 | from __future__ import print_function 29 | import os 30 | import re 31 | import json 32 | from shutit_patterns import shutitfile 33 | 34 | 35 | def create_skeleton(shutit): 36 | """Creates module based on a pattern supplied as a git repo. 37 | """ 38 | skel_path = shutit.cfg['skeleton']['path'] 39 | skel_module_name = shutit.cfg['skeleton']['module_name'] 40 | skel_domain = shutit.cfg['skeleton']['domain'] 41 | skel_domain_hash = shutit.cfg['skeleton']['domain_hash'] 42 | skel_depends = shutit.cfg['skeleton']['depends'] 43 | skel_shutitfiles = shutit.cfg['skeleton']['shutitfiles'] 44 | skel_delivery = shutit.cfg['skeleton']['delivery'] 45 | skel_pattern = shutit.cfg['skeleton']['pattern'] 46 | # For vagrant only 47 | skel_vagrant_num_machines = shutit.cfg['skeleton']['vagrant_num_machines'] 48 | skel_vagrant_machine_prefix = shutit.cfg['skeleton']['vagrant_machine_prefix'] 49 | skel_vagrant_ssh_access = shutit.cfg['skeleton']['vagrant_ssh_access'] 50 | skel_vagrant_docker = shutit.cfg['skeleton']['vagrant_docker'] 51 | skel_vagrant_snapshot = shutit.cfg['skeleton']['vagrant_snapshot'] 52 | skel_vagrant_upload = shutit.cfg['skeleton']['vagrant_upload'] 53 | skel_vagrant_disk_size = shutit.cfg['skeleton']['vagrant_disk_size'] 54 | skel_vagrant_image_name = shutit.cfg['skeleton']['vagrant_image_name'] 55 | 56 | # Check setup 57 | if not skel_path or skel_path[0] != '/': 58 | shutit.fail('Must supply a directory and it must be absolute') # pragma: no cover 59 | if os.path.exists(skel_path): 60 | shutit.fail(skel_path + ' already exists') # pragma: no cover 61 | if not skel_module_name: 62 | shutit.fail('Must supply a name for your module, eg mymodulename') # pragma: no cover 63 | if not re.match('^[a-zA-z_][0-9a-zA-Z_]+$', skel_module_name): 64 | shutit.fail('Module names must comply with python classname standards: cf: http://stackoverflow.com/questions/10120295/valid-characters-in-a-python-class-name name: ' + skel_module_name) # pragma: no cover 65 | if not skel_domain: 66 | shutit.fail('Must supply a domain for your module, eg com.yourname.madeupdomainsuffix') # pragma: no cover 67 | 68 | # Create folders and process pattern 69 | os.makedirs(skel_path) 70 | os.chdir(skel_path) 71 | # If it's shutitfile and vagrant 72 | if shutit.cfg['skeleton']['pattern'] == 'bash': 73 | from shutit_patterns import bash 74 | bash.setup_bash_pattern(shutit, 75 | skel_path=skel_path, 76 | skel_delivery=skel_delivery, 77 | skel_domain=skel_domain, 78 | skel_module_name=skel_module_name, 79 | skel_shutitfiles=skel_shutitfiles, 80 | skel_domain_hash=skel_domain_hash, 81 | skel_depends=skel_depends) 82 | elif shutit.cfg['skeleton']['pattern'] == 'docker': 83 | from shutit_patterns import docker 84 | docker.setup_docker_pattern(shutit, 85 | skel_path=skel_path, 86 | skel_delivery=skel_delivery, 87 | skel_domain=skel_domain, 88 | skel_module_name=skel_module_name, 89 | skel_shutitfiles=skel_shutitfiles, 90 | skel_domain_hash=skel_domain_hash, 91 | skel_depends=skel_depends) 92 | elif shutit.cfg['skeleton']['pattern'] == 'vagrant': # pragma: no cover 93 | from shutit_patterns import vagrant 94 | vagrant.setup_vagrant_pattern(shutit, 95 | skel_path=skel_path, 96 | skel_delivery=skel_delivery, 97 | skel_domain=skel_domain, 98 | skel_module_name=skel_module_name, 99 | skel_shutitfiles=skel_shutitfiles, 100 | skel_domain_hash=skel_domain_hash, 101 | skel_depends=skel_depends, 102 | skel_vagrant_num_machines=skel_vagrant_num_machines, 103 | skel_vagrant_machine_prefix=skel_vagrant_machine_prefix, 104 | skel_vagrant_ssh_access=skel_vagrant_ssh_access, 105 | skel_vagrant_docker=skel_vagrant_docker, 106 | skel_vagrant_snapshot=skel_vagrant_snapshot, 107 | skel_vagrant_upload=skel_vagrant_upload, 108 | skel_vagrant_disk_size=skel_vagrant_disk_size, 109 | skel_vagrant_image_name=skel_vagrant_image_name) 110 | elif shutit.cfg['skeleton']['pattern'] == 'shutitfile': 111 | shutitfile.setup_shutitfile_pattern(shutit, 112 | skel_path=skel_path, 113 | skel_delivery=skel_delivery, 114 | skel_pattern=skel_pattern, 115 | skel_domain=skel_domain, 116 | skel_module_name=skel_module_name, 117 | skel_vagrant_num_machines=skel_vagrant_num_machines, 118 | skel_vagrant_machine_prefix=skel_vagrant_machine_prefix, 119 | skel_vagrant_ssh_access=skel_vagrant_ssh_access, 120 | skel_vagrant_docker=skel_vagrant_docker) 121 | elif shutit.cfg['skeleton']['pattern'] == 'docker_tutorial': # pragma: no cover 122 | shutit.fail('docker_tutorial not yet supported') 123 | 124 | 125 | def process_shutitfile(shutit, shutitfile_contents): 126 | # Wipe the command as we expect one in the file. 127 | shutitfile_representation = {'shutitfile': {}} 128 | shutitfile_representation['shutitfile']['cmd'] = '' 129 | shutitfile_representation['shutitfile']['maintainer'] = '' 130 | shutitfile_representation['shutitfile']['description'] = '' 131 | shutitfile_representation['shutitfile']['module_id'] = '' 132 | shutitfile_representation['shutitfile']['script'] = [] 133 | shutitfile_representation['shutitfile']['config'] = [] 134 | shutitfile_representation['shutitfile']['onbuild'] = [] 135 | shutitfile_representation['shutitfile']['volume'] = [] 136 | shutitfile_representation['shutitfile']['expose'] = [] 137 | shutitfile_representation['shutitfile']['entrypoint'] = [] 138 | shutitfile_representation['shutitfile']['env'] = [] 139 | shutitfile_representation['shutitfile']['depends'] = [] 140 | shutitfile_representation['shutitfile']['delivery'] = [] 141 | shutitfile_representation['shutitfile']['base_image'] = [] 142 | # Whether to build this module by default (defaults to 'yes/true' 143 | shutitfile_representation['shutitfile']['default_include'] = 'true' 144 | shutitfile_list, ok = shutitfile.parse_shutitfile(shutitfile_contents) 145 | if not ok: # pragma: no cover 146 | return [], False 147 | # Set defaults from given shutitfile 148 | last_shutitfile_command = '' 149 | shutitfile_state = 'NONE' 150 | inline_script = '' 151 | for item in shutitfile_list: 152 | # These items are not order-dependent and don't affect the build, so we collect them here: 153 | shutitfile_command = item[0].upper() 154 | # List of handled shutitfile_commands 155 | if shutitfile_state != 'SCRIPT_DURING': 156 | assert shutitfile_command in ('SCRIPT_END','SCRIPT_BEGIN','SCRIPT_END','FROM','ONBUILD','VOLUME','DESCRIPTION','MAINTAINER','EXPOSE','ENTRYPOINT','CMD','USER','LOGIN','LOGOUT','GET_PASSWORD','ENV','RUN','SEND','ASSERT_OUTPUT','PAUSE_POINT','EXPECT','EXPECT_MULTI','EXPECT_REACT','UNTIL','ADD','COPY','WORKDIR','COMMENT','NOTE','INSTALL','REMOVE','DEPENDS','DELIVERY','MODULE_ID','REPLACE_LINE','ENSURE_LINE','START_BEGIN','START_END','STOP_BEGIN','STOP_END','TEST_BEGIN','TEST_END','BUILD_BEGIN','BUILD_END','ISINSTALLED_BEGIN','ISINSTALLED_END','IF','IF_NOT','ELIF_NOT','ELIF','ELSE','ENDIF','COMMIT','PUSH','DEFAULT_INCLUDE','LOG','CONFIG','CONFIG_SECRET','QUIT','STORE_RUN','VAGRANT_LOGIN','VAGRANT_LOGOUT'), shutit_util.print_debug(msg='%r is not a handled ShutItFile command' % shutitfile_command) 157 | if shutitfile_command != 'SCRIPT_END' and shutitfile_state == 'SCRIPT_DURING': 158 | inline_script += '\n' + ' '.join(item) 159 | elif shutitfile_command == 'SCRIPT_BEGIN': 160 | shutitfile_state = 'SCRIPT_DURING' 161 | elif shutitfile_command == 'SCRIPT_END': 162 | shutitfile_representation['shutitfile']['script'].append(['RUN_SCRIPT', inline_script]) 163 | shutitfile_state = 'NONE' 164 | inline_script = '' 165 | elif shutitfile_command == 'FROM': 166 | if shutitfile_representation['shutitfile']['base_image'] == []: 167 | shutitfile_representation['shutitfile']['base_image'] = item[1] 168 | shutit.shutitfile['base_image'] = item[1] 169 | else: 170 | shutit_global.shutit_global_object.shutit_print('Ignoring FROM line as this it has already been set.') # pragma: no cover 171 | elif shutitfile_command == 'ONBUILD': 172 | # TESTED? NO 173 | # Maps to finalize :) - can we have more than one of these? assume yes 174 | # This contains within it one of the above commands, so we need to abstract this out. 175 | shutitfile_representation['shutitfile']['onbuild'].append(item[1]) 176 | elif shutitfile_command == 'MAINTAINER': 177 | shutitfile_representation['shutitfile']['maintainer'] = item[1] 178 | elif shutitfile_command == 'DESCRIPTION': 179 | shutitfile_representation['shutitfile']['description'] = item[1] 180 | elif shutitfile_command == 'VOLUME': 181 | # TESTED? NO 182 | # Put in the run.sh. 183 | try: 184 | shutitfile_representation['shutitfile']['volume'].append(' '.join(json.loads(item[1]))) 185 | except Exception: 186 | shutitfile_representation['shutitfile']['volume'].append(item[1]) 187 | elif shutitfile_command == 'EXPOSE': 188 | # TESTED? NO 189 | # Put in the run.sh. 190 | shutitfile_representation['shutitfile']['expose'].append(item[1]) 191 | elif shutitfile_command == 'ENTRYPOINT': 192 | # TESTED? NO 193 | # Put in the run.sh? Yes, if it exists it goes at the front of cmd 194 | try: 195 | shutitfile_representation['shutitfile']['entrypoint'] = ' '.join(json.loads(item[1])) 196 | except Exception: 197 | shutitfile_representation['shutitfile']['entrypoint'] = item[1] 198 | elif shutitfile_command == 'CMD': 199 | # TESTED? NO 200 | # Put in the run.sh 201 | try: 202 | shutitfile_representation['shutitfile']['cmd'] = ' '.join(json.loads(item[1])) 203 | except Exception: 204 | shutitfile_representation['shutitfile']['cmd'] = item[1] 205 | # Other items to be run through sequentially (as they are part of the script) 206 | elif shutitfile_command == 'GET_PASSWORD': 207 | # If we are directed to get the password, change the previous directive internally. 208 | if last_shutitfile_command not in ('LOGIN','USER'): 209 | shutit.fail('GET_PASSWORD line not after a USER or LOGIN line: ' + shutitfile_command + ' ' + item[1]) # pragma: no cover 210 | if last_shutitfile_command in ('LOGIN','USER'): 211 | if last_shutitfile_command == 'LOGIN': 212 | shutitfile_representation['shutitfile']['script'][-1][0] = 'LOGIN_WITH_PASSWORD' 213 | elif last_shutitfile_command == 'USER': 214 | shutitfile_representation['shutitfile']['script'][-1][0] = 'USER_WITH_PASSWORD' 215 | shutitfile_representation['shutitfile']['script'][-1].append(item[1]) 216 | elif shutitfile_command == 'ENV': 217 | # Put in the run.sh. 218 | shutitfile_representation['shutitfile']['script'].append([shutitfile_command, item[1]]) 219 | # Set in the build 220 | shutitfile_representation['shutitfile']['env'].append(item[1]) 221 | elif shutitfile_command in ('RUN','SEND'): 222 | # Only handle simple commands for now and ignore the fact that shutitfiles run with /bin/sh -c rather than bash. 223 | try: 224 | shutitfile_representation['shutitfile']['script'].append([shutitfile_command, ' '.join(json.loads(item[1]))]) 225 | except Exception: 226 | shutitfile_representation['shutitfile']['script'].append([shutitfile_command, item[1]]) 227 | elif shutitfile_command == 'ASSERT_OUTPUT': 228 | if last_shutitfile_command not in ('RUN','SEND'): 229 | shutit.fail('ASSERT_OUTPUT line not after a RUN/SEND line: ' + shutitfile_command + ' ' + item[1]) # pragma: no cover 230 | shutitfile_representation['shutitfile']['script'][-1][0] = 'ASSERT_OUTPUT_SEND' 231 | shutitfile_representation['shutitfile']['script'].append([shutitfile_command, item[1]]) 232 | elif shutitfile_command == 'EXPECT': 233 | if last_shutitfile_command not in ('RUN','SEND','GET_PASSWORD'): 234 | shutit.fail('EXPECT line not after a RUN, SEND or GET_PASSWORD line: ' + shutitfile_command + ' ' + item[1]) # pragma: no cover 235 | shutitfile_representation['shutitfile']['script'][-1][0] = 'SEND_EXPECT' 236 | shutitfile_representation['shutitfile']['script'].append([shutitfile_command, item[1]]) 237 | elif shutitfile_command == 'EXPECT_MULTI': 238 | if last_shutitfile_command not in ('RUN','SEND','GET_PASSWORD'): 239 | shutit.fail('EXPECT_MULTI line not after a RUN, SEND or GET_PASSWORD line: ' + shutitfile_command + ' ' + item[1]) # pragma: no cover 240 | shutitfile_representation['shutitfile']['script'][-1][0] = 'SEND_EXPECT_MULTI' 241 | shutitfile_representation['shutitfile']['script'].append([shutitfile_command, item[1]]) 242 | elif shutitfile_command == 'EXPECT_REACT': 243 | if last_shutitfile_command not in ('RUN','SEND','GET_PASSWORD'): 244 | shutit.fail('EXPECT_REACT line not after a RUN, SEND or GET_PASSWORD line: ' + shutitfile_command + ' ' + item[1]) # pragma: no cover 245 | shutitfile_representation['shutitfile']['script'][-1][0] = 'SEND_EXPECT_REACT' 246 | shutitfile_representation['shutitfile']['script'].append([shutitfile_command, item[1]]) 247 | elif shutitfile_command == 'UNTIL': 248 | if last_shutitfile_command not in ('RUN','SEND'): 249 | shutit.fail('UNTIL line not after a RUN, SEND: ' + shutitfile_command + ' ' + item[1]) # pragma: no cover 250 | shutitfile_representation['shutitfile']['script'][-1][0] = 'SEND_UNTIL' 251 | shutitfile_representation['shutitfile']['script'].append([shutitfile_command, item[1]]) 252 | elif shutitfile_command == 'DEPENDS': 253 | shutitfile_representation['shutitfile']['depends'].append([shutitfile_command, item[1]]) 254 | elif shutitfile_command == 'DELIVERY': 255 | shutitfile_representation['shutitfile']['delivery'].append([shutitfile_command, item[1]]) 256 | elif shutitfile_command == 'MODULE_ID': 257 | # Only one item allowed. 258 | shutitfile_representation['shutitfile']['module_id'] = item[1] 259 | elif shutitfile_command == 'DEFAULT_INCLUDE': 260 | shutitfile_representation['shutitfile']['default_include'] = item[1] 261 | elif shutitfile_command == 'CONFIG': 262 | shutitfile_representation['shutitfile']['config'].append([shutitfile_command, item[1]]) 263 | elif shutitfile_command == 'CONFIG_SECRET': 264 | shutitfile_representation['shutitfile']['config'].append([shutitfile_command, item[1]]) 265 | elif shutitfile_command in ('ADD','COPY','WORKDIR','COMMENT','INSTALL','REMOVE','REPLACE_LINE','ENSURE_LINE','LOG','COMMIT','PUSH','QUIT','PAUSE_POINT','USER','LOGIN','LOGOUT','VAGRANT_LOGIN','VAGRANT_LOGOUT'): 266 | shutitfile_representation['shutitfile']['script'].append([shutitfile_command, item[1]]) 267 | elif shutitfile_command in ('IF','IF_NOT','ELIF_NOT','ELIF','STORE_RUN'): 268 | # Parser retrieved two items here 269 | shutitfile_representation['shutitfile']['script'].append([shutitfile_command, item[1], item[2]]) 270 | elif shutitfile_command in ('ELSE','ENDIF','START_BEGIN','START_END','STOP_BEGIN','STOP_END','TEST_BEGIN','TEST_END','BUILD_BEGIN','BUILD_END','ISINSTALLED_BEGIN','ISINSTALLED_END'): 271 | shutitfile_representation['shutitfile']['script'].append([shutitfile_command]) 272 | else: 273 | shutit.fail('shutitfile command: ' + shutitfile_command + ' not processed') # pragma: no cover 274 | last_shutitfile_command = shutitfile_command 275 | return shutitfile_representation, True 276 | -------------------------------------------------------------------------------- /shutit_threads.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import time 3 | import threading 4 | import traceback 5 | import sys 6 | import os 7 | from curtsies.input import Input 8 | 9 | # There are two threads running in ShutIt. The 'main' one, which drives the 10 | # automation, and the 'watcher' one, which manages either the different view 11 | # panes, or outputs a stack trace of the main thread if 'nothing happens' on it. 12 | 13 | # Boolean indicating whether we've already set up a tracker. 14 | tracker_setup = False 15 | 16 | 17 | # TODO: reject tmux sessions - it does not seem to play nice 18 | # TODO: keep a time counter after the line 19 | # TODO: show context of line (ie lines around) 20 | # TODO: put the lines into an array of objects and mark the lines as inverted/not 21 | def gather_module_paths(): 22 | import shutit_global 23 | shutit_global_object = shutit_global.shutit_global_object 24 | owd = shutit_global_object.owd 25 | shutit_module_paths = set() 26 | for shutit_object in shutit_global.shutit_global_object.shutit_objects: 27 | shutit_module_paths = shutit_module_paths.union(set(shutit_object.host['shutit_module_path'])) 28 | if '.' in shutit_module_paths: 29 | shutit_module_paths.remove('.') 30 | shutit_module_paths.add(owd) 31 | for path in shutit_module_paths: 32 | if path[0] != '/': 33 | shutit_module_paths.remove(path) 34 | shutit_module_paths.add(owd + '/' + path) 35 | return shutit_module_paths 36 | 37 | 38 | def managing_thread_main(): 39 | import shutit_global 40 | from shutit_global import SessionPaneLine 41 | shutit_global.shutit_global_object.global_thread_lock.acquire() 42 | shutit_module_paths = gather_module_paths() 43 | shutit_global.shutit_global_object.global_thread_lock.release() 44 | shutit_global.shutit_global_object.stacktrace_lines_arr = [SessionPaneLine('',time.time(),'log'),] 45 | last_code = [] 46 | draw_type = 'default' 47 | zoom_state = None 48 | while True: 49 | # We have acquired the lock, so read in input 50 | with Input() as input_generator: 51 | input_char = input_generator.send(0.001) 52 | if input_char == 'r': 53 | # Rotate sessions at the bottom 54 | shutit_global.shutit_global_object.lower_pane_rotate_count += 1 55 | elif input_char == '1': 56 | if zoom_state == 1: 57 | draw_type = 'default' 58 | zoom_state = None 59 | else: 60 | draw_type = 'zoomed1' 61 | zoom_state = 1 62 | elif input_char == '2': 63 | if zoom_state == 2: 64 | draw_type = 'default' 65 | zoom_state = None 66 | else: 67 | draw_type = 'zoomed2' 68 | zoom_state = 2 69 | elif input_char == '3': 70 | if zoom_state == 3: 71 | draw_type = 'default' 72 | zoom_state = None 73 | else: 74 | draw_type = 'zoomed3' 75 | zoom_state = 3 76 | elif input_char == '4': 77 | if zoom_state == 4: 78 | draw_type = 'default' 79 | zoom_state = None 80 | else: 81 | draw_type = 'zoomed4' 82 | zoom_state = 4 83 | elif input_char == 'q': 84 | draw_type = 'clearscreen' 85 | shutit_global.shutit_global_object.pane_manager.draw_screen(draw_type=draw_type) 86 | os.system('reset') 87 | os._exit(1) 88 | # Acquire lock to write screen. Prevents nasty race conditions. 89 | # Different depending PY2/3 90 | if shutit_global.shutit_global_object.ispy3: 91 | if not shutit_global.shutit_global_object.global_thread_lock.acquire(blocking=False): 92 | time.sleep(0.01) 93 | continue 94 | else: 95 | if not shutit_global.shutit_global_object.global_thread_lock.acquire(False): 96 | time.sleep(0.01) 97 | continue 98 | code = [] 99 | for thread_id, stack in sys._current_frames().items(): 100 | # ignore own thread: 101 | if thread_id == threading.current_thread().ident: 102 | continue 103 | for filename, lineno, name, line in traceback.extract_stack(stack): 104 | # if the file is in the same folder or subfolder as a folder in: self.host['shutit_module_path'] 105 | # then show that context 106 | for shutit_module_path in shutit_module_paths: 107 | if filename.find(shutit_module_path) == 0: 108 | if len(shutit_global.shutit_global_object.stacktrace_lines_arr) == 0 or shutit_global.shutit_global_object.stacktrace_lines_arr[-1] != line: 109 | linearrow = '===> ' + str(line) 110 | code.append('_' * 80) 111 | code.append('=> %s:%d:%s' % (filename, lineno, name)) 112 | code.append('%s' % (linearrow,)) 113 | from_lineno = lineno - 5 114 | if from_lineno < 0: 115 | from_lineno = 0 116 | to_lineno = 10 117 | else: 118 | to_lineno = lineno + 5 119 | lineno_count = from_lineno 120 | with open(filename, "r") as f: 121 | for line in itertools.islice(f, from_lineno, to_lineno): 122 | line = line.replace('\t',' ') 123 | lineno_count += 1 124 | if lineno_count == lineno: 125 | code.append('***' + str(lineno_count) + '> ' + line.rstrip()) 126 | else: 127 | code.append('===' + str(lineno_count) + '> ' + line.rstrip()) 128 | code.append('_' * 80) 129 | if code != last_code: 130 | for line in code: 131 | shutit_global.shutit_global_object.stacktrace_lines_arr.append(SessionPaneLine(line,time.time(),'log')) 132 | last_code = code 133 | shutit_global.shutit_global_object.pane_manager.draw_screen(draw_type=draw_type) 134 | shutit_global.shutit_global_object.global_thread_lock.release() 135 | 136 | 137 | 138 | def managing_thread_main_simple(): 139 | """Simpler thread to track whether main thread has been quiet for long enough 140 | that a thread dump should be printed. 141 | """ 142 | import shutit_global 143 | last_msg = '' 144 | while True: 145 | printed_anything = False 146 | if shutit_global.shutit_global_object.log_trace_when_idle and time.time() - shutit_global.shutit_global_object.last_log_time > 10: 147 | this_msg = '' 148 | this_header = '' 149 | for thread_id, stack in sys._current_frames().items(): 150 | # ignore own thread: 151 | if thread_id == threading.current_thread().ident: 152 | continue 153 | printed_thread_started = False 154 | for filename, lineno, name, line in traceback.extract_stack(stack): 155 | if not printed_anything: 156 | printed_anything = True 157 | this_header += '\n='*80 + '\n' 158 | this_header += 'STACK TRACES PRINTED ON IDLE: THREAD_ID: ' + str(thread_id) + ' at ' + time.strftime('%c') + '\n' 159 | this_header += '='*80 + '\n' 160 | if not printed_thread_started: 161 | printed_thread_started = True 162 | this_msg += '%s:%d:%s' % (filename, lineno, name) + '\n' 163 | if line: 164 | this_msg += ' %s' % (line,) + '\n' 165 | if printed_anything: 166 | this_msg += '='*80 + '\n' 167 | this_msg += 'STACK TRACES DONE\n' 168 | this_msg += '='*80 + '\n' 169 | if this_msg != last_msg: 170 | print(this_header + this_msg) 171 | last_msg = this_msg 172 | time.sleep(5) 173 | 174 | 175 | def track_main_thread(): 176 | global tracker_setup 177 | if not tracker_setup: 178 | tracker_setup = True 179 | t = threading.Thread(target=managing_thread_main) 180 | t.daemon = True 181 | t.start() 182 | 183 | 184 | def track_main_thread_simple(): 185 | global tracker_setup 186 | if not tracker_setup: 187 | tracker_setup = True 188 | t = threading.Thread(target=managing_thread_main_simple) 189 | t.daemon = True 190 | t.start() 191 | -------------------------------------------------------------------------------- /shutit_util.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env pythen 2 | 3 | """ShutIt utility functions. 4 | """ 5 | 6 | 7 | # The MIT License (MIT) 8 | # 9 | # Copyright (C) 2014 OpenBet Limited 10 | # 11 | # Permission is hereby granted, free of charge, to any person obtaining a copy of 12 | # this software and associated documentation files (the "Software"), to deal in 13 | # the Software without restriction, including without limitation the rights to 14 | # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 15 | # of the Software, and to permit persons to whom the Software is furnished to do 16 | # so, subject to the following conditions: 17 | # 18 | # The above copyright notice and this permission notice shall be included in all 19 | # copies or substantial portions of the Software. 20 | # 21 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 22 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 23 | # ITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 24 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 25 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 26 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 27 | # SOFTWARE. 28 | 29 | from __future__ import print_function 30 | import binascii 31 | import getpass 32 | import logging 33 | import os 34 | import random 35 | import re 36 | import readline 37 | import signal 38 | import socket 39 | import stat 40 | import string 41 | import sys 42 | import threading 43 | import time 44 | import traceback 45 | import shutit_assets 46 | import shutit_class 47 | import shutit_global 48 | import shutit 49 | 50 | 51 | if shutit_global.shutit_global_object.ispy3: 52 | from builtins import input 53 | else: 54 | input=raw_input 55 | 56 | 57 | def is_file_secure(file_name): 58 | """Returns false if file is considered insecure, true if secure. 59 | If file doesn't exist, it's considered secure! 60 | """ 61 | if not os.path.isfile(file_name): 62 | return True 63 | file_mode = os.stat(file_name).st_mode 64 | if file_mode & (stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH): 65 | return False 66 | return True 67 | 68 | 69 | def colorise(code, msg): 70 | """Colorize the given string for a terminal. 71 | See https://misc.flogisoft.com/bash/tip_colors_and_formatting 72 | """ 73 | return '\033[%sm%s\033[0m' % (code, msg) if code else msg 74 | 75 | 76 | def emblinken(msg): 77 | """Blink the message for a terminal 78 | """ 79 | return '\033[5m%s\033[0m' % msg 80 | 81 | 82 | def random_id(size=8, chars=string.ascii_letters + string.digits): 83 | """Generates a random string of given size from the given chars. 84 | 85 | @param size: The size of the random string. 86 | @param chars: Constituent pool of characters to draw random characters from. 87 | @type size: number 88 | @type chars: string 89 | @rtype: string 90 | @return: The string of random characters. 91 | """ 92 | return ''.join(random.choice(chars) for _ in range(size)) 93 | 94 | 95 | def random_word(size=6): 96 | """Returns a random word in lower case. 97 | """ 98 | words = shutit_assets.get_words().splitlines() 99 | word = '' 100 | while len(word) != size or "'" in word: 101 | word = words[int(random.random() * (len(words) - 1))] 102 | return word.lower() 103 | 104 | 105 | def get_hash(string_to_hash): 106 | """Helper function to get preceding integer 107 | eg com.openbet == 1003189494 108 | >>> import binascii 109 | >>> abs(binascii.crc32(b'shutit.tk')) 110 | 782914092 111 | 112 | Recommended means of determining run order integer part. 113 | """ 114 | return abs(binascii.crc32(string_to_hash.encode())) 115 | 116 | 117 | # get the ordinal for a given char, in a friendly way 118 | def get_wide_hex(char): 119 | if len(char) != 2: 120 | return r'\x' + hex(ord(char))[2:] 121 | return r'\u' + hex(0x10000 + (ord(char[0]) - 0xD800) * 0x400 + (ord(char[1]) - 0xDC00))[2:] 122 | 123 | # CTRL-\ HANDLING CODE STARTS 124 | def ctrl_quit_signal_handler(_, frame): 125 | shutit_global.shutit_global_object.shutit_print(r'CRTL-\ caught, hard-exiting ShutIt') 126 | shutit_frame = get_shutit_frame(frame) 127 | if shutit_frame: 128 | shutit_class.do_finalize() 129 | shutit_global.shutit_global_object.handle_exit(exit_code=1) 130 | # CTRL-\ HANDLING CODE ENDS 131 | 132 | # CTRL-C HANDLING CODE STARTS 133 | in_ctrlc = False 134 | def ctrlc_background(): 135 | global ctrl_c_calls 136 | global in_ctrlc 137 | ctrl_c_calls += 1 138 | if ctrl_c_calls > 10: 139 | shutit_global.shutit_global_object.handle_exit(exit_code=1) 140 | in_ctrlc = True 141 | time.sleep(1) 142 | in_ctrlc = False 143 | 144 | 145 | def ctrl_c_signal_handler(_, frame): 146 | """CTRL-c signal handler - enters a pause point if it can. 147 | """ 148 | global ctrl_c_calls 149 | ctrl_c_calls += 1 150 | if ctrl_c_calls > 10: 151 | shutit_global.shutit_global_object.handle_exit(exit_code=1) 152 | shutit_frame = get_shutit_frame(frame) 153 | if in_ctrlc: 154 | msg = 'CTRL-C hit twice, quitting' 155 | if shutit_frame: 156 | shutit_global.shutit_global_object.shutit_print('\n') 157 | shutit = shutit_frame.f_locals['shutit'] 158 | shutit.log(msg, level=logging.CRITICAL) 159 | else: 160 | shutit_global.shutit_global_object.shutit_print(msg) 161 | shutit_global.shutit_global_object.handle_exit(exit_code=1) 162 | if shutit_frame: 163 | shutit = shutit_frame.f_locals['shutit'] 164 | if shutit.build['ctrlc_passthrough']: 165 | shutit.self.get_current_shutit_pexpect_session().pexpect_child.sendline(r'') 166 | return 167 | shutit_global.shutit_global_object.shutit_print(colorise(31, "\r" + r"You may need to wait for a command to complete before a pause point is available. Alternatively, CTRL-\ to quit.")) 168 | shutit.build['ctrlc_stop'] = True 169 | t = threading.Thread(target=ctrlc_background) 170 | t.daemon = True 171 | t.start() 172 | # Reset the ctrl-c calls 173 | ctrl_c_calls = 0 174 | return 175 | shutit_global.shutit_global_object.shutit_print(colorise(31, '\n' + '*' * 80)) 176 | shutit_global.shutit_global_object.shutit_print(colorise(31, "CTRL-c caught, CTRL-c twice to quit.")) 177 | shutit_global.shutit_global_object.shutit_print(colorise(31, '*' * 80)) 178 | t = threading.Thread(target=ctrlc_background) 179 | t.daemon = True 180 | t.start() 181 | # Reset the ctrl-c calls 182 | ctrl_c_calls = 0 183 | 184 | 185 | def get_shutit_frame(frame): 186 | global ctrl_c_calls 187 | ctrl_c_calls += 1 188 | if ctrl_c_calls > 10: 189 | shutit_global.shutit_global_object.handle_exit(exit_code=1) 190 | if not frame.f_back: 191 | return None 192 | else: 193 | if 'shutit' in frame.f_locals: 194 | return frame 195 | return get_shutit_frame(frame.f_back) 196 | ctrl_c_calls = 0 197 | # CTRL-C HANDLING CODE ENDS 198 | 199 | 200 | def print_frame_recurse(frame): 201 | if frame.f_back: 202 | shutit_global.shutit_global_object.shutit_print('=' * 77) 203 | shutit_global.shutit_global_object.shutit_print(frame.f_locals) 204 | print_frame_recurse(frame.f_back) 205 | 206 | 207 | def check_regexp(regex): 208 | if regex is None: 209 | # Is this ok? 210 | return True 211 | try: 212 | re.compile(regex) 213 | return True 214 | except re.error: 215 | return False 216 | 217 | 218 | def sendline(child, line): 219 | """Handles sending of line to pexpect object. 220 | """ 221 | child.sendline(line) 222 | 223 | 224 | def sanitize_terminal(): 225 | os.system('stty sane') 226 | 227 | def exit_cleanup(): 228 | countdown = range(60, 1, -1) 229 | sys.stdout.write('\n\r') 230 | for s in countdown: 231 | sys.stdout.write('ShutIt has exited, resetting terminal in ' + str(s) + ' unless you CTRL-C...\n\r') 232 | time.sleep(1) 233 | sys.stdout.flush() 234 | os.system('reset') 235 | 236 | 237 | def util_raw_input(prompt='', default=None, ispass=False, use_readline=True): 238 | """Handles raw_input calls, and switches off interactivity if there is apparently 239 | no controlling terminal (or there are any other problems) 240 | """ 241 | if use_readline: 242 | try: 243 | readline.read_init_file('/etc/inputrc') 244 | except IOError: 245 | pass 246 | readline.parse_and_bind('tab: complete') 247 | prompt = '\r\n' + prompt 248 | if ispass: 249 | prompt += '\r\nInput Secret: ' 250 | sanitize_terminal() 251 | if shutit_global.shutit_global_object.interactive == 0: 252 | return default 253 | ## See: https//github.com/ianmiell/shutit/issues/299 - python3 made input == python 2's raw_input 254 | #if not shutit_global.shutit_global_object.ispy3: 255 | # input = raw_input 256 | #try: 257 | # input 258 | #except NameError: 259 | # shutit_global.shutit_global_object.shutit_print('input not available, printing debug') 260 | # print_debug() 261 | # sys.exit(1) 262 | if not shutit_global.shutit_global_object.determine_interactive(): 263 | return default 264 | while True: 265 | try: 266 | if ispass: 267 | return getpass.getpass(prompt=prompt) 268 | else: 269 | return input(prompt).strip() or default 270 | except KeyboardInterrupt: 271 | continue 272 | except IOError: 273 | msg = 'Problems getting raw input, assuming no controlling terminal.' 274 | if ispass: 275 | return getpass.getpass(prompt=prompt) 276 | else: 277 | return input(prompt).strip() or default 278 | shutit_global.shutit_global_object.set_noninteractive(msg=msg) 279 | return default 280 | 281 | 282 | def get_input(msg, default='', valid=None, boolean=False, ispass=False, color=None): 283 | """Gets input from the user, and returns the answer. 284 | 285 | @param msg: message to send to user 286 | @param default: default value if nothing entered 287 | @param valid: valid input values (default == empty list == anything allowed) 288 | @param boolean: whether return value should be boolean 289 | @param ispass: True if this is a password (ie whether to not echo input) 290 | @param color: Color code to colorize with (eg 32 = green) 291 | """ 292 | # switch off log tracing when in get_input 293 | log_trace_when_idle_original_value = shutit_global.shutit_global_object.log_trace_when_idle 294 | shutit_global.shutit_global_object.log_trace_when_idle = False 295 | if boolean and valid is None: 296 | valid = ('yes', 'y', 'Y', '1', 'true', 'no', 'n', 'N', '0', 'false') 297 | if color: 298 | answer = util_raw_input(prompt=colorise(color, msg),ispass=ispass) 299 | else: 300 | answer = util_raw_input(msg, ispass=ispass) 301 | if boolean and answer in ('', None) and default != '': 302 | # Revert log trace value to original 303 | shutit_global.shutit_global_object.log_trace_when_idle = log_trace_when_idle_original_value 304 | return default 305 | if valid is not None: 306 | while answer not in valid: 307 | shutit_global.shutit_global_object.shutit_print('Answer must be one of: ' + str(valid),transient=True) 308 | if color: 309 | answer = util_raw_input(prompt=colorise(color, msg),ispass=ispass) 310 | else: 311 | answer = util_raw_input(msg, ispass=ispass) 312 | if boolean: 313 | if answer.lower() in ('yes','y','1','true','t'): 314 | # Revert log trace value to original 315 | shutit_global.shutit_global_object.log_trace_when_idle = log_trace_when_idle_original_value 316 | return True 317 | elif answer.lower() in ('no','n','0','false','f'): 318 | # Revert log trace value to original 319 | shutit_global.shutit_global_object.log_trace_when_idle = log_trace_when_idle_original_value 320 | return False 321 | # Revert log trace value to original 322 | shutit_global.shutit_global_object.log_trace_when_idle = log_trace_when_idle_original_value 323 | return answer or default 324 | 325 | 326 | def print_debug(exc_info=None, msg=''): 327 | # TODO: put the file in a smarter place 328 | f = open('/tmp/shutit_debug.txt', 'w') 329 | if msg: 330 | shutit_global.shutit_global_object.shutit_print('Message: ' + msg, debugfile=f) 331 | environ_string = '' 332 | for env in os.environ: 333 | environ_string += 'export ' + env + '=' + str(os.environ[env]) + ';' 334 | shutit_global.shutit_global_object.shutit_print('\n=============================== DEBUG INFO START ==============================', debugfile=f) 335 | shutit_global.shutit_global_object.shutit_print('This file: ' + os.path.dirname(os.path.realpath(__file__)), debugfile=f) 336 | shutit_global.shutit_global_object.shutit_print('Python version: ' + 'sys.version_info: ' + str(sys.version_info) + ', sys.version: ' + str(sys.version), debugfile=f) 337 | shutit_global.shutit_global_object.shutit_print('Shutit version: ' + shutit.shutit_version, debugfile=f) 338 | shutit_global.shutit_global_object.shutit_print('Server: ' + socket.gethostname(), debugfile=f) 339 | shutit_global.shutit_global_object.shutit_print('Environment: ' + environ_string.replace(';', '\n'), debugfile=f) 340 | shutit_global.shutit_global_object.shutit_print('Command was: ' + sys.executable + (' ').join(sys.argv), debugfile=f) 341 | shutit_global.shutit_global_object.shutit_print('ShutIt global state: ' + str(shutit_global.shutit_global_object), debugfile=f) 342 | if exc_info: 343 | stack_trace = '' 344 | for line in traceback.format_exception(*exc_info): 345 | stack_trace += line 346 | shutit_global.shutit_global_object.shutit_print('Stacktrace:\n' + stack_trace, debugfile=f) 347 | shutit_global.shutit_global_object.shutit_print('\n=============================== DEBUG INFO DONE ==============================', debugfile=f) 348 | --------------------------------------------------------------------------------