├── LICENSE ├── README.md ├── cent7-afterreboot.sh ├── cent7-afterspacewalk.sh ├── cent7-upgrade.sh └── playbook_cent7_upgrade.yaml /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # centos7-upgrade-scripts 2 | Ansible playbook and supporting scripts for upgrading OpenStack compute/hypervisor hosts from CentOS 6 to 7 3 | 4 | **NOTE: You almost certainly cannot drop this directly into your environment and have it work right!!** This is what we used for our configuration and it is very specific to our setup and particular versions of packages and the kernel. 5 | 6 | (There is a lot of OpenStack-specific stuff here, so if you've found this repo while looking for how to upgrade a generic CentOS 6 host, be aware of that.) 7 | 8 | # Playbook and Scripts 9 | 10 | - **`playbook_cent7_upgrade.yaml`** - playbook for orchestrating everything 11 | - **`cent7-upgrade.sh`** - Phase 1 script for configuring prerequisites and getting everything lined up 12 | - **`cent7-afterreboot.sh`** - Phase 2 script that runs after the system first boots after the upgrade is done. It's main purpose is to upgrade the Spacewalk/yum client packages and prepare the server to transition to the CentOS 7 channels in Spacewalk 13 | - **`cent7-afterspacewalk.sh`** - Phase 3 script which handles doing all the final cleanup and upgrade steps 14 | 15 | See the source of each of those scripts for more details and comments to see what they're doing. 16 | 17 | # Upgrade Procedure 18 | 19 | The playbook will do all the work (as long as nothing goes wrong!): 20 | 21 | ``` 22 | ./playbook_cent7_upgrade.yaml -e 'hosts=' 23 | ``` 24 | 25 | Basic task flow: 26 | 27 | 1. Sanity check the current state of packages on the machine (you'll need to provide a sorted package list file for the comparison) 28 | 2. Set maintenance mode in monitoring systems (these commands are removed here since they are very specific to Go Daddy) 29 | 3. Disable the nova-compute agent 30 | 4. Record a list of VMs running on the hypervisor 31 | 5. Run the `cent7-upgrade.sh` phase 1 script 32 | 6. Sanity check that everything was configured properly to kick off the upgrade 33 | 7. 5 minute sanity check pause (OK to cancel early with ctrl-c + c) 34 | 8. Reboot the server to kick off the upgrade and wait for server to come back online 35 | 9. Run the `cent7-afterreboot.sh` phase 2 script 36 | 10. Call Spacewalk API to move server to CentOS 7 base channel 37 | 11. Run the `cent7-afterspacewalk.sh` phase 3 script 38 | 12. Sanity check the GRUB 2 bootloader was properly installed and configured 39 | 13. 5 minute sanity check pause (OK to cancel early with ctrl-c + c) 40 | 14. Reboot the server to boot into the latest kernel and wait for server to come back online 41 | 15. Run Puppet (twice) to lay down all the OpenStack bits for CentOS 7 42 | 16. Run server spec tests to validate everything is good 43 | 17. Pause for 180 seconds to allow time for nova-compute agent to restart all the VMs (DO NOT cancel this one early!) 44 | 18. Wait for the nova-compute agent to come back online (check in with the control plane) 45 | 19. Verify that all VMs have been restarted 46 | 20. Enable the nova-compute agent 47 | 21. Clear maintenance mode in monitoring systems (these commands are removed here since they are very specific to Go Daddy) 48 | 49 | # Background 50 | 51 | We originally looked to the [CentOS upgrade tool](https://wiki.centos.org/TipsAndTricks/CentOSUpgradeTool) to handle the upgrade from 6 to 7, and the procedure here does use that tool. This is a good starting point, but there is more work you will have to do. It'll do it's best to figure things out, but most likely you will have to do some cleanup work for yourself. 52 | 53 | Basically I'd recommend going through the process described on the [CentOS upgrade tool wiki](https://wiki.centos.org/TipsAndTricks/CentOSUpgradeTool). The pre-upgrade script will do a pretty good job of showing you stuff you'll need to fix before, or after, the upgrade. More or less you just have to try it on one system, figure out what breaks, how to fix it, and then script together the command history so you can automate it for other hosts. 54 | 55 | # Problems we had that you should watch for 56 | 57 | ### Upgrade tool doesn't configure an initrd in grub for the el7 kernel 58 | 59 | This is a big problem, because it'll cause the system to hang after the reboot when the upgrade is done. What we did was drop in a script to `/root/preupgrade/postupgrade.d` which runs after the upgrade work has been done, but before the reboot. The script fixes up grub.conf to have the appropriate initrd setting, so the system will actually boot properly. 60 | 61 | ### systemd Ethernet device naming 62 | 63 | Under CentOS 7/systemd, the naming of ethernet NICs changes from the ethN scheme to "predictable device naming" (which in my opinion is not that predictable.) (See [these](http://www.freedesktop.org/wiki/Software/systemd/PredictableNetworkInterfaceNames/) two [posts](https://major.io/2015/08/21/understanding-systemds-predictable-network-device-names/) for details) In any case, you need to change the network configuration with the new device names to handle this. 64 | 65 | We handled this by shimming in the necessary network reconfiguration commands into `/etc/rc.local`, which get ran when the host reboots after the upgrade is completed. See [cent7-upgrade.sh](https://github.com/godaddy/centos7-upgrade-scripts/blob/master/cent7-upgrade.sh#L119-L127) for details. Note also that our hosts are using Open vSwitch (since they are OpenStack hypervisors), so you'll need to adjust the network configuration according to your particular setup. You will probably want to make sure the `biosdevname` package is installed to make the device naming a little simpler. 66 | 67 | ### VMs go to SHUTDOWN state after upgrade is completed 68 | 69 | This one is OpenStack specific. Make sure that the `nova-compute` agent is stopped on the host before suspending the VMs via stopping the `libvirt-guests` service. Otherwise, `nova-compute` will see the VMs shutting down, and will change their state within nova. This means that after the upgrade is said and done, nova will not restart those VMs and you'll have to do it manually. 70 | 71 | As long as you stop `nova-compute` before shutting down/suspending the VMs when rebooting for the upgrade, and you have `resume_guests_state_on_host_boot=True` in `nova.conf`, then `nova-compute` will handle starting the VMs back up automatically when the upgrade is all done. 72 | 73 | ### Maintain symlinks for el6 versions of some libraries 74 | 75 | Some libraries from CentOS 6 will need to be maintained, at least temporarily, to support binaries that don't get properly updated to the CentOS 7 version (see below.) In particular, I saw this happen for `libsasl2.so.2` and `libpcre.so.0`. What you can do is just symlink these to the newer CentOS 7 versions. (And hopefully you'll get lucky and this will just work. It does for libsasl2 and libpcre.): 76 | 77 | ``` 78 | cd /lib64 79 | [ ! -e libsasl2.so.2 ] && ln -s libsasl2.so.3 libsasl2.so.2 80 | [ ! -e libpcre.so.0 ] && ln -s libpcre.so.1.2.0 libpcre.so.0 81 | ``` 82 | 83 | ### Some CentOS 6 packages not upgraded due to lower version number in base CentOS 7 repo 84 | 85 | The [CentOS upgrade tool wiki](https://wiki.centos.org/TipsAndTricks/CentOSUpgradeTool) calls this out, too, but there are several packages which currently have higher version numbers under the latest CentOS 6 updates than they do in the base CentOS 7 repo. Therefore, they are not upgraded, and you're left with the old CentOS 6 versions. 86 | 87 | The best way I've been able to deal with this is by identifying these packages with `rpm -qa | grep el6`, removing them with `rpm -e --nodeps`, and reinstalling using `yum install`, which will get the CentOS 7 versions. 88 | 89 | But, you have to be a little careful with this, because you will seriously break things by removing a package which provides libraries that other things depend on. (For example, elfutils.) You can accomplish this this `rpm -e --justdb --nodeps` and then reinstalling with `yum install` 90 | 91 | You'll have to do some trial and error on this to figure out the right ordering to do this work such that to keep everything working and get all the dependencies resolved. 92 | 93 | ### Some CentOS 6 packages not upgraded because they do not exist in CentOS 7 94 | 95 | You'll notice some CentOS 6 packages left over that don't have equivalent packages in CentOS 7. As long as those are not needed to fulfill dependencies for something else, you should be safe to just remove them. This may also unblock dependencies on other CentOS 6 packages that you would like to upgrade to CentOS 7, but can't because of the dependencies on the old CentOS 6 packages. 96 | 97 | ### Use `yum check dependencies` as a sanity check 98 | 99 | If you can get to a point where `yum check dependencies` shows up as clear, then you're probably mostly good. That command can give you some good clues as far as which packages to look at next for fixing. 100 | 101 | ### Need to install the grub 2 bootloader 102 | 103 | CentOS 7 uses grub version 2, and that package does get installed by this upgrade. But, it does not actually install the bootloader bits onto the MBR of the disk. So you'll need to do this manually. Otherwise, any time you upgrade kernels, the legacy grub 0.94 config will not be updated. 104 | 105 | Details on how to install grub 2 are detailed in this post: http://www.dorm.org/blog/installing-grub2-on-gpt-disks-after-el6-to-el7-upgrade/ 106 | -------------------------------------------------------------------------------- /cent7-afterreboot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | # 3 | # Copyright 2015 Go Daddy 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # 19 | # PHASE 2 20 | # 21 | 22 | # This stuff runs once the box has been rebooted into Cent 7 after the upgrade 23 | 24 | export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/opt/dell/srvadmin/bin:/opt/dell/srvadmin/sbin:/root/bin 25 | 26 | DATESTART=`date` 27 | 28 | # Remove all the extra stuff from rc.local that we put in there with cent7-upgrade.sh 29 | head -n -15 /etc/rc.d/rc.local > /tmp/rc.local 30 | mv -f /tmp/rc.local /etc/rc.d/rc.local 31 | chmod 755 /etc/rc.d/rc.local 32 | 33 | # Fix up the /usr/bin/python symlink, it'll still point to python2.6 at this point 34 | rm -f /usr/bin/python 35 | ln -s /usr/bin/python2.7 /usr/bin/python 36 | 37 | # Configure the yum repo for the v7 Spacewalk stuff 38 | cat >/etc/yum.repos.d/spacewalk-v7.repo </etc/yum.repos.d/upg.repo <>/root/preupg-output.txt 2>&1 || exit 1 51 | 52 | # Import the repo's GPG key 53 | rpm --import http://mirror.centos.org/centos/RPM-GPG-KEY-CentOS-7 || exit 1 54 | 55 | # Selinux must be turned off-ish for this all to work 56 | semodule -r sandbox 57 | 58 | # Make sure the SCL repo is disabled 59 | sed -i -r 's/enabled=.+/enabled=0/' /etc/yum.repos.d/CentOS-SCL.repo 60 | 61 | # Start fresh, again! 62 | yum clean all 63 | 64 | # Clear out any extra kernels 65 | rpm -q kernel | grep -v `uname -r` | xargs rpm -e 66 | 67 | # Remove SCL's (don't worry, after we upgrade to Cent 7, we'll have the needed 68 | # Ruby and Python stuff) 69 | # (Note: neutron-openvswitch-agent will start logging tons of errors after this 70 | # point. That's ok, but messy. So we stop the ovs agent here before, since 71 | # after python27 is removed, it can't do anything anyway) 72 | service openstack-neutron-openvswitch-agent stop 73 | yum -y remove ruby193\* 74 | yum -y remove python27\* 75 | 76 | # Fix upgrade tool code (this is getting very hacky) 77 | # Not 100% sure what's going on here, but without this the tool fails on trying 78 | # to resolve all the dependencies. 79 | cwd=${PWD} 80 | cd /usr/lib/python2.6/site-packages/redhat_upgrade_tool/ 81 | patch <>/root/upgrade-tool-output.txt 2>&1 101 | upgres=$? 102 | # Break out of the loop if we succeeded 103 | [ $upgres -eq 0 ] && break 104 | done 105 | 106 | # Exit with error if the upgrade tool failed 107 | [ $upgres -ne 0 ] && exit 1 108 | 109 | # Stick stuff into rc.local that we need to happen right at bootup after the upgrade 110 | # - Maintain old sasl2 and pcre libraries that are still needed temporarily 111 | # - Retool the network config for the new bios names under systemd 112 | cat >>/etc/rc.d/rc.local </root/preupgrade/postupgrade.d/zz_grub_fixup.sh </boot/grub/grub.conf < 37 | rpm -qa | sort | 38 | grep -v ^pbis-enterprise | 39 | grep -v ^cpm-graphite-collector | 40 | grep -v ^netxen-firmware | 41 | grep -v ^authconfig | 42 | grep -v ^libitm | 43 | grep -v ^system-config-firewall-base | 44 | diff -u - /root/cent7-preupgrade-package-list.txt 45 | when: skip_package_check == false 46 | 47 | # Set maintenance mode in monitoring systems 48 | # (Go Daddy-specific commands removed) 49 | 50 | # Disable the nova-compute agent on this host 51 | - name: Disable nova-compute agent 52 | shell: . /root/keystonerc_admin && nova service-disable 'api!phx_private_cell_001@'{{ inventory_hostname }} nova-compute 53 | delegate_to: nova_api_host 54 | 55 | # Record list of running VMs on this host 56 | - name: Record running VMs list 57 | shell: . /root/keystonerc_admin && openstack server list --all-projects --status ACTIVE --host {{ inventory_hostname }} -c ID -f csv | grep -v ID 58 | delegate_to: nova_api_host 59 | register: vmslist 60 | 61 | # Dump list of VM UUIDs in case we need to start them manually later 62 | - name: List running VMs 63 | debug: msg={{ item }} 64 | with_items: vmslist.stdout_lines 65 | 66 | 67 | ################################ 68 | # Phase One 69 | ################################ 70 | 71 | # Run the first stage of the upgrade scripts 72 | - name: Upgrade scripts stage one (should take 15-20 minutes) (if errors, see logs at /root/upgrade-tool-output.txt and /root/preupg-output.txt) 73 | script: cent7-upgrade.sh creates=/root/preupgrade/postupgrade.d/zz_grub_fixup.sh 74 | register: result_stage1 75 | 76 | # Make sure biosdevname is really installed 77 | - name: Install biosdevname 78 | yum: name=biosdevname 79 | register: result_biosdevname_install 80 | until: result_biosdevname_install|success 81 | delay: 5 82 | 83 | # Sanity checks, and pause 84 | - name: Sanity check contents of rc.local 85 | shell: grep libsasl2 /etc/rc.local && grep libpcre /etc/rc.local && grep ovs-vsctl /etc/rc.local && grep em1 /etc/rc.local && grep em3 /etc/rc.local && grep python2.7 /etc/rc.local 86 | when: not result_stage1|skipped 87 | 88 | - name: Sanity check biosdevname is installed and executable 89 | shell: "rpm -q biosdevname && [ -x /sbin/biosdevname ]" 90 | 91 | - name: Sanity check grub config for upgrade tool 92 | shell: /sbin/grubby --default-kernel | grep redhat-upgrade-tool 93 | when: not result_stage1|skipped 94 | 95 | - name: Sanity check postupgrade.d/zz_grub_fixup.sh script 96 | shell: cat /root/preupgrade/postupgrade.d/zz_grub_fixup.sh | wc -l | grep '^28$' 97 | 98 | - name: Sanity check nova-compute is stopped 99 | shell: "[ `ps awuxf | grep nova-compute | grep -v grep | wc -l` -eq 0 ]" 100 | register: result_nova_compute_stopped 101 | until: result_nova_compute_stopped|success 102 | delay: 5 103 | 104 | - name: Sanity check pause, 5 minutes (cancel early if good to go) 105 | pause: minutes={{ 5 }} 106 | when: result_nova_compute_stopped|success 107 | 108 | # Reboot to trigger the upgrade 109 | - name: reboot stage one 110 | command: /sbin/shutdown -r 1 111 | async: 0 112 | poll: 0 113 | ignore_errors: yes 114 | when: not result_stage1|skipped 115 | 116 | # Wait for box to go down 117 | - name: Waiting for host shutdown (can take 20-30 minutes depending on how long it takes to suspend all the running VMs) 118 | wait_for: host={{ inventory_hostname }} port={{ 22 }} timeout={{ 3600 }} state=stopped 119 | connection: local 120 | sudo: false 121 | when: not result_stage1|skipped 122 | 123 | # Wait for box to come back up 124 | - name: Waiting for host bootup (this will take 15-20 minutes because the CentOS 7 upgrade is happening now) 125 | wait_for: host={{ inventory_hostname }} port={{ 22 }} timeout={{ 7200 }} state=started 126 | connection: local 127 | sudo: false 128 | when: result_stage1|success 129 | 130 | # Verify we can actually log in to the box 131 | - name: Validate ability to login 1 132 | command: /bin/true 133 | register: result_validate_ssh_1 134 | until: result_validate_ssh_1|success 135 | delay: 5 136 | retries: 100 137 | 138 | 139 | ################################ 140 | # Phase Two 141 | ################################ 142 | 143 | # Run second stage of the upgrade scripts 144 | - name: Upgrade scripts stage two 145 | script: cent7-afterreboot.sh creates=/etc/yum.repos.d/spacewalk-v7.repo 146 | when: result_stage1|success 147 | register: result_stage2 148 | 149 | # Change base channel in spacewalk, retry until it succeeds 150 | - name: Change base Spacewalk channel 151 | script: update_base_channel.py {{ inventory_hostname }} 152 | register: result_base_channel 153 | delegate_to: spacewalk_api_host 154 | when: not result_stage2|skipped 155 | until: result_base_channel|success 156 | delay: 5 157 | 158 | # Wait until the base channel actually updates 159 | - name: Wait for base channel update 160 | shell: "/sbin/rhn-channel --list | grep base-v7-64bit" 161 | register: result_rhn_channel 162 | until: result_rhn_channel|success 163 | delay: 5 164 | 165 | 166 | ################################ 167 | # Phase Three 168 | ################################ 169 | 170 | # Run second stage of the upgrade scripts 171 | - name: Upgrade scripts stage three 172 | script: cent7-afterspacewalk.sh creates=/boot/grub2/grub.cfg 173 | when: result_rhn_channel|success 174 | register: result_stage3 175 | 176 | # Sanity checks, and pause 177 | - name: Sanity check of default kernel 178 | shell: /sbin/grubby --default-kernel | grep vmlinuz-3.10.0-229.20.1.el7.x86_64 179 | 180 | - name: Sanity check of grub bootloader 181 | shell: /bin/file -s /dev/sda | grep 'GRand Unified Bootloader' | grep -v 'GRUB version 0.94' 182 | 183 | - name: Sanity check pause, 5 minutes (cancel early if good to go) 184 | pause: minutes={{ 5 }} 185 | 186 | # Reboot to finalize everything 187 | - name: reboot two 188 | command: /sbin/shutdown -r 1 189 | async: 0 190 | poll: 0 191 | ignore_errors: yes 192 | when: not result_stage3|skipped 193 | 194 | # Wait for box to go down 195 | - name: Waiting for host shutdown 196 | wait_for: host={{ inventory_hostname }} port={{ 22 }} timeout={{ 3600 }} state=stopped 197 | connection: local 198 | sudo: false 199 | when: not result_stage3|skipped 200 | 201 | # Wait for box to come back up 202 | - name: Waiting for host bootup (typically takes about 5 minutes) 203 | wait_for: host={{ inventory_hostname }} port={{ 22 }} timeout={{ 3600 }} state=started 204 | connection: local 205 | sudo: false 206 | when: result_stage3|success 207 | 208 | # Verify we can actually log in to the box 209 | - name: Validate ability to login 2 210 | command: /bin/true 211 | register: result_validate_ssh_2 212 | until: result_validate_ssh_2|success 213 | delay: 5 214 | retries: 100 215 | 216 | # Wait for a minute to make sure everything is fully started 217 | - name: Pausing 60s to allow for full bootup (do not cancel this one early!) 218 | pause: minutes={{ 1 }} 219 | 220 | 221 | ################################ 222 | # Puppet & Server Spec 223 | ################################ 224 | 225 | # Run Puppet 226 | - name: Running puppet deployment script 227 | shell: "/etc/puppet/environments/master/tools/deploy-puppet 1>/var/log/puppet/ansible-puppet-run-{{ ansible_date_time.iso8601 }}a.log 2>&1" 228 | register: result_puppet1 229 | 230 | - name: Running puppet deployment script again 231 | shell: "/etc/puppet/environments/master/tools/deploy-puppet 1>/var/log/puppet/ansible-puppet-run-{{ ansible_date_time.iso8601 }}b.log 2>&1" 232 | register: result_puppet2 233 | when: result_puppet1|success 234 | 235 | # Run server spec tests 236 | - name: Running server spec tests 237 | shell: "cd /usr/local/serverspec && /bin/bundle exec rake spec:common && /bin/bundle exec rake spec:hypervisor" 238 | register: result_serverspec 239 | when: result_puppet2|success 240 | 241 | 242 | ################################ 243 | # VMs Start & Enable Agent 244 | ################################ 245 | 246 | # Wait for nova-compute to catch up with the state of things 247 | - name: Waiting 180s for nova-compute to restart VMs (do not cancel this one early!) 248 | pause: minutes={{ 3 }} 249 | when: result_serverspec|success 250 | 251 | # Wait for nova-compute agent to come back online (this should mean all the VMs are started) 252 | - name: Validate nova-compute agent is online 253 | shell: ". /root/keystonerc_admin && while [ `nova service-list | grep {{ inventory_hostname }} | awk '{print $12}'` == 'down' ] ; do sleep 5 ; done" 254 | delegate_to: nova_api_host 255 | register: result_nova_compute_online 256 | until: result_nova_compute_online|success 257 | delay: 5 258 | retries: 100 259 | when: result_serverspec|success 260 | 261 | # Verify all the VMs started 262 | - name: Verify VMs are active 263 | shell: . /root/keystonerc_admin && openstack server show {{ item }} -c status -f value | grep ACTIVE 264 | delegate_to: nova_api_host 265 | register: result_check_vm_active 266 | until: result_check_vm_active|success 267 | delay: 5 268 | retries: 100 269 | with_items: vmslist.stdout_lines 270 | 271 | # Enable nova-compute agent 272 | - name: Enable nova-compute agent 273 | shell: . /root/keystonerc_admin && nova service-enable 'api!phx_private_cell_001@'{{ inventory_hostname }} nova-compute 274 | delegate_to: nova_api_host 275 | register: result_service_enable 276 | until: result_service_enable|success 277 | delay: 2 278 | when: result_check_vm_active|success 279 | 280 | 281 | ################################ 282 | # Teardown 283 | ################################ 284 | 285 | # Clear maintenance mode in monitoring systems 286 | # (Go Daddy-specific commands removed) 287 | --------------------------------------------------------------------------------