├── Dockerfile ├── LICENSE.txt ├── README.md ├── bin ├── build_cluster ├── build_image ├── housekeeping └── start_cluster ├── clusterdock.sh ├── clusterdock ├── __init__.py ├── cluster.py ├── constants.cfg ├── docker_utils.py ├── images │ └── centos6.6_nodebase │ │ ├── Dockerfile │ │ └── ssh │ │ └── id_rsa.pub ├── ssh.py ├── topologies │ ├── __init__.py │ ├── cdh │ │ ├── __init__.py │ │ ├── actions.py │ │ ├── cm.py │ │ ├── cm_api │ │ │ ├── __init__.py │ │ │ ├── api_client.py │ │ │ ├── endpoints │ │ │ │ ├── __init__.py │ │ │ │ ├── batch.py │ │ │ │ ├── clusters.py │ │ │ │ ├── cms.py │ │ │ │ ├── dashboards.py │ │ │ │ ├── events.py │ │ │ │ ├── host_templates.py │ │ │ │ ├── hosts.py │ │ │ │ ├── parcels.py │ │ │ │ ├── role_config_groups.py │ │ │ │ ├── roles.py │ │ │ │ ├── services.py │ │ │ │ ├── timeseries.py │ │ │ │ ├── tools.py │ │ │ │ ├── types.py │ │ │ │ └── users.py │ │ │ ├── http_client.py │ │ │ └── resource.py │ │ ├── cm_utils.py │ │ ├── profile.cfg │ │ └── ssh │ │ │ └── id_rsa │ ├── nodebase │ │ ├── __init__.py │ │ ├── actions.py │ │ ├── profile.cfg │ │ └── ssh │ │ │ ├── id_rsa │ │ │ └── id_rsa.pub │ └── parsing.py └── utils.py ├── pylintrc └── requirements.txt /Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012 - 2018 Cloudera, Inc. 2 | # All Rights Reserved. 3 | 4 | FROM debian:wheezy 5 | 6 | # Use a label indicating that a container is running the clusterdock framework to allow the 7 | # framework to handle things like stopping containers on the host machine without accidentally 8 | # killing itself. 9 | LABEL org.apache.hbase.is-clusterdock= 10 | 11 | ENV DOCKER_BUCKET get.docker.com 12 | ENV DOCKER_VERSION 1.11.1 13 | 14 | # Install Docker, just to have the client available; the framework assumes /var/run/docker.sock 15 | # will be volume mounted from the host. That is, executing `docker run` inside a container created 16 | # from this image will start a container on the host machine, not inside said container. 17 | RUN apt-get -y update \ 18 | && apt-get -y install curl \ 19 | git \ 20 | libffi-dev \ 21 | libssl-dev \ 22 | libxml2-dev \ 23 | libxslt1-dev \ 24 | libz-dev \ 25 | python-dev \ 26 | python-pip \ 27 | && curl -fSL "https://${DOCKER_BUCKET}/builds/Linux/x86_64/docker-${DOCKER_VERSION}.tgz" \ 28 | -o docker.tgz \ 29 | && tar -xzvf docker.tgz \ 30 | && mv docker/* /usr/local/bin/ \ 31 | && rmdir docker \ 32 | && rm docker.tgz \ 33 | && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 34 | ADD . /root/clusterdock 35 | 36 | # Make sure the SSH private key for each topology has the correct permissions. 37 | RUN find /root/clusterdock -type f -name id_rsa -exec chmod 600 {} \; \ 38 | && pip install --upgrade -r /root/clusterdock/requirements.txt \ 39 | && rm -rf /root/.cache/pip/* 40 | 41 | WORKDIR /root/clusterdock 42 | ENTRYPOINT ["python"] 43 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 19 | # clusterdock 20 | 21 | ## Overview 22 | *clusterdock* is a framework for creating Docker-based container clusters. Unlike regular Docker 23 | containers, which tend to run single processes and then exit once the process terminates, these 24 | container clusters are characterized by the execution of an init process in daemon mode. As such, 25 | the containers act more like "fat containers" or "light VMs;" entities with accessible IP addresses 26 | which emulate standalone hosts. 27 | 28 | ## Usage 29 | The *clusterdock* framework has been designed to be run out of its own container while affecting 30 | operations on the host. To do this, the framework is started by invoking `docker run` with an option 31 | of `-v /var/run/docker.sock:/var/run/docker.sock` required to ensure that containers launched by the 32 | framework are started on the host machine. To avoid problems that might result from incorrectly 33 | formatting this framework invocation, a Bash helper script (`clusterdock.sh`) can be sourced on a 34 | host that has Docker installed. Afterwards, invocation of any of the binaries intended to carry 35 | out *clusterdock* actions can be done using the `clusterdock_run` command. As an example, assuming 36 | Docker is already installed and the working directory is the root of this Git repository: 37 | ``` 38 | source ./clusterdock.sh 39 | clusterdock_run ./bin/start_cluster cdh --help 40 | ``` 41 | -------------------------------------------------------------------------------- /bin/build_cluster: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | import argparse 19 | import importlib 20 | import logging 21 | import os 22 | import sys 23 | from os.path import dirname, abspath 24 | from time import gmtime, strftime, time 25 | 26 | from docker import Client 27 | 28 | # This path hack allows this script to be run from any folder. Otherwise, running it from within 29 | # the bin directory would lead to problems importing modules in the clusterdock package. 30 | sys.path.insert(0, dirname(dirname(abspath(__file__)))) 31 | 32 | from clusterdock import Constants 33 | from clusterdock.docker_utils import login, push_image 34 | from clusterdock.topologies.parsing import get_profile_config_item, parse_profiles 35 | 36 | logger = logging.getLogger(__name__) 37 | logger.setLevel(logging.INFO) 38 | 39 | DEFAULT_DOCKER_REGISTRY_URL = Constants.DEFAULT.docker_registry_url # pylint: disable=no-member 40 | 41 | if __name__ == '__main__': 42 | parser = argparse.ArgumentParser(description='Build cluster images', 43 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 44 | 45 | # Arguments that should be present for all possible topologies go here. This should be a very 46 | # short list... 47 | parser.add_argument('-o', '--operating-system', help='Operating system of the cluster', 48 | metavar="os", default="centos6.6") 49 | parser.add_argument('--namespace', metavar='ns', 50 | help='The namespace to use when tagging the image') 51 | parser.add_argument('-n', '--network', help='User-defined network to use', 52 | metavar="network", default='cluster') 53 | parser.add_argument('-p', '--push', help='Push Docker images', action='store_true') 54 | parser.add_argument('-r', '--registry-url', metavar='url', default=DEFAULT_DOCKER_REGISTRY_URL, 55 | help='URL of the Docker registry to use when naming and pushing the image') 56 | 57 | # What args we parse depends on the topology we pass as the first position argument to this 58 | # script. This is handled by the topologies module. 59 | parse_profiles(parser, action='build') 60 | args = parser.parse_args() 61 | 62 | # To actually start the cluster for the specified topology, we dynamically import the specified 63 | # topology and then call its start function. 64 | actions = importlib.import_module("clusterdock.topologies.{0}.actions".format(args.topology)) 65 | 66 | # Keep time in a variable to output how long cluster deployment took at the end. 67 | start = time() 68 | images = actions.build(args) 69 | end = time() 70 | time_diff = gmtime(end-start) 71 | topology_name = get_profile_config_item(args.topology, 'general', 'name') 72 | logger.info("%s cluster built in %s.", topology_name, strftime("%M min, %S sec", time_diff)) 73 | 74 | if args.push: 75 | if os.environ.get('DOCKER_REGISTRY_INSECURE') != 'true': 76 | login(username=os.environ['DOCKER_REGISTRY_USERNAME'], 77 | password=os.environ['DOCKER_REGISTRY_PASSWORD'], 78 | registry=args.registry_url) 79 | for image in images: 80 | push_image(image) 81 | -------------------------------------------------------------------------------- /bin/build_image: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | import argparse 19 | import logging 20 | import os 21 | import sys 22 | from os import listdir 23 | from os.path import dirname, abspath, isdir, join 24 | 25 | from docker import Client 26 | 27 | # This path hack allows this script to be run from any folder. Otherwise, running it from within 28 | # the bin directory would lead to problems importing modules in the clusterdock package. 29 | sys.path.insert(0, dirname(dirname(abspath(__file__)))) 30 | 31 | from clusterdock import Constants 32 | from clusterdock.docker_utils import build_image, login, push_image 33 | 34 | logger = logging.getLogger(__name__) 35 | logger.setLevel(logging.INFO) 36 | 37 | images_folder = join(dirname(dirname(__file__)), 'clusterdock', 'images') 38 | 39 | DEFAULT_DOCKER_REGISTRY_URL = Constants.DEFAULT.docker_registry_url 40 | 41 | def main(args): 42 | image = '/'.join([item 43 | for item in (args.registry_url, args.namespace, 44 | "clusterdock:{0}".format(args.image)) 45 | if item]) 46 | dockerfile = join(images_folder, args.image, 'Dockerfile') 47 | build_image(dockerfile=dockerfile, tag=image) 48 | 49 | if args.push: 50 | if os.environ.get('DOCKER_REGISTRY_INSECURE') != 'true': 51 | login(username=os.environ['DOCKER_REGISTRY_USERNAME'], 52 | password=os.environ['DOCKER_REGISTRY_PASSWORD'], 53 | registry=args.registry_url) 54 | push_image(image) 55 | 56 | if __name__ == '__main__': 57 | parser = argparse.ArgumentParser(description='Build a Docker image in the images subfolder.', 58 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 59 | parser.add_argument('--namespace', metavar='ns', 60 | help='The namespace to use when tagging the image') 61 | parser.add_argument('-p', '--push', help='Push Docker image to registry after building', 62 | action='store_true') 63 | parser.add_argument('-r', '--registry-url', metavar='url', default=DEFAULT_DOCKER_REGISTRY_URL, 64 | help='URL of the Docker registry to use when naming and pushing the image') 65 | parser.add_argument('image', choices=[image for image in listdir(images_folder) if 66 | isdir(join(images_folder, image))], 67 | help='Folder under ./image from which to build a Docker image') 68 | args = parser.parse_args() 69 | 70 | main(args) 71 | -------------------------------------------------------------------------------- /bin/housekeeping: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | import argparse 19 | import logging 20 | import sys 21 | from os import environ 22 | from os.path import dirname, abspath 23 | 24 | from fabric.api import local, quiet 25 | 26 | # This path hack allows this script to be run from any folder. Otherwise, running it from within 27 | # the bin directory would lead to problems importing modules in the clusterdock package. 28 | sys.path.insert(0, dirname(dirname(abspath(__file__)))) 29 | 30 | from clusterdock.docker_utils import (get_container_id, get_network_container_hostnames, 31 | kill_all_containers, remove_all_containers, 32 | remove_all_images, remove_all_networks, 33 | remove_container, remove_network) 34 | 35 | logger = logging.getLogger('housekeeping') 36 | logger.setLevel(logging.INFO) 37 | 38 | def main(): 39 | parser = argparse.ArgumentParser(description='Perform common Docker housekeeping actions') 40 | subparsers = parser.add_subparsers(help='Action to carry out') 41 | 42 | nuke_parser = subparsers.add_parser('nuke') 43 | nuke_parser.set_defaults(action='nuke') 44 | 45 | remove_all_images_parser = subparsers.add_parser('remove_all_images') 46 | remove_all_images_parser.set_defaults(action='remove_all_images') 47 | 48 | remove_parser = subparsers.add_parser('remove') 49 | remove_parser.add_argument('-n', '--network', help='Network to be removed', action='append') 50 | remove_parser.add_argument('-c', '--container', help='Container to be removed', action='append') 51 | remove_parser.set_defaults(action='remove') 52 | 53 | args = parser.parse_args() 54 | 55 | if args.action == 'nuke': 56 | logger.info('Removing all containers on this host...') 57 | remove_all_containers() 58 | logger.info('Successfully removed all containers on this host.') 59 | logger.info('Removing all user-defined networks on this host...') 60 | remove_all_networks() 61 | logger.info('Successfully removed all user-defined networks on this host.') 62 | with open('/etc/hosts', 'r') as etc_hosts: 63 | host_lines = etc_hosts.readlines() 64 | clusterdock_signature = '# Added by clusterdock' 65 | if any(clusterdock_signature in line for line in host_lines): 66 | logger.info('Clearing container entries from /etc/hosts...') 67 | with open('/etc/hosts', 'w') as etc_hosts: 68 | etc_hosts.writelines([line for line in host_lines if 69 | clusterdock_signature not in line]) 70 | logger.info('Successfully cleared container entries from /etc/hosts.') 71 | if not environ.get('CLUSTERDOCK'): 72 | logger.info('Restarting Docker daemon...') 73 | with quiet(): 74 | local('service docker restart') 75 | logger.info('Successfully nuked this host.') 76 | elif args.action == 'remove_all_images': 77 | logger.info('Removing all images on this host...') 78 | remove_all_images() 79 | logger.info('Successfully removed all images.') 80 | elif args.action == 'remove': 81 | if args.network: 82 | for network in args.network: 83 | containers = get_network_container_hostnames(network) 84 | if containers: 85 | logger.info('Removing all containers in network "%s" ...', network) 86 | for container in containers: 87 | remove_container(get_container_id(container, network)) 88 | logger.info('Successfully removed all containers in network "%s".', network) 89 | logger.info('Removing network "%s" itself...', network) 90 | remove_network(network) 91 | logger.info('Successfully removed network "%s".', network) 92 | if args.container: 93 | logger.info('Removing all specified containers...') 94 | for container in args.container: 95 | remove_container(container) 96 | logger.info('Successfully removed all specified containers.') 97 | 98 | if __name__ == '__main__': 99 | main() 100 | -------------------------------------------------------------------------------- /bin/start_cluster: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | import argparse 19 | import importlib 20 | import logging 21 | import sys 22 | from os.path import dirname, abspath 23 | from time import gmtime, strftime, time 24 | 25 | logger = logging.getLogger('start_cluster') 26 | logger.setLevel(logging.INFO) 27 | 28 | # This path hack allows this script to be run from any folder. Otherwise, running it from within 29 | # the bin directory would lead to problems importing modules in the clusterdock package. 30 | sys.path.insert(0, dirname(dirname(abspath(__file__)))) 31 | 32 | from clusterdock import Constants 33 | from clusterdock.topologies.parsing import get_profile_config_item, parse_profiles 34 | 35 | DEFAULT_DOCKER_REGISTRY_URL = Constants.DEFAULT.docker_registry_url # pylint: disable=no-member 36 | 37 | def main(): 38 | parser = argparse.ArgumentParser(description='Start a cluster with pre-built images', 39 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 40 | 41 | # Arguments that should be present for all possible topologies go here. This should be a very 42 | # short list... 43 | parser.add_argument('--always-pull', help='Pull latest images, even if available locally', 44 | action='store_true') 45 | parser.add_argument('--namespace', metavar='ns', 46 | help='The namespace to use when creating the image') 47 | parser.add_argument('-n', '--network', help="User-defined network to use", 48 | metavar="network", default="cluster") 49 | parser.add_argument('-o', '--operating-system', help="Operating system of the cluster", 50 | metavar="os", default="centos6.6") 51 | parser.add_argument('-r', '--registry-url', metavar='url', default=DEFAULT_DOCKER_REGISTRY_URL, 52 | help='URL of the Docker registry to use when pulling images') 53 | 54 | # What args we parse depends on the topology we pass as the first position argument to this 55 | # script. This is handled by clusterdock.topologies.parsing's parse_profiles function. 56 | parse_profiles(parser, action='start') 57 | args = parser.parse_args() 58 | 59 | # To actually start the cluster for the specified topology, we dynamically import the specified 60 | # topology's action module and then call its start function. 61 | actions = importlib.import_module("clusterdock.topologies.{0}.actions".format(args.topology)) 62 | start = time() 63 | actions.start(args) 64 | end = time() 65 | time_diff = gmtime(end-start) 66 | topology_name = get_profile_config_item(args.topology, 'general', 'name') 67 | logger.info("%s cluster started in %s.", topology_name, strftime("%M min, %S sec", time_diff)) 68 | 69 | if __name__ == '__main__': 70 | main() 71 | -------------------------------------------------------------------------------- /clusterdock.sh: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012 - 2018 Cloudera, Inc. 2 | # All Rights Reserved. 3 | 4 | #!/usr/bin/env bash 5 | 6 | # This helper script is designed to be sourced, at which time its functions are made available 7 | # to the user. Most functions defined have functionality defined by environmental variables, which 8 | # can be set during invocation. For example, 9 | # 10 | # CLUSTERDOCK_PULL=false clusterdock_run ./bin/... 11 | 12 | ## @description Run a clusterdock Python script within a Docker container 13 | ## @audience public 14 | ## @stability stable 15 | ## @replaceable no 16 | ## @param Python script to run relative to the ./dev-support/clusterdock/clusterdock folder 17 | clusterdock_run() { 18 | # Supported environmental variables: 19 | # - CLUSTERDOCK_DOCKER_REGISTRY_INSECURE: whether the Docker registry is insecure (either true or 20 | # false) 21 | # - CLUSTERDOCK_DOCKER_REGISTRY_USERNAME: username needed to login to any secure Docker registry 22 | # - CLUSTERDOCK_DOCKER_REGISTRY_PASSWORD: password needed to login to any secure Docker registry 23 | # - CLUSTERDOCK_IMAGE: the Docker image from which to start the clusterdock framework container 24 | # - CLUSTERDOCK_PULL: whether to pull the clusterdock image and CLUSTERDOCK_TOPOLOGY_IMAGE, if 25 | # specified (either true or false; defaults to true) 26 | # - CLUSTERDOCK_TARGET_DIR: a folder on the host to mount into /root/target in the clusterdock 27 | # container 28 | # - CLUSTERDOCK_TOPOLOGY_IMAGE: a Docker image to mount into clusterdock's topology folder and 29 | # make available to users 30 | 31 | if [ -z "${CLUSTERDOCK_IMAGE}" ]; then 32 | local CONSTANTS_CONFIG_URL='https://raw.githubusercontent.com/cloudera/clusterdock/master/clusterdock/constants.cfg' 33 | 34 | # awk -F argument allows for any number of spaces around equal sign. 35 | local DOCKER_REGISTRY_URL=$(curl -s "${CONSTANTS_CONFIG_URL}" \ 36 | | awk -F " *= *" '/^docker_registry_url/ {print $2}') 37 | local CLOUDERA_NAMESPACE=$(curl -s "${CONSTANTS_CONFIG_URL}" \ 38 | | awk -F " *= *" '/^cloudera_namespace/ {print $2}') 39 | 40 | CLUSTERDOCK_IMAGE="${DOCKER_REGISTRY_URL}/${CLOUDERA_NAMESPACE}/clusterdock:latest" 41 | fi 42 | 43 | if [ "${CLUSTERDOCK_PULL}" != "false" ]; then 44 | sudo docker pull "${CLUSTERDOCK_IMAGE}" &> /dev/null 45 | fi 46 | 47 | if [ -n "${CLUSTERDOCK_TARGET_DIR}" ]; then 48 | local TARGET_DIR_MOUNT="-v ${CLUSTERDOCK_TARGET_DIR}:/root/target" 49 | fi 50 | 51 | if [ -n "${CLUSTERDOCK_DOCKER_REGISTRY_INSECURE}" ]; then 52 | local REGISTRY_INSECURE="-e DOCKER_REGISTRY_INSECURE=${CLUSTERDOCK_DOCKER_REGISTRY_INSECURE}" 53 | fi 54 | 55 | if [ -n "${CLUSTERDOCK_DOCKER_REGISTRY_USERNAME}" ]; then 56 | local REGISTRY_USERNAME="-e DOCKER_REGISTRY_USERNAME=${CLUSTERDOCK_DOCKER_REGISTRY_USERNAME}" 57 | fi 58 | 59 | if [ -n "${CLUSTERDOCK_DOCKER_REGISTRY_PASSWORD}" ]; then 60 | local REGISTRY_PASSWORD="-e DOCKER_REGISTRY_PASSWORD=${CLUSTERDOCK_DOCKER_REGISTRY_PASSWORD}" 61 | fi 62 | 63 | if [ -n "${CLUSTERDOCK_TOPOLOGY_IMAGE}" ]; then 64 | if [ "${CLUSTERDOCK_PULL}" != "false" ]; then 65 | sudo docker pull "${CLUSTERDOCK_TOPOLOGY_IMAGE}" &> /dev/null 66 | fi 67 | 68 | local TOPOLOGY_CONTAINER_ID=$(sudo docker create "${CLUSTERDOCK_TOPOLOGY_IMAGE}") 69 | local TOPOLOGY_VOLUME="--volumes-from=${TOPOLOGY_CONTAINER_ID}" 70 | fi 71 | 72 | # The /etc/hosts bind-mount allows clusterdock to update /etc/hosts on the host machine for 73 | # better access to internal container addresses. 74 | sudo docker run --net=host -t \ 75 | --privileged \ 76 | ${TARGET_DIR_MOUNT} \ 77 | ${TOPOLOGY_VOLUME} \ 78 | ${REGISTRY_INSECURE} \ 79 | ${REGISTRY_USERNAME} \ 80 | ${REGISTRY_PASSWORD} \ 81 | -v /tmp/clusterdock \ 82 | -v /etc/hosts:/etc/hosts \ 83 | -v /etc/localtime:/etc/localtime \ 84 | -v /var/run/docker.sock:/var/run/docker.sock \ 85 | "${CLUSTERDOCK_IMAGE}" $@ 86 | 87 | if [ -n "${TOPOLOGY_CONTAINER_ID}" ]; then 88 | sudo docker rm -v "${TOPOLOGY_CONTAINER_ID}" &> /dev/null 89 | fi 90 | } 91 | 92 | ## @description SSH to a clusterdock-created container cluster node 93 | ## @audience public 94 | ## @stability stable 95 | ## @replaceable no 96 | ## @param Fully-qualified domain name of container to which to connect 97 | ## @param An optional command to run 98 | clusterdock_ssh() { 99 | local NODE=${1} 100 | # Shift away arguments by 1. If anything is left, it's a command to run over SSH. 101 | shift 1 102 | 103 | if [ -z "${CLUSTERDOCK_IMAGE}" ]; then 104 | local CONSTANTS_CONFIG_URL='https://raw.githubusercontent.com/cloudera/clusterdock/master/clusterdock/constants.cfg' 105 | 106 | # awk -F argument allows for any number of spaces around equal sign. 107 | local DOCKER_REGISTRY_URL=$(curl -s "${CONSTANTS_CONFIG_URL}" \ 108 | | awk -F " *= *" '/^docker_registry_url/ {print $2}') 109 | local CLOUDERA_NAMESPACE=$(curl -s "${CONSTANTS_CONFIG_URL}" \ 110 | | awk -F " *= *" '/^cloudera_namespace/ {print $2}') 111 | 112 | CLUSTERDOCK_IMAGE="${DOCKER_REGISTRY_URL}/${CLOUDERA_NAMESPACE}/clusterdock:latest" 113 | fi 114 | 115 | if [ "${CLUSTERDOCK_PULL}" != "false" ]; then 116 | sudo docker pull "${CLUSTERDOCK_IMAGE}" &> /dev/null 117 | fi 118 | 119 | if [ -n "${CLUSTERDOCK_TOPOLOGY_IMAGE}" ]; then 120 | if [ "${CLUSTERDOCK_PULL}" != "false" ]; then 121 | sudo docker pull "${CLUSTERDOCK_TOPOLOGY_IMAGE}" &> /dev/null 122 | fi 123 | 124 | local TOPOLOGY_CONTAINER_ID=$(sudo docker create "${CLUSTERDOCK_TOPOLOGY_IMAGE}") 125 | local TOPOLOGY_VOLUME="--volumes-from=${TOPOLOGY_CONTAINER_ID}" 126 | fi 127 | 128 | local ID 129 | for ID in $(docker ps -qa); do 130 | if [ "$(docker inspect --format '{{.Config.Hostname}}' ${ID})" = "${NODE}" ]; then 131 | break 132 | fi 133 | done 134 | 135 | # In order to get a proper login shell with expected files sourced (e.g. /etc/environment), we 136 | # get into the container using docker exec running ssh (instead of Bash). 137 | sudo docker exec -it ${ID} ssh localhost "$*" 138 | } 139 | -------------------------------------------------------------------------------- /clusterdock/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """The clusterdock package houses the core clusterdock code that handles Docker container cluster 18 | orchestration, as well as topologies, the abstraction the defines the behavior of these clusters. 19 | It also contains a number of utility modules to wrap common Docker API functionality.""" 20 | 21 | import logging 22 | from ConfigParser import SafeConfigParser 23 | from os.path import dirname, join 24 | 25 | logging.basicConfig(level=logging.ERROR) 26 | 27 | class Constants(object): 28 | """A class just designed to make the contents of constants.cfg available to clusterdock modules 29 | in a pretty way. Yes, this could have been done as a nested dictionary, but accessing 30 | Constants.docker_images.maven looks less gross to me than constants['docker_images']['maven']. 31 | 32 | Note that, to keep Pylint happy, we'll have to add 33 | # pylint: disable=no-member 34 | at the end of every line in which we reference this class. 35 | """ 36 | 37 | # pylint: disable=too-few-public-methods 38 | 39 | _config = SafeConfigParser() 40 | _config.read(join(dirname(__file__), 'constants.cfg')) 41 | for section in _config.sections() + ['DEFAULT']: 42 | locals()[section] = type(section, (), {item[0]: item[1] for item in _config.items(section)}) 43 | -------------------------------------------------------------------------------- /clusterdock/cluster.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """This module implements the main abstractions used to model distributed clusters on a single 18 | host through the use of networked Docker containers. In particular, classes for clusters (Cluster), 19 | nodes (Node), and groups of nodes that may be convenient referenced together (NodeGroup) are 20 | implemented here. 21 | """ 22 | 23 | import logging 24 | import re 25 | import threading 26 | from os.path import dirname, join 27 | from time import time, sleep 28 | 29 | from docker import Client 30 | from docker.errors import APIError 31 | from docker.utils import create_ipam_pool 32 | 33 | from clusterdock.docker_utils import (get_container_ip_address, 34 | get_network_container_hostnames, get_network_subnet, 35 | get_available_network_subnet, is_container_reachable, 36 | is_network_present, NetworkNotFoundException) 37 | from clusterdock.ssh import ssh 38 | 39 | # We disable a couple of Pylint conventions because it assumes that module level variables must be 40 | # named as if they're constants (which isn't the case here). 41 | logger = logging.getLogger(__name__) # pylint: disable=invalid-name 42 | logger.setLevel(logging.INFO) 43 | 44 | client = Client() # pylint: disable=invalid-name 45 | 46 | class Cluster(object): 47 | """The central abstraction for dealing with Docker container clusters. Instances of this class 48 | can be created as needed, but no Docker-specific behavior is done until start() is invoked. 49 | """ 50 | def __init__(self, topology, node_groups, network_name): 51 | """Creates a cluster instance from a given topology name, list of NodeGroups, and network 52 | name.""" 53 | self.topology = topology 54 | self.ssh_key = join(dirname(__file__), 'topologies', self.topology, 'ssh', 'id_rsa') 55 | 56 | self.node_groups = node_groups 57 | self.network_name = network_name 58 | 59 | self.nodes = [node for node_group in self.node_groups for node in node_group.nodes] 60 | 61 | def setup_network(self): 62 | """If the network doesn't already exist, create it, being careful to pick a subnet that 63 | doesn't collide with that of any other Docker networks already present.""" 64 | if not is_network_present(self.network_name): 65 | logger.info("Network (%s) not present, creating it...", self.network_name) 66 | next_network_subnet = get_available_network_subnet() 67 | while True: 68 | try: 69 | client.create_network(name=self.network_name, driver='bridge', ipam={ 70 | 'Config': [create_ipam_pool(subnet=next_network_subnet)] 71 | }) 72 | except APIError as api_error: 73 | if 'networks have overlapping IPv4' not in api_error.explanation: 74 | raise api_error 75 | else: 76 | # The hash after "conflicts with network" is the name with the overlapping 77 | # subnet. Save this to speed up finding the next available subnet the next 78 | # time around in the while loop. 79 | conflicting_network = re.findall(r'conflicts with network (\S+)', 80 | api_error.explanation)[0] 81 | logger.info("Conflicting network:(%s)", conflicting_network) 82 | # Try up get the next network subnet up to 5 times (looks like there's a 83 | # race where the conflicting network is known, but not yet visible through 84 | # the API). 85 | for _ in range(0, 5): 86 | try: 87 | next_network_subnet = get_available_network_subnet( 88 | get_network_subnet(conflicting_network) 89 | ) 90 | except NetworkNotFoundException as network_not_found_exception: 91 | if 'Cannot find network' not in network_not_found_exception.message: 92 | raise network_not_found_exception 93 | sleep(1) 94 | else: 95 | break 96 | else: 97 | logger.info("Successfully setup network (name: %s).", self.network_name) 98 | break 99 | 100 | def ssh(self, command, nodes=None): 101 | """Execute command on all nodes (unless a list of Node instances is passed) in parallel.""" 102 | ssh(command=command, 103 | hosts=[node.ip_address for node in self.nodes if not nodes or node in nodes], 104 | ssh_key=self.ssh_key) 105 | 106 | def start(self): 107 | """Actually start Docker containers, mimicking the cluster layout specified in the Cluster 108 | instance.""" 109 | start = time() 110 | self.setup_network() 111 | 112 | # Before starting any containers, make sure that there aren't any containers in the 113 | # network with the same hostname. 114 | network_container_hostnames = ( 115 | get_network_container_hostnames(self.network_name)) 116 | for node in self.nodes: 117 | # Set the Node instance's cluster attribute to the Cluster instance to give the node 118 | # access to the topology's SSH keys. 119 | node.cluster = self 120 | 121 | if node.hostname in network_container_hostnames: 122 | raise Exception( 123 | "A container with hostname {0} already exists in network {1}".format( 124 | node.hostname, self.network_name)) 125 | threads = [threading.Thread(target=node.start) for 126 | node in self.nodes] 127 | for thread in threads: 128 | thread.start() 129 | # Sleep shortly between node starts to bring some determinacy to the order of the IP 130 | # addresses that we get. 131 | sleep(0.25) 132 | for thread in threads: 133 | thread.join() 134 | etc_hosts_string = ''.join("{0} {1}.{2} # Added by clusterdock\n".format(node.ip_address, 135 | node.hostname, 136 | node.network) for 137 | node in self.nodes) 138 | with open('/etc/hosts', 'a') as etc_hosts: 139 | etc_hosts.write(etc_hosts_string) 140 | 141 | end = time() 142 | logger.info("Started cluster in %.2f seconds.", end - start) 143 | 144 | def __iter__(self): 145 | for node in self.nodes: 146 | yield node 147 | 148 | def __len__(self): 149 | return len(self.nodes) 150 | 151 | 152 | class NodeGroup(object): 153 | """A node group denotes a set of Nodes that share some characteristic so as to make it desirable 154 | to refer to them separately from other sets of Nodes. For example, in a typical HDFS cluster, 155 | one node would run the HDFS NameNode while the remaining nodes would run HDFS DataNodes. In 156 | this case, the former might comprise the "primary" node group while the latter may be part of 157 | the "secondary" node group. 158 | """ 159 | 160 | def __init__(self, name, nodes=None): 161 | """Initialize a Group instance called name with a list of nodes.""" 162 | self.name = name 163 | self.nodes = nodes 164 | 165 | def __iter__(self): 166 | for node in self.nodes: 167 | yield node 168 | 169 | def add_node(self, node): 170 | """Add a Node instance to the list of nodes in the NodeGroup.""" 171 | self.nodes.append(node) 172 | 173 | def ssh(self, command): 174 | """Run command over SSH across all nodes in the NodeGroup in parallel.""" 175 | ssh_key = self[0].cluster.ssh_key 176 | ssh(command=command, hosts=[node.ip_address for node in self.nodes], ssh_key=ssh_key) 177 | 178 | class Node(object): 179 | """The abstraction will eventually be actualized as a running Docker container. This container, 180 | unlike the typical Docker container, does not house a single process, but tends to run an 181 | init to make the container act more or less like a regular cluster node. 182 | """ 183 | 184 | # pylint: disable=too-many-instance-attributes 185 | # 11 instance attributes to keep track of node properties isn't too many (Pylint sets the limit 186 | # at 7), and while we could create a single dictionary attribute, that doesn't really improve 187 | # readability. 188 | 189 | def __init__(self, hostname, network, image, **kwargs): 190 | """volumes must be a list of dictionaries with keys being the directory on the host and the 191 | values being the corresponding directory in the container to mount.""" 192 | self.hostname = hostname 193 | self.network = network 194 | self.fqdn = "{0}.{1}".format(hostname, network) 195 | self.image = image 196 | 197 | # Optional arguments are relegated to the kwargs dictionary, in part to keep Pylint happy. 198 | self.command = kwargs.get('command') 199 | self.ports = kwargs.get('ports') 200 | # /etc/localtime is always volume mounted so that containers have the same timezone as their 201 | # host machines. 202 | self.volumes = [{'/etc/localtime': '/etc/localtime'}] + kwargs.get('volumes', []) 203 | 204 | # Define a number of instance attributes that will get assigned proper values when the node 205 | # starts. 206 | self.cluster = None 207 | self.container_id = None 208 | self.host_config = None 209 | self.ip_address = None 210 | 211 | def _get_binds(self): 212 | """docker-py takes binds in the form "/host/dir:/container/dir:rw" as host configs. This 213 | method returns a list of binds in that form.""" 214 | return ["{0}:{1}:rw".format(host_location, volume[host_location]) for volume in self.volumes 215 | for host_location in volume] 216 | 217 | def start(self): 218 | """Actually start a Docker container-based node on the host.""" 219 | 220 | # Create a host_configs dictionary to populate and then pass to Client.create_host_config(). 221 | host_configs = {} 222 | # To make them act like real hosts, Nodes must have all Linux capabilities enabled. For 223 | # some reason, we discovered that doing this causes less trouble than starting containers in 224 | # privileged mode (see KITCHEN-10073). We also disable the default seccomp profile (see #3) 225 | # and pass in the volumes list at this point. 226 | host_configs['cap_add'] = ['ALL'] 227 | host_configs['security_opt'] = ['seccomp:unconfined'] 228 | host_configs['publish_all_ports'] = True 229 | 230 | if self.volumes: 231 | host_configs['binds'] = self._get_binds() 232 | 233 | self.host_config = client.create_host_config(**host_configs) 234 | 235 | # docker-py runs containers in a two-step process: first it creates a container and then 236 | # it starts the container using the container ID. 237 | container_configs = { 238 | 'hostname': self.fqdn, 239 | 'image': self.image, 240 | 'host_config': self.host_config, 241 | 'detach': True, 242 | 'command': self.command, 243 | 'ports': self.ports, 244 | 'volumes': [volume[host_location] for volume in self.volumes 245 | for host_location in volume if self.volumes], 246 | 'labels': {"volume{0}".format(i): volume 247 | for i, volume in enumerate([volume.keys()[0] 248 | for volume in self.volumes 249 | if volume.keys()[0] not in ['/etc/localtime']], 250 | start=1) 251 | } 252 | } 253 | self.container_id = client.create_container(**container_configs)['Id'] 254 | 255 | # Don't start up containers on the default 'bridge' network for better isolation. 256 | client.disconnect_container_from_network(container=self.container_id, net_id='bridge') 257 | client.connect_container_to_network(container=self.container_id, net_id=self.network, 258 | aliases=[self.hostname]) 259 | client.start(container=self.container_id) 260 | 261 | self.ip_address = get_container_ip_address(container_id=self.container_id, 262 | network=self.network) 263 | if not is_container_reachable(container_id=self.container_id, network=self.network, 264 | ssh_key=self.cluster.ssh_key): 265 | raise Exception("Timed out waiting for {0} to become reachable.".format(self.hostname)) 266 | else: 267 | logger.info("Successfully started %s (IP address: %s).", self.fqdn, self.ip_address) 268 | 269 | def ssh(self, command): 270 | """Run command over SSH on the node.""" 271 | ssh(command=command, hosts=[self.ip_address], ssh_key=self.cluster.ssh_key) 272 | -------------------------------------------------------------------------------- /clusterdock/constants.cfg: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Values stored in the DEFAULT section can be used for interpolation in any other section. 18 | [DEFAULT] 19 | docker_registry_url = docker.io 20 | apache_namespace = apache 21 | cloudera_namespace = cloudera 22 | 23 | [docker_images] 24 | maven = maven:3.3.3-jdk-8 25 | 26 | [network] 27 | subnet_start = 192.168.123.0/24 28 | -------------------------------------------------------------------------------- /clusterdock/docker_utils.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """A hodgepodge collection of utility functions that interact with or use Docker.""" 18 | 19 | import logging 20 | from os.path import dirname, join 21 | from sys import stdout 22 | from time import time 23 | 24 | from docker import Client 25 | from docker.errors import NotFound 26 | from fabric.api import local, quiet 27 | from netaddr import IPNetwork 28 | 29 | from clusterdock import Constants 30 | from clusterdock.ssh import quiet_ssh 31 | from clusterdock.utils import get_nested_value 32 | 33 | logger = logging.getLogger(__name__) # pylint: disable=invalid-name 34 | logger.setLevel(logging.INFO) 35 | 36 | DEFAULT_NETWORKS = ["bridge", "host", "none"] 37 | 38 | # Change timeout for Docker client commands from default (60 s) to 30 min. This prevents timeouts 39 | # seen when removing really large containers. 40 | DOCKER_CLIENT_TIMEOUT = 1800 41 | 42 | NETWORK_SUBNET_START = Constants.network.subnet_start # pylint: disable=no-member 43 | 44 | client = Client(timeout=DOCKER_CLIENT_TIMEOUT) # pylint: disable=invalid-name 45 | 46 | class ContainerNotFoundException(Exception): 47 | """An exception to raise when a particular Docker container cannot be found.""" 48 | pass 49 | 50 | 51 | class ContainerExitCodeException(Exception): 52 | """An exception to raise when a container exits with a non-zero exit code.""" 53 | pass 54 | 55 | 56 | class NetworkNotFoundException(Exception): 57 | """An exception to raise when a particular Docker network cannot be found.""" 58 | pass 59 | 60 | 61 | def build_image(dockerfile, tag): 62 | """Python wrapper for the docker build command line argument. Could also be implemented using 63 | docker-py, but then we lose the progress indicators that the Docker command line gives us.""" 64 | local("docker build -t {0} --no-cache {1}".format(tag, dirname(dockerfile))) 65 | 66 | def get_all_containers(): 67 | """Returns a list of Docker containers on the host. This list contains dictionaries full of 68 | container metadata.""" 69 | return client.containers(all=True) 70 | 71 | def get_available_network_subnet(start_subnet=NETWORK_SUBNET_START): 72 | """Returns the next unused network subnet available to a Docker network in CIDR format.""" 73 | subnet = IPNetwork(start_subnet) 74 | while overlaps_network_subnet(str(subnet)): 75 | subnet = subnet.next(1) 76 | return str(subnet) 77 | 78 | def get_clusterdock_container_id(): 79 | """Returns the container ID of the Docker container running clusterdock. 80 | """ 81 | with quiet(): 82 | for cgroup in local('cat /proc/self/cgroup', capture=True).stdout.split('\n'): 83 | if 'docker' in cgroup: 84 | return cgroup.rsplit('/')[-1] 85 | 86 | # If we get through the loop and never find the cgroup, something has gone very wrong. 87 | raise ContainerNotFoundException('Could not find container name from /proc/self/cgroup.') 88 | 89 | def get_container_attribute(container_id, dot_separated_key): 90 | """Helper function that gets a specified container's attribute, as requested in the form of a 91 | dot-separated string. That is, every level of nesting in the container's metadata is denoted 92 | by a period.""" 93 | container_attributes = _get_container_attributes(container_id) 94 | return get_nested_value(container_attributes, dot_separated_key) 95 | 96 | def get_container_hostname(container_id): 97 | """Returns the hostname of the specified container. Note that this is not the fully qualified 98 | domain name.""" 99 | return get_container_attribute(container_id, "Config.Hostname") 100 | 101 | def get_container_id(hostname, network_name): 102 | """Returns the container ID corresponding to a given hostname in a particular Docker network. 103 | This is relevant because a single Docker host can happily run containers duplicate hostnames 104 | as long as they are isolated by Docker networks.""" 105 | for network in get_networks(): 106 | if network['Name'] == network_name: 107 | for container in network['Containers']: 108 | if get_container_hostname(container) == hostname: 109 | return container 110 | 111 | def get_container_ip_address(container_id, network=None): 112 | """Returns the [internally-accessible] IP address of a particular container. If a Docker network 113 | is also specified, it will return the IP address assigned from within that network.""" 114 | ip_address_attribute = ("NetworkSettings.Networks.{0}.IPAddress".format(network) if network else 115 | "NetworkSettings.IPAddress") 116 | ip_address = get_container_attribute(container_id, ip_address_attribute) 117 | return ip_address 118 | 119 | def get_container_ip_from_hostname(hostname, network='bridge'): 120 | """Returns the IP address of a container given its hostname within the Docker network.""" 121 | container_id = get_container_id(hostname, network) 122 | return get_container_ip_address(container_id, network) 123 | 124 | def get_host_port_binding(container_id, container_port): 125 | """Return the port on the Docker host to which a particular container's port is being 126 | redirected.""" 127 | ports = get_container_attribute(container_id, 128 | "NetworkSettings.Ports.{0}/tcp".format(container_port)) 129 | return ports[0].get('HostPort') if ports else None 130 | 131 | def get_network_container_hostnames(name): 132 | """Returns a list of every container hostname in the specified network.""" 133 | for network in get_networks(): 134 | if network["Name"] == name: 135 | return [get_container_hostname(container) for container in network["Containers"]] 136 | 137 | def get_network_id(name): 138 | """Returns the Docker network ID corresponding to a particular network name on the host.""" 139 | for network in get_networks(): 140 | if network["Name"] == name: 141 | return network["Id"] 142 | 143 | def get_network_names(): 144 | """Returns a list of every Docker network name on the host.""" 145 | return [network["Name"] for network in get_networks()] 146 | 147 | def get_network_subnet(network_id): 148 | """Get a particular Docker network's subnet.""" 149 | networks = get_networks() 150 | for network in networks: 151 | if network['Id'] == network_id: 152 | return network['IPAM']['Config'][0]['Subnet'] 153 | # If we get through the loop and never find the network, something has gone very wrong. 154 | raise NetworkNotFoundException( 155 | "Cannot find network (Id: {0}). Networks present: {1}".format(network_id, networks) 156 | ) 157 | 158 | def get_network_subnets(): 159 | """Returns a list of all the subnets to which Docker networks on the host have been assigned.""" 160 | return [network["IPAM"]["Config"][0]["Subnet"] for network in get_networks() if 161 | network["IPAM"]["Config"]] 162 | 163 | def get_networks(): 164 | """Returns a list of Docker networks present on the host.""" 165 | return [network for network in client.networks()] 166 | 167 | def is_container_reachable(container_id, ssh_key, network): 168 | """Return true if a container can be reached via SSH (timeout after 60 s), false otherwise.""" 169 | try: 170 | start = time() 171 | quiet_ssh(command="whoami", hosts=get_container_ip_address(container_id, network), 172 | ssh_key=ssh_key) 173 | end = time() 174 | logger.debug("Verified SSH connectivity in %s seconds.", end-start) 175 | return True 176 | except BaseException: 177 | return False 178 | 179 | def is_container_running(name): 180 | """Return true if a Docker container is running, false otherwise.""" 181 | try: 182 | running = get_container_attribute(name, "State.Running") 183 | return running 184 | # If docker.errors.NotFound is raised, container is not running. 185 | except NotFound: 186 | return False 187 | 188 | def is_image_available_locally(name): 189 | """Return true if the Docker image 'name' is present on the Docker host, false otherwise.""" 190 | 191 | # Docker's API hides the registry URL if it's docker.io (e.g. docker.io/example/image:latest 192 | # becomes example/image:latest). 193 | if name.startswith('docker.io/'): 194 | name = name[len('docker.io/'):] 195 | return any(name in image['RepoTags'] for image in client.images() if image['RepoTags']) 196 | 197 | def is_network_present(name): 198 | """Returns true if Docker network 'name' is present on the Docker host, false otherwise.""" 199 | return name in get_network_names() 200 | 201 | def kill_all_containers(): 202 | """Kills all running Docker containers on the host.""" 203 | for container in _get_running_containers(): 204 | kill_container(name=container["Id"]) 205 | 206 | def kill_container(name): 207 | """Kills a particular running Docker container on the host.""" 208 | logger.info("Killing container %s...", name) 209 | return client.kill(container=name) 210 | 211 | def login(username, password, registry): 212 | """Python wrapper for the docker login command line argument. This is required since we do 213 | docker pulls using a similar command line argument and need to have credentials cached locally. 214 | """ 215 | local("docker login -u {0} -p {1} {2}".format(username, password, registry)) 216 | 217 | def overlaps_network_subnet(subnet): 218 | """Takes subnet as string in CIDR format (e.g. 192.168.123.0/24) and returns true if it overlaps 219 | any existing Docker network subnets.""" 220 | subnets = [IPNetwork(docker_subnet) for docker_subnet in get_network_subnets()] 221 | return IPNetwork(subnet) in subnets 222 | 223 | def pull_image(name): 224 | """Python wrapper for the docker pull command line argument. Could also be implemented using 225 | docker-py, but then we lose the progress indicators that the Docker command line gives us.""" 226 | local("docker pull {0}".format(name)) 227 | 228 | def pull_image_if_missing(name): 229 | """Simple wrapper function that will pull the Docker image 'name' if it's not present on the 230 | Docker host.""" 231 | if not is_image_available_locally(name=name): 232 | pull_image(name=name) 233 | 234 | def push_image(name): 235 | """Python wrapper for the docker push command line argument. Could also be implemented using 236 | docker-py, but then we lose the progress indicators that the Docker command line gives us.""" 237 | local("docker push {0}".format(name)) 238 | 239 | def raise_for_exit_code(container_id): 240 | """Raises ContainerExitCodeException if a particular container exits with a non-zero exit 241 | code.""" 242 | exit_code = get_container_attribute(container_id, "State.ExitCode") 243 | if exit_code != 0: 244 | raise ContainerExitCodeException( 245 | "Container {0} exited with code {1}.".format(container_id, exit_code) 246 | ) 247 | 248 | def remove_all_containers(): 249 | """Removes all containers on the Docker host. This will also handle cleanup of any volumes 250 | mounted to the host.""" 251 | clusterdock_container_id = get_clusterdock_container_id() 252 | 253 | for container in get_all_containers(): 254 | # Before removing containers, get a list of host folders being mounted inside (denoted by 255 | # a label during container creation (e.g. "volume0=/directory/on/host") and delete them. 256 | host_folders_to_delete = [value 257 | for key, value in container['Labels'].iteritems() 258 | if 'volume' in key 259 | # Don't delete any host folder like '/name' (eventually we may 260 | # want to allow this, but we want to avoid accidentally removing 261 | # / if someone is careless...). By checking rsplit('/',1)[0] and 262 | # stopping if it's an empty string, we can do this. 263 | and value.rsplit('/', 1)[0]] 264 | if host_folders_to_delete: 265 | # Since clusterdock is intended to be run out of a Docker container, we delete host 266 | # folders by mounting their parent into another container and then simply running rm -r. 267 | binds = ["{0}:/tmp{1}".format(mount.rsplit('/', 1)[0], i) 268 | for i, mount in enumerate(host_folders_to_delete, start=1)] 269 | volumes = [mount.rsplit(':', 1)[-1] for mount in binds] 270 | rm_command = ['rm', '-r'] + [join("/tmp{0}".format(i), mount.rsplit('/', 1)[-1]) 271 | for i, mount in enumerate(host_folders_to_delete, start=1)] 272 | logger.info("Removing host volumes (%s)...", host_folders_to_delete) 273 | 274 | utility_image = 'busybox:latest' 275 | pull_image_if_missing(name=utility_image) 276 | container_configs = { 277 | 'image': utility_image, 278 | 'command': rm_command, 279 | 'volumes': volumes, 280 | 'host_config': client.create_host_config(binds=binds) 281 | } 282 | container_id = client.create_container(**container_configs)['Id'] 283 | client.start(container=container_id) 284 | for line in client.logs(container=container_id, stream=True): 285 | stdout.write(line) 286 | stdout.flush() 287 | delete_host_folders_exit_code = client.wait(container=container_id) 288 | if delete_host_folders_exit_code != 0: 289 | logger.warning("Exit code of %d encountered when deleting folders %s. " 290 | "Continuing...", delete_host_folders_exit_code, 291 | host_folders_to_delete) 292 | client.remove_container(container=container_id) 293 | 294 | if container['Id'] != clusterdock_container_id: 295 | remove_container(name=container['Id']) 296 | 297 | def remove_all_images(): 298 | """Removes all Docker images on the host, using force to handle any images currently being run 299 | as containers. This will skip any clusterdock images, to avoid killing the process running the 300 | function itself.""" 301 | for image in client.images(): 302 | if 'org.apache.hbase.is-clusterdock' not in image['Labels']: 303 | client.remove_image(image, force=True) 304 | 305 | def remove_all_networks(): 306 | """Removes all Docker networks from the host (except for the DEFAULT_NETWORKS).""" 307 | for network in get_network_names(): 308 | if network not in DEFAULT_NETWORKS: 309 | remove_network(network) 310 | 311 | def remove_container(name): 312 | """Removes a particular Docker container on the host. If it is currently running, it will first 313 | be killed (via force).""" 314 | client.remove_container(container=name, force=True) 315 | 316 | def remove_network(name): 317 | """Removes the specified Docker network from the host.""" 318 | client.remove_network(name) 319 | 320 | def _get_container_attributes(container_id): 321 | """Return a dictionary containing all of a container's attributes.""" 322 | if not container_id: 323 | raise Exception("Tried to inspect container with null id.") 324 | return client.inspect_container(container=container_id) 325 | 326 | def _get_images(): 327 | return client.images(quiet=True) 328 | 329 | def _get_running_containers(): 330 | return client.containers(all=False, quiet=True) -------------------------------------------------------------------------------- /clusterdock/images/centos6.6_nodebase/Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | FROM centos:6.6 17 | 18 | # Install useful things that are missing from the centos:6.6 image. 19 | RUN yum install -y openssh-clients \ 20 | openssh-server \ 21 | rsyslog \ 22 | sudo \ 23 | tar \ 24 | wget \ 25 | which 26 | 27 | # Add pre-created SSH keys into .ssh folder. 28 | ADD ssh /root/.ssh/ 29 | 30 | # Copy public key into authorized_keys and limit access to the private key to ensure SSH can use it. 31 | RUN cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys && \ 32 | chmod 600 /root/.ssh/id_rsa 33 | 34 | # The official CentOS Docker images retain a lot of things that only apply on a real machine. 35 | # Among the most problematic is starting udev, which was seen to hang when containers were started 36 | # simultaneously. Simple solution (and one done by the lxc-centos.in template) is to simply 37 | # not trigger it. The other is the inclusion of the 90-nproc.conf file, which overrides reasonable 38 | # defaults for things like the maximum number of user processes when running commands as a non-root 39 | # user. Get rid of it (and see tinyurl.com/zqdfzpg). 40 | RUN sed -i 's|/sbin/start_udev||' /etc/rc.d/rc.sysinit && \ 41 | rm /etc/security/limits.d/90-nproc.conf 42 | 43 | # Disable strict host key checking and set the known hosts file to /dev/null to make 44 | # SSH between containers less of a pain. 45 | RUN sed -i -r "s|\s*Host \*\s*|&\n StrictHostKeyChecking no|" /etc/ssh/ssh_config && \ 46 | sed -i -r "s|\s*Host \*\s*|&\n UserKnownHostsFile=/dev/null|" /etc/ssh/ssh_config 47 | 48 | CMD ["/sbin/init"] 49 | -------------------------------------------------------------------------------- /clusterdock/images/centos6.6_nodebase/ssh/id_rsa.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2P/Jlk0X5u/bp6paqzH68V6FKXEEmn9CEisVYqmMdylAnb2jqEk32ahTcah+3YLnDoWhl2MLVNaXOHFSdFLzF6SwWdbci2+SPJIsDUirqNyFjE119ncfBN0fSu5szW4n8KeRqKOjEGoYGY9IvRPF+py15onwA+GyXCXRZYnpv60EIcNWlMsBN5u2CAIB+EqZK4BiafdAionsMWhIaTCGTGlQe65UJnFQfcIjEsjD7qekCuv9LmC0WjnSiLoVc+a+/qYmDrIuqrdWUpxCHp38I5omikef6ct53aQQIRJSWlOu4ymvohwYRKDPiGeSaGinyhNjjaxAmE88XYCGr1rdp clusterdock 2 | -------------------------------------------------------------------------------- /clusterdock/ssh.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """This module contains some basic wrappers of the Fabric API to facilitate using SSH to execute 18 | commands on cluster nodes.""" 19 | 20 | import os 21 | 22 | import fabric.api 23 | from fabric.api import env, execute, run, parallel 24 | from fabric.context_managers import quiet, settings, show 25 | import fabric.state 26 | 27 | SSH_TIMEOUT_IN_SECONDS = 1 28 | SSH_MAX_RETRIES = 60 29 | 30 | env.disable_known_hosts = True 31 | fabric.state.output['running'] = False 32 | 33 | @parallel(pool_size=8) 34 | @fabric.api.task 35 | def _quiet_task(command, ssh_key): 36 | with settings(quiet(), always_use_pty=False, output_prefix=False, key_filename=ssh_key, 37 | connection_attempts=SSH_MAX_RETRIES, timeout=SSH_TIMEOUT_IN_SECONDS): 38 | return run(command) 39 | 40 | @parallel(pool_size=8) 41 | @fabric.api.task 42 | def _task(command, ssh_key): 43 | with settings(show('stdout'), always_use_pty=False, output_prefix=False, key_filename=ssh_key, 44 | connection_attempts=SSH_MAX_RETRIES, timeout=SSH_TIMEOUT_IN_SECONDS): 45 | return run(command) 46 | 47 | def quiet_ssh(command, hosts, ssh_key): 48 | """Execute command over SSH on hosts, suppressing all output. This is useful for instances where 49 | you may only want to see if a command succeeds or times out, since stdout is otherwise 50 | discarded.""" 51 | return execute(_quiet_task, command=command, hosts=hosts, ssh_key=ssh_key) 52 | 53 | def ssh(command, hosts, ssh_key): 54 | """Execute command over SSH on hosts.""" 55 | return execute(_task, command=command, hosts=hosts, ssh_key=ssh_key) 56 | -------------------------------------------------------------------------------- /clusterdock/topologies/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /clusterdock/topologies/cdh/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /clusterdock/topologies/cdh/actions.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import logging 18 | from argparse import Namespace 19 | from collections import OrderedDict 20 | from datetime import datetime 21 | from os import makedirs 22 | from os.path import dirname, join 23 | from socket import getfqdn 24 | from sys import stdout 25 | from time import sleep 26 | from uuid import uuid4 27 | 28 | from docker import Client 29 | 30 | from clusterdock import Constants 31 | from clusterdock.cluster import Cluster, Node, NodeGroup 32 | from clusterdock.docker_utils import (get_host_port_binding, is_image_available_locally, 33 | pull_image) 34 | from clusterdock.topologies.cdh.cm import ClouderaManagerDeployment 35 | from clusterdock.utils import wait_for_port_open 36 | 37 | logger = logging.getLogger(__name__) 38 | logger.setLevel(logging.INFO) 39 | 40 | DEFAULT_CLOUDERA_NAMESPACE = Constants.DEFAULT.cloudera_namespace # pylint: disable=no-member 41 | 42 | def start(args): 43 | primary_node_image = "{0}/{1}/clusterdock:{2}_{3}_primary-node".format( 44 | args.registry_url, args.namespace or DEFAULT_CLOUDERA_NAMESPACE, 45 | args.cdh_string, args.cm_string 46 | ) 47 | 48 | secondary_node_image = "{0}/{1}/clusterdock:{2}_{3}_secondary-node".format( 49 | args.registry_url, args.namespace or DEFAULT_CLOUDERA_NAMESPACE, 50 | args.cdh_string, args.cm_string 51 | ) 52 | 53 | for image in [primary_node_image, secondary_node_image]: 54 | if args.always_pull or not is_image_available_locally(image): 55 | logger.info("Pulling image %s. This might take a little while...", image) 56 | pull_image(image) 57 | 58 | CM_SERVER_PORT = 7180 59 | HUE_SERVER_PORT = 8888 60 | 61 | primary_node = Node(hostname=args.primary_node[0], network=args.network, 62 | image=primary_node_image, ports=[CM_SERVER_PORT, HUE_SERVER_PORT]) 63 | 64 | secondary_nodes = [Node(hostname=hostname, network=args.network, image=secondary_node_image) 65 | for hostname in args.secondary_nodes] 66 | 67 | secondary_node_group = NodeGroup(name='secondary', nodes=secondary_nodes) 68 | node_groups = [NodeGroup(name='primary', nodes=[primary_node]), 69 | secondary_node_group] 70 | 71 | cluster = Cluster(topology='cdh', node_groups=node_groups, network_name=args.network) 72 | cluster.start() 73 | 74 | ''' 75 | A hack is needed here. In short, Docker mounts a number of files from the host into 76 | the container (and so do we). As such, when CM runs 'mount' inside of the containers 77 | during setup, it sees these ext4 files as suitable places in which to install things. 78 | Unfortunately, CM doesn't have a blacklist to ignore filesystem types and only including 79 | our containers' filesystem in the agents' config.ini whitelist is insufficient, since CM 80 | merges that list with the contents of /proc/filesystems. To work around this, we copy 81 | the culprit files inside of the container, which creates those files in aufs. We then 82 | unmount the volumes within the container and then move the files back to their original 83 | locations. By doing this, we preserve the contents of the files (which is necessary for 84 | things like networking to work properly) and keep CM happy. 85 | ''' 86 | filesystem_fix_commands = [] 87 | for file in ['/etc/hosts', '/etc/resolv.conf', '/etc/hostname', '/etc/localtime']: 88 | filesystem_fix_commands.append("cp {0} {0}.1; umount {0}; mv {0}.1 {0};".format(file)) 89 | filesystem_fix_command = ' '.join(filesystem_fix_commands) 90 | cluster.ssh(filesystem_fix_command) 91 | 92 | change_cm_server_host(cluster, primary_node.fqdn) 93 | if len(secondary_nodes) > 1: 94 | additional_nodes = [node for node in secondary_nodes[1:]] 95 | remove_files(cluster, files=['/var/lib/cloudera-scm-agent/uuid', 96 | '/dfs*/dn/current/*'], 97 | nodes=additional_nodes) 98 | 99 | # It looks like there may be something buggy when it comes to restarting the CM agent. Keep 100 | # going if this happens while we work on reproducing the problem. 101 | try: 102 | restart_cm_agents(cluster) 103 | except: 104 | pass 105 | 106 | logger.info('Waiting for Cloudera Manager server to come online...') 107 | cm_server_startup_time = wait_for_port_open(primary_node.ip_address, 108 | CM_SERVER_PORT, timeout_sec=180) 109 | logger.info("Detected Cloudera Manager server after %.2f seconds.", cm_server_startup_time) 110 | cm_server_web_ui_host_port = get_host_port_binding(primary_node.container_id, 111 | CM_SERVER_PORT) 112 | 113 | logger.info("CM server is now accessible at http://%s:%s", 114 | getfqdn(), cm_server_web_ui_host_port) 115 | 116 | deployment = ClouderaManagerDeployment(cm_server_address=primary_node.ip_address) 117 | deployment.setup_api_resources() 118 | 119 | if len(cluster) > 2: 120 | deployment.add_hosts_to_cluster(secondary_node_fqdn=secondary_nodes[0].fqdn, 121 | all_fqdns=[node.fqdn for node in cluster]) 122 | 123 | deployment.update_database_configs() 124 | deployment.update_hive_metastore_namenodes() 125 | 126 | if args.include_service_types: 127 | # CM maintains service types in CAPS, so make sure our args.include_service_types list 128 | # follows the same convention. 129 | service_types_to_leave = args.include_service_types.upper().split(',') 130 | for service in deployment.cluster.get_all_services(): 131 | if service.type not in service_types_to_leave: 132 | logger.info('Removing service %s from %s...', service.name, deployment.cluster.displayName) 133 | deployment.cluster.delete_service(service.name) 134 | elif args.exclude_service_types: 135 | service_types_to_remove = args.exclude_service_types.upper().split(',') 136 | for service in deployment.cluster.get_all_services(): 137 | if service.type in service_types_to_remove: 138 | logger.info('Removing service %s from %s...', service.name, deployment.cluster.displayName) 139 | deployment.cluster.delete_service(service.name) 140 | 141 | hue_server_host_port = get_host_port_binding(primary_node.container_id, HUE_SERVER_PORT) 142 | for service in deployment.cluster.get_all_services(): 143 | if service.type == 'HUE': 144 | logger.info("Once its service starts, Hue server will be accessible at http://%s:%s", 145 | getfqdn(), hue_server_host_port) 146 | break 147 | 148 | logger.info("Deploying client configuration...") 149 | deployment.cluster.deploy_client_config().wait() 150 | 151 | if not args.dont_start_cluster: 152 | logger.info('Starting cluster...') 153 | if not deployment.cluster.start().wait().success: 154 | raise Exception('Failed to start cluster.') 155 | logger.info('Starting Cloudera Management service...') 156 | if not deployment.cm.get_service().start().wait().success: 157 | raise Exception('Failed to start Cloudera Management service.') 158 | 159 | deployment.validate_services_started() 160 | 161 | logger.info("We'd love to know what you think of our CDH topology for clusterdock! Please " 162 | "direct any feedback to our community forum at " 163 | "http://tiny.cloudera.com/hadoop-101-forum.") 164 | 165 | def restart_cm_agents(cluster): 166 | logger.info('Restarting CM agents...') 167 | cluster.ssh('service cloudera-scm-agent restart') 168 | 169 | def change_cm_server_host(cluster, server_host): 170 | change_server_host_command = ( 171 | r'sed -i "s/\(server_host\).*/\1={0}/" /etc/cloudera-scm-agent/config.ini'.format( 172 | server_host 173 | ) 174 | ) 175 | logger.info("Changing server_host to %s in /etc/cloudera-scm-agent/config.ini...", 176 | server_host) 177 | cluster.ssh(change_server_host_command) 178 | 179 | def remove_files(cluster, files, nodes): 180 | logger.info("Removing files (%s) from hosts (%s)...", 181 | ', '.join(files), ', '.join([node.fqdn for node in nodes])) 182 | cluster.ssh('rm -rf {0}'.format(' '.join(files)), nodes=nodes) 183 | -------------------------------------------------------------------------------- /clusterdock/topologies/cdh/cm.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import logging 18 | import sys 19 | from ConfigParser import ConfigParser 20 | from os.path import dirname, join 21 | from time import sleep, time 22 | 23 | import requests 24 | 25 | # We use this PYTHONPATH hack because we want to use the cm_api package without installing it into 26 | # our virtualenv (since we want to leave out Cloudera-specific artifacts from the clusterdock 27 | # installation process). As a result, to get the cm_api in, we need to put the parent folder of 28 | # cm_api on the path so that imports done within the module (e.g. "from cm_api.api_client 29 | # import...") resolve properly. Otherwise, they would need to be changed to include a prefix 30 | # referencing clusterdock. 31 | sys.path.insert(0, dirname(__file__)) 32 | from cm_api.api_client import ApiResource 33 | 34 | from clusterdock.topologies.cdh import cm_utils 35 | from clusterdock.utils import XmlConfiguration 36 | 37 | logger = logging.getLogger(__name__) 38 | logger.setLevel(logging.INFO) 39 | 40 | DEFAULT_CM_PORT = '7180' 41 | DEFAULT_CM_USERNAME = 'admin' 42 | DEFAULT_CM_PASSWORD = 'admin' 43 | 44 | def xml(properties): 45 | return XmlConfiguration(properties=properties).to_string(hide_root=True) 46 | 47 | class ClouderaManagerDeployment(object): 48 | def __init__(self, cm_server_address, cm_server_port=DEFAULT_CM_PORT, 49 | username=DEFAULT_CM_USERNAME, password=DEFAULT_CM_PASSWORD): 50 | self.cm_server_address = cm_server_address 51 | self.cm_server_port = cm_server_port 52 | self.username = username 53 | self.password = password 54 | 55 | def setup_api_resources(self): 56 | self.api = ApiResource(server_host=self.cm_server_address, server_port=self.cm_server_port, 57 | username=self.username, password=self.password, 58 | version=self._get_api_version()) 59 | 60 | self.cm = self.api.get_cloudera_manager() 61 | self.cluster = self.api.get_cluster('Cluster 1 (clusterdock)') 62 | 63 | def prep_for_start(self): 64 | pass 65 | 66 | def validate_services_started(self, timeout_min=10, healthy_time_threshold_sec=30): 67 | start_validating_time = time() 68 | healthy_time = None 69 | 70 | logger.info('Beginning service health validation...') 71 | while healthy_time is None or (time() - healthy_time < healthy_time_threshold_sec): 72 | if (time() - start_validating_time < timeout_min * 60): 73 | all_services = list(self.cluster.get_all_services()) + [self.cm.get_service()] 74 | at_fault_services = list() 75 | for service in all_services: 76 | if (service.serviceState != "NA" and service.serviceState != "STARTED"): 77 | at_fault_services.append([service.name, "NOT STARTED"]) 78 | elif (service.serviceState != "NA" and service.healthSummary != "GOOD"): 79 | checks = list() 80 | for check in service.healthChecks: 81 | if (check["summary"] not in ("GOOD", "DISABLED")): 82 | checks.append(check["name"]) 83 | at_fault_services.append([service.name, 84 | "Failed health checks: {0}".format(checks)]) 85 | 86 | if not healthy_time or at_fault_services: 87 | healthy_time = time() if not at_fault_services else None 88 | sleep(3) 89 | else: 90 | raise Exception(("Timed out after waiting {0} minutes for services to start " 91 | "(at fault: {1}).").format(timeout_min, at_fault_services)) 92 | logger.info("Validated that all services started (time: %.2f s).", 93 | time() - start_validating_time) 94 | 95 | def add_hosts_to_cluster(self, secondary_node_fqdn, all_fqdns): 96 | cm_utils.add_hosts_to_cluster(api=self.api, cluster=self.cluster, 97 | secondary_node_fqdn=secondary_node_fqdn, 98 | all_fqdns=all_fqdns) 99 | 100 | def update_hive_metastore_namenodes(self): 101 | for service in self.cluster.get_all_services(): 102 | if service.type == 'HIVE': 103 | logger.info('Updating NameNode references in Hive metastore...') 104 | update_metastore_namenodes_cmd = service.update_metastore_namenodes().wait() 105 | if not update_metastore_namenodes_cmd.success: 106 | logger.warning(("Failed to update NameNode references in Hive metastore " 107 | "(command returned %s)."), update_metastore_namenodes_cmd) 108 | 109 | def update_database_configs(self): 110 | cm_utils.update_database_configs(api=self.api, cluster=self.cluster) 111 | 112 | def _get_api_version(self): 113 | api_version_response = requests.get( 114 | "http://{0}:{1}/api/version".format(self.cm_server_address, 115 | self.cm_server_port), 116 | auth=(self.username, self.password)) 117 | api_version_response.raise_for_status() 118 | api_version = api_version_response.content 119 | if 'v' not in api_version: 120 | raise Exception("/api/version returned unexpected result (%s).", api_version) 121 | else: 122 | logger.info("Detected CM API %s.", api_version) 123 | return api_version.strip('v') 124 | -------------------------------------------------------------------------------- /clusterdock/topologies/cdh/cm_api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudera/clusterdock/222ac2d5a9aa2fef7b84b3cf704f9ee54c0cc5d2/clusterdock/topologies/cdh/cm_api/__init__.py -------------------------------------------------------------------------------- /clusterdock/topologies/cdh/cm_api/api_client.py: -------------------------------------------------------------------------------- 1 | # Licensed to Cloudera, Inc. under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. Cloudera, Inc. licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import logging 18 | try: 19 | import json 20 | except ImportError: 21 | import simplejson as json 22 | 23 | from cm_api.http_client import HttpClient, RestException 24 | from cm_api.endpoints import batch, cms, clusters, events, hosts, tools 25 | from cm_api.endpoints import types, users, timeseries 26 | from cm_api.resource import Resource 27 | 28 | __docformat__ = "epytext" 29 | 30 | LOG = logging.getLogger(__name__) 31 | 32 | API_AUTH_REALM = "Cloudera Manager" 33 | API_CURRENT_VERSION = 12 34 | 35 | class ApiException(RestException): 36 | """ 37 | Any error result from the API is converted into this exception type. 38 | This handles errors from the HTTP level as well as the API level. 39 | """ 40 | def __init__(self, error): 41 | # The parent class will set up _code and _message 42 | RestException.__init__(self, error) 43 | try: 44 | # See if the body is json 45 | json_body = json.loads(self._message) 46 | self._message = json_body['message'] 47 | except (ValueError, KeyError): 48 | pass # Ignore json parsing error 49 | 50 | 51 | class ApiResource(Resource): 52 | """ 53 | Resource object that provides methods for managing the top-level API resources. 54 | """ 55 | 56 | def __init__(self, server_host, server_port=None, 57 | username="admin", password="admin", 58 | use_tls=False, version=API_CURRENT_VERSION): 59 | """ 60 | Creates a Resource object that provides API endpoints. 61 | 62 | @param server_host: The hostname of the Cloudera Manager server. 63 | @param server_port: The port of the server. Defaults to 7180 (http) or 64 | 7183 (https). 65 | @param username: Login name. 66 | @param password: Login password. 67 | @param use_tls: Whether to use tls (https). 68 | @param version: API version. 69 | @return: Resource object referring to the root. 70 | """ 71 | self._version = version 72 | protocol = use_tls and "https" or "http" 73 | if server_port is None: 74 | server_port = use_tls and 7183 or 7180 75 | base_url = "%s://%s:%s/api/v%s" % \ 76 | (protocol, server_host, server_port, version) 77 | 78 | client = HttpClient(base_url, exc_class=ApiException) 79 | client.set_basic_auth(username, password, API_AUTH_REALM) 80 | client.set_headers( { "Content-Type" : "application/json" } ) 81 | Resource.__init__(self, client) 82 | 83 | @property 84 | def version(self): 85 | """ 86 | Returns the API version (integer) being used. 87 | """ 88 | return self._version 89 | 90 | # CMS ops. 91 | 92 | def get_cloudera_manager(self): 93 | """ 94 | Returns a Cloudera Manager object. 95 | """ 96 | return cms.ClouderaManager(self) 97 | 98 | # Cluster ops. 99 | 100 | def create_cluster(self, name, version=None, fullVersion=None): 101 | """ 102 | Create a new cluster. 103 | 104 | @param name: Cluster name. 105 | @param version: Cluster major CDH version, e.g. 'CDH5'. Ignored if 106 | fullVersion is specified. 107 | @param fullVersion: Complete CDH version, e.g. '5.1.2'. Overrides major 108 | version if both specified. 109 | @return: The created cluster. 110 | """ 111 | return clusters.create_cluster(self, name, version, fullVersion) 112 | 113 | def delete_cluster(self, name): 114 | """ 115 | Delete a cluster by name. 116 | 117 | @param name: Cluster name 118 | @return: The deleted ApiCluster object 119 | """ 120 | return clusters.delete_cluster(self, name) 121 | 122 | def get_all_clusters(self, view = None): 123 | """ 124 | Retrieve a list of all clusters. 125 | @param view: View to materialize ('full' or 'summary'). 126 | @return: A list of ApiCluster objects. 127 | """ 128 | return clusters.get_all_clusters(self, view) 129 | 130 | def get_cluster(self, name): 131 | """ 132 | Look up a cluster by name. 133 | 134 | @param name: Cluster name. 135 | @return: An ApiCluster object. 136 | """ 137 | return clusters.get_cluster(self, name) 138 | 139 | # Host ops. 140 | 141 | def create_host(self, host_id, name, ipaddr, rack_id = None): 142 | """ 143 | Create a host. 144 | 145 | @param host_id: The host id. 146 | @param name: Host name 147 | @param ipaddr: IP address 148 | @param rack_id: Rack id. Default None. 149 | @return: An ApiHost object 150 | """ 151 | return hosts.create_host(self, host_id, name, ipaddr, rack_id) 152 | 153 | def delete_host(self, host_id): 154 | """ 155 | Delete a host by id. 156 | 157 | @param host_id: Host id 158 | @return: The deleted ApiHost object 159 | """ 160 | return hosts.delete_host(self, host_id) 161 | 162 | def get_all_hosts(self, view = None): 163 | """ 164 | Get all hosts 165 | 166 | @param view: View to materialize ('full' or 'summary'). 167 | @return: A list of ApiHost objects. 168 | """ 169 | return hosts.get_all_hosts(self, view) 170 | 171 | def get_host(self, host_id): 172 | """ 173 | Look up a host by id. 174 | 175 | @param host_id: Host id 176 | @return: An ApiHost object 177 | """ 178 | return hosts.get_host(self, host_id) 179 | 180 | # Users 181 | 182 | def get_all_users(self, view = None): 183 | """ 184 | Get all users. 185 | 186 | @param view: View to materialize ('full' or 'summary'). 187 | @return: A list of ApiUser objects. 188 | """ 189 | return users.get_all_users(self, view) 190 | 191 | def get_user(self, username): 192 | """ 193 | Look up a user by username. 194 | 195 | @param username: Username to look up 196 | @return: An ApiUser object 197 | """ 198 | return users.get_user(self, username) 199 | 200 | def create_user(self, username, password, roles): 201 | """ 202 | Create a user. 203 | 204 | @param username: Username 205 | @param password: Password 206 | @param roles: List of roles for the user. This should be [] for a 207 | regular user, or ['ROLE_ADMIN'] for an admin. 208 | @return: An ApiUser object 209 | """ 210 | return users.create_user(self, username, password, roles) 211 | 212 | def delete_user(self, username): 213 | """ 214 | Delete user by username. 215 | 216 | @param username: Username 217 | @return: An ApiUser object 218 | """ 219 | return users.delete_user(self, username) 220 | 221 | # Events 222 | 223 | def query_events(self, query_str = None): 224 | """ 225 | Query events. 226 | @param query_str: Query string. 227 | @return: A list of ApiEvent. 228 | """ 229 | return events.query_events(self, query_str) 230 | 231 | def get_event(self, event_id): 232 | """ 233 | Retrieve a particular event by ID. 234 | @param event_id: The event ID. 235 | @return: An ApiEvent. 236 | """ 237 | return events.get_event(self, event_id) 238 | 239 | # Tools 240 | 241 | def echo(self, message): 242 | """Have the server echo a message back.""" 243 | return tools.echo(self, message) 244 | 245 | def echo_error(self, message): 246 | """Generate an error, but we get to set the error message.""" 247 | return tools.echo_error(self, message) 248 | 249 | # Metrics 250 | 251 | def get_metrics(self, path, from_time, to_time, metrics, view, params=None): 252 | """ 253 | Generic function for querying metrics. 254 | 255 | @param from_time: A datetime; start of the period to query (optional). 256 | @param to_time: A datetime; end of the period to query (default = now). 257 | @param metrics: List of metrics to query (default = all). 258 | @param view: View to materialize ('full' or 'summary') 259 | @param params: Other query parameters. 260 | @return: List of metrics and their readings. 261 | """ 262 | if not params: 263 | params = { } 264 | if from_time: 265 | params['from'] = from_time.isoformat() 266 | if to_time: 267 | params['to'] = to_time.isoformat() 268 | if metrics: 269 | params['metrics'] = metrics 270 | if view: 271 | params['view'] = view 272 | resp = self.get(path, params=params) 273 | return types.ApiList.from_json_dict(resp, self, types.ApiMetric) 274 | 275 | def query_timeseries(self, query, from_time=None, to_time=None, by_post=False): 276 | """ 277 | Query time series. 278 | @param query: Query string. 279 | @param from_time: Start of the period to query (optional). 280 | @param to_time: End of the period to query (default = now). 281 | @return: A list of ApiTimeSeriesResponse. 282 | """ 283 | return timeseries.query_timeseries(self, query, from_time, to_time, by_post=by_post) 284 | 285 | def get_metric_schema(self): 286 | """ 287 | Get the schema for all of the metrics. 288 | @return: A list of ApiMetricSchema. 289 | """ 290 | return timeseries.get_metric_schema(self) 291 | 292 | # Batch 293 | 294 | def do_batch(self, elements): 295 | """ 296 | Execute a batch request with one or more elements. If any element fails, 297 | the entire request is rolled back and subsequent elements are ignored. 298 | @param elements: A list of ApiBatchRequestElements 299 | @return: 2-tuple (overall success, list of ApiBatchResponseElements). 300 | """ 301 | return batch.do_batch(self, elements) 302 | 303 | def get_root_resource(server_host, server_port=None, 304 | username="admin", password="admin", 305 | use_tls=False, version=API_CURRENT_VERSION): 306 | """ 307 | See ApiResource. 308 | """ 309 | return ApiResource(server_host, server_port, username, password, use_tls, 310 | version) 311 | -------------------------------------------------------------------------------- /clusterdock/topologies/cdh/cm_api/endpoints/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudera/clusterdock/222ac2d5a9aa2fef7b84b3cf704f9ee54c0cc5d2/clusterdock/topologies/cdh/cm_api/endpoints/__init__.py -------------------------------------------------------------------------------- /clusterdock/topologies/cdh/cm_api/endpoints/batch.py: -------------------------------------------------------------------------------- 1 | # Licensed to Cloudera, Inc. under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. Cloudera, Inc. licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from cm_api.endpoints.types import * 18 | 19 | __docformat__ = "epytext" 20 | 21 | BATCH_PATH = "/batch" 22 | 23 | def do_batch(resource_root, elements): 24 | """ 25 | Execute a batch request with one or more elements. If any element fails, 26 | the entire request is rolled back and subsequent elements are ignored. 27 | 28 | @param elements: A list of ApiBatchRequestElements 29 | @return: an ApiBatchResponseList 30 | @since: API v6 31 | """ 32 | return call(resource_root.post, BATCH_PATH, ApiBatchResponseList, 33 | data=elements, api_version=6) 34 | -------------------------------------------------------------------------------- /clusterdock/topologies/cdh/cm_api/endpoints/dashboards.py: -------------------------------------------------------------------------------- 1 | # Licensed to Cloudera, Inc. under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. Cloudera, Inc. licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from cm_api.endpoints.types import * 18 | 19 | __docformat__ = "epytext" 20 | 21 | DASHBOARDS_PATH = "/timeseries/dashboards" 22 | 23 | def _get_dashboard_path(dashboard_name): 24 | return DASHBOARDS_PATH + "/%s" % (dashboard_name) 25 | 26 | def create_dashboards(resource_root, dashboard_list): 27 | """ 28 | Creates the list of dashboards. If any of the dashboards already exist 29 | this whole command will fail and no dashboards will be created. 30 | @since: API v6 31 | @return: The list of dashboards created. 32 | """ 33 | return call(resource_root.post, DASHBOARDS_PATH, ApiDashboard, \ 34 | ret_is_list=True, data=dashboard_list) 35 | 36 | def get_dashboards(resource_root): 37 | """ 38 | Returns the list of all dashboards. 39 | @since: API v6 40 | @return: A list of API dashboard objects. 41 | """ 42 | return call(resource_root.get, DASHBOARDS_PATH, ApiDashboard, \ 43 | ret_is_list=True) 44 | 45 | def get_dashboard(resource_root, dashboard_name): 46 | """ 47 | Returns a dashboard definition for the specified name. This dashboard 48 | can be imported with the createDashboards API. 49 | @since: API v6 50 | @return: An API dasbhboard object. 51 | """ 52 | return call(resource_root.get, _get_dashboard_path(dashboard_name), \ 53 | ApiDashboard) 54 | 55 | def delete_dashboard(resource_root, dashboard_name): 56 | """ 57 | Deletes a dashboard. 58 | @since: API v6 59 | @return: The deleted dashboard. 60 | """ 61 | return call(resource_root.delete, _get_dashboard_path(dashboard_name), \ 62 | ApiDashboard) 63 | 64 | class ApiDashboard(BaseApiResource): 65 | _ATTRIBUTES = { 66 | 'name' : None, 67 | 'json' : None 68 | } 69 | 70 | def __init__(self, resource_root, name=None, json=None): 71 | BaseApiObject.init(self, resource_root, locals()) 72 | 73 | def __str__(self): 74 | return ": %s" % (self.name) 75 | 76 | def _path(self): 77 | return _get_dashboard_path(self.name) 78 | -------------------------------------------------------------------------------- /clusterdock/topologies/cdh/cm_api/endpoints/events.py: -------------------------------------------------------------------------------- 1 | # Licensed to Cloudera, Inc. under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. Cloudera, Inc. licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from cm_api.endpoints.types import * 18 | 19 | __docformat__ = "epytext" 20 | 21 | EVENTS_PATH = "/events" 22 | 23 | def query_events(resource_root, query_str=None): 24 | """ 25 | Search for events. 26 | @param query_str: Query string. 27 | @return: A list of ApiEvent. 28 | """ 29 | params = None 30 | if query_str: 31 | params = dict(query=query_str) 32 | return call(resource_root.get, EVENTS_PATH, ApiEventQueryResult, 33 | params=params) 34 | 35 | def get_event(resource_root, event_id): 36 | """ 37 | Retrieve a particular event by ID. 38 | @param event_id: The event ID. 39 | @return: An ApiEvent. 40 | """ 41 | return call(resource_root.get, "%s/%s" % (EVENTS_PATH, event_id), ApiEvent) 42 | 43 | 44 | class ApiEvent(BaseApiObject): 45 | _ATTRIBUTES = { 46 | 'id' : ROAttr(), 47 | 'content' : ROAttr(), 48 | 'timeOccurred' : ROAttr(datetime.datetime), 49 | 'timeReceived' : ROAttr(datetime.datetime), 50 | 'category' : ROAttr(), 51 | 'severity' : ROAttr(), 52 | 'alert' : ROAttr(), 53 | 'attributes' : ROAttr(), 54 | } 55 | 56 | class ApiEventQueryResult(ApiList): 57 | _ATTRIBUTES = { 58 | 'totalResults' : ROAttr(), 59 | } 60 | _MEMBER_CLASS = ApiEvent 61 | -------------------------------------------------------------------------------- /clusterdock/topologies/cdh/cm_api/endpoints/host_templates.py: -------------------------------------------------------------------------------- 1 | # Licensed to Cloudera, Inc. under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. Cloudera, Inc. licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import copy 18 | from cm_api.endpoints.types import * 19 | 20 | __docformat__ = "epytext" 21 | 22 | HOST_TEMPLATES_PATH = "/clusters/%s/hostTemplates" 23 | HOST_TEMPLATE_PATH = "/clusters/%s/hostTemplates/%s" 24 | APPLY_HOST_TEMPLATE_PATH = HOST_TEMPLATE_PATH + "/commands/applyHostTemplate" 25 | 26 | def create_host_template(resource_root, name, cluster_name): 27 | """ 28 | Create a host template. 29 | @param resource_root: The root Resource object. 30 | @param name: Host template name 31 | @param cluster_name: Cluster name 32 | @return: An ApiHostTemplate object for the created host template. 33 | @since: API v3 34 | """ 35 | apitemplate = ApiHostTemplate(resource_root, name, []) 36 | return call(resource_root.post, 37 | HOST_TEMPLATES_PATH % (cluster_name,), 38 | ApiHostTemplate, True, data=[apitemplate], api_version=3)[0] 39 | 40 | def get_host_template(resource_root, name, cluster_name): 41 | """ 42 | Lookup a host template by name in the specified cluster. 43 | @param resource_root: The root Resource object. 44 | @param name: Host template name. 45 | @param cluster_name: Cluster name. 46 | @return: An ApiHostTemplate object. 47 | @since: API v3 48 | """ 49 | return call(resource_root.get, 50 | HOST_TEMPLATE_PATH % (cluster_name, name), 51 | ApiHostTemplate, api_version=3) 52 | 53 | def get_all_host_templates(resource_root, cluster_name="default"): 54 | """ 55 | Get all host templates in a cluster. 56 | @param cluster_name: Cluster name. 57 | @return: ApiList of ApiHostTemplate objects for all host templates in a cluster. 58 | @since: API v3 59 | """ 60 | return call(resource_root.get, 61 | HOST_TEMPLATES_PATH % (cluster_name,), 62 | ApiHostTemplate, True, api_version=3) 63 | 64 | def delete_host_template(resource_root, name, cluster_name): 65 | """ 66 | Delete a host template identified by name in the specified cluster. 67 | @param resource_root: The root Resource object. 68 | @param name: Host template name. 69 | @param cluster_name: Cluster name. 70 | @return: The deleted ApiHostTemplate object. 71 | @since: API v3 72 | """ 73 | return call(resource_root.delete, 74 | HOST_TEMPLATE_PATH % (cluster_name, name), 75 | ApiHostTemplate, api_version=3) 76 | 77 | def update_host_template(resource_root, name, cluster_name, api_host_template): 78 | """ 79 | Update a host template identified by name in the specified cluster. 80 | @param resource_root: The root Resource object. 81 | @param name: Host template name. 82 | @param cluster_name: Cluster name. 83 | @param api_host_template: The updated host template. 84 | @return: The updated ApiHostTemplate. 85 | @since: API v3 86 | """ 87 | return call(resource_root.put, 88 | HOST_TEMPLATE_PATH % (cluster_name, name), 89 | ApiHostTemplate, data=api_host_template, api_version=3) 90 | 91 | def apply_host_template(resource_root, name, cluster_name, host_ids, start_roles): 92 | """ 93 | Apply a host template identified by name on the specified hosts and 94 | optionally start them. 95 | @param resource_root: The root Resource object. 96 | @param name: Host template name. 97 | @param cluster_name: Cluster name. 98 | @param host_ids: List of host ids. 99 | @param start_roles: Whether to start the created roles or not. 100 | @return: An ApiCommand object. 101 | @since: API v3 102 | """ 103 | host_refs = [] 104 | for host_id in host_ids: 105 | host_refs.append(ApiHostRef(resource_root, host_id)) 106 | 107 | params = {"startRoles" : start_roles} 108 | return call(resource_root.post, 109 | APPLY_HOST_TEMPLATE_PATH % (cluster_name, name), 110 | ApiCommand, data=host_refs, params=params, api_version=3) 111 | 112 | 113 | class ApiHostTemplate(BaseApiResource): 114 | _ATTRIBUTES = { 115 | 'name' : None, 116 | 'roleConfigGroupRefs' : Attr(ApiRoleConfigGroupRef), 117 | 'clusterRef' : ROAttr(ApiClusterRef), 118 | } 119 | 120 | def __init__(self, resource_root, name=None, roleConfigGroupRefs=None): 121 | BaseApiObject.init(self, resource_root, locals()) 122 | 123 | def __str__(self): 124 | return ": %s (cluster %s)" % (self.name, self.clusterRef.clusterName) 125 | 126 | def _api_version(self): 127 | return 3 128 | 129 | def _path(self): 130 | return HOST_TEMPLATE_PATH % (self.clusterRef.clusterName, self.name) 131 | 132 | def _do_update(self, update): 133 | self._update(self._put('', ApiHostTemplate, data=update)) 134 | return self 135 | 136 | def rename(self, new_name): 137 | """ 138 | Rename a host template. 139 | @param new_name: New host template name. 140 | @return: An ApiHostTemplate object. 141 | """ 142 | update = copy.copy(self) 143 | update.name = new_name 144 | return self._do_update(update) 145 | 146 | def set_role_config_groups(self, role_config_group_refs): 147 | """ 148 | Updates the role config groups in a host template. 149 | @param role_config_group_refs: List of role config group refs. 150 | @return: An ApiHostTemplate object. 151 | """ 152 | update = copy.copy(self) 153 | update.roleConfigGroupRefs = role_config_group_refs 154 | return self._do_update(update) 155 | 156 | def apply_host_template(self, host_ids, start_roles): 157 | """ 158 | Apply a host template identified by name on the specified hosts and 159 | optionally start them. 160 | @param host_ids: List of host ids. 161 | @param start_roles: Whether to start the created roles or not. 162 | @return: An ApiCommand object. 163 | """ 164 | return apply_host_template(self._get_resource_root(), self.name, self.clusterRef.clusterName, host_ids, start_roles) 165 | -------------------------------------------------------------------------------- /clusterdock/topologies/cdh/cm_api/endpoints/hosts.py: -------------------------------------------------------------------------------- 1 | # Licensed to Cloudera, Inc. under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. Cloudera, Inc. licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import datetime 18 | 19 | from cm_api.endpoints.types import * 20 | 21 | __docformat__ = "epytext" 22 | 23 | HOSTS_PATH = "/hosts" 24 | 25 | def create_host(resource_root, host_id, name, ipaddr, rack_id=None): 26 | """ 27 | Create a host 28 | @param resource_root: The root Resource object. 29 | @param host_id: Host id 30 | @param name: Host name 31 | @param ipaddr: IP address 32 | @param rack_id: Rack id. Default None 33 | @return: An ApiHost object 34 | """ 35 | apihost = ApiHost(resource_root, host_id, name, ipaddr, rack_id) 36 | return call(resource_root.post, HOSTS_PATH, ApiHost, True, data=[apihost])[0] 37 | 38 | def get_host(resource_root, host_id): 39 | """ 40 | Lookup a host by id 41 | @param resource_root: The root Resource object. 42 | @param host_id: Host id 43 | @return: An ApiHost object 44 | """ 45 | return call(resource_root.get, "%s/%s" % (HOSTS_PATH, host_id), ApiHost) 46 | 47 | def get_all_hosts(resource_root, view=None): 48 | """ 49 | Get all hosts 50 | @param resource_root: The root Resource object. 51 | @return: A list of ApiHost objects. 52 | """ 53 | return call(resource_root.get, HOSTS_PATH, ApiHost, True, 54 | params=view and dict(view=view) or None) 55 | 56 | def delete_host(resource_root, host_id): 57 | """ 58 | Delete a host by id 59 | @param resource_root: The root Resource object. 60 | @param host_id: Host id 61 | @return: The deleted ApiHost object 62 | """ 63 | return call(resource_root.delete, "%s/%s" % (HOSTS_PATH, host_id), ApiHost) 64 | 65 | 66 | class ApiHost(BaseApiResource): 67 | _ATTRIBUTES = { 68 | 'hostId' : None, 69 | 'hostname' : None, 70 | 'ipAddress' : None, 71 | 'rackId' : None, 72 | 'status' : ROAttr(), 73 | 'lastHeartbeat' : ROAttr(datetime.datetime), 74 | 'roleRefs' : ROAttr(ApiRoleRef), 75 | 'healthSummary' : ROAttr(), 76 | 'healthChecks' : ROAttr(), 77 | 'hostUrl' : ROAttr(), 78 | 'commissionState' : ROAttr(), 79 | 'maintenanceMode' : ROAttr(), 80 | 'maintenanceOwners' : ROAttr(), 81 | 'numCores' : ROAttr(), 82 | 'numPhysicalCores' : ROAttr(), 83 | 'totalPhysMemBytes' : ROAttr(), 84 | 'entityStatus' : ROAttr(), 85 | 'clusterRef' : ROAttr(ApiClusterRef), 86 | } 87 | 88 | def __init__(self, resource_root, hostId=None, hostname=None, 89 | ipAddress=None, rackId=None): 90 | BaseApiObject.init(self, resource_root, locals()) 91 | 92 | def __str__(self): 93 | return ": %s (%s)" % (self.hostId, self.ipAddress) 94 | 95 | def _path(self): 96 | return HOSTS_PATH + '/' + self.hostId 97 | 98 | def _put_host(self): 99 | """ 100 | Update this resource. 101 | @return: The updated object. 102 | """ 103 | return self._put('', ApiHost, data=self) 104 | 105 | def get_config(self, view=None): 106 | """ 107 | Retrieve the host's configuration. 108 | 109 | The 'summary' view contains strings as the dictionary values. The full 110 | view contains ApiConfig instances as the values. 111 | 112 | @param view: View to materialize ('full' or 'summary') 113 | @return: Dictionary with configuration data. 114 | """ 115 | return self._get_config("config", view) 116 | 117 | def update_config(self, config): 118 | """ 119 | Update the host's configuration. 120 | 121 | @param config: Dictionary with configuration to update. 122 | @return: Dictionary with updated configuration. 123 | """ 124 | return self._update_config("config", config) 125 | 126 | def get_metrics(self, from_time=None, to_time=None, metrics=None, 127 | ifs=[], storageIds=[], view=None): 128 | """ 129 | This endpoint is not supported as of v6. Use the timeseries API 130 | instead. To get all metrics for a host with the timeseries API use 131 | the query: 132 | 133 | 'select * where hostId = $HOST_ID'. 134 | 135 | To get specific metrics for a host use a comma-separated list of 136 | the metric names as follows: 137 | 138 | 'select $METRIC_NAME1, $METRIC_NAME2 where hostId = $HOST_ID'. 139 | 140 | For more information see http://tiny.cloudera.com/tsquery_doc 141 | @param from_time: A datetime; start of the period to query (optional). 142 | @param to_time: A datetime; end of the period to query (default = now). 143 | @param metrics: List of metrics to query (default = all). 144 | @param ifs: network interfaces to query. Default all, use None to disable. 145 | @param storageIds: storage IDs to query. Default all, use None to disable. 146 | @param view: View to materialize ('full' or 'summary') 147 | @return: List of metrics and their readings. 148 | """ 149 | params = { } 150 | if ifs: 151 | params['ifs'] = ifs 152 | elif ifs is None: 153 | params['queryNw'] = 'false' 154 | if storageIds: 155 | params['storageIds'] = storageIds 156 | elif storageIds is None: 157 | params['queryStorage'] = 'false' 158 | return self._get_resource_root().get_metrics(self._path() + '/metrics', 159 | from_time, to_time, metrics, view, params) 160 | 161 | def enter_maintenance_mode(self): 162 | """ 163 | Put the host in maintenance mode. 164 | 165 | @return: Reference to the completed command. 166 | @since: API v2 167 | """ 168 | cmd = self._cmd('enterMaintenanceMode') 169 | if cmd.success: 170 | self._update(get_host(self._get_resource_root(), self.hostId)) 171 | return cmd 172 | 173 | def exit_maintenance_mode(self): 174 | """ 175 | Take the host out of maintenance mode. 176 | 177 | @return: Reference to the completed command. 178 | @since: API v2 179 | """ 180 | cmd = self._cmd('exitMaintenanceMode') 181 | if cmd.success: 182 | self._update(get_host(self._get_resource_root(), self.hostId)) 183 | return cmd 184 | 185 | def migrate_roles(self, role_names_to_migrate, destination_host_id, 186 | clear_stale_role_data): 187 | """ 188 | Migrate roles from this host to a different host. 189 | 190 | Currently, this command applies only to HDFS NameNode, JournalNode, 191 | and Failover Controller roles. In order to migrate these roles: 192 | 193 | - HDFS High Availability must be enabled, using quorum-based storage. 194 | - HDFS must not be configured to use a federated nameservice. 195 | 196 | I{B{Migrating a NameNode role requires cluster downtime.}} HDFS, along 197 | with all of its dependent services, will be stopped at the beginning 198 | of the migration process, and restarted at its conclusion. 199 | 200 | If the active NameNode is selected for migration, a manual failover 201 | will be performed before the role is migrated. The role will remain in 202 | standby mode after the migration is complete. 203 | 204 | When migrating a NameNode role, the co-located Failover Controller 205 | role must be migrated as well. The Failover Controller role name must 206 | be included in the list of role names to migrate specified in the 207 | arguments to this command (it will not be included implicitly). This 208 | command does not allow a Failover Controller role to be moved by itself, 209 | although it is possible to move a JournalNode independently. 210 | 211 | @param role_names_to_migrate: list of role names to migrate. 212 | @param destination_host_id: the id of the host to which the roles 213 | should be migrated. 214 | @param clear_stale_role_data: true to delete existing stale role data, 215 | if any. For example, when migrating a 216 | NameNode, if the destination host has 217 | stale data in the NameNode data 218 | directories (possibly because a NameNode 219 | role was previously located there), this 220 | stale data will be deleted before migrating 221 | the role. 222 | @return: Reference to the submitted command. 223 | @since: API v10 224 | """ 225 | args = dict( 226 | roleNamesToMigrate = role_names_to_migrate, 227 | destinationHostId = destination_host_id, 228 | clearStaleRoleData = clear_stale_role_data) 229 | return self._cmd('migrateRoles', data=args, api_version=10) 230 | 231 | def set_rack_id(self, rackId): 232 | """ 233 | Update the rack ID of this host. 234 | """ 235 | self.rackId = rackId 236 | self._put_host() 237 | -------------------------------------------------------------------------------- /clusterdock/topologies/cdh/cm_api/endpoints/parcels.py: -------------------------------------------------------------------------------- 1 | # Licensed to Cloudera, Inc. under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. Cloudera, Inc. licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from cm_api.endpoints.types import * 18 | 19 | __docformat__ = "epytext" 20 | 21 | PARCELS_PATH = "/clusters/%s/parcels" 22 | PARCEL_PATH = "/clusters/%s/parcels/products/%s/versions/%s" 23 | 24 | def get_parcel(resource_root, product, version, cluster_name="default"): 25 | """ 26 | Lookup a parcel by name 27 | @param resource_root: The root Resource object. 28 | @param product: Parcel product name 29 | @param version: Parcel version 30 | @param cluster_name: Cluster name 31 | @return: An ApiService object 32 | """ 33 | return _get_parcel(resource_root, PARCEL_PATH % (cluster_name, product, version)) 34 | 35 | def _get_parcel(resource_root, path): 36 | return call(resource_root.get, path, ApiParcel, api_version=3) 37 | 38 | def get_all_parcels(resource_root, cluster_name="default", view=None): 39 | """ 40 | Get all parcels 41 | @param resource_root: The root Resource object. 42 | @param cluster_name: Cluster name 43 | @return: A list of ApiParcel objects. 44 | @since: API v3 45 | """ 46 | return call(resource_root.get, PARCELS_PATH % (cluster_name,), 47 | ApiParcel, True, params=view and dict(view=view) or None, api_version=3) 48 | 49 | class ApiParcelState(BaseApiObject): 50 | """ 51 | An object that represents the state of a parcel. 52 | """ 53 | _ATTRIBUTES = { 54 | 'progress' : ROAttr(), 55 | 'totalProgress' : ROAttr(), 56 | 'count' : ROAttr(), 57 | 'totalCount' : ROAttr(), 58 | 'warnings' : ROAttr(), 59 | 'errors' : ROAttr(), 60 | } 61 | 62 | def __init__(self, resource_root): 63 | BaseApiObject.init(self, resource_root) 64 | 65 | def __str__(self): 66 | return ": (progress: %s) (totalProgress: %s) (count: %s) (totalCount: %s)" % ( 67 | self.progress, self.totalProgress, self.count, self.totalCount) 68 | 69 | class ApiParcel(BaseApiResource): 70 | """ 71 | An object that represents a parcel and allows administrative operations. 72 | 73 | @since: API v3 74 | """ 75 | _ATTRIBUTES = { 76 | 'product' : ROAttr(), 77 | 'version' : ROAttr(), 78 | 'stage' : ROAttr(), 79 | 'state' : ROAttr(ApiParcelState), 80 | 'clusterRef' : ROAttr(ApiClusterRef), 81 | } 82 | 83 | def __init__(self, resource_root): 84 | BaseApiObject.init(self, resource_root) 85 | 86 | def __str__(self): 87 | return ": %s-%s (stage: %s) (state: %s) (cluster: %s)" % ( 88 | self.product, self.version, self.stage, self.state, self._get_cluster_name()) 89 | 90 | def _api_version(self): 91 | return 3 92 | 93 | def _path(self): 94 | """ 95 | Return the API path for this service. 96 | """ 97 | return PARCEL_PATH % (self._get_cluster_name(), self.product, self.version) 98 | 99 | def _get_cluster_name(self): 100 | if self.clusterRef: 101 | return self.clusterRef.clusterName 102 | return None 103 | 104 | def start_download(self): 105 | """ 106 | Start the download of the parcel 107 | 108 | @return: Reference to the completed command. 109 | """ 110 | return self._cmd('startDownload') 111 | 112 | def cancel_download(self): 113 | """ 114 | Cancels the parcel download. If the parcel is not 115 | currently downloading an exception is raised. 116 | 117 | @return: Reference to the completed command. 118 | """ 119 | return self._cmd('cancelDownload') 120 | 121 | def remove_download(self): 122 | """ 123 | Removes the downloaded parcel 124 | 125 | @return: Reference to the completed command. 126 | """ 127 | return self._cmd('removeDownload') 128 | 129 | def start_distribution(self): 130 | """ 131 | Start the distribution of the parcel to all hosts 132 | in the cluster. 133 | 134 | @return: Reference to the completed command. 135 | """ 136 | return self._cmd('startDistribution') 137 | 138 | def cancel_distribution(self): 139 | """ 140 | Cancels the parcel distrubution. If the parcel is not 141 | currently distributing an exception is raised. 142 | 143 | @return: Reference to the completed command 144 | """ 145 | return self._cmd('cancelDistribution') 146 | 147 | def start_removal_of_distribution(self): 148 | """ 149 | Start the removal of the distribution of the parcel 150 | from all the hosts in the cluster. 151 | 152 | @return: Reference to the completed command. 153 | """ 154 | return self._cmd('startRemovalOfDistribution') 155 | 156 | def activate(self): 157 | """ 158 | Activate the parcel on all the hosts in the cluster. 159 | 160 | @return: Reference to the completed command. 161 | """ 162 | return self._cmd('activate') 163 | 164 | def deactivate(self): 165 | """ 166 | Deactivates the parcel on all the hosts in the cluster. 167 | 168 | @return: Reference to the completed command. 169 | """ 170 | return self._cmd('deactivate') 171 | 172 | -------------------------------------------------------------------------------- /clusterdock/topologies/cdh/cm_api/endpoints/role_config_groups.py: -------------------------------------------------------------------------------- 1 | # Licensed to Cloudera, Inc. under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. Cloudera, Inc. licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from cm_api.endpoints.types import * 18 | from cm_api.endpoints.roles import ApiRole 19 | 20 | __docformat__ = "epytext" 21 | 22 | ROLE_CONFIG_GROUPS_PATH = "/clusters/%s/services/%s/roleConfigGroups" 23 | CM_ROLE_CONFIG_GROUPS_PATH = "/cm/service/roleConfigGroups" 24 | 25 | def _get_role_config_groups_path(cluster_name, service_name): 26 | if cluster_name: 27 | return ROLE_CONFIG_GROUPS_PATH % (cluster_name, service_name) 28 | else: 29 | return CM_ROLE_CONFIG_GROUPS_PATH 30 | 31 | def _get_role_config_group_path(cluster_name, service_name, name): 32 | path = _get_role_config_groups_path(cluster_name, service_name) 33 | return "%s/%s" % (path, name) 34 | 35 | def create_role_config_groups(resource_root, service_name, apigroup_list, 36 | cluster_name="default"): 37 | """ 38 | Create role config groups. 39 | @param resource_root: The root Resource object. 40 | @param service_name: Service name. 41 | @param apigroup_list: List of role config groups to create. 42 | @param cluster_name: Cluster name. 43 | @return: New ApiRoleConfigGroup object. 44 | @since: API v3 45 | """ 46 | return call(resource_root.post, 47 | _get_role_config_groups_path(cluster_name, service_name), 48 | ApiRoleConfigGroup, True, data=apigroup_list, api_version=3) 49 | 50 | def create_role_config_group(resource_root, service_name, name, display_name, 51 | role_type, cluster_name="default"): 52 | """ 53 | Create a role config group. 54 | @param resource_root: The root Resource object. 55 | @param service_name: Service name. 56 | @param name: The name of the new group. 57 | @param display_name: The display name of the new group. 58 | @param role_type: The role type of the new group. 59 | @param cluster_name: Cluster name. 60 | @return: List of created role config groups. 61 | """ 62 | apigroup = ApiRoleConfigGroup(resource_root, name, display_name, role_type) 63 | return create_role_config_groups(resource_root, service_name, [apigroup], 64 | cluster_name)[0] 65 | 66 | def get_role_config_group(resource_root, service_name, name, 67 | cluster_name="default"): 68 | """ 69 | Find a role config group by name. 70 | @param resource_root: The root Resource object. 71 | @param service_name: Service name. 72 | @param name: Role config group name. 73 | @param cluster_name: Cluster name. 74 | @return: An ApiRoleConfigGroup object. 75 | """ 76 | return _get_role_config_group(resource_root, _get_role_config_group_path( 77 | cluster_name, service_name, name)) 78 | 79 | def _get_role_config_group(resource_root, path): 80 | return call(resource_root.get, path, ApiRoleConfigGroup, api_version=3) 81 | 82 | def get_all_role_config_groups(resource_root, service_name, 83 | cluster_name="default"): 84 | """ 85 | Get all role config groups in the specified service. 86 | @param resource_root: The root Resource object. 87 | @param service_name: Service name. 88 | @param cluster_name: Cluster name. 89 | @return: A list of ApiRoleConfigGroup objects. 90 | @since: API v3 91 | """ 92 | return call(resource_root.get, 93 | _get_role_config_groups_path(cluster_name, service_name), 94 | ApiRoleConfigGroup, True, api_version=3) 95 | 96 | def update_role_config_group(resource_root, service_name, name, apigroup, 97 | cluster_name="default"): 98 | """ 99 | Update a role config group by name. 100 | @param resource_root: The root Resource object. 101 | @param service_name: Service name. 102 | @param name: Role config group name. 103 | @param apigroup: The updated role config group. 104 | @param cluster_name: Cluster name. 105 | @return: The updated ApiRoleConfigGroup object. 106 | @since: API v3 107 | """ 108 | return call(resource_root.put, 109 | _get_role_config_group_path(cluster_name, service_name, name), 110 | ApiRoleConfigGroup, data=apigroup, api_version=3) 111 | 112 | def delete_role_config_group(resource_root, service_name, name, 113 | cluster_name="default"): 114 | """ 115 | Delete a role config group by name. 116 | @param resource_root: The root Resource object. 117 | @param service_name: Service name. 118 | @param name: Role config group name. 119 | @param cluster_name: Cluster name. 120 | @return: The deleted ApiRoleConfigGroup object. 121 | @since: API v3 122 | """ 123 | return call(resource_root.delete, 124 | _get_role_config_group_path(cluster_name, service_name, name), 125 | ApiRoleConfigGroup, api_version=3) 126 | 127 | def move_roles(resource_root, service_name, name, role_names, 128 | cluster_name="default"): 129 | """ 130 | Moves roles to the specified role config group. 131 | 132 | The roles can be moved from any role config group belonging 133 | to the same service. The role type of the destination group 134 | must match the role type of the roles. 135 | 136 | @param name: The name of the group the roles will be moved to. 137 | @param role_names: The names of the roles to move. 138 | @return: List of roles which have been moved successfully. 139 | @since: API v3 140 | """ 141 | return call(resource_root.put, 142 | _get_role_config_group_path(cluster_name, service_name, name) + '/roles', 143 | ApiRole, True, data=role_names, api_version=3) 144 | 145 | def move_roles_to_base_role_config_group(resource_root, service_name, 146 | role_names, cluster_name="default"): 147 | """ 148 | Moves roles to the base role config group. 149 | 150 | The roles can be moved from any role config group belonging to the same 151 | service. The role type of the roles may vary. Each role will be moved to 152 | its corresponding base group depending on its role type. 153 | 154 | @param role_names: The names of the roles to move. 155 | @return: List of roles which have been moved successfully. 156 | @since: API v3 157 | """ 158 | return call(resource_root.put, 159 | _get_role_config_groups_path(cluster_name, service_name) + '/roles', 160 | ApiRole, True, data=role_names, api_version=3) 161 | 162 | 163 | class ApiRoleConfigGroup(BaseApiResource): 164 | """ 165 | name is RW only temporarily; once all RCG names are unique, 166 | this property will be auto-generated and Read-only 167 | 168 | @since: API v3 169 | """ 170 | _ATTRIBUTES = { 171 | 'name' : None, 172 | 'displayName' : None, 173 | 'roleType' : None, 174 | 'config' : Attr(ApiConfig), 175 | 'base' : ROAttr(), 176 | 'serviceRef' : ROAttr(ApiServiceRef), 177 | } 178 | 179 | def __init__(self, resource_root, name=None, displayName=None, roleType=None, 180 | config=None): 181 | BaseApiObject.init(self, resource_root, locals()) 182 | 183 | def __str__(self): 184 | return ": %s (cluster: %s; service: %s)" % ( 185 | self.name, self.serviceRef.clusterName, self.serviceRef.serviceName) 186 | 187 | def _api_version(self): 188 | return 3 189 | 190 | def _path(self): 191 | return _get_role_config_group_path(self.serviceRef.clusterName, 192 | self.serviceRef.serviceName, 193 | self.name) 194 | 195 | def get_config(self, view = None): 196 | """ 197 | Retrieve the group's configuration. 198 | 199 | The 'summary' view contains strings as the dictionary values. The full 200 | view contains ApiConfig instances as the values. 201 | 202 | @param view: View to materialize ('full' or 'summary'). 203 | @return: Dictionary with configuration data. 204 | """ 205 | path = self._path() + '/config' 206 | resp = self._get_resource_root().get(path, 207 | params = view and dict(view=view) or None) 208 | return json_to_config(resp, view == 'full') 209 | 210 | def update_config(self, config): 211 | """ 212 | Update the group's configuration. 213 | 214 | @param config: Dictionary with configuration to update. 215 | @return: Dictionary with updated configuration. 216 | """ 217 | path = self._path() + '/config' 218 | resp = self._get_resource_root().put(path, data = config_to_json(config)) 219 | return json_to_config(resp) 220 | 221 | def get_all_roles(self): 222 | """ 223 | Retrieve the roles in this role config group. 224 | 225 | @return: List of roles in this role config group. 226 | """ 227 | return self._get("roles", ApiRole, True) 228 | 229 | def move_roles(self, roles): 230 | """ 231 | Moves roles to this role config group. 232 | 233 | The roles can be moved from any role config group belonging 234 | to the same service. The role type of the destination group 235 | must match the role type of the roles. 236 | 237 | @param roles: The names of the roles to move. 238 | @return: List of roles which have been moved successfully. 239 | """ 240 | return move_roles(self._get_resource_root(), self.serviceRef.serviceName, 241 | self.name, roles, self.serviceRef.clusterName) 242 | -------------------------------------------------------------------------------- /clusterdock/topologies/cdh/cm_api/endpoints/roles.py: -------------------------------------------------------------------------------- 1 | # Licensed to Cloudera, Inc. under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. Cloudera, Inc. licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from cm_api.endpoints.types import * 18 | 19 | __docformat__ = "epytext" 20 | 21 | ROLES_PATH = "/clusters/%s/services/%s/roles" 22 | CM_ROLES_PATH = "/cm/service/roles" 23 | 24 | def _get_roles_path(cluster_name, service_name): 25 | if cluster_name: 26 | return ROLES_PATH % (cluster_name, service_name) 27 | else: 28 | return CM_ROLES_PATH 29 | 30 | def _get_role_path(cluster_name, service_name, role_name): 31 | path = _get_roles_path(cluster_name, service_name) 32 | return "%s/%s" % (path, role_name) 33 | 34 | def create_role(resource_root, 35 | service_name, 36 | role_type, 37 | role_name, 38 | host_id, 39 | cluster_name="default"): 40 | """ 41 | Create a role 42 | @param resource_root: The root Resource object. 43 | @param service_name: Service name 44 | @param role_type: Role type 45 | @param role_name: Role name 46 | @param cluster_name: Cluster name 47 | @return: An ApiRole object 48 | """ 49 | apirole = ApiRole(resource_root, role_name, role_type, 50 | ApiHostRef(resource_root, host_id)) 51 | return call(resource_root.post, 52 | _get_roles_path(cluster_name, service_name), 53 | ApiRole, True, data=[apirole])[0] 54 | 55 | def get_role(resource_root, service_name, name, cluster_name="default"): 56 | """ 57 | Lookup a role by name 58 | @param resource_root: The root Resource object. 59 | @param service_name: Service name 60 | @param name: Role name 61 | @param cluster_name: Cluster name 62 | @return: An ApiRole object 63 | """ 64 | return _get_role(resource_root, _get_role_path(cluster_name, service_name, name)) 65 | 66 | def _get_role(resource_root, path): 67 | return call(resource_root.get, path, ApiRole) 68 | 69 | def get_all_roles(resource_root, service_name, cluster_name="default", view=None): 70 | """ 71 | Get all roles 72 | @param resource_root: The root Resource object. 73 | @param service_name: Service name 74 | @param cluster_name: Cluster name 75 | @return: A list of ApiRole objects. 76 | """ 77 | return call(resource_root.get, 78 | _get_roles_path(cluster_name, service_name), 79 | ApiRole, True, params=view and dict(view=view) or None) 80 | 81 | def get_roles_by_type(resource_root, service_name, role_type, 82 | cluster_name="default", view=None): 83 | """ 84 | Get all roles of a certain type in a service 85 | @param resource_root: The root Resource object. 86 | @param service_name: Service name 87 | @param role_type: Role type 88 | @param cluster_name: Cluster name 89 | @return: A list of ApiRole objects. 90 | """ 91 | roles = get_all_roles(resource_root, service_name, cluster_name, view) 92 | return [ r for r in roles if r.type == role_type ] 93 | 94 | def delete_role(resource_root, service_name, name, cluster_name="default"): 95 | """ 96 | Delete a role by name 97 | @param resource_root: The root Resource object. 98 | @param service_name: Service name 99 | @param name: Role name 100 | @param cluster_name: Cluster name 101 | @return: The deleted ApiRole object 102 | """ 103 | return call(resource_root.delete, 104 | _get_role_path(cluster_name, service_name, name), ApiRole) 105 | 106 | 107 | class ApiRole(BaseApiResource): 108 | _ATTRIBUTES = { 109 | 'name' : None, 110 | 'type' : None, 111 | 'hostRef' : Attr(ApiHostRef), 112 | 'roleState' : ROAttr(), 113 | 'healthSummary' : ROAttr(), 114 | 'healthChecks' : ROAttr(), 115 | 'serviceRef' : ROAttr(ApiServiceRef), 116 | 'configStale' : ROAttr(), 117 | 'configStalenessStatus' : ROAttr(), 118 | 'haStatus' : ROAttr(), 119 | 'roleUrl' : ROAttr(), 120 | 'commissionState' : ROAttr(), 121 | 'maintenanceMode' : ROAttr(), 122 | 'maintenanceOwners' : ROAttr(), 123 | 'roleConfigGroupRef' : ROAttr(ApiRoleConfigGroupRef), 124 | 'zooKeeperServerMode' : ROAttr(), 125 | 'entityStatus' : ROAttr(), 126 | } 127 | 128 | def __init__(self, resource_root, name=None, type=None, hostRef=None): 129 | BaseApiObject.init(self, resource_root, locals()) 130 | 131 | def __str__(self): 132 | return ": %s (cluster: %s; service: %s)" % ( 133 | self.name, self.serviceRef.clusterName, self.serviceRef.serviceName) 134 | 135 | def _path(self): 136 | return _get_role_path(self.serviceRef.clusterName, 137 | self.serviceRef.serviceName, 138 | self.name) 139 | 140 | def _get_log(self, log): 141 | path = "%s/logs/%s" % (self._path(), log) 142 | return self._get_resource_root().get(path) 143 | 144 | def get_commands(self, view=None): 145 | """ 146 | Retrieve a list of running commands for this role. 147 | 148 | @param view: View to materialize ('full' or 'summary') 149 | @return: A list of running commands. 150 | """ 151 | return self._get("commands", ApiCommand, True, 152 | params = view and dict(view=view) or None) 153 | 154 | def get_config(self, view = None): 155 | """ 156 | Retrieve the role's configuration. 157 | 158 | The 'summary' view contains strings as the dictionary values. The full 159 | view contains ApiConfig instances as the values. 160 | 161 | @param view: View to materialize ('full' or 'summary') 162 | @return: Dictionary with configuration data. 163 | """ 164 | return self._get_config("config", view) 165 | 166 | def update_config(self, config): 167 | """ 168 | Update the role's configuration. 169 | 170 | @param config: Dictionary with configuration to update. 171 | @return: Dictionary with updated configuration. 172 | """ 173 | return self._update_config("config", config) 174 | 175 | def get_full_log(self): 176 | """ 177 | Retrieve the contents of the role's log file. 178 | 179 | @return: Contents of log file. 180 | """ 181 | return self._get_log('full') 182 | 183 | def get_stdout(self): 184 | """ 185 | Retrieve the contents of the role's standard output. 186 | 187 | @return: Contents of stdout. 188 | """ 189 | return self._get_log('stdout') 190 | 191 | def get_stderr(self): 192 | """ 193 | Retrieve the contents of the role's standard error. 194 | 195 | @return: Contents of stderr. 196 | """ 197 | return self._get_log('stderr') 198 | 199 | def get_stacks_log(self): 200 | """ 201 | Retrieve the contents of the role's stacks log file. 202 | 203 | @return: Contents of stacks log file. 204 | @since: API v8 205 | """ 206 | return self._get_log('stacks') 207 | 208 | def get_stacks_logs_bundle(self): 209 | """ 210 | Retrieve a zip file of the role's stacks log files. 211 | 212 | @return: A zipfile of stacks log files. 213 | @since: API v8 214 | """ 215 | return self._get_log('stacksBundle') 216 | 217 | def get_metrics(self, from_time=None, to_time=None, metrics=None, view=None): 218 | """ 219 | This endpoint is not supported as of v6. Use the timeseries API 220 | instead. To get all metrics for a role with the timeseries API use 221 | the query: 222 | 223 | 'select * where roleName = $ROLE_NAME'. 224 | 225 | To get specific metrics for a role use a comma-separated list of 226 | the metric names as follows: 227 | 228 | 'select $METRIC_NAME1, $METRIC_NAME2 where roleName = $ROLE_NAME'. 229 | 230 | For more information see http://tiny.cloudera.com/tsquery_doc 231 | @param from_time: A datetime; start of the period to query (optional). 232 | @param to_time: A datetime; end of the period to query (default = now). 233 | @param metrics: List of metrics to query (default = all). 234 | @param view: View to materialize ('full' or 'summary') 235 | @return: List of metrics and their readings. 236 | """ 237 | return self._get_resource_root().get_metrics(self._path() + '/metrics', 238 | from_time, to_time, metrics, view) 239 | 240 | def enter_maintenance_mode(self): 241 | """ 242 | Put the role in maintenance mode. 243 | 244 | @return: Reference to the completed command. 245 | @since: API v2 246 | """ 247 | cmd = self._cmd('enterMaintenanceMode') 248 | if cmd.success: 249 | self._update(_get_role(self._get_resource_root(), self._path())) 250 | return cmd 251 | 252 | def exit_maintenance_mode(self): 253 | """ 254 | Take the role out of maintenance mode. 255 | 256 | @return: Reference to the completed command. 257 | @since: API v2 258 | """ 259 | cmd = self._cmd('exitMaintenanceMode') 260 | if cmd.success: 261 | self._update(_get_role(self._get_resource_root(), self._path())) 262 | return cmd 263 | 264 | def list_commands_by_name(self): 265 | """ 266 | Lists all the commands that can be executed by name 267 | on the provided role. 268 | 269 | @return: A list of command metadata objects 270 | @since: API v6 271 | """ 272 | return self._get("commandsByName", ApiCommandMetadata, True, api_version=6) 273 | 274 | 275 | -------------------------------------------------------------------------------- /clusterdock/topologies/cdh/cm_api/endpoints/timeseries.py: -------------------------------------------------------------------------------- 1 | # Licensed to Cloudera, Inc. under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. Cloudera, Inc. licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import datetime 18 | 19 | from cm_api.endpoints.types import * 20 | 21 | __docformat__ = "epytext" 22 | 23 | TIME_SERIES_PATH = "/timeseries" 24 | METRIC_SCHEMA_PATH = "/timeseries/schema" 25 | METRIC_ENTITY_TYPE_PATH = "/timeseries/entityTypes" 26 | METRIC_ENTITY_ATTR_PATH = "/timeseries/entityTypeAttributes" 27 | 28 | def query_timeseries(resource_root, query, from_time=None, to_time=None, 29 | desired_rollup=None, must_use_desired_rollup=None, by_post=False): 30 | """ 31 | Query for time series data from the CM time series data store. 32 | @param query: Query string. 33 | @param from_time: Start of the period to query (optional). 34 | @type from_time: datetime.datetime Note the that datetime must either be time 35 | zone aware or specified in the server time zone. See the 36 | python datetime documentation for more details about python's 37 | time zone handling. 38 | @param to_time: End of the period to query (default = now). 39 | This may be an ISO format string, or a datetime object. 40 | @type to_time: datetime.datetime Note the that datetime must either be time 41 | zone aware or specified in the server time zone. See the 42 | python datetime documentation for more details about python's 43 | time zone handling. 44 | @param desired_rollup: The aggregate rollup to get data for. This can be 45 | RAW, TEN_MINUTELY, HOURLY, SIX_HOURLY, DAILY, or 46 | WEEKLY. Note that rollup desired is only a hint unless 47 | must_use_desired_rollup is set to true. 48 | @param must_use_desired_rollup: Indicates that the monitoring server should 49 | return the data at the rollup desired. 50 | @param by_post: If true, an HTTP POST request will be made to server. This 51 | allows longer query string to be accepted compared to HTTP 52 | GET request. 53 | @return: List of ApiTimeSeriesResponse 54 | """ 55 | data = None 56 | params = {} 57 | request_method = resource_root.get 58 | if by_post: 59 | request = ApiTimeSeriesRequest(resource_root, 60 | query=query) 61 | data = request 62 | request_method = resource_root.post 63 | elif query: 64 | params['query'] = query 65 | 66 | if from_time: 67 | params['from'] = from_time.isoformat() 68 | if to_time: 69 | params['to'] = to_time.isoformat() 70 | if desired_rollup: 71 | params['desiredRollup'] = desired_rollup 72 | if must_use_desired_rollup: 73 | params['mustUseDesiredRollup'] = must_use_desired_rollup 74 | return call(request_method, TIME_SERIES_PATH, 75 | ApiTimeSeriesResponse, True, params=params, data=data) 76 | 77 | def get_metric_schema(resource_root): 78 | """ 79 | Get the schema for all of the metrics. 80 | @return: List of metric schema. 81 | """ 82 | return call(resource_root.get, METRIC_SCHEMA_PATH, 83 | ApiMetricSchema, True) 84 | 85 | def get_entity_types(resource_root): 86 | """ 87 | Get the time series entity types that CM monitors. 88 | @return: List of time series entity type. 89 | """ 90 | return call(resource_root.get, METRIC_ENTITY_TYPE_PATH, 91 | ApiTimeSeriesEntityType, True) 92 | 93 | def get_entity_attributes(resource_root): 94 | """ 95 | Get the time series entity attributes that CM monitors. 96 | @return: List of time series entity attribute. 97 | """ 98 | return call(resource_root.get, METRIC_ENTITY_ATTR_PATH, 99 | ApiTimeSeriesEntityAttribute, True) 100 | 101 | class ApiTimeSeriesCrossEntityMetadata(BaseApiObject): 102 | _ATTRIBUTES = { 103 | 'maxEntityDisplayName' : ROAttr(), 104 | 'minEntityDisplayName' : ROAttr(), 105 | 'maxEntityName' : ROAttr(), 106 | 'minEntityName' : ROAttr(), 107 | 'numEntities' : ROAttr() 108 | } 109 | 110 | class ApiTimeSeriesAggregateStatistics(BaseApiObject): 111 | _ATTRIBUTES = { 112 | 'sampleTime' : ROAttr(datetime.datetime), 113 | 'sampleValue' : ROAttr(), 114 | 'count' : ROAttr(), 115 | 'min' : ROAttr(), 116 | 'minTime' : ROAttr(datetime.datetime), 117 | 'max' : ROAttr(), 118 | 'maxTime' : ROAttr(datetime.datetime), 119 | 'mean' : ROAttr(), 120 | 'stdDev' : ROAttr(), 121 | 'crossEntityMetadata' : ROAttr(ApiTimeSeriesCrossEntityMetadata) 122 | } 123 | 124 | class ApiTimeSeriesData(BaseApiObject): 125 | _ATTRIBUTES = { 126 | 'timestamp' : ROAttr(datetime.datetime), 127 | 'value' : ROAttr(), 128 | 'type' : ROAttr(), 129 | 'aggregateStatistics' : ROAttr(ApiTimeSeriesAggregateStatistics) 130 | } 131 | 132 | class ApiTimeSeriesMetadata(BaseApiObject): 133 | _ATTRIBUTES = { 134 | 'metricName' : ROAttr(), 135 | 'entityName' : ROAttr(), 136 | 'startTime' : ROAttr(datetime.datetime), 137 | 'endTime' : ROAttr(datetime.datetime), 138 | 'attributes' : ROAttr(), 139 | 'unitNumerators' : ROAttr(), 140 | 'unitDenominators' : ROAttr(), 141 | 'expression' : ROAttr(), 142 | 'alias' : ROAttr(), 143 | 'metricCollectionFrequencyMs': ROAttr(), 144 | 'rollupUsed' : ROAttr() 145 | } 146 | 147 | class ApiTimeSeries(BaseApiObject): 148 | _ATTRIBUTES = { 149 | 'metadata' : ROAttr(ApiTimeSeriesMetadata), 150 | 'data' : ROAttr(ApiTimeSeriesData), 151 | } 152 | 153 | class ApiTimeSeriesResponse(BaseApiObject): 154 | _ATTRIBUTES = { 155 | 'timeSeries' : ROAttr(ApiTimeSeries), 156 | 'warnings' : ROAttr(), 157 | 'errors' : ROAttr(), 158 | 'timeSeriesQuery' : ROAttr(), 159 | } 160 | 161 | class ApiMetricSchema(BaseApiObject): 162 | _ATTRIBUTES = { 163 | 'name' : ROAttr(), 164 | 'displayName' : ROAttr(), 165 | 'description' : ROAttr(), 166 | 'isCounter' : ROAttr(), 167 | 'unitNumerator' : ROAttr(), 168 | 'unitDenominator' : ROAttr(), 169 | 'aliases' : ROAttr(), 170 | 'sources' : ROAttr(), 171 | } 172 | 173 | class ApiTimeSeriesEntityAttribute(BaseApiObject): 174 | _ATTRIBUTES = { 175 | 'name' : ROAttr(), 176 | 'displayName' : ROAttr(), 177 | 'description' : ROAttr(), 178 | 'isValueCaseSensitive' : ROAttr() 179 | } 180 | 181 | class ApiTimeSeriesEntityType(BaseApiObject): 182 | _ATTRIBUTES = { 183 | 'name' : ROAttr(), 184 | 'category' : ROAttr(), 185 | 'displayName' : ROAttr(), 186 | 'description' : ROAttr(), 187 | 'nameForCrossEntityAggregateMetrics' : ROAttr(), 188 | 'immutableAttributeNames' : ROAttr(), 189 | 'mutableAttributeNames' : ROAttr(), 190 | 'entityNameFormat' : ROAttr(), 191 | 'entityDisplayNameForamt' : ROAttr(), 192 | 'parentMetricEntityTypeNames' : ROAttr() 193 | } 194 | -------------------------------------------------------------------------------- /clusterdock/topologies/cdh/cm_api/endpoints/tools.py: -------------------------------------------------------------------------------- 1 | # Licensed to Cloudera, Inc. under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. Cloudera, Inc. licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | __docformat__ = "epytext" 18 | 19 | 20 | ECHO_PATH = "/tools/echo" 21 | ECHO_ERROR_PATH = "/tools/echoError" 22 | 23 | def echo(root_resource, message): 24 | """Have the server echo our message back.""" 25 | params = dict(message=message) 26 | return root_resource.get(ECHO_PATH, params) 27 | 28 | def echo_error(root_resource, message): 29 | """Generate an error, but we get to set the error message.""" 30 | params = dict(message=message) 31 | return root_resource.get(ECHO_ERROR_PATH, params) 32 | -------------------------------------------------------------------------------- /clusterdock/topologies/cdh/cm_api/endpoints/users.py: -------------------------------------------------------------------------------- 1 | # Licensed to Cloudera, Inc. under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. Cloudera, Inc. licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from cm_api.endpoints.types import * 18 | 19 | USERS_PATH = "/users" 20 | 21 | def get_all_users(resource_root, view=None): 22 | """ 23 | Get all users. 24 | 25 | @param resource_root: The root Resource object 26 | @param view: View to materialize ('full' or 'summary'). 27 | @return: A list of ApiUser objects. 28 | """ 29 | return call(resource_root.get, USERS_PATH, ApiUser, True, 30 | params=view and dict(view=view) or None) 31 | 32 | def get_user(resource_root, username): 33 | """ 34 | Look up a user by username. 35 | 36 | @param resource_root: The root Resource object 37 | @param username: Username to look up 38 | @return: An ApiUser object 39 | """ 40 | return call(resource_root.get, 41 | '%s/%s' % (USERS_PATH, username), ApiUser) 42 | 43 | def create_user(resource_root, username, password, roles): 44 | """ 45 | Create a user. 46 | 47 | @param resource_root: The root Resource object 48 | @param username: Username 49 | @param password: Password 50 | @param roles: List of roles for the user. This should be [] or ['ROLE_USER'] 51 | for a regular user, ['ROLE_ADMIN'] for an admin, or 52 | ['ROLE_LIMITED'] for a limited admin. 53 | @return: An ApiUser object 54 | """ 55 | apiuser = ApiUser(resource_root, username, password=password, roles=roles) 56 | return call(resource_root.post, USERS_PATH, ApiUser, True, 57 | data=[apiuser])[0] 58 | 59 | def delete_user(resource_root, username): 60 | """ 61 | Delete user by username. 62 | 63 | @param resource_root: The root Resource object 64 | @param username: Username 65 | @return: An ApiUser object 66 | """ 67 | return call(resource_root.delete, 68 | '%s/%s' % (USERS_PATH, username), ApiUser) 69 | 70 | def update_user(resource_root, user): 71 | """ 72 | Update a user. 73 | 74 | Replaces the user's details with those provided. 75 | 76 | @param resource_root: The root Resource object 77 | @param user: An ApiUser object 78 | @return: An ApiUser object 79 | """ 80 | return call(resource_root.put, 81 | '%s/%s' % (USERS_PATH, user.name), ApiUser, data=user) 82 | 83 | class ApiUser(BaseApiResource): 84 | _ATTRIBUTES = { 85 | 'name' : None, 86 | 'password' : None, 87 | 'roles' : None, 88 | } 89 | 90 | def __init__(self, resource_root, name=None, password=None, roles=None): 91 | BaseApiObject.init(self, resource_root, locals()) 92 | 93 | def _path(self): 94 | return '%s/%s' % (USERS_PATH, self.name) 95 | 96 | def grant_admin_role(self): 97 | """ 98 | Grant admin access to a user. If the user already has admin access, this 99 | does nothing. If the user currently has a non-admin role, it will be replaced 100 | with the admin role. 101 | 102 | @return: An ApiUser object 103 | """ 104 | apiuser = ApiUser(self._get_resource_root(), self.name, roles=['ROLE_ADMIN']) 105 | return self._put('', ApiUser, data=apiuser) 106 | 107 | def revoke_admin_role(self): 108 | """ 109 | Revoke admin access from a user. If the user does not have admin access, 110 | this does nothing. After revocation, the user will have the un-privileged 111 | regular user role. 112 | 113 | @return: An ApiUser object 114 | """ 115 | apiuser = ApiUser(self._get_resource_root(), self.name, roles=[]) 116 | return self._put('', ApiUser, data=apiuser) 117 | -------------------------------------------------------------------------------- /clusterdock/topologies/cdh/cm_api/http_client.py: -------------------------------------------------------------------------------- 1 | # Licensed to Cloudera, Inc. under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. Cloudera, Inc. licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import os 18 | import cookielib 19 | import logging 20 | import posixpath 21 | import types 22 | import urllib 23 | 24 | try: 25 | import socks 26 | import socket 27 | socks_server = os.environ.get("SOCKS_SERVER", None) 28 | if socks_server: 29 | host, port = socks_server.split(":") 30 | socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, host, int(port)) 31 | socket.socket = socks.socksocket 32 | except ImportError: 33 | pass 34 | 35 | import urllib2 36 | 37 | __docformat__ = "epytext" 38 | 39 | LOG = logging.getLogger(__name__) 40 | 41 | class RestException(Exception): 42 | """ 43 | Any error result from the Rest API is converted into this exception type. 44 | """ 45 | def __init__(self, error): 46 | Exception.__init__(self, error) 47 | self._error = error 48 | self._code = None 49 | self._message = str(error) 50 | # See if there is a code or a message. (For urllib2.HTTPError.) 51 | try: 52 | self._code = error.code 53 | self._message = error.read() 54 | except AttributeError: 55 | pass 56 | 57 | def __str__(self): 58 | res = self._message or "" 59 | if self._code is not None: 60 | res += " (error %s)" % (self._code,) 61 | return res 62 | 63 | def get_parent_ex(self): 64 | if isinstance(self._error, Exception): 65 | return self._error 66 | return None 67 | 68 | @property 69 | def code(self): 70 | return self._code 71 | 72 | @property 73 | def message(self): 74 | return self._message 75 | 76 | 77 | class HttpClient(object): 78 | """ 79 | Basic HTTP client tailored for rest APIs. 80 | """ 81 | def __init__(self, base_url, exc_class=None, logger=None): 82 | """ 83 | @param base_url: The base url to the API. 84 | @param exc_class: An exception class to handle non-200 results. 85 | 86 | Creates an HTTP(S) client to connect to the Cloudera Manager API. 87 | """ 88 | self._base_url = base_url.rstrip('/') 89 | self._exc_class = exc_class or RestException 90 | self._logger = logger or LOG 91 | self._headers = { } 92 | 93 | # Make a basic auth handler that does nothing. Set credentials later. 94 | self._passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm() 95 | authhandler = urllib2.HTTPBasicAuthHandler(self._passmgr) 96 | 97 | # Make a cookie processor 98 | cookiejar = cookielib.CookieJar() 99 | 100 | self._opener = urllib2.build_opener( 101 | HTTPErrorProcessor(), 102 | urllib2.HTTPCookieProcessor(cookiejar), 103 | authhandler) 104 | 105 | 106 | def set_basic_auth(self, username, password, realm): 107 | """ 108 | Set up basic auth for the client 109 | @param username: Login name. 110 | @param password: Login password. 111 | @param realm: The authentication realm. 112 | @return: The current object 113 | """ 114 | self._passmgr.add_password(realm, self._base_url, username, password) 115 | return self 116 | 117 | def set_headers(self, headers): 118 | """ 119 | Add headers to the request 120 | @param headers: A dictionary with the key value pairs for the headers 121 | @return: The current object 122 | """ 123 | self._headers = headers 124 | return self 125 | 126 | 127 | @property 128 | def base_url(self): 129 | return self._base_url 130 | 131 | @property 132 | def logger(self): 133 | return self._logger 134 | 135 | def _get_headers(self, headers): 136 | res = self._headers.copy() 137 | if headers: 138 | res.update(headers) 139 | return res 140 | 141 | def execute(self, http_method, path, params=None, data=None, headers=None): 142 | """ 143 | Submit an HTTP request. 144 | @param http_method: GET, POST, PUT, DELETE 145 | @param path: The path of the resource. 146 | @param params: Key-value parameter data. 147 | @param data: The data to attach to the body of the request. 148 | @param headers: The headers to set for this request. 149 | 150 | @return: The result of urllib2.urlopen() 151 | """ 152 | # Prepare URL and params 153 | url = self._make_url(path, params) 154 | if http_method in ("GET", "DELETE"): 155 | if data is not None: 156 | self.logger.warn( 157 | "GET method does not pass any data. Path '%s'" % (path,)) 158 | data = None 159 | 160 | # Setup the request 161 | request = urllib2.Request(url, data) 162 | # Hack/workaround because urllib2 only does GET and POST 163 | request.get_method = lambda: http_method 164 | 165 | headers = self._get_headers(headers) 166 | for k, v in headers.items(): 167 | request.add_header(k, v) 168 | 169 | # Call it 170 | self.logger.debug("%s %s" % (http_method, url)) 171 | try: 172 | return self._opener.open(request) 173 | except urllib2.HTTPError, ex: 174 | raise self._exc_class(ex) 175 | 176 | def _make_url(self, path, params): 177 | res = self._base_url 178 | if path: 179 | res += posixpath.normpath('/' + path.lstrip('/')) 180 | if params: 181 | param_str = urllib.urlencode(params, True) 182 | res += '?' + param_str 183 | return iri_to_uri(res) 184 | 185 | 186 | class HTTPErrorProcessor(urllib2.HTTPErrorProcessor): 187 | """ 188 | Python 2.4 only recognize 200 and 206 as success. It's broken. So we install 189 | the following processor to catch the bug. 190 | """ 191 | def http_response(self, request, response): 192 | if 200 <= response.code < 300: 193 | return response 194 | return urllib2.HTTPErrorProcessor.http_response(self, request, response) 195 | 196 | https_response = http_response 197 | 198 | # 199 | # Method copied from Django 200 | # 201 | def iri_to_uri(iri): 202 | """ 203 | Convert an Internationalized Resource Identifier (IRI) portion to a URI 204 | portion that is suitable for inclusion in a URL. 205 | 206 | This is the algorithm from section 3.1 of RFC 3987. However, since we are 207 | assuming input is either UTF-8 or unicode already, we can simplify things a 208 | little from the full method. 209 | 210 | Returns an ASCII string containing the encoded result. 211 | """ 212 | # The list of safe characters here is constructed from the "reserved" and 213 | # "unreserved" characters specified in sections 2.2 and 2.3 of RFC 3986: 214 | # reserved = gen-delims / sub-delims 215 | # gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@" 216 | # sub-delims = "!" / "$" / "&" / "'" / "(" / ")" 217 | # / "*" / "+" / "," / ";" / "=" 218 | # unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" 219 | # Of the unreserved characters, urllib.quote already considers all but 220 | # the ~ safe. 221 | # The % character is also added to the list of safe characters here, as the 222 | # end of section 3.1 of RFC 3987 specifically mentions that % must not be 223 | # converted. 224 | if iri is None: 225 | return iri 226 | return urllib.quote(smart_str(iri), safe="/#%[]=:;$&()+,!?*@'~") 227 | 228 | # 229 | # Method copied from Django 230 | # 231 | def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'): 232 | """ 233 | Returns a bytestring version of 's', encoded as specified in 'encoding'. 234 | 235 | If strings_only is True, don't convert (some) non-string-like objects. 236 | """ 237 | if strings_only and isinstance(s, (types.NoneType, int)): 238 | return s 239 | elif not isinstance(s, basestring): 240 | try: 241 | return str(s) 242 | except UnicodeEncodeError: 243 | if isinstance(s, Exception): 244 | # An Exception subclass containing non-ASCII data that doesn't 245 | # know how to print itself properly. We shouldn't raise a 246 | # further exception. 247 | return ' '.join([smart_str(arg, encoding, strings_only, 248 | errors) for arg in s]) 249 | return unicode(s).encode(encoding, errors) 250 | elif isinstance(s, unicode): 251 | return s.encode(encoding, errors) 252 | elif s and encoding != 'utf-8': 253 | return s.decode('utf-8', errors).encode(encoding, errors) 254 | else: 255 | return s 256 | 257 | -------------------------------------------------------------------------------- /clusterdock/topologies/cdh/cm_api/resource.py: -------------------------------------------------------------------------------- 1 | # Licensed to Cloudera, Inc. under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. Cloudera, Inc. licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import os 18 | try: 19 | import json 20 | except ImportError: 21 | import simplejson as json 22 | import logging 23 | import posixpath 24 | import time 25 | import socket 26 | try: 27 | import socks 28 | socks_server = os.environ.get("SOCKS_SERVER", None) 29 | if socks_server: 30 | host, port = socks_server.split(":") 31 | socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, host, int(port)) 32 | socket.socket = socks.socksocket 33 | except ImportError: 34 | pass 35 | import urllib2 36 | 37 | LOG = logging.getLogger(__name__) 38 | 39 | 40 | class Resource(object): 41 | """ 42 | Encapsulates a resource, and provides actions to invoke on it. 43 | """ 44 | def __init__(self, client, relpath=""): 45 | """ 46 | @param client: A Client object. 47 | @param relpath: The relative path of the resource. 48 | """ 49 | self._client = client 50 | self._path = relpath.strip('/') 51 | self.retries = 3 52 | self.retry_sleep = 3 53 | 54 | @property 55 | def base_url(self): 56 | return self._client.base_url 57 | 58 | def _join_uri(self, relpath): 59 | if relpath is None: 60 | return self._path 61 | return self._path + posixpath.normpath('/' + relpath) 62 | 63 | def invoke(self, method, relpath=None, params=None, data=None, headers=None): 64 | """ 65 | Invoke an API method. 66 | @return: Raw body or JSON dictionary (if response content type is JSON). 67 | """ 68 | path = self._join_uri(relpath) 69 | resp = self._client.execute(method, 70 | path, 71 | params=params, 72 | data=data, 73 | headers=headers) 74 | try: 75 | body = resp.read() 76 | except Exception, ex: 77 | raise Exception("Command '%s %s' failed: %s" % 78 | (method, path, ex)) 79 | 80 | self._client.logger.debug( 81 | "%s Got response: %s%s" % 82 | (method, body[:32], len(body) > 32 and "..." or "")) 83 | 84 | # Is the response application/json? 85 | if len(body) != 0 and \ 86 | resp.info().getmaintype() == "application" and \ 87 | resp.info().getsubtype() == "json": 88 | try: 89 | json_dict = json.loads(body) 90 | return json_dict 91 | except Exception, ex: 92 | self._client.logger.exception('JSON decode error: %s' % (body,)) 93 | raise ex 94 | else: 95 | return body 96 | 97 | 98 | def get(self, relpath=None, params=None): 99 | """ 100 | Invoke the GET method on a resource. 101 | @param relpath: Optional. A relative path to this resource's path. 102 | @param params: Key-value data. 103 | 104 | @return: A dictionary of the JSON result. 105 | """ 106 | for retry in xrange(self.retries + 1): 107 | if retry: 108 | time.sleep(self.retry_sleep) 109 | try: 110 | return self.invoke("GET", relpath, params) 111 | except (socket.error, urllib2.URLError) as e: 112 | if "timed out" in str(e).lower(): 113 | log_message = "Timeout issuing GET request for %s." \ 114 | % (self._join_uri(relpath), ) 115 | if retry < self.retries: 116 | log_message += " Will retry." 117 | else: 118 | log_message += " No retries left." 119 | LOG.warn(log_message, exc_info=True) 120 | else: 121 | raise 122 | else: 123 | raise e 124 | 125 | 126 | def delete(self, relpath=None, params=None): 127 | """ 128 | Invoke the DELETE method on a resource. 129 | @param relpath: Optional. A relative path to this resource's path. 130 | @param params: Key-value data. 131 | 132 | @return: A dictionary of the JSON result. 133 | """ 134 | return self.invoke("DELETE", relpath, params) 135 | 136 | 137 | def post(self, relpath=None, params=None, data=None, contenttype=None): 138 | """ 139 | Invoke the POST method on a resource. 140 | @param relpath: Optional. A relative path to this resource's path. 141 | @param params: Key-value data. 142 | @param data: Optional. Body of the request. 143 | @param contenttype: Optional. 144 | 145 | @return: A dictionary of the JSON result. 146 | """ 147 | return self.invoke("POST", relpath, params, data, 148 | self._make_headers(contenttype)) 149 | 150 | 151 | def put(self, relpath=None, params=None, data=None, contenttype=None): 152 | """ 153 | Invoke the PUT method on a resource. 154 | @param relpath: Optional. A relative path to this resource's path. 155 | @param params: Key-value data. 156 | @param data: Optional. Body of the request. 157 | @param contenttype: Optional. 158 | 159 | @return: A dictionary of the JSON result. 160 | """ 161 | return self.invoke("PUT", relpath, params, data, 162 | self._make_headers(contenttype)) 163 | 164 | 165 | def _make_headers(self, contenttype=None): 166 | if contenttype: 167 | return { 'Content-Type': contenttype } 168 | return None 169 | -------------------------------------------------------------------------------- /clusterdock/topologies/cdh/cm_utils.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import logging 18 | from time import sleep, time 19 | 20 | logger = logging.getLogger(__name__) 21 | logger.setLevel(logging.INFO) 22 | 23 | def add_hosts_to_cluster(api, cluster, secondary_node_fqdn, all_fqdns): 24 | """Add all CM hosts to cluster.""" 25 | 26 | # Wait up to 60 seconds for CM to see all hosts. 27 | TIMEOUT_IN_SECS = 60 28 | TIMEOUT_TIME = time() + TIMEOUT_IN_SECS 29 | while time() < TIMEOUT_TIME: 30 | all_hosts = api.get_all_hosts() 31 | # Once hostname changes have propagated through CM, we switch all_hosts to be a list of 32 | # hostIds (since that's what CM uses). 33 | if set([host.hostname for host in all_hosts]) == set(all_fqdns): 34 | all_hosts = [host.hostId for host in all_hosts] 35 | break 36 | sleep(1) 37 | else: 38 | raise Exception("Timed out waiting for CM to recognize all hosts (saw: {0}).".format( 39 | ', '.join(all_hosts) 40 | )) 41 | 42 | hosts_in_cluster = [host.hostId for host in cluster.list_hosts()] 43 | # Use Set.difference to get all_hosts - hosts_in_cluster. 44 | hosts_to_add = list(set(all_hosts).difference(hosts_in_cluster)) 45 | logger.info("Adding hosts (Ids: %s) to %s...", ', '.join(hosts_to_add), cluster.displayName) 46 | cluster.add_hosts(hosts_to_add) 47 | 48 | secondary_node_template = get_secondary_node_template( 49 | api=api, cluster=cluster, secondary_node_fqdn=secondary_node_fqdn 50 | ) 51 | logger.info('Sleeping for 30 seconds to ensure that parcels are activated...') 52 | sleep(30) 53 | 54 | logger.info('Applying secondary host template...') 55 | secondary_node_template.apply_host_template(host_ids=hosts_to_add, start_roles=False) 56 | 57 | def get_secondary_node_template(api, cluster, secondary_node_fqdn): 58 | template = cluster.create_host_template("template") 59 | hosts = api.get_all_hosts(view='full') 60 | for host in hosts: 61 | if host.hostname == secondary_node_fqdn: 62 | logger.info('Creating secondary node host template...') 63 | secondary_node_role_group_refs = [] 64 | for role_ref in host.roleRefs: 65 | service = cluster.get_service(role_ref.serviceName) 66 | role = service.get_role(role_ref.roleName) 67 | secondary_node_role_group_refs.append(role.roleConfigGroupRef) 68 | template.set_role_config_groups(secondary_node_role_group_refs) 69 | return template 70 | else: 71 | raise Exception("Could not find secondary node ({0}) among hosts ({1}).".format( 72 | secondary_node_fqdn, ', '.join([host.hostname for host in hosts]) 73 | )) 74 | 75 | def set_hdfs_replication_configs(cluster): 76 | HDFS_SERVICE_NAME = 'HDFS-1' 77 | hdfs = cluster.get_service(HDFS_SERVICE_NAME) 78 | hdfs.update_config({ 79 | 'dfs_replication': len(cluster.list_hosts()) - 1, 80 | 81 | # Change dfs.replication.max, this helps ACCUMULO and HBASE to start. 82 | # If this configuration is not changed both services will complain about the Requested 83 | # replication factor. 84 | 'dfs_replication_max': len(cluster.list_hosts()) 85 | }) 86 | 87 | def update_database_configs(api, cluster): 88 | # In our case, the databases are always co-located with the CM host, so we grab that from the 89 | # ApiResource object and then update various configurations accordingly. 90 | logger.info('Updating database configurations...') 91 | cm_service = api.get_cloudera_manager().get_service() 92 | cm_host_id = cm_service.get_all_roles()[0].hostRef.hostId 93 | # Called hostname, actually a fully-qualified domain name. 94 | cm_hostname = api.get_host(cm_host_id).hostname 95 | 96 | for service in cluster.get_all_services(): 97 | if service.type == 'HIVE': 98 | service.update_config({'hive_metastore_database_host': cm_hostname}) 99 | elif service.type == 'OOZIE': 100 | for role in service.get_roles_by_type('OOZIE_SERVER'): 101 | role.update_config({'oozie_database_host': "{0}:7432".format(cm_hostname)}) 102 | elif service.type == 'HUE': 103 | service.update_config({'database_host': cm_hostname}) 104 | elif service.type == 'SENTRY': 105 | service.update_config({'sentry_server_database_host': cm_hostname}) 106 | 107 | for role in cm_service.get_roles_by_type('ACTIVITYMONITOR'): 108 | role.update_config({'firehose_database_host': "{0}:7432".format(cm_hostname)}) 109 | for role in cm_service.get_roles_by_type('REPORTSMANAGER'): 110 | role.update_config({'headlamp_database_host': "{0}:7432".format(cm_hostname)}) 111 | for role in cm_service.get_roles_by_type('NAVIGATOR'): 112 | role.update_config({'navigator_database_host': "{0}:7432".format(cm_hostname)}) 113 | for role in cm_service.get_roles_by_type('NAVIGATORMETASERVER'): 114 | role.update_config({'nav_metaserver_database_host': "{0}:7432".format(cm_hostname)}) 115 | -------------------------------------------------------------------------------- /clusterdock/topologies/cdh/profile.cfg: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | [general] 18 | name = CDH 19 | description = A basic CDH cluster with 1 primary node and n-1 secondary nodes 20 | 21 | [node_groups] 22 | # Define node groups and specify which nodes of each to start during the build process. 23 | primary-node = node-1 24 | secondary-nodes = node-2 25 | 26 | [start] 27 | arg.cdh-string = cdh580 28 | arg.cdh-string.help = CDH version to use 29 | arg.cdh-string.metavar = cdh 30 | 31 | arg.cm-string = cm581 32 | arg.cm-string.help = CM version to use 33 | arg.cm-string.metavar = cm 34 | 35 | arg.dont-start-cluster = False 36 | arg.dont-start-cluster.help = Don't start clusters/services in Cloudera Manager 37 | 38 | arg.exclude-service-types 39 | arg.exclude-service-types.help = If specified, a comma-separated list of service types to exclude from the CDH cluster 40 | arg.exclude-service-types.metavar = svc1,svc2,... 41 | 42 | arg.include-service-types 43 | arg.include-service-types.help = If specified, a comma-separated list of service types to include in the CDH cluster 44 | arg.include-service-types.metavar = svc1,svc2,... 45 | -------------------------------------------------------------------------------- /clusterdock/topologies/cdh/ssh/id_rsa: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpAIBAAKCAQEAtj/yZZNF+bv26eqWqsx+vFehSlxBJp/QhIrFWKpjHcpQJ29o 3 | 6hJN9moU3Goft2C5w6FoZdjC1TWlzhxUnRS8xeksFnW3ItvkjySLA1Iq6jchYxNd 4 | fZ3HwTdH0rubM1uJ/CnkaijoxBqGBmPSL0TxfqcteaJ8APhslwl0WWJ6b+tBCHDV 5 | pTLATebtggCAfhKmSuAYmn3QIqJ7DFoSGkwhkxpUHuuVCZxUH3CIxLIw+6npArr/ 6 | S5gtFo50oi6FXPmvv6mJg6yLqq3VlKcQh6d/COaJopHn+nLed2kECESUlpTruMpr 7 | 6IcGESgz4hnkmhop8oTY42sQJhPPF2Ahq9a3aQIDAQABAoIBADSbxPb5SkvKrH3d 8 | j9yB51uq2A5FDzF9FI4OGOV9WdsxmW2oxVo8KnElMhxmLf2bWERWhXJQ3fz53YDf 9 | wLUPVWaz5lwdYt4XJ6UCYXZ185lkjKiy4FvwfccSlBMKwMRUekJmPV8/q+Ff3qxd 10 | iEDI4AU1cPUZqD4HeCEpQ4LB4KIJhCdLkCgoWxxaCwwuB6DnwB4P4VLeAfsm2NEX 11 | k9dld87q/miOmuw9QsmSv9wYiQqoPdV5Qj7KYqHBAa6syqUfFni3Ibmw1WBzMydp 12 | 8YyP9HvrzDBMnPPzkmp6od0fAgGafIlkIxz/9sCKOSISnuuqahbNAJK/rIiJzLY3 13 | Pi49M+ECgYEA2vCFObmM/hpKUPNxG841nQScFfeMg1q1z1krEmfjqDTELXyq9AOS 14 | fGiVTxfWagOGoAWIwg3ZfgGEmxsKrOSxkFvabWdhN1Wn98Zf8fJG8YAwLYg8JOgf 15 | gZ5pkxFW4FwrAeFDyJyKvFJVrbDw1PM41yvTmRzf3NjcaqJrBE2fgKUCgYEA1RmF 16 | XjfMlBoMLZ4pXS1zF91WgOF4SNdJJj9RCGzqqdy+DzPKNAZVa0HBhaIHCZXL3Hcv 17 | zqgEb6RSMysyVYjZPwfSwevsSuxpfriVpYux5MN3AEXX5Ysv51WWStWgt9+iQlfo 18 | xAdxxukOa++PZ4Z+TIIEDAFS47rnKEQUh+ZNfHUCgYEA0amTa3wtcQlsMalvn9kR 19 | rpRDhSXTAddUVIRnovCqKuKdG5JPg+4H0eu1UFDbnBpUSdoC5RKuPOTnQEHdL0Sy 20 | ZjQQMMTXbE4y1Cy8pM4G8i539KKKNi20PkSdhaENOT4KUXqPlwWSNlYChprzhnqE 21 | 7EmkEPR9zNg//D4djbloDaECgYANOJIfsFKO9ba/tcpXL5SubFsLj/GIg2LUbqU2 22 | YpuEgl+ATfRDmgj+qIu7ILxTCeol+XcL2Ty9OHKpHgr3Z5Ai6vdWdK6qT1SUOht+ 23 | s9YLnVzqtWqZoTMNpS+34N0hy0wj1ZRpZRTYBGmSpMA+6gc38/EQVZyw6E2jH+Yu 24 | MEmqaQKBgQDGh9uXCl/WjhBOF9VLrX/Aeaa2Mzrh21Ic1dw6aWrE4EW6k1LvSP36 25 | evrvrs2jQuzRMGH6DKX8ImnVEWjK+gZfgf2MuyDSW7KYR5zxkdZtRkotF6X0fu6N 26 | 8uLa7CN8UmS4FiAMLwNbTJ6zA6ohny7r+AiOqNGlP9vBFMhpGs3NFg== 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /clusterdock/topologies/nodebase/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /clusterdock/topologies/nodebase/actions.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import importlib 18 | import logging 19 | import os 20 | from os.path import dirname, join 21 | 22 | from clusterdock import Constants 23 | from clusterdock.cluster import Cluster, Node, NodeGroup 24 | from clusterdock.docker_utils import is_image_available_locally, pull_image 25 | 26 | logger = logging.getLogger(__name__) 27 | logger.setLevel(logging.INFO) 28 | 29 | DEFAULT_CLOUDERA_NAMESPACE = Constants.DEFAULT.cloudera_namespace # pylint: disable=no-member 30 | 31 | def start(args): 32 | image = "{0}/{1}/clusterdock:{2}_nodebase".format(args.registry_url, 33 | args.namespace or DEFAULT_CLOUDERA_NAMESPACE, 34 | args.operating_system) 35 | if args.always_pull or not is_image_available_locally(image): 36 | pull_image(image) 37 | 38 | node_groups = [NodeGroup(name='nodes', nodes=[Node(hostname=hostname, network=args.network, 39 | image=image) 40 | for hostname in args.nodes])] 41 | cluster = Cluster(topology='nodebase', node_groups=node_groups, network_name=args.network) 42 | cluster.start() 43 | -------------------------------------------------------------------------------- /clusterdock/topologies/nodebase/profile.cfg: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | [general] 18 | name = nodebase 19 | description = A basic cluster with n nodes 20 | 21 | [node_groups] 22 | # Define node groups and specify which nodes of each to start during the build process. 23 | nodes = node-{1,2} 24 | -------------------------------------------------------------------------------- /clusterdock/topologies/nodebase/ssh/id_rsa: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpAIBAAKCAQEAtj/yZZNF+bv26eqWqsx+vFehSlxBJp/QhIrFWKpjHcpQJ29o 3 | 6hJN9moU3Goft2C5w6FoZdjC1TWlzhxUnRS8xeksFnW3ItvkjySLA1Iq6jchYxNd 4 | fZ3HwTdH0rubM1uJ/CnkaijoxBqGBmPSL0TxfqcteaJ8APhslwl0WWJ6b+tBCHDV 5 | pTLATebtggCAfhKmSuAYmn3QIqJ7DFoSGkwhkxpUHuuVCZxUH3CIxLIw+6npArr/ 6 | S5gtFo50oi6FXPmvv6mJg6yLqq3VlKcQh6d/COaJopHn+nLed2kECESUlpTruMpr 7 | 6IcGESgz4hnkmhop8oTY42sQJhPPF2Ahq9a3aQIDAQABAoIBADSbxPb5SkvKrH3d 8 | j9yB51uq2A5FDzF9FI4OGOV9WdsxmW2oxVo8KnElMhxmLf2bWERWhXJQ3fz53YDf 9 | wLUPVWaz5lwdYt4XJ6UCYXZ185lkjKiy4FvwfccSlBMKwMRUekJmPV8/q+Ff3qxd 10 | iEDI4AU1cPUZqD4HeCEpQ4LB4KIJhCdLkCgoWxxaCwwuB6DnwB4P4VLeAfsm2NEX 11 | k9dld87q/miOmuw9QsmSv9wYiQqoPdV5Qj7KYqHBAa6syqUfFni3Ibmw1WBzMydp 12 | 8YyP9HvrzDBMnPPzkmp6od0fAgGafIlkIxz/9sCKOSISnuuqahbNAJK/rIiJzLY3 13 | Pi49M+ECgYEA2vCFObmM/hpKUPNxG841nQScFfeMg1q1z1krEmfjqDTELXyq9AOS 14 | fGiVTxfWagOGoAWIwg3ZfgGEmxsKrOSxkFvabWdhN1Wn98Zf8fJG8YAwLYg8JOgf 15 | gZ5pkxFW4FwrAeFDyJyKvFJVrbDw1PM41yvTmRzf3NjcaqJrBE2fgKUCgYEA1RmF 16 | XjfMlBoMLZ4pXS1zF91WgOF4SNdJJj9RCGzqqdy+DzPKNAZVa0HBhaIHCZXL3Hcv 17 | zqgEb6RSMysyVYjZPwfSwevsSuxpfriVpYux5MN3AEXX5Ysv51WWStWgt9+iQlfo 18 | xAdxxukOa++PZ4Z+TIIEDAFS47rnKEQUh+ZNfHUCgYEA0amTa3wtcQlsMalvn9kR 19 | rpRDhSXTAddUVIRnovCqKuKdG5JPg+4H0eu1UFDbnBpUSdoC5RKuPOTnQEHdL0Sy 20 | ZjQQMMTXbE4y1Cy8pM4G8i539KKKNi20PkSdhaENOT4KUXqPlwWSNlYChprzhnqE 21 | 7EmkEPR9zNg//D4djbloDaECgYANOJIfsFKO9ba/tcpXL5SubFsLj/GIg2LUbqU2 22 | YpuEgl+ATfRDmgj+qIu7ILxTCeol+XcL2Ty9OHKpHgr3Z5Ai6vdWdK6qT1SUOht+ 23 | s9YLnVzqtWqZoTMNpS+34N0hy0wj1ZRpZRTYBGmSpMA+6gc38/EQVZyw6E2jH+Yu 24 | MEmqaQKBgQDGh9uXCl/WjhBOF9VLrX/Aeaa2Mzrh21Ic1dw6aWrE4EW6k1LvSP36 25 | evrvrs2jQuzRMGH6DKX8ImnVEWjK+gZfgf2MuyDSW7KYR5zxkdZtRkotF6X0fu6N 26 | 8uLa7CN8UmS4FiAMLwNbTJ6zA6ohny7r+AiOqNGlP9vBFMhpGs3NFg== 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /clusterdock/topologies/nodebase/ssh/id_rsa.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2P/Jlk0X5u/bp6paqzH68V6FKXEEmn9CEisVYqmMdylAnb2jqEk32ahTcah+3YLnDoWhl2MLVNaXOHFSdFLzF6SwWdbci2+SPJIsDUirqNyFjE119ncfBN0fSu5szW4n8KeRqKOjEGoYGY9IvRPF+py15onwA+GyXCXRZYnpv60EIcNWlMsBN5u2CAIB+EqZK4BiafdAionsMWhIaTCGTGlQe65UJnFQfcIjEsjD7qekCuv9LmC0WjnSiLoVc+a+/qYmDrIuqrdWUpxCHp38I5omikef6ct53aQQIRJSWlOu4ymvohwYRKDPiGeSaGinyhNjjaxAmE88XYCGr1rdp clusterdock 2 | -------------------------------------------------------------------------------- /clusterdock/topologies/parsing.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """This module handles the parsing of topologies' profile.cfg files into the command line 18 | arguments used by some of the scripts in the ./bin folder.""" 19 | 20 | import argparse 21 | import ConfigParser 22 | import os 23 | from collections import OrderedDict 24 | 25 | from braceexpand import braceexpand 26 | 27 | def get_profile_config_item(topology, section, item): 28 | """Return a string represenation of a particular topology's section's item's value.""" 29 | config_filename = os.path.join(os.path.dirname(__file__), topology, TOPOLOGIES_CONFIG_NAME) 30 | config = ConfigParser.ConfigParser(allow_no_value=True) 31 | config.read(config_filename) 32 | return config.get(section, item) 33 | 34 | ARG_PREFIX = 'arg.' 35 | ARG_HELP_SUFFIX = '.help' 36 | ARG_METAVAR_SUFFIX = '.metavar' 37 | 38 | def parse_args_from_config(parser, config, section): 39 | """Iterate through a particular ConfigParser instance, feeding its contents into an argparse 40 | ArgumentParser instance.""" 41 | 42 | # By creating an argument group (without a title), we clean up our help messages by 43 | # separating groups by a blank line. 44 | group = parser.add_argument_group() 45 | config_args = OrderedDict() 46 | if config.has_section(section): 47 | for option in config.options(section): 48 | if option.startswith(ARG_PREFIX): 49 | if option.endswith(ARG_HELP_SUFFIX): 50 | help_message = config.get(section, option) 51 | stripped_option = option[len(ARG_PREFIX):-len(ARG_HELP_SUFFIX)] 52 | config_args[stripped_option]['help'] = help_message 53 | elif option.endswith(ARG_METAVAR_SUFFIX): 54 | metavar = config.get(section, option) 55 | stripped_option = option[len(ARG_PREFIX):-len(ARG_METAVAR_SUFFIX)] 56 | config_args[stripped_option]['metavar'] = metavar 57 | else: 58 | stripped_option = option[len(ARG_PREFIX):] 59 | config_args[stripped_option] = dict.fromkeys(['default', 'help', 'metavar']) 60 | default = config.get(section, option) 61 | config_args[stripped_option]['default'] = default 62 | 63 | # If the default arg is a boolean, the presence of the argument should set a boolean (i.e. 64 | # it doesn't expect to store the string following the argument). 65 | for arg in config_args: 66 | add_argument_options = dict() 67 | for option in ['default', 'help', 'metavar']: 68 | if config_args[arg].get(option): 69 | add_argument_options[option] = config_args[arg].get(option) 70 | 71 | if config_args[arg].get('default'): 72 | if config_args[arg].get('default').lower() == 'false': 73 | add_argument_options['action'] = 'store_true' 74 | del add_argument_options['default'] 75 | elif config_args[arg].get('default').lower() == 'true': 76 | add_argument_options['action'] = 'store_false' 77 | del add_argument_options['default'] 78 | 79 | group.add_argument("--{0}".format(arg), **add_argument_options) 80 | 81 | TOPOLOGIES_CONFIG_NAME = 'profile.cfg' 82 | def parse_profiles(parser, action='start'): 83 | """Given an argparse parser and a cluster action, generate subparsers for each topology.""" 84 | topologies_directory = os.path.dirname(__file__) 85 | 86 | subparsers = parser.add_subparsers(help='The topology to use when starting the cluster', 87 | dest='topology') 88 | 89 | parsers = dict() 90 | for topology in os.listdir(topologies_directory): 91 | if os.path.isdir(os.path.join(topologies_directory, topology)): 92 | # Generate help and optional arguments based on the options under our topology's 93 | # profile.cfg file's node_groups section. 94 | config_filename = os.path.join(os.path.dirname(__file__), topology, 95 | TOPOLOGIES_CONFIG_NAME) 96 | config = ConfigParser.ConfigParser(allow_no_value=True) 97 | config.read(config_filename) 98 | 99 | parsers[topology] = subparsers.add_parser( 100 | topology, help=config.get('general', 'description'), 101 | formatter_class=argparse.ArgumentDefaultsHelpFormatter 102 | ) 103 | 104 | # Arguments in the [all] group should be available to all actions. 105 | parse_args_from_config(parsers[topology], config, 'all') 106 | 107 | if action == 'start': 108 | for option in config.options('node_groups'): 109 | # While we use our custom StoreBraceExpandedAction to process the given values, 110 | # we need to separately brace-expand the default to make it show up correctly 111 | # in help messages. 112 | default = list(braceexpand(config.get('node_groups', option))) 113 | parsers[topology].add_argument("--{0}".format(option), metavar='NODES', 114 | default=default, action=StoreBraceExpandedAction, 115 | help="Nodes of the {0} group".format(option)) 116 | parse_args_from_config(parsers[topology], config, 'start') 117 | elif action == 'build': 118 | parse_args_from_config(parsers[topology], config, 'build') 119 | 120 | class StoreBraceExpandedAction(argparse.Action): 121 | """A custom argparse Action that brace-expands values using the braceexpands module before 122 | storing them in dest as a list. This lets us not have to do any post-processing of strings like 123 | 'node-{1..4}.internal' later on.""" 124 | 125 | # pylint: disable=too-few-public-methods 126 | 127 | def __init__(self, option_strings, dest, nargs=None, **kwargs): 128 | if nargs is not None: 129 | raise ValueError("nargs not allowed") 130 | super(StoreBraceExpandedAction, self).__init__(option_strings, dest, 131 | **kwargs) 132 | 133 | def __call__(self, parser, namespace, values, option_string=None): 134 | setattr(namespace, self.dest, list(braceexpand(values))) 135 | -------------------------------------------------------------------------------- /clusterdock/utils.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """This module contains utility functions that may be relevant to more than one topology.""" 18 | 19 | import operator 20 | from socket import socket 21 | from time import sleep, time 22 | 23 | from lxml import etree 24 | 25 | def get_nested_value(the_map, dot_separated_key): 26 | """Give a nested dictionary map, get the value specified by a dot-separated key where dots 27 | denote an additional depth. Taken from stack overflow (http://stackoverflow.com/a/12414913). 28 | """ 29 | keys = dot_separated_key.split(".") 30 | return reduce(operator.getitem, keys[:-1], the_map)[keys[-1]] 31 | 32 | def strip_components_from_tar(tar, leading_elements_to_remove=1): 33 | """Designed to feed tarfile.extractall's members parameter.""" 34 | for the_file in tar: 35 | split_on_first_dir = the_file.name.split('/', leading_elements_to_remove) 36 | if len(split_on_first_dir) > leading_elements_to_remove: 37 | the_file.name = split_on_first_dir[leading_elements_to_remove] 38 | yield the_file 39 | 40 | def wait_for_port_open(address, port, timeout_sec=60): 41 | """Check the accessibility of address:port in a loop until it succeeds or times out.""" 42 | start_waiting_time = time() 43 | stop_waiting_time = start_waiting_time + timeout_sec 44 | 45 | while time() < stop_waiting_time: 46 | if port_is_open(address=address, port=port): 47 | return time() - start_waiting_time 48 | sleep(1) 49 | 50 | # If we get here without having returned, we've timed out. 51 | raise Exception("Timed out after {0} seconds waiting for {1}:{2} to be open.".format( 52 | timeout_sec, address, port 53 | )) 54 | 55 | def port_is_open(address, port): 56 | """Returns True if port at address is open.""" 57 | return socket().connect_ex((address, port)) == 0 58 | 59 | 60 | class XmlConfiguration(object): 61 | """A class to handle the creation of XML configuration files.""" 62 | def __init__(self, properties=None, root_name='configuration', source_file=None): 63 | if source_file: 64 | parser = etree.XMLParser(remove_blank_text=True) 65 | self.tree = etree.parse(source_file, parser) 66 | self.root = self.tree.getroot() 67 | else: 68 | self.root = etree.Element(root_name) 69 | self.tree = etree.ElementTree(self.root) 70 | 71 | if properties: 72 | for the_property in properties: 73 | self.add_property(the_property, properties[the_property]) 74 | 75 | def __str__(self): 76 | return self.to_string() 77 | 78 | def add_property(self, name, value): 79 | """Adds a property to the XML configuration.""" 80 | the_property = etree.SubElement(self.root, 'property') 81 | etree.SubElement(the_property, 'name').text = name 82 | etree.SubElement(the_property, 'value').text = value 83 | 84 | def to_string(self, hide_root=False): 85 | """Converts the XmlConfiguration instance into a string.""" 86 | if hide_root: 87 | # We build this string with concatenation instead of by doing a join on a list 88 | # because StackOverflow says it's faster this way. I'll take its word for it. 89 | properties = str() 90 | for the_property in self.root: 91 | properties += etree.tostring(the_property, pretty_print=True) 92 | return properties 93 | else: 94 | return etree.tostring(self.tree, pretty_print=True) 95 | 96 | def write_to_file(self, filename): 97 | """Writes a string representation of the XmlConfiguration instance into a file.""" 98 | self.tree.write(filename, pretty_print=True) 99 | -------------------------------------------------------------------------------- /pylintrc: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012 - 2018 Cloudera, Inc. 2 | # All Rights Reserved. 3 | 4 | # This config file is generated by running Pylint --generate-rcfile and redirecting the output to 5 | # a file called pylintrc. With this file present in the working directory from which Pylint is 6 | # invoked, it will be used to add configuration options and make scans repeatable across different 7 | # environments. In particular, the following options were added to the configuration: 8 | # 9 | # --unsafe-load-any-extension=y 10 | # loads C extensions (e.g. lxml). 11 | 12 | [MASTER] 13 | 14 | # Specify a configuration file. 15 | #rcfile= 16 | 17 | # Python code to execute, usually for sys.path manipulation such as 18 | # pygtk.require(). 19 | #init-hook= 20 | 21 | # Add files or directories to the blacklist. They should be base names, not 22 | # paths. 23 | ignore=CVS 24 | 25 | # Pickle collected data for later comparisons. 26 | persistent=yes 27 | 28 | # List of plugins (as comma separated values of python modules names) to load, 29 | # usually to register additional checkers. 30 | load-plugins= 31 | 32 | # Use multiple processes to speed up Pylint. 33 | jobs=1 34 | 35 | # Allow loading of arbitrary C extensions. Extensions are imported into the 36 | # active Python interpreter and may run arbitrary code. 37 | unsafe-load-any-extension=yes 38 | 39 | # A comma-separated list of package or module names from where C extensions may 40 | # be loaded. Extensions are loading into the active Python interpreter and may 41 | # run arbitrary code 42 | extension-pkg-whitelist= 43 | 44 | # Allow optimization of some AST trees. This will activate a peephole AST 45 | # optimizer, which will apply various small optimizations. For instance, it can 46 | # be used to obtain the result of joining multiple strings with the addition 47 | # operator. Joining a lot of strings can lead to a maximum recursion error in 48 | # Pylint and this flag can prevent that. It has one side effect, the resulting 49 | # AST will be different than the one from reality. 50 | optimize-ast=no 51 | 52 | 53 | [MESSAGES CONTROL] 54 | 55 | # Only show warnings with the listed confidence levels. Leave empty to show 56 | # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED 57 | confidence= 58 | 59 | # Enable the message, report, category or checker with the given id(s). You can 60 | # either give multiple identifier separated by comma (,) or put this option 61 | # multiple time (only on the command line, not in the configuration file where 62 | # it should appear only once). See also the "--disable" option for examples. 63 | #enable= 64 | 65 | # Disable the message, report, category or checker with the given id(s). You 66 | # can either give multiple identifiers separated by comma (,) or put this 67 | # option multiple times (only on the command line, not in the configuration 68 | # file where it should appear only once).You can also use "--disable=all" to 69 | # disable everything first and then reenable specific checks. For example, if 70 | # you want to run only the similarities checker, you can use "--disable=all 71 | # --enable=similarities". If you want to run only the classes checker, but have 72 | # no Warning level messages displayed, use"--disable=all --enable=classes 73 | # --disable=W" 74 | disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating 75 | 76 | 77 | [REPORTS] 78 | 79 | # Set the output format. Available formats are text, parseable, colorized, msvs 80 | # (visual studio) and html. You can also give a reporter class, eg 81 | # mypackage.mymodule.MyReporterClass. 82 | output-format=text 83 | 84 | # Put messages in a separate file for each module / package specified on the 85 | # command line instead of printing them on stdout. Reports (if any) will be 86 | # written in a file name "pylint_global.[txt|html]". 87 | files-output=no 88 | 89 | # Tells whether to display a full report or only the messages 90 | reports=yes 91 | 92 | # Python expression which should return a note less than 10 (10 is the highest 93 | # note). You have access to the variables errors warning, statement which 94 | # respectively contain the number of errors / warnings messages and the total 95 | # number of statements analyzed. This is used by the global evaluation report 96 | # (RP0004). 97 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 98 | 99 | # Template used to display messages. This is a python new-style format string 100 | # used to format the message information. See doc for all details 101 | #msg-template= 102 | 103 | 104 | [BASIC] 105 | 106 | # List of builtins function names that should not be used, separated by a comma 107 | bad-functions=map,filter,input 108 | 109 | # Good variable names which should always be accepted, separated by a comma 110 | good-names=i,j,k,ex,Run,_ 111 | 112 | # Bad variable names which should always be refused, separated by a comma 113 | bad-names=foo,bar,baz,toto,tutu,tata 114 | 115 | # Colon-delimited sets of names that determine each other's naming style when 116 | # the name regexes allow several styles. 117 | name-group= 118 | 119 | # Include a hint for the correct naming format with invalid-name 120 | include-naming-hint=no 121 | 122 | # Regular expression matching correct function names 123 | function-rgx=[a-z_][a-z0-9_]{2,30}$ 124 | 125 | # Naming hint for function names 126 | function-name-hint=[a-z_][a-z0-9_]{2,30}$ 127 | 128 | # Regular expression matching correct variable names 129 | variable-rgx=[a-z_][a-z0-9_]{2,30}$ 130 | 131 | # Naming hint for variable names 132 | variable-name-hint=[a-z_][a-z0-9_]{2,30}$ 133 | 134 | # Regular expression matching correct constant names 135 | const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 136 | 137 | # Naming hint for constant names 138 | const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 139 | 140 | # Regular expression matching correct attribute names 141 | attr-rgx=[a-z_][a-z0-9_]{2,30}$ 142 | 143 | # Naming hint for attribute names 144 | attr-name-hint=[a-z_][a-z0-9_]{2,30}$ 145 | 146 | # Regular expression matching correct argument names 147 | argument-rgx=[a-z_][a-z0-9_]{2,30}$ 148 | 149 | # Naming hint for argument names 150 | argument-name-hint=[a-z_][a-z0-9_]{2,30}$ 151 | 152 | # Regular expression matching correct class attribute names 153 | class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 154 | 155 | # Naming hint for class attribute names 156 | class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 157 | 158 | # Regular expression matching correct inline iteration names 159 | inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ 160 | 161 | # Naming hint for inline iteration names 162 | inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ 163 | 164 | # Regular expression matching correct class names 165 | class-rgx=[A-Z_][a-zA-Z0-9]+$ 166 | 167 | # Naming hint for class names 168 | class-name-hint=[A-Z_][a-zA-Z0-9]+$ 169 | 170 | # Regular expression matching correct module names 171 | module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 172 | 173 | # Naming hint for module names 174 | module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 175 | 176 | # Regular expression matching correct method names 177 | method-rgx=[a-z_][a-z0-9_]{2,30}$ 178 | 179 | # Naming hint for method names 180 | method-name-hint=[a-z_][a-z0-9_]{2,30}$ 181 | 182 | # Regular expression which should only match function or class names that do 183 | # not require a docstring. 184 | no-docstring-rgx=^_ 185 | 186 | # Minimum line length for functions/classes that require docstrings, shorter 187 | # ones are exempt. 188 | docstring-min-length=-1 189 | 190 | 191 | [ELIF] 192 | 193 | # Maximum number of nested blocks for function / method body 194 | max-nested-blocks=5 195 | 196 | 197 | [FORMAT] 198 | 199 | # Maximum number of characters on a single line. 200 | max-line-length=100 201 | 202 | # Regexp for a line that is allowed to be longer than the limit. 203 | ignore-long-lines=^\s*(# )??$ 204 | 205 | # Allow the body of an if to be on the same line as the test if there is no 206 | # else. 207 | single-line-if-stmt=no 208 | 209 | # List of optional constructs for which whitespace checking is disabled. `dict- 210 | # separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. 211 | # `trailing-comma` allows a space between comma and closing bracket: (a, ). 212 | # `empty-line` allows space-only lines. 213 | no-space-check=trailing-comma,dict-separator 214 | 215 | # Maximum number of lines in a module 216 | max-module-lines=1000 217 | 218 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 219 | # tab). 220 | indent-string=' ' 221 | 222 | # Number of spaces of indent required inside a hanging or continued line. 223 | indent-after-paren=4 224 | 225 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 226 | expected-line-ending-format= 227 | 228 | 229 | [LOGGING] 230 | 231 | # Logging modules to check that the string format arguments are in logging 232 | # function parameter format 233 | logging-modules=logging 234 | 235 | 236 | [MISCELLANEOUS] 237 | 238 | # List of note tags to take in consideration, separated by a comma. 239 | notes=FIXME,XXX,TODO 240 | 241 | 242 | [SIMILARITIES] 243 | 244 | # Minimum lines number of a similarity. 245 | min-similarity-lines=4 246 | 247 | # Ignore comments when computing similarities. 248 | ignore-comments=yes 249 | 250 | # Ignore docstrings when computing similarities. 251 | ignore-docstrings=yes 252 | 253 | # Ignore imports when computing similarities. 254 | ignore-imports=no 255 | 256 | 257 | [SPELLING] 258 | 259 | # Spelling dictionary name. Available dictionaries: none. To make it working 260 | # install python-enchant package. 261 | spelling-dict= 262 | 263 | # List of comma separated words that should not be checked. 264 | spelling-ignore-words= 265 | 266 | # A path to a file that contains private dictionary; one word per line. 267 | spelling-private-dict-file= 268 | 269 | # Tells whether to store unknown words to indicated private dictionary in 270 | # --spelling-private-dict-file option instead of raising a message. 271 | spelling-store-unknown-words=no 272 | 273 | 274 | [TYPECHECK] 275 | 276 | # Tells whether missing members accessed in mixin class should be ignored. A 277 | # mixin class is detected if its name ends with "mixin" (case insensitive). 278 | ignore-mixin-members=yes 279 | 280 | # List of module names for which member attributes should not be checked 281 | # (useful for modules/projects where namespaces are manipulated during runtime 282 | # and thus existing member attributes cannot be deduced by static analysis. It 283 | # supports qualified module names, as well as Unix pattern matching. 284 | ignored-modules= 285 | 286 | # List of classes names for which member attributes should not be checked 287 | # (useful for classes with attributes dynamically set). This supports can work 288 | # with qualified names. 289 | ignored-classes= 290 | 291 | # List of members which are set dynamically and missed by pylint inference 292 | # system, and so shouldn't trigger E1101 when accessed. Python regular 293 | # expressions are accepted. 294 | generated-members= 295 | 296 | 297 | [VARIABLES] 298 | 299 | # Tells whether we should check for unused import in __init__ files. 300 | init-import=no 301 | 302 | # A regular expression matching the name of dummy variables (i.e. expectedly 303 | # not used). 304 | dummy-variables-rgx=_$|dummy 305 | 306 | # List of additional names supposed to be defined in builtins. Remember that 307 | # you should avoid to define new builtins when possible. 308 | additional-builtins= 309 | 310 | # List of strings which can identify a callback function by name. A callback 311 | # name must start or end with one of those strings. 312 | callbacks=cb_,_cb 313 | 314 | 315 | [CLASSES] 316 | 317 | # List of method names used to declare (i.e. assign) instance attributes. 318 | defining-attr-methods=__init__,__new__,setUp 319 | 320 | # List of valid names for the first argument in a class method. 321 | valid-classmethod-first-arg=cls 322 | 323 | # List of valid names for the first argument in a metaclass class method. 324 | valid-metaclass-classmethod-first-arg=mcs 325 | 326 | # List of member names, which should be excluded from the protected access 327 | # warning. 328 | exclude-protected=_asdict,_fields,_replace,_source,_make 329 | 330 | 331 | [DESIGN] 332 | 333 | # Maximum number of arguments for function / method 334 | max-args=5 335 | 336 | # Argument names that match this expression will be ignored. Default to name 337 | # with leading underscore 338 | ignored-argument-names=_.* 339 | 340 | # Maximum number of locals for function / method body 341 | max-locals=15 342 | 343 | # Maximum number of return / yield for function / method body 344 | max-returns=6 345 | 346 | # Maximum number of branch for function / method body 347 | max-branches=12 348 | 349 | # Maximum number of statements in function / method body 350 | max-statements=50 351 | 352 | # Maximum number of parents for a class (see R0901). 353 | max-parents=7 354 | 355 | # Maximum number of attributes for a class (see R0902). 356 | max-attributes=7 357 | 358 | # Minimum number of public methods for a class (see R0903). 359 | min-public-methods=2 360 | 361 | # Maximum number of public methods for a class (see R0904). 362 | max-public-methods=20 363 | 364 | # Maximum number of boolean expressions in a if statement 365 | max-bool-expr=5 366 | 367 | 368 | [IMPORTS] 369 | 370 | # Deprecated modules which should not be used, separated by a comma 371 | deprecated-modules=regsub,TERMIOS,Bastion,rexec 372 | 373 | # Create a graph of every (i.e. internal and external) dependencies in the 374 | # given file (report RP0402 must not be disabled) 375 | import-graph= 376 | 377 | # Create a graph of external dependencies in the given file (report RP0402 must 378 | # not be disabled) 379 | ext-import-graph= 380 | 381 | # Create a graph of internal dependencies in the given file (report RP0402 must 382 | # not be disabled) 383 | int-import-graph= 384 | 385 | 386 | [EXCEPTIONS] 387 | 388 | # Exceptions that will emit a warning when being caught. Defaults to 389 | # "Exception" 390 | overgeneral-exceptions=Exception 391 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012 - 2018 Cloudera, Inc. 2 | # All Rights Reserved. 3 | 4 | backports.ssl-match-hostname==3.5.0.1 5 | braceexpand==0.1.2 6 | cffi==1.6.0 7 | colorama==0.2.5 8 | cryptography==1.3.4 9 | docker-py==1.8.1 10 | ecdsa==0.13 11 | enum34==1.1.6 12 | Fabric==1.11.1 13 | html5lib==0.999 14 | idna==2.1 15 | inflection==0.3.1 16 | ipaddress==1.0.16 17 | lxml==3.6.0 18 | Mako==1.0.4 19 | MarkupSafe==0.23 20 | ndg-httpsclient==0.4.0 21 | netaddr==0.7.18 22 | paramiko==2.0.0 23 | pyasn1==0.1.9 24 | pycparser==2.14 25 | pycrypto==2.6.1 26 | pyOpenSSL==16.0.0 27 | requests==2.10.0 28 | six==1.10.0 29 | urllib3==1.7.1 30 | websocket-client==0.37.0 31 | --------------------------------------------------------------------------------