├── .coveragerc ├── .gitignore ├── .readthedocs.yml ├── AUTHORS.rst ├── CHANGELOG.rst ├── LICENSE.txt ├── NOTICE.txt ├── README.rst ├── docs ├── Makefile ├── _static │ └── .gitignore ├── authors.rst ├── changelog.rst ├── conf.py ├── index.rst ├── license.rst ├── readme.rst └── requirements.txt ├── pyproject.toml ├── setup.cfg ├── setup.py ├── src └── cdpy │ ├── __init__.py │ ├── cdpy.py │ ├── common.py │ ├── datahub.py │ ├── datalake.py │ ├── de.py │ ├── df.py │ ├── dw.py │ ├── environments.py │ ├── iam.py │ ├── ml.py │ └── opdb.py └── tests ├── __init__.py ├── conftest.py ├── test_credentials.py ├── test_environments.py └── test_iam.py /.coveragerc: -------------------------------------------------------------------------------- 1 | # .coveragerc to control coverage.py 2 | [run] 3 | branch = True 4 | source = cdpy 5 | # omit = bad_file.py 6 | 7 | [paths] 8 | source = 9 | src/ 10 | */site-packages/ 11 | 12 | [report] 13 | # Regexes for lines to exclude from consideration 14 | exclude_lines = 15 | # Have to re-enable the standard pragma 16 | pragma: no cover 17 | 18 | # Don't complain about missing debug-only code: 19 | def __repr__ 20 | if self\.debug 21 | 22 | # Don't complain if tests don't hit defensive assertion code: 23 | raise AssertionError 24 | raise NotImplementedError 25 | 26 | # Don't complain if non-runnable code isn't run: 27 | if 0: 28 | if __name__ == .__main__.: 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .pytest_cache 2 | .tox 3 | build 4 | .coverage -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | 4 | # Required 5 | version: 2 6 | 7 | # Build documentation in the docs/ directory with Sphinx 8 | sphinx: 9 | configuration: docs/conf.py 10 | 11 | # Build documentation with MkDocs 12 | #mkdocs: 13 | # configuration: mkdocs.yml 14 | 15 | # Optionally build your docs in additional formats such as PDF 16 | formats: 17 | - pdf 18 | 19 | build: 20 | os: ubuntu-22.04 21 | tools: 22 | python: "3.11" 23 | 24 | python: 25 | install: 26 | - requirements: docs/requirements.txt 27 | - {path: ., method: pip} 28 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Contributors 3 | ============ 4 | 5 | * Daniel Chaffelson 6 | * Webster Mudge 7 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | ========= 2 | Changelog 3 | ========= 4 | 5 | Version 0.1.0 6 | ============= 7 | 8 | - Initial Release 9 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2023 Cloudera Inc. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | cdpy Python library 2 | Copyright 2023 Cloudera Inc. All Rights Reserved. 3 | 4 | 5 | OVERVIEW 6 | Items appearing in the following notices are provided for the notification and 7 | guidance of the recipient in accordance with the listed license. The listing 8 | may include various dependencies applicable only to development, testing, and 9 | evaluation, which may not be included in the distribution and may have been 10 | deprecated in favor of alternatives. Accordingly, this listing should not be 11 | construed as an admission that the listed component appears in the distribution 12 | nor in the manner indicated. Similarly, fulfilling a license's compliance 13 | requirements for a particular form of distribution is not an admission that the 14 | software is in fact distributed in that form. Distributions and packaging vary 15 | and not all third party components are relied upon in the same manner or degree. 16 | In addition, some compliance documentation, such as embedded notices files in 17 | source code, may be found within the distribution itself rather than below. 18 | "Third Party Software" and "Separately Licensed Code" as defined in the 19 | applicable Cloudera or Hortonworks agreement, respectively, have the same 20 | meaning for the purposes of this notice. 21 | 22 | MOST PERMISSIVE LICENSE APPLIES TO THIRD PARTY SOFTWARE 23 | Where the Third Party Software contains dual or multiple licensing options, the 24 | most permissive license compatible with your Cloudera software license applies. 25 | 26 | CONTAINERIZED OPERATING SYSTEM 27 | The Cloudera software runs on certain operating systems, such as the Linux 28 | operating system, which may be delivered to you with a containerized 29 | installation of the operating system for your convenience. Cloudera software 30 | constitutes a separate and independent work from the operating system, and the 31 | provided container is an aggregation of the operating system and the Cloudera 32 | software. 33 | 34 | NO WARRANTY; NO LIABILITY; NO INDEMNIFICATION/HOLD HARMLESS 35 | Notwithstanding any agreement recipient may have with Cloudera: (A) CLOUDERA 36 | PROVIDES THRID PARTY SOFTWARE TO RECIPIENT WITHOUT WARRANTIES OF ANY KIND; (B) 37 | CLOUDERA DISCLAIMS ANY AND ALL EXPRESS AND IMPLIED WARRANTIES WITH RESPECT TO 38 | THIRD PARTY SOFTWARE, INCLUDING BUT NOT LIMITED TO IMPLIED WARRANTIES OF TITLE, 39 | NON-INFRINGEMENT, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE; (C) 40 | CLOUDERA IS NOT LIABLE TO RECIPIENT, AND WILL NOT DEFEND, INDEMNIFY, NOR HOLD 41 | RECIPIENT HARMLESS FOR ANY CLAIMS ARISING FROM OR RELATED TO THIRD PARTY 42 | SOFTWARE; AND (D) WITH RESPECT TO THE THIRD PARTY SOFTWARE, CLOUDERA IS NOT 43 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, PUNITIVE OR 44 | CONSEQUENTIAL DAMAGES INCLUDING, BUT NOT LIMITED TO, DAMAGES RELATED TO LOST 45 | REVENUE, LOST PROFITS, LOSS OF INCOME, LOSS OF BUSINESS ADVANTAGE OR 46 | UNAVAILABILITY, LOSS OR CORRUPTION OF DATA. 47 | 48 | SOURCE CODE OFFER 49 | To the extent required by the licenses associated with the Third Party Software, 50 | Cloudera, Inc. will provide a copy of the Third Party Software's source code 51 | upon request and at a reasonable fee. All requests for source code should be 52 | made in a letter clearly identifying the components for which the source code is 53 | requested, and mailed to "Cloudera, Inc., ATTN: Legal Dept., 5470 Great America 54 | Pkwy, Santa Clara, CA 95054" 55 | 56 | Further, to the extent Cloudera delivers a containerized installation of an 57 | operating system, such operating system contains open source software 58 | constituting Third Party Software, including software licensed under the GNU 59 | General Public License (GPL). Such Third Party Software may or may not be 60 | identified separately in the notices below. To the extent required by the GPL 61 | or other licenses associated with the Third Party Software under which you 62 | receive only binaries, Cloudera will provide a copy of the Third Party 63 | Software's source code upon request and at a reasonable fee. All requests for 64 | source code should be made in a letter clearly identifying the components for 65 | which the source code is requested, typically sent within three (3) years from 66 | the date you received the covered binary, and mailed to "Cloudera, Inc., ATTN: 67 | Legal Dept., 5470 Great America Pkwy, Santa Clara, CA 95054". Your written 68 | request should include: (i) the name and version number of the covered binary, 69 | (ii) the version number of the Cloudera product containing the covered binary, 70 | (iii) your name, (iv) your company or organization name (if applicable), (v) the 71 | license under which the source code must be provided, and (vi) your return 72 | mailing and email address (if available). 73 | 74 | 75 | Components: 76 | 77 | pyskyq 0.2.1 : MIT License 78 | 79 | Licenses: 80 | 81 | MIT License 82 | (pyskyq 0.2.1) 83 | 84 | The MIT License 85 | =============== 86 | 87 | Copyright (c) 88 | 89 | Permission is hereby granted, free of charge, to any person obtaining a copy of 90 | this software and associated documentation files (the "Software"), to deal in the 91 | Software without restriction, including without limitation the rights to use, 92 | copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the 93 | Software, and to permit persons to whom the Software is furnished to do so, 94 | subject to the following conditions: 95 | 96 | The above copyright notice and this permission notice shall be included in all 97 | copies or substantial portions of the Software. 98 | 99 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 100 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 101 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 102 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN 103 | AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 104 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ==== 2 | cdpy 3 | ==== 4 | 5 | 6 | A Simple Pythonic Client wrapper for Cloudera CDP CLI, designed for use with the Ansible framework 7 | 8 | Installation 9 | ============ 10 | 11 | To install directly from latest commits, in python requirements.txt :: 12 | 13 | git+https://github.com/cloudera-labs/cdpy@main#egg=cdpy 14 | 15 | For General usage, installed from cmdline :: 16 | 17 | pip install cdpy 18 | 19 | To install the development branch instead of main :: 20 | 21 | git+https://github.com/cloudera-labs/cdpy@devel#egg=cdpy 22 | 23 | Usage 24 | ===== 25 | Note that this readme covers usage of this wrapper library only, for details of CDPCLI and underlying commands please see the CDPCLI documentation. 26 | 27 | Simple Usage 28 | ------------ 29 | 30 | Basic function call with defaults where credentials are already available in user profile :: 31 | 32 | from cdpy.cdpy import Cdpy 33 | Cdpy().iam.get_user() 34 | > {'userId': 'de740ef9-b40e-4497-8b3b-c137481c7633', 'crn': 'crn:altus:iam:us-west-1:558bc1d2-8867-4357-8524-311d51259233:user:de740ef9-b40e-4497-8b3b-c137481c7633', 'email': 'dchaffey@cloudera.com', 'firstName': 'Daniel', 'lastName': 'Chaffelson', 'creationDate': datetime.datetime(2019, 11, 4, 11, 54, 27, 581000, tzinfo=tzutc()), 'accountAdmin': False, 'identityProviderCrn': 'crn:altus:iam:us-west-1:558bc1d2-8867-4357-8524-311d51259233:samlProvider:cloudera-okta-production/a0afd6e3-ffc1-48bd-953a-60003d82f8ae', 'lastInteractiveLogin': datetime.datetime(2020, 12, 1, 11, 32, 38, 901000, tzinfo=tzutc()), 'workloadUsername': 'dchaffey'} 35 | 36 | Typical function calls in CDP CLI are shadowed within the relevant Classes when wrapped by cdpy, or may be called directly using the call function shown further down. 37 | 38 | Basic function call with target that does not exist (404) :: 39 | 40 | from cdpy.cdpy import Cdpy 41 | Cdpy().iam.get_user('fakeusername') 42 | >> UserWarning:CDP User could not be retrieved 43 | > None 44 | 45 | This Class function has been wrapped to automatically handle the NOT_FOUND error generated when requesting an unknown user, note the use of self.sdk.call within the Class :: 46 | 47 | def get_user(self, name=None): 48 | return self.sdk.call( 49 | # Describe base function calls 50 | svc='iam', # Name of the client service 51 | func='get_user', # Name of the Function within the service to call 52 | ret_field='user', # Optional child field to return, often CDP CLI responses are wrapped like this 53 | squelch=[ # List of below Client Error Handlers 54 | # Describe any Client Error responses using the provided Squelch class 55 | Squelch( 56 | field='error_code', # CdpError Field to test 57 | value='NOT_FOUND', # String to check for in Field 58 | warning='CDP User could not be retrieved', # Warning to throw if encountered 59 | default=None # Value to return instead of Error 60 | ) 61 | ], 62 | # Include any keyword args that may be used in the function call, None/'' args will be ignored 63 | userId=name # As name is None by default, it will be ignored unless provided 64 | ) 65 | 66 | Basic function call with invalid value :: 67 | 68 | from cdpy.cdpy import Cdpy 69 | Cdpy().iam.get_user('') 70 | >> UserWarning:Removing empty string arg userId from submission 71 | > {'userId': 'de740ef9-b40e-4497-8b3b-c137481c7633', 'crn': 'crn:altus:iam:us-west-1:558bc1d2-8867-4357-8524-311d51259233:user:de740ef9-b40e-4497-8b3b-c137481c7633', 'email': 'dchaffey@cloudera.com', 'firstName': 'Daniel', 'lastName': 'Chaffelson', 'creationDate': datetime.datetime(2019, 11, 4, 11, 54, 27, 581000, tzinfo=tzutc()), 'accountAdmin': False, 'identityProviderCrn': 'crn:altus:iam:us-west-1:558bc1d2-8867-4357-8524-311d51259233:samlProvider:cloudera-okta-production/a0afd6e3-ffc1-48bd-953a-60003d82f8ae', 'lastInteractiveLogin': datetime.datetime(2020, 12, 1, 11, 32, 38, 901000, tzinfo=tzutc()), 'workloadUsername': 'dchaffey'} 72 | 73 | Here the invalid Parameter for for typical Ansible parameter `name` is mapped to `userId` within CDPCLI and is scrubbed from the submission ( may not submit zero length strings), a user warning is issued. 74 | The command is run by default with that Parameter removed, this triggering the 'list all' logic of the underlying call. 75 | 76 | This behavior may be bypassed using the `scrub_inputs` switch and thus raise a native Python Error :: 77 | 78 | Cdpy(scrub_inputs=False).iam.get_user('') 79 | Traceback (most recent call last): 80 | ... 81 | 82 | More Complex Usage 83 | ------------------ 84 | 85 | Call Wrapper execution directly for arbitrary CDP Service and Function with arbitrary keyword param. This `call` function is the same that is exposed in the more abstract Classes in earlier examples, this is the more direct usage method allowing developers to work at varying levels within the layered abstractions with relative ease :: 86 | 87 | from cdpy.common import CdpcliWrapper 88 | 89 | CdpcliWrapper().call(svc='iam', func='set_workload_password_policy', maxPasswordLifetimeDays=lifetime) 90 | 91 | Define function to call wrapped method with prebuild payload and use custom error handling logic. 92 | Note that the `env_config` arguments are unpacked and passed to the CDP API directly, allowing developers to work with additional parameters without waiting for the higher level abstractions to support them 93 | This example also demonstrates bypassing the Squelch error handler to implement custom error handling logic, and the ability to revert to the provided `throw_error` capabilities (which are superable) :: 94 | 95 | from cdpy.common import CdpcliWrapper 96 | from cdpy.common import CdpError 97 | 98 | wrap = CdpcliWrapper() 99 | env_config = dict(valueOne=value, valueTwo=value) 100 | 101 | resp = wrap.call( 102 | svc='environments', func='create_aws_environment', ret_field='environment', ret_error=True, 103 | **env_config 104 | ) 105 | if isinstance(resp, CdpError): 106 | if resp.error_code == 'INVALID_ARGUMENT': 107 | if 'constraintViolations' not in str(resp.violations): 108 | resp.update(message="Received violation warning:\n%s" % self.sdk.dumps(str(resp.violations))) 109 | self.sdk.throw_warning(resp) 110 | self.sdk.throw_error(resp) 111 | return resp 112 | 113 | Declare custom error handling function and instantiate with it. This abstraction is specifically to allow developers to replace native Python error handling with framework specific handling, such as the typical Ansible module `fail_json` seen here :: 114 | 115 | from cdpy.common import CdpError 116 | 117 | class CdpModule(object) 118 | def _cdp_module_throw_error(self, error: 'CdpError'): 119 | """Wraps throwing Errors when used as Ansible module""" 120 | self.module.fail_json(msg=str(error.__dict__)) 121 | 122 | self.sdk = Cdpy(error_handler=self._cdp_module_throw_error) 123 | 124 | Ideally for extensive development you would make use of the metaclass Cdpy, this is currently used as the basis for the Cloudera CDP Public Cloud Ansible Collection :: 125 | 126 | from cdpy.cdpy import Cdpy 127 | client = Cdpy(debug=self.debug, tls_verify=self.tls, strict_errors=self.strict, error_handler=self._cdp_module_throw_error, warning_handler=self._cdp_module_throw_warning) 128 | client.sdk.call(...) 129 | client.sdk.iam.(...) 130 | client.sdk.TERMINATION_STATES 131 | etc. 132 | 133 | 134 | Development 135 | ===================== 136 | 137 | Contributing 138 | ------------ 139 | 140 | Please create a feature branch from the current development Branch then submit a PR referencing an Issue for discussion. 141 | 142 | Please note that we require signed commits inline with Developer Certificate of Origin best-practices for Open Source Collaboration. 143 | 144 | PyScaffold Note 145 | =============== 146 | 147 | This project has been set up using PyScaffold 4. For details and usage 148 | information on PyScaffold see https://pyscaffold.org/. 149 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | AUTODOCDIR = api 11 | 12 | # User-friendly check for sphinx-build 13 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $?), 1) 14 | $(error "The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from https://sphinx-doc.org/") 15 | endif 16 | 17 | .PHONY: help clean Makefile 18 | 19 | # Put it first so that "make" without argument is like "make help". 20 | help: 21 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 22 | 23 | clean: 24 | rm -rf $(BUILDDIR)/* $(AUTODOCDIR) 25 | 26 | # Catch-all target: route all unknown targets to Sphinx using the new 27 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 28 | %: Makefile 29 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 30 | -------------------------------------------------------------------------------- /docs/_static/.gitignore: -------------------------------------------------------------------------------- 1 | # Empty directory 2 | -------------------------------------------------------------------------------- /docs/authors.rst: -------------------------------------------------------------------------------- 1 | .. _authors: 2 | .. include:: ../AUTHORS.rst 3 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | .. _changes: 2 | .. include:: ../CHANGELOG.rst 3 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # This file is execfile()d with the current directory set to its containing dir. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | # 7 | # All configuration values have a default; values that are commented out 8 | # serve to show the default. 9 | 10 | import os 11 | import sys 12 | import shutil 13 | 14 | # -- Path setup -------------------------------------------------------------- 15 | 16 | __location__ = os.path.dirname(__file__) 17 | 18 | # If extensions (or modules to document with autodoc) are in another directory, 19 | # add these directories to sys.path here. If the directory is relative to the 20 | # documentation root, use os.path.abspath to make it absolute, like shown here. 21 | sys.path.insert(0, os.path.join(__location__, "../src")) 22 | 23 | # -- Run sphinx-apidoc ------------------------------------------------------- 24 | # This hack is necessary since RTD does not issue `sphinx-apidoc` before running 25 | # `sphinx-build -b html . _build/html`. See Issue: 26 | # https://github.com/readthedocs/readthedocs.org/issues/1139 27 | # DON'T FORGET: Check the box "Install your project inside a virtualenv using 28 | # setup.py install" in the RTD Advanced Settings. 29 | # Additionally it helps us to avoid running apidoc manually 30 | 31 | try: # for Sphinx >= 1.7 32 | from sphinx.ext import apidoc 33 | except ImportError: 34 | from sphinx import apidoc 35 | 36 | output_dir = os.path.join(__location__, "api") 37 | module_dir = os.path.join(__location__, "../src/cdpy") 38 | try: 39 | shutil.rmtree(output_dir) 40 | except FileNotFoundError: 41 | pass 42 | 43 | try: 44 | import sphinx 45 | 46 | cmd_line = f"sphinx-apidoc --implicit-namespaces -f -o {output_dir} {module_dir}" 47 | 48 | args = cmd_line.split(" ") 49 | if tuple(sphinx.__version__.split(".")) >= ("1", "7"): 50 | # This is a rudimentary parse_version to avoid external dependencies 51 | args = args[1:] 52 | 53 | apidoc.main(args) 54 | except Exception as e: 55 | print("Running `sphinx-apidoc` failed!\n{}".format(e)) 56 | 57 | # -- General configuration --------------------------------------------------- 58 | 59 | # If your documentation needs a minimal Sphinx version, state it here. 60 | # needs_sphinx = '1.0' 61 | 62 | # Add any Sphinx extension module names here, as strings. They can be extensions 63 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 64 | extensions = [ 65 | "sphinx.ext.autodoc", 66 | "sphinx.ext.intersphinx", 67 | "sphinx.ext.todo", 68 | "sphinx.ext.autosummary", 69 | "sphinx.ext.viewcode", 70 | "sphinx.ext.coverage", 71 | "sphinx.ext.doctest", 72 | "sphinx.ext.ifconfig", 73 | "sphinx.ext.mathjax", 74 | "sphinx.ext.napoleon", 75 | ] 76 | 77 | # Add any paths that contain templates here, relative to this directory. 78 | templates_path = ["_templates"] 79 | 80 | # The suffix of source filenames. 81 | source_suffix = ".rst" 82 | 83 | # The encoding of source files. 84 | # source_encoding = 'utf-8-sig' 85 | 86 | # The master toctree document. 87 | master_doc = "index" 88 | 89 | # General information about the project. 90 | project = "cdpy" 91 | copyright = "2023, Cloudera Inc." 92 | 93 | # The version info for the project you're documenting, acts as replacement for 94 | # |version| and |release|, also used in various other places throughout the 95 | # built documents. 96 | # 97 | # version: The short X.Y version. 98 | # release: The full version, including alpha/beta/rc tags. 99 | # If you don’t need the separation provided between version and release, 100 | # just set them both to the same value. 101 | try: 102 | from cdpy import __version__ as version 103 | except ImportError: 104 | version = "" 105 | 106 | if not version or version.lower() == "unknown": 107 | version = os.getenv("READTHEDOCS_VERSION", "unknown") # automatically set by RTD 108 | 109 | release = version 110 | 111 | # The language for content autogenerated by Sphinx. Refer to documentation 112 | # for a list of supported languages. 113 | # language = None 114 | 115 | # There are two options for replacing |today|: either, you set today to some 116 | # non-false value, then it is used: 117 | # today = '' 118 | # Else, today_fmt is used as the format for a strftime call. 119 | # today_fmt = '%B %d, %Y' 120 | 121 | # List of patterns, relative to source directory, that match files and 122 | # directories to ignore when looking for source files. 123 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", ".venv"] 124 | 125 | # The reST default role (used for this markup: `text`) to use for all documents. 126 | # default_role = None 127 | 128 | # If true, '()' will be appended to :func: etc. cross-reference text. 129 | # add_function_parentheses = True 130 | 131 | # If true, the current module name will be prepended to all description 132 | # unit titles (such as .. function::). 133 | # add_module_names = True 134 | 135 | # If true, sectionauthor and moduleauthor directives will be shown in the 136 | # output. They are ignored by default. 137 | # show_authors = False 138 | 139 | # The name of the Pygments (syntax highlighting) style to use. 140 | pygments_style = "sphinx" 141 | 142 | # A list of ignored prefixes for module index sorting. 143 | # modindex_common_prefix = [] 144 | 145 | # If true, keep warnings as "system message" paragraphs in the built documents. 146 | # keep_warnings = False 147 | 148 | # If this is True, todo emits a warning for each TODO entries. The default is False. 149 | todo_emit_warnings = True 150 | 151 | 152 | # -- Options for HTML output ------------------------------------------------- 153 | 154 | # The theme to use for HTML and HTML Help pages. See the documentation for 155 | # a list of builtin themes. 156 | html_theme = "alabaster" 157 | 158 | # Theme options are theme-specific and customize the look and feel of a theme 159 | # further. For a list of options available for each theme, see the 160 | # documentation. 161 | html_theme_options = { 162 | "sidebar_width": "300px", 163 | "page_width": "1200px" 164 | } 165 | 166 | # Add any paths that contain custom themes here, relative to this directory. 167 | # html_theme_path = [] 168 | 169 | # The name for this set of Sphinx documents. If None, it defaults to 170 | # " v documentation". 171 | # html_title = None 172 | 173 | # A shorter title for the navigation bar. Default is the same as html_title. 174 | # html_short_title = None 175 | 176 | # The name of an image file (relative to this directory) to place at the top 177 | # of the sidebar. 178 | # html_logo = "" 179 | 180 | # The name of an image file (within the static path) to use as favicon of the 181 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 182 | # pixels large. 183 | # html_favicon = None 184 | 185 | # Add any paths that contain custom static files (such as style sheets) here, 186 | # relative to this directory. They are copied after the builtin static files, 187 | # so a file named "default.css" will overwrite the builtin "default.css". 188 | html_static_path = ["_static"] 189 | 190 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 191 | # using the given strftime format. 192 | # html_last_updated_fmt = '%b %d, %Y' 193 | 194 | # If true, SmartyPants will be used to convert quotes and dashes to 195 | # typographically correct entities. 196 | # html_use_smartypants = True 197 | 198 | # Custom sidebar templates, maps document names to template names. 199 | # html_sidebars = {} 200 | 201 | # Additional templates that should be rendered to pages, maps page names to 202 | # template names. 203 | # html_additional_pages = {} 204 | 205 | # If false, no module index is generated. 206 | # html_domain_indices = True 207 | 208 | # If false, no index is generated. 209 | # html_use_index = True 210 | 211 | # If true, the index is split into individual pages for each letter. 212 | # html_split_index = False 213 | 214 | # If true, links to the reST sources are added to the pages. 215 | # html_show_sourcelink = True 216 | 217 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 218 | # html_show_sphinx = True 219 | 220 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 221 | # html_show_copyright = True 222 | 223 | # If true, an OpenSearch description file will be output, and all pages will 224 | # contain a tag referring to it. The value of this option must be the 225 | # base URL from which the finished HTML is served. 226 | # html_use_opensearch = '' 227 | 228 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 229 | # html_file_suffix = None 230 | 231 | # Output file base name for HTML help builder. 232 | htmlhelp_basename = "cdpy-doc" 233 | 234 | 235 | # -- Options for LaTeX output ------------------------------------------------ 236 | 237 | latex_elements = { 238 | # The paper size ("letterpaper" or "a4paper"). 239 | # "papersize": "letterpaper", 240 | # The font size ("10pt", "11pt" or "12pt"). 241 | # "pointsize": "10pt", 242 | # Additional stuff for the LaTeX preamble. 243 | # "preamble": "", 244 | } 245 | 246 | # Grouping the document tree into LaTeX files. List of tuples 247 | # (source start file, target name, title, author, documentclass [howto/manual]). 248 | latex_documents = [ 249 | ("index", "user_guide.tex", "cdpy Documentation", "Daniel Chaffelson", "manual") 250 | ] 251 | 252 | # The name of an image file (relative to this directory) to place at the top of 253 | # the title page. 254 | # latex_logo = "" 255 | 256 | # For "manual" documents, if this is true, then toplevel headings are parts, 257 | # not chapters. 258 | # latex_use_parts = False 259 | 260 | # If true, show page references after internal links. 261 | # latex_show_pagerefs = False 262 | 263 | # If true, show URL addresses after external links. 264 | # latex_show_urls = False 265 | 266 | # Documents to append as an appendix to all manuals. 267 | # latex_appendices = [] 268 | 269 | # If false, no module index is generated. 270 | # latex_domain_indices = True 271 | 272 | # -- External mapping -------------------------------------------------------- 273 | python_version = ".".join(map(str, sys.version_info[0:2])) 274 | intersphinx_mapping = { 275 | "sphinx": ("https://www.sphinx-doc.org/en/master", None), 276 | "python": ("https://docs.python.org/" + python_version, None), 277 | "matplotlib": ("https://matplotlib.org", None), 278 | "numpy": ("https://numpy.org/doc/stable", None), 279 | "sklearn": ("https://scikit-learn.org/stable", None), 280 | "pandas": ("https://pandas.pydata.org/pandas-docs/stable", None), 281 | "scipy": ("https://docs.scipy.org/doc/scipy/reference", None), 282 | "setuptools": ("https://setuptools.pypa.io/en/stable/", None), 283 | "pyscaffold": ("https://pyscaffold.org/en/stable", None), 284 | } 285 | 286 | print(f"loading configurations for {project} {version} ...", file=sys.stderr) 287 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ==== 2 | cdpy 3 | ==== 4 | 5 | This is the documentation of **cdpy**. 6 | 7 | Contents 8 | ======== 9 | 10 | .. toctree:: 11 | :maxdepth: 2 12 | 13 | Readme 14 | License 15 | Authors 16 | Changelog 17 | Module Reference 18 | 19 | 20 | Indices and tables 21 | ================== 22 | 23 | * :ref:`genindex` 24 | * :ref:`modindex` 25 | * :ref:`search` 26 | 27 | .. _toctree: http://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html 28 | .. _reStructuredText: http://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html 29 | .. _references: http://www.sphinx-doc.org/en/stable/markup/inline.html 30 | .. _Python domain syntax: http://sphinx-doc.org/domains.html#the-python-domain 31 | .. _Sphinx: http://www.sphinx-doc.org/ 32 | .. _Python: http://docs.python.org/ 33 | .. _Numpy: http://docs.scipy.org/doc/numpy 34 | .. _SciPy: http://docs.scipy.org/doc/scipy/reference/ 35 | .. _matplotlib: https://matplotlib.org/contents.html# 36 | .. _Pandas: http://pandas.pydata.org/pandas-docs/stable 37 | .. _Scikit-Learn: http://scikit-learn.org/stable 38 | .. _autodoc: http://www.sphinx-doc.org/en/stable/ext/autodoc.html 39 | .. _Google style: https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings 40 | .. _NumPy style: https://numpydoc.readthedocs.io/en/latest/format.html 41 | .. _classical style: http://www.sphinx-doc.org/en/stable/domains.html#info-field-lists 42 | -------------------------------------------------------------------------------- /docs/license.rst: -------------------------------------------------------------------------------- 1 | .. _license: 2 | 3 | ======= 4 | License 5 | ======= 6 | 7 | .. include:: ../LICENSE.txt 8 | -------------------------------------------------------------------------------- /docs/readme.rst: -------------------------------------------------------------------------------- 1 | .. _readme: 2 | .. include:: ../README.rst 3 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | # Requirements file for ReadTheDocs, check .readthedocs.yml. 2 | # To build the module reference correctly, make sure every external package 3 | # under `install_requires` in `setup.cfg` is also listed here! 4 | sphinx>=4 5 | # sphinx_rtd_theme 6 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | # AVOID CHANGING REQUIRES: IT WILL BE UPDATED BY PYSCAFFOLD! 3 | requires = ["setuptools>=46.1.0", "setuptools_scm[toml]>=5"] 4 | build-backend = "setuptools.build_meta" 5 | 6 | [tool.setuptools_scm] 7 | # For smarter version schemes and other configuration options, 8 | # check out https://github.com/pypa/setuptools_scm 9 | version_scheme = "no-guess-dev" 10 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | # This file is used to configure your project. 2 | # Read more about the various options under: 3 | # http://setuptools.readthedocs.io/en/latest/setuptools.html#configuring-setup-using-setup-cfg-files 4 | 5 | [metadata] 6 | name = cdpy 7 | version = 1.3.0 8 | description = A Pythonic client wrapper for Cloudera CDP CLI 9 | author = Cloudera Labs 10 | author_email = cloudera-labs@cloudera.com 11 | license = apache 12 | long_description = file: README.rst 13 | long_description_content_type = text/x-rst; charset=UTF-8 14 | url = https://github.com/cloudera-labs/cdpy 15 | project_urls = 16 | Documentation = https://github.com/cloudera-labs/cdpy/readme.rst 17 | # Change if running only on Windows, Mac or Linux (comma-separated) 18 | platforms = any 19 | # Add here all kinds of additional classifiers as defined under 20 | # https://pypi.python.org/pypi?%3Aaction=list_classifiers 21 | classifiers = 22 | Development Status :: 4 - Beta 23 | Programming Language :: Python 24 | 25 | [options] 26 | zip_safe = False 27 | packages = find_namespace: 28 | include_package_data = True 29 | package_dir = 30 | =src 31 | # Add here dependencies of your project (semicolon/line-separated), e.g. 32 | install_requires = 33 | importlib-metadata 34 | cdpcli-beta>=0.9.101 35 | # The usage of test_requires is discouraged, see `Dependency Management` docs 36 | # tests_require = pytest; pytest-cov 37 | # Require a specific Python version, e.g. Python 2.7 or >= 3.4 38 | python_requires = >=3.8 39 | 40 | [options.packages.find] 41 | where = src 42 | exclude = 43 | tests 44 | 45 | [options.extras_require] 46 | # Add here additional requirements for extra features, to install with: 47 | # `pip install cdpy[PDF]` like: 48 | # PDF = ReportLab; RXP 49 | # Add here test requirements (semicolon/line-separated) 50 | testing = 51 | pytest 52 | pytest-cov 53 | wheel 54 | flake8 55 | sphinx 56 | tox 57 | pre-commit 58 | 59 | [test] 60 | # py.test options when running `python setup.py test` 61 | # addopts = --verbose 62 | extras = True 63 | 64 | [tool:pytest] 65 | # Options for py.test: 66 | # Specify command line options as you would do when invoking py.test directly. 67 | # e.g. --cov-report html (or xml) for html/xml output or --junitxml junit.xml 68 | # in order to write a coverage file that can be read by Jenkins. 69 | addopts = 70 | #--cov=cdpy --cov-report=term-missing 71 | --verbose 72 | norecursedirs = 73 | dist 74 | build 75 | .tox 76 | testpaths = tests 77 | 78 | [aliases] 79 | dists = bdist_wheel 80 | 81 | [bdist_wheel] 82 | # Use this option if your package is pure-python 83 | universal = 1 84 | 85 | [build_sphinx] 86 | source_dir = docs 87 | build_dir = build/sphinx 88 | 89 | [devpi:upload] 90 | # Options for the devpi: PyPI server and packaging tool 91 | # VCS export must be deactivated since we are using setuptools-scm 92 | no-vcs = 1 93 | formats = bdist_wheel 94 | 95 | [flake8] 96 | # Some sane defaults for the code style checker flake8 97 | exclude = 98 | .tox 99 | build 100 | dist 101 | .eggs 102 | docs/conf.py 103 | 104 | [pyscaffold] 105 | # PyScaffold's parameters when the project was created. 106 | # This will be used when updating. Do not change! 107 | version = 4.4 108 | package = cdpy 109 | extensions = 110 | no_skeleton 111 | tox 112 | 113 | [tox:tox] 114 | env_list = 115 | py38 116 | py39 117 | py310 118 | py311 119 | minversion = 4 120 | 121 | [testenv] 122 | description = run the tests with pytest 123 | package = wheel 124 | wheel_build_env = .pkg 125 | deps = 126 | pytest>=6 127 | pytest-sugar 128 | commands = 129 | pytest {tty:--color=yes} {posargs} 130 | 131 | [testenv:{build,clean}] 132 | description = 133 | build: Build the package in isolation according to PEP517, see https://github.com/pypa/build 134 | clean: Remove old distribution files and temporary build artifacts (./build and ./dist) 135 | # https://setuptools.pypa.io/en/stable/build_meta.html#how-to-use-it 136 | skip_install = True 137 | changedir = {toxinidir} 138 | deps = 139 | build: build[virtualenv] 140 | passenv = 141 | SETUPTOOLS_* 142 | commands = 143 | clean: python -c 'import shutil; [shutil.rmtree(p, True) for p in ("build", "dist", "docs/_build")]' 144 | clean: python -c 'import pathlib, shutil; [shutil.rmtree(p, True) for p in pathlib.Path("src").glob("*.egg-info")]' 145 | build: python -m build {posargs} 146 | 147 | [testenv:{docs,doctests,linkcheck}] 148 | description = 149 | docs: Invoke sphinx-build to build the docs 150 | doctests: Invoke sphinx-build to run doctests 151 | linkcheck: Check for broken links in the documentation 152 | passenv = 153 | SETUPTOOLS_* 154 | setenv = 155 | DOCSDIR = {toxinidir}/docs 156 | BUILDDIR = {toxinidir}/docs/_build 157 | docs: BUILD = html 158 | doctests: BUILD = doctest 159 | linkcheck: BUILD = linkcheck 160 | # skip_install = true 161 | allowlist_externals=echo 162 | deps = 163 | -r {toxinidir}/docs/requirements.txt 164 | # ^ requirements.txt shared with Read The Docs 165 | # See https://github.com/Kopfstein/pyscaffold-gh-pages-deploy/blob/main/action.yml 166 | commands = 167 | echo "Docs build chain is unavailable via tox - blocked by a hard requirement of docutils in cdpcli. Use the Sphinx 'make' as an alternative." 168 | exit 169 | sphinx-build --color -b {env:BUILD} -d "{env:BUILDDIR}/doctrees" "{env:DOCSDIR}" "{env:BUILDDIR}/{env:BUILD}" {posargs} 170 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | Setup file for cdpy. 3 | Use setup.cfg to configure your project. 4 | 5 | This file was generated with PyScaffold 4.4. 6 | PyScaffold helps you to put up the scaffold of your new Python project. 7 | Learn more under: https://pyscaffold.org/ 8 | """ 9 | from setuptools import setup 10 | 11 | if __name__ == "__main__": 12 | try: 13 | setup(use_scm_version={"version_scheme": "no-guess-dev"}) 14 | except: # noqa 15 | print( 16 | "\n\nAn error occurred while building the project, " 17 | "please ensure you have the most updated version of setuptools, " 18 | "setuptools_scm and wheel with:\n" 19 | " pip install -U setuptools setuptools_scm wheel\n\n" 20 | ) 21 | raise 22 | -------------------------------------------------------------------------------- /src/cdpy/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | if sys.version_info[:2] >= (3, 8): 4 | # TODO: Import directly (no need for conditional) when `python_requires = >= 3.8` 5 | from importlib.metadata import PackageNotFoundError, version # pragma: no cover 6 | else: 7 | from importlib_metadata import PackageNotFoundError, version # pragma: no cover 8 | 9 | try: 10 | # Change here if project is renamed and does not equal the package name 11 | dist_name = __name__ 12 | __version__ = version(dist_name) 13 | except PackageNotFoundError: # pragma: no cover 14 | __version__ = "unknown" 15 | finally: 16 | del version, PackageNotFoundError 17 | -------------------------------------------------------------------------------- /src/cdpy/cdpy.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | For Wrapping interactions with the Cloudera CDP CLI 5 | """ 6 | 7 | from cdpy.common import CdpSdkBase 8 | from cdpy.iam import CdpyIam 9 | from cdpy.environments import CdpyEnvironments 10 | from cdpy.datahub import CdpyDatahub 11 | from cdpy.datalake import CdpyDatalake 12 | from cdpy.ml import CdpyMl 13 | from cdpy.de import CdpyDe 14 | from cdpy.opdb import CdpyOpdb 15 | from cdpy.dw import CdpyDw 16 | from cdpy.df import CdpyDf 17 | 18 | 19 | class Cdpy(CdpSdkBase): 20 | def __init__(self, *args, **kwargs): 21 | super().__init__(*args, **kwargs) 22 | 23 | self.iam = CdpyIam(*args, **kwargs) 24 | self.environments = CdpyEnvironments(*args, **kwargs) 25 | self.datahub = CdpyDatahub(*args, **kwargs) 26 | self.datalake = CdpyDatalake(*args, **kwargs) 27 | self.ml = CdpyMl(*args, **kwargs) 28 | self.de = CdpyDe(*args, **kwargs) 29 | self.opdb = CdpyOpdb(*args, **kwargs) 30 | self.dw = CdpyDw(*args, **kwargs) 31 | self.df = CdpyDf(*args, **kwargs) 32 | self.de = CdpyDe(*args, **kwargs) 33 | -------------------------------------------------------------------------------- /src/cdpy/common.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from datetime import datetime 4 | import pkg_resources 5 | from time import time, sleep 6 | import html 7 | import io 8 | import json 9 | import logging 10 | import platform 11 | import re 12 | import warnings 13 | import traceback 14 | import urllib3 15 | from urllib.parse import urljoin 16 | from urllib3.exceptions import InsecureRequestWarning 17 | from json import JSONDecodeError 18 | from typing import Union 19 | import urllib.parse as urlparse 20 | from os import path 21 | 22 | from cdpcli import VERSION as CDPCLI_VERSION 23 | from cdpcli.client import ClientCreator, Context 24 | from cdpcli.credentials import Credentials 25 | from cdpcli.endpoint import EndpointCreator, EndpointResolver 26 | from cdpcli.exceptions import ClientError, ParamValidationError, ValidationError 27 | from cdpcli.loader import Loader 28 | from cdpcli.parser import ResponseParserFactory 29 | from cdpcli.retryhandler import create_retry_handler 30 | from cdpcli.translate import build_retry_config 31 | 32 | 33 | class CdpWarning(UserWarning): 34 | """Class for deriving custom warnings from UserWarning""" 35 | def __init__(self, message): 36 | self.message = message 37 | 38 | 39 | class CdpError(Exception): 40 | """Parser Class for Errors returned from CDP CLI SDK""" 41 | def __init__(self, base_error, *args): 42 | self.base_error = base_error 43 | self.ext_traceback = traceback.format_stack()[:-1] 44 | self.error_code = None 45 | self.violations = None 46 | self.message = None 47 | self.status_code = None 48 | self.rc = None 49 | self.service = None 50 | self.operation = None 51 | self.request_id = None 52 | 53 | if isinstance(self.base_error, AttributeError): 54 | self.rc = 1 55 | self.status_code = '404' 56 | self.error_code = "LOCAL_NOT_IMPLEMENTED" 57 | self.violations = self.message = str(self.base_error) + ". The installed CDPCLI does not support this call." 58 | 59 | if isinstance(self.base_error, ValidationError): 60 | self.message = self.violations = str(self.base_error) + ". The installed CDPCLI does not support this call." 61 | self.rc = 1 62 | self.status_code = '404' 63 | self.operation = self.base_error.kwargs['param'] 64 | self.service = self.base_error.kwargs['value'] 65 | self.error_code = 'LOCAL_NOT_IMPLEMENTED' 66 | 67 | if isinstance(self.base_error, ClientError): 68 | _CLIENT_ERROR_PATTERN = re.compile( 69 | r"Status Code: (.*?); Error Code: (.*?); Service: " 70 | r"(.*?); Operation: (.*?); Request ID: (.*?);" 71 | ) 72 | _payload = re.search(_CLIENT_ERROR_PATTERN, str(self.base_error)) 73 | try: 74 | _violations = json.loads(html.unescape(self.base_error.response['error']['message'])) 75 | except JSONDecodeError: 76 | try: 77 | _violations = self.base_error.response['error']['message'] 78 | except KeyError: 79 | _violations = self.base_error.args[0] 80 | try: 81 | self.error_code = self.base_error.response['error']['code'] 82 | except KeyError: 83 | self.error_code = '' 84 | self.violations = _violations 85 | self.message = "Client request error" 86 | self.status_code = _payload.group(1) 87 | self.rc = 1 88 | self.service = _payload.group(3) 89 | self.operation = _payload.group(4) 90 | self.request_id = _payload.group(5) 91 | 92 | if isinstance(self.base_error, ParamValidationError): 93 | _PARAM_ERROR_PATTERN = re.compile( 94 | r"Parameter validation failed:\n([\s\S]*)" 95 | ) 96 | _payload = re.search(_PARAM_ERROR_PATTERN, str(self.base_error)) 97 | _violations = _payload.group(1).split('\n') 98 | self.violations = _violations 99 | self.message = "Parameter validation error" 100 | self.error_code = 'PARAMETER_VALIDATION_ERROR' 101 | 102 | # Handle instance where client calls function not found in remote CDP Control Plane 103 | if self.status_code == '404' \ 104 | and self.error_code == 'UNKNOWN_ERROR' \ 105 | and 'HTTP ERROR 404 Not Found' in self.message: 106 | self.error_code = "REMOTE_NOT_IMPLEMENTED" 107 | self.violations = "Function {0} in Remote Service {1} was not found. " \ 108 | "Your connected CDP Control Plane may not support this call. " \ 109 | "Rerun this call with strict_errors enabled to get full traceback." \ 110 | .format(self.operation, self.service) 111 | 112 | super().__init__(base_error, *args) 113 | 114 | def update(self, *args, **kwargs): 115 | return self.__dict__.update(*args, **kwargs) 116 | 117 | def __str__(self): 118 | return self.__repr__() 119 | 120 | 121 | class Squelch(dict): 122 | def __init__(self, value, field='error_code', default=None, warning=None): 123 | super().__init__() 124 | self.field = field 125 | self.value = value 126 | self.default = default 127 | self.warning = warning 128 | 129 | 130 | class StaticCredentials(Credentials): 131 | """A credential class that simply takes a set of static credentials.""" 132 | 133 | def __init__(self, access_key_id='', private_key='', access_token='', method='static'): 134 | super(StaticCredentials, self).__init__( 135 | access_key_id=access_key_id, private_key=private_key, 136 | access_token=access_token, method=method 137 | ) 138 | 139 | 140 | class CdpcliWrapper(object): 141 | def __init__(self, debug=False, tls_verify=False, strict_errors=False, tls_warnings=False, client_endpoint=None, 142 | cdp_credentials=None, error_handler=None, warning_handler=None, scrub_inputs=True, cp_region='default', 143 | agent_header=None): 144 | # Init Params 145 | self.debug = debug 146 | self.tls_verify = tls_verify 147 | self.strict_errors = strict_errors 148 | self.tls_warnings = tls_warnings 149 | self.client_endpoint = client_endpoint 150 | self.cdp_credentials = cdp_credentials 151 | self.scrub_inputs = scrub_inputs 152 | self.cp_region = cp_region 153 | self.agent_header = agent_header if agent_header is not None else 'CDPY' 154 | 155 | # Setup 156 | self.throw_error = error_handler if error_handler else self._default_throw_error 157 | self.throw_warning = warning_handler if warning_handler else self._default_throw_warning 158 | self._clients = {} 159 | self.DEFAULT_PAGE_SIZE = 100 160 | 161 | _loader = Loader() 162 | _user_agent = self._make_user_agent_header() 163 | 164 | self._client_creator = ClientCreator( 165 | _loader, 166 | Context(), 167 | EndpointCreator(EndpointResolver()), 168 | _user_agent, 169 | ResponseParserFactory(), 170 | create_retry_handler(self._load_retry_config(_loader))) 171 | 172 | # Logging 173 | _log_format = '%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s' 174 | if debug: 175 | self._setup_logger(logging.DEBUG, _log_format) 176 | self.logger.debug("CDP SDK version: %s", _user_agent) 177 | else: 178 | self._setup_logger(logging.ERROR, _log_format) 179 | 180 | if self.tls_warnings is False: 181 | urllib3.disable_warnings(InsecureRequestWarning) 182 | 183 | # Warnings 184 | def _warning_format(message, category, filename, lineno, line=None): 185 | return ' %s:%s: %s:%s' % (filename, lineno, category.__name__, message) 186 | 187 | warnings.formatwarning = _warning_format 188 | 189 | # State listings 190 | # https://github.com/hortonworks/cloudbreak/blob/master/cluster-api/src/main/java/com/sequenceiq/ 191 | # cloudbreak/cluster/status/ClusterStatus.java#L8-L18 192 | # https://github.com/hortonworks/cloudbreak/blob/master/core-api/src/main/java/com/sequenceiq/ 193 | # cloudbreak/api/endpoint/v4/common/Status.java#L14-L53 194 | self.CREATION_STATES = [ 195 | 'REQUESTED', 196 | 'EXTERNAL_DATABASE_CREATION_IN_PROGRESS', 197 | 'STACK_CREATION_IN_PROGRESS', 198 | 'CREATION_INITIATED', 199 | 'FREEIPA_CREATION_IN_PROGRESS', 200 | 'STARTING', 201 | 'ENABLING', # DF 202 | 'provision:started', # ML 203 | 'installation:started' # ML 204 | ] 205 | 206 | self.TERMINATION_STATES = [ 207 | 'EXTERNAL_DATABASE_DELETION_IN_PROGRESS', 208 | 'STACK_DELETION_IN_PROGRESS', 209 | 'FREEIPA_DELETE_IN_PROGRESS', 210 | 'STOPPING', 211 | 'deprovision:started', # ML 212 | 'DISABLING' # DF 213 | ] 214 | 215 | self.STARTED_STATES = [ 216 | 'EXTERNAL_DATABASE_START_IN_PROGRESS', 217 | 'AVAILABLE', 218 | 'START_IN_PROGRESS', 219 | 'RUNNING', 220 | 'installation:finished', # ML 221 | 'Running', # DW 222 | 'GOOD_HEALTH', # DF 223 | 'ClusterCreationCompleted' #DE 224 | ] 225 | 226 | self.STOPPED_STATES = [ 227 | 'EXTERNAL_DATABASE_STOP_IN_PROGRESS', 228 | 'STOP_IN_PROGRESS', 229 | 'STOPPED', 230 | 'ENV_STOPPED', 231 | 'Stopped', # DW 232 | 'NOT_ENABLED', # DF 233 | 'ClusterDeletionCompleted', 'AppDeleted' # DE 234 | 235 | ] 236 | 237 | self.FAILED_STATES = [ 238 | 'PROVISIONING_FAILED', 239 | 'CREATE_FAILED', 240 | 'REJECTED', 241 | 'FAILED', 242 | 'TIMEDOUT', 243 | 'DELETE_FAILED', 244 | 'Error', # DW 245 | 'installation:failed', # ML 246 | 'provision:failed', # ML 247 | 'deprovision:failed', # ML 248 | 'BAD_HEALTH', # DF 249 | # DE service (all intermediate failure states, until CDE exposes a higher-level summary state) 250 | 'ClusterChartInstallationFailed', 'ClusterDNSCreationFailed', 'ClusterDNSDeletionFailed', 251 | 'ClusterIngressCreationFailed', 'ClusterProvisioningFailed', 'DBProvisioningFailed', 252 | 'FSMountTargetsCreationFailed', 'FSProvisioningFailed', 'ClusterTLSCertCreationFailed', 253 | 'ClusterServiceMeshProvisioningFailed', 'ClusterMonitoringConfigurationFailed', 254 | 'ClusterChartDeletionFailed', 'ClusterDeletionFailed', 'ClusterNamespaceDeletionFailed', 255 | 'DBDeletionFailed', 'FSMountTargetsDeletionFailed', 'FSDeletionFailed', 256 | 'ClusterTLSCertDeletionFailed', 'ClusterServiceMeshDeletionFailed', 257 | 'ClusterAccessGroupCreationFailed', 'ClusterAccessGroupDeletionFailed', 258 | 'ClusterUserSyncCheckFailed', 'ClusterCreationFailed', 'ClusterDeleteFromDBFailed', 259 | 'ClusterMaintenanceFailed', 'ClusterTLSCertRenewalFailed', 260 | # DE virtual cluster 261 | 'AppInstallationFailed', 'AppDeletionFailed' 262 | ] 263 | 264 | self.REMOVABLE_STATES = [ 265 | 'AVAILABLE', 'UPDATE_FAILED', 'CREATE_FAILED', 'ENABLE_SECURITY_FAILED', 'DELETE_FAILED', 266 | 'DELETE_COMPLETED', 'DELETED_ON_PROVIDER_SIDE', 'STOPPED', 'START_FAILED', 'STOP_FAILED', 267 | 'installation:failed', 'deprovision:failed', 'installation:finished', 'modify:finished', # ML 268 | 'Error', 'Running', 'Stopped', 'Deleting', # DW 269 | 'GOOD_HEALTH', 'CONCERNING_HEALTH', 'BAD_HEALTH', # DF 270 | 'ClusterCreationCompleted', 'AppInstalled', 'ClusterProvisioningFailed' #DE 271 | ] 272 | 273 | # common regex patterns 274 | self.DATAHUB_NAME_PATTERN = re.compile(r'[^a-z0-9-]') 275 | self.DATALAKE_NAME_PATTERN = re.compile(r'[^a-z0-9-]') 276 | self.ENV_NAME_PATTERN = re.compile(r'(^[^a-z0-9]|[^a-z0-9-]|^.{,4}$|^.{29,}$)') 277 | self.CREDENTIAL_NAME_PATTERN = re.compile(r'[^a-z0-9-]') 278 | self.OPERATION_REGEX = re.compile(r'operation ([0-9a-zA-Z-]{36}) running') 279 | 280 | # Workload services with special credential and endpoint handling 281 | self.WORKLOAD_SERVICES = ['dfworkload'] 282 | 283 | # substrings to check for in different CRNs 284 | self.CRN_STRINGS = { 285 | 'generic': ['crn:'], 286 | 'env': [':environments:', ':environment:'], 287 | 'df': [':df:', ':service:'], 288 | 'flow': [':df:', ':flow:'], 289 | 'readyflow': [':df:', 'readyFlow'], 290 | 'deployment': [':df:', ':deployment:'] 291 | } 292 | 293 | def _make_user_agent_header(self): 294 | cdpy_version = pkg_resources.get_distribution('cdpy').version 295 | return '%s CDPY/%s CDPCLI/%s Python/%s %s/%s' % ( 296 | self.agent_header, 297 | cdpy_version, 298 | CDPCLI_VERSION, 299 | platform.python_version(), 300 | platform.system(), 301 | platform.release()) 302 | 303 | @staticmethod 304 | def _load_retry_config(loader): 305 | original_config = loader.load_json('_retry.json') 306 | retry_config = build_retry_config( 307 | original_config['retry'], 308 | original_config.get('definitions', {})) 309 | return retry_config 310 | 311 | def _setup_logger(self, log_level, log_format): 312 | self.logger = logging.getLogger('CdpSdk') 313 | self.logger.setLevel(log_level) 314 | 315 | self.__log_capture = io.StringIO() 316 | handler = logging.StreamHandler(self.__log_capture) 317 | handler.setLevel(log_level) 318 | 319 | formatter = logging.Formatter(log_format) 320 | handler.setFormatter(formatter) 321 | 322 | self.logger.addHandler(handler) 323 | 324 | def _build_client(self, service, parameters=None): 325 | if service in self.WORKLOAD_SERVICES: 326 | if service == 'dfworkload': 327 | workload_name = 'DF' 328 | else: 329 | workload_name = None 330 | self.throw_error(CdpError("Workload %s not recognised for client generation" % service)) 331 | if 'environmentCrn' not in parameters: 332 | self.throw_error(CdpError("environmentCrn must be supplied when connecting to %s" % service)) 333 | df_access_token = self.call( 334 | svc='iam', func='generate_workload_auth_token', 335 | workloadName=workload_name, environmentCrn=parameters['environmentCrn'] 336 | ) 337 | token = df_access_token['token'] 338 | if not token.startswith('Bearer '): 339 | token = 'Bearer ' + token 340 | credentials = StaticCredentials(access_token=token) 341 | endpoint_url = urljoin(df_access_token['endpointUrl'], '/') 342 | else: 343 | if not self.cdp_credentials: 344 | self.cdp_credentials = self._client_creator.context.get_credentials() 345 | credentials = self.cdp_credentials 346 | endpoint_url = self.client_endpoint 347 | try: 348 | # region introduced in client version 0.9.42 349 | client = self._client_creator.create_client( 350 | service_name=service, 351 | region=self.cp_region, 352 | explicit_endpoint_url=endpoint_url, 353 | tls_verification=self.tls_verify, 354 | credentials=credentials 355 | ) 356 | except TypeError: 357 | client = self._client_creator.create_client( 358 | service_name=service, 359 | explicit_endpoint_url=endpoint_url, 360 | tls_verification=self.tls_verify, 361 | credentials=credentials 362 | ) 363 | return client 364 | 365 | @staticmethod 366 | def _default_throw_error(error: 'CdpError'): 367 | """ 368 | Default Error Handler if not supplied during init 369 | Args: 370 | error (CdpError): The Error to raise, expects a CdpError 371 | 372 | Returns: 373 | None 374 | 375 | Raises: 376 | CdpError: The supplied Error 377 | """ 378 | raise error 379 | 380 | @staticmethod 381 | def _default_throw_warning(warning: 'CdpWarning'): 382 | """ 383 | Default Warning Handler if not supplied during init 384 | Args: 385 | warning (str): The Warning string to process 386 | 387 | Returns: 388 | None 389 | """ 390 | warnings.warn(message=warning.message) 391 | 392 | # Public convenience Methods 393 | @staticmethod 394 | def regex_search(pattern, obj): 395 | return re.search(pattern, obj) 396 | 397 | def validate_crn(self, obj: str, crn_type='generic'): 398 | for substring in self.CRN_STRINGS[crn_type]: 399 | if substring not in obj: 400 | self.throw_error(CdpError("Supplied crn %s of proposed type %s is missing substring %s" 401 | % (str(obj), crn_type, substring))) 402 | 403 | @staticmethod 404 | def sleep(seconds): 405 | sleep(seconds) 406 | 407 | @staticmethod 408 | def first_item_if_exists(obj): 409 | """Accepts an iterable like a list, and returns the first item if there is one""" 410 | return next(iter(obj), obj) 411 | 412 | @staticmethod 413 | def filter_by_key(obj, key): 414 | """Accepts a list of dicts and a key, returns a flat list of that key from the dicts""" 415 | return list(map(lambda f: f[key], obj)) 416 | 417 | @staticmethod 418 | def dumps(data): 419 | """Perform a json.dumps, but handle datetime objects.""" 420 | 421 | def _convert(o): 422 | if isinstance(o, datetime): 423 | return o.__str__() 424 | 425 | return json.dumps(data, indent=2, default=_convert) 426 | 427 | def _client(self, service, parameters=None): 428 | """Builds a CDP Endpoint client of a given type, and caches it against later reuse""" 429 | if service not in self._clients: 430 | self._clients[service] = self._build_client(service, parameters) 431 | return self._clients[service] 432 | 433 | def read_file(self, file_path): 434 | try: 435 | with open(str(file_path), 'r') as f: 436 | return f.read() 437 | except IOError as err: 438 | parsed_err = CdpError(err) 439 | if self.debug: 440 | log = self.get_log() 441 | parsed_err.update(sdk_out=log, sdk_out_lines=log.splitlines()) 442 | self.throw_error(parsed_err) 443 | 444 | def get_log(self): 445 | contents = self.__log_capture.getvalue() 446 | self.__log_capture.truncate(0) 447 | return contents 448 | 449 | @staticmethod 450 | def _get_path(obj, path): 451 | value = obj 452 | for p in path: 453 | if isinstance(value, dict): 454 | value = value.get(p) 455 | else: 456 | value = None 457 | if value is None: 458 | return None 459 | return value 460 | 461 | @staticmethod 462 | def encode_value(value): 463 | if value: 464 | return urlparse.quote(value) 465 | return None 466 | 467 | def expand_file_path(self, file_path): 468 | if path.exists(file_path): 469 | return path.expandvars(path.expanduser(file_path)) 470 | else: 471 | self.throw_error( 472 | CdpError('Path [{}] not found'.format(file_path)) 473 | ) 474 | 475 | def wait_for_state(self, describe_func, params: dict, field: Union[str, None, list] = 'status', 476 | state: Union[list, str, None] = None, delay: int = 15, timeout: int = 3600, 477 | ignore_failures: bool = False): 478 | """ 479 | Proceses a loop waiting for a given function to achieve a given state or known failure states 480 | 481 | Args: 482 | describe_func (func): The status check function to call as it relates to the sdk object, 483 | e.g self.cdpy.opdb.describe_database 484 | params (dict): Parameters the describe_func requires to poll the status, e.g. { name=myname, env=myenv } 485 | field (str, None, list): The field to check in the describe_func output for the state. Use None to check for 486 | listing removal during deletion. Provide a list of strings for nested structures. Defaults to 'status' 487 | state (list, str, None): The state or list of states valid for return from wait function, list of states 488 | may include None for object removal. Defaults to None. 489 | delay (int): Delay in seconds between each poll of the describe_func. Default is 15 490 | timeout (int): Total wait time in seconds before the function should return a timeout. Default is 3600 491 | ignore_failures (bool): Whether to ignore failed states when waiting for a forced deletion 492 | 493 | Returns: Output of describe function received during last polling attempt. 494 | """ 495 | self.logger.info("Waiting for function {0} on params [{1}] to have field {2} with state {3}" 496 | .format(describe_func.__name__, str(params), field, str(state))) 497 | state = state if isinstance(state, list) else [state] 498 | if field is not None: 499 | field = field if isinstance(field, list) else [field] 500 | start_time = time() 501 | while time() < start_time + timeout: 502 | current = describe_func(**params) 503 | if current is None: 504 | if field is None or None in state: 505 | return current 506 | else: 507 | self.logger.info("Waiting for identity {0} to be returned by function {1}") 508 | else: 509 | if field is not None: 510 | current_status = self._get_path(current, field) 511 | else: # field not provided, therefore seek default status fields to check for failures 512 | default_status_fields = [ 513 | ['status'], # Datalake, DW, OpDB, Datahub, DE 514 | ['instanceStatus'], # ML 515 | ['status', 'state'], # DF, DE 516 | ] 517 | possible_status = [ 518 | self._get_path(current, x) for x in default_status_fields 519 | if x[0] in current 520 | ] 521 | selected_status = [x for x in possible_status if x is not None] 522 | if len(selected_status) > 0: 523 | current_status = selected_status[0] 524 | else: 525 | current_status = None 526 | self.throw_error( 527 | CdpError("Could not determine default status field in response {0}".format(current)) 528 | ) 529 | if current_status is None: 530 | self.logger.info("Waiting to find field {0} in function {1} response" 531 | .format(field, describe_func)) 532 | elif current_status in state: 533 | return current 534 | elif current_status in self.FAILED_STATES: 535 | status_reason = 'None provided' 536 | for fail_msg_field in ['statusReason', 'failureMessage']: 537 | if fail_msg_field in current: 538 | status_reason = current[fail_msg_field] 539 | if ignore_failures: 540 | self.throw_warning( 541 | CdpWarning("Ignored Failure status '%s' while waiting" % current_status) 542 | ) 543 | else: 544 | self.throw_error( 545 | CdpError("Function {0} with params [{1}] encountered failed state {2} with reason {3}" 546 | .format(describe_func.__name__, str(params), current_status, status_reason))) 547 | else: 548 | self.logger.info("Waiting for change in {0}: [{1}], current is {2}: {3}" 549 | .format(describe_func.__name__, str(params), field, current_status)) 550 | sleep(delay) 551 | else: 552 | self.throw_error( 553 | CdpError("Timeout waiting for function {0} with params [{1}] to return field {2} with state {3}" 554 | .format(describe_func.__name__, str(params), field, str(state)))) 555 | 556 | def _scrub_inputs(self, inputs): 557 | # Used in main call() function 558 | logging.debug("Scrubbing inputs in payload") 559 | # Remove unused submission values as the API rejects them 560 | payload = {x: y for x, y in inputs.items() if y is not None} 561 | # Remove and issue warning for empty string submission values as the API rejects them 562 | _ = [self.throw_warning( 563 | CdpWarning('Removing empty string arg %s from submission' % x)) 564 | for x, y in payload.items() if y == ''] 565 | payload = {x: y for x, y in payload.items() if y != ''} 566 | return payload 567 | 568 | def _handle_paging(self, response, call_function, payload): 569 | # Used in main call() function 570 | while 'nextToken' in response: 571 | token = response.pop('nextToken') 572 | next_page = call_function( 573 | **payload, startingToken=token, pageSize=self.DEFAULT_PAGE_SIZE) 574 | for key in next_page.keys(): 575 | if isinstance(next_page[key], str): 576 | response[key] = next_page[key] 577 | elif isinstance(next_page[key], list): 578 | response[key] += (next_page[key]) 579 | return response 580 | 581 | def _handle_call_errors(self, err, squelch): 582 | # Used in main call() function 583 | # Note that the cascade of behaviors here is designed to be convenient for Ansible module development 584 | parsed_err = CdpError(err) 585 | if self.debug: 586 | log = self.get_log() 587 | parsed_err.update(sdk_out=log, sdk_out_lines=log.splitlines()) 588 | if self.strict_errors is True: 589 | self.throw_error(parsed_err) 590 | if isinstance(err, ClientError): 591 | if squelch is not None: 592 | for item in squelch: 593 | if item.value in str(parsed_err.__getattribute__(item.field)): 594 | warning = item.warning if item.warning is not None else str(parsed_err.violations) 595 | self.throw_warning(CdpWarning(warning)) 596 | return item.default 597 | return parsed_err 598 | 599 | def _handle_redirect_call(self, client, call_function, payload, headers): 600 | # cdpcli/extensions/redirect.py 601 | http, resp = client.make_api_call( 602 | client.meta.method_to_api_mapping[call_function], 603 | payload, 604 | allow_redirects=False 605 | ) 606 | if not http.is_redirect: 607 | self.throw_error(CdpError("Redirect headers supplied but no redirect URL from API call")) 608 | redirect_url = http.headers.get('Location', None) 609 | 610 | if redirect_url is not None: 611 | with open(self.expand_file_path(payload['file']), 'rb') as f: 612 | http, full_response = client.make_request( 613 | operation_name=client.meta.method_to_api_mapping[call_function], 614 | method='post', 615 | url_path=redirect_url, 616 | headers=self._scrub_inputs(inputs=headers), 617 | body=f 618 | ) 619 | else: 620 | self.throw_error(CdpError("Redirect call attempted but redirect URL was empty")) 621 | return full_response 622 | 623 | def _handle_std_call(self, client, call_function, payload): 624 | func_to_call = getattr(client, call_function) 625 | raw_response = func_to_call(**payload) 626 | if raw_response is not None and 'nextToken' in raw_response: 627 | logging.debug("Found paged results in %s" % call_function) 628 | full_response = self._handle_paging(raw_response, func_to_call, payload) 629 | else: 630 | full_response = raw_response 631 | return full_response 632 | 633 | def call(self, svc: str, func: str, ret_field: str = None, squelch: ['Squelch'] = None, ret_error: bool = False, 634 | redirect_headers: dict = None, **kwargs: Union[dict, bool, str, list]) -> Union[list, dict, 'CdpError']: 635 | """ 636 | Wraps the call to an underlying CDP CLI Service, handles common errors, and parses output 637 | 638 | Args: 639 | svc (str): Name of the service, ex. iam 640 | func (str): Name of the function to call, ex. get-user 641 | ret_field (str, None): Name of the top level child field to return from results, ex. user 642 | squelch (list(Squelch)): list of Descriptions of Error squelching options 643 | ret_error (bool): Whether to return the error object if generated, 644 | defaults to False and raise instead 645 | redirect_headers (dict): Dict of http submission headers for the call, triggers redirected upload call. 646 | **kwargs (dict): Keyword Args to be supplied to the Function, e.g. userId 647 | 648 | Returns (dict, list, None): Output of CDP CLI Call 649 | """ 650 | try: 651 | if self.scrub_inputs: 652 | payload = self._scrub_inputs(inputs=kwargs) 653 | else: 654 | payload = kwargs 655 | 656 | svc_client = self._client(service=svc, parameters=payload) 657 | 658 | if redirect_headers is not None: 659 | full_response = self._handle_redirect_call(svc_client, func, payload, redirect_headers) 660 | else: 661 | full_response = self._handle_std_call(svc_client, func, payload) 662 | 663 | if ret_field is not None: 664 | if not full_response: 665 | self.throw_warning(CdpWarning('Call Response is empty, cannot return child field %s' % ret_field)) 666 | else: 667 | return full_response[ret_field] 668 | return full_response 669 | 670 | except Exception as err: 671 | parsed_err = self._handle_call_errors(err, squelch) 672 | if ret_error is True or not isinstance(parsed_err, CdpError): 673 | return parsed_err 674 | self.throw_error(parsed_err) 675 | 676 | 677 | class CdpSdkBase(object): 678 | """A base class to use for explicitly namespacing child service calls alongside the sdk""" 679 | def __init__(self, *args, **kwargs): 680 | self.sdk = CdpcliWrapper(*args, **kwargs) 681 | -------------------------------------------------------------------------------- /src/cdpy/datahub.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from cdpy.common import CdpSdkBase, Squelch, CdpError, CdpWarning 4 | 5 | ENTITLEMENT_DISABLED='Datahubs not enabled on CDP Tenant' 6 | 7 | 8 | class CdpyDatahub(CdpSdkBase): 9 | def __init__(self, *args, **kwargs): 10 | super().__init__(*args, **kwargs) 11 | 12 | def describe_cluster(self, name): 13 | return self.sdk.call( 14 | svc='datahub', func='describe_cluster', ret_field='cluster', squelch=[ 15 | Squelch('NOT_FOUND'), 16 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 17 | ], 18 | clusterName=name 19 | ) 20 | 21 | def list_clusters(self, environment_name=None): 22 | return self.sdk.call( 23 | svc='datahub', func='list_clusters', ret_field='clusters', squelch=[ 24 | Squelch(value='INVALID_ARGUMENT', default=list(), 25 | warning='No Datahubs found in Tenant or provided Environment %s' % str(environment_name)), 26 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 27 | ], 28 | environmentName=environment_name 29 | ) 30 | 31 | def describe_all_clusters(self, environment_name=None): 32 | clusters_listing = self.list_clusters(environment_name) 33 | if clusters_listing: 34 | return [self.describe_cluster(cluster['clusterName']) for cluster in clusters_listing] 35 | return clusters_listing 36 | 37 | def list_cluster_templates(self, retries=3, delay=5): 38 | # Intermittent timeout issue in CDP 7.2.10, should be reverted to bare listing in 7.2.12 39 | resp = self.sdk.call( 40 | svc='datahub', func='list_cluster_templates', ret_field='clusterTemplates', squelch=[ 41 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 42 | ], 43 | ret_error=True # if not a Squelch-able error, return for further review 44 | ) 45 | if isinstance(resp, CdpError): 46 | if retries > 0: 47 | if str(resp.status_code) == '500' and resp.error_code == 'UNKNOWN': 48 | retries = retries - 1 49 | self.sdk.throw_warning( 50 | CdpWarning('Got likely CDP Control Plane eventual consistency error, %d retries left...' 51 | % (retries)) 52 | ) 53 | self.sdk.sleep(delay) 54 | return self.list_cluster_templates(retries, delay) 55 | else: 56 | self.sdk.throw_error(resp) 57 | return resp 58 | 59 | def describe_cluster_template(self, name): 60 | return self.sdk.call( 61 | svc='datahub', func='describe_cluster_template', squelch=[ 62 | Squelch(value='NOT_FOUND'), 63 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 64 | ], 65 | ret_field='clusterTemplate', clusterTemplateName=name 66 | ) 67 | 68 | def delete_cluster(self, name): 69 | return self.sdk.call( 70 | svc='datahub', func='delete_cluster', squelch=[ 71 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 72 | ], 73 | clusterName=name 74 | ) 75 | 76 | def delete_cluster_templates(self, names): 77 | names = names if isinstance(names, list) else [names] 78 | return self.sdk.call( 79 | svc='datahub', func='delete_cluster_templates', squelch=[ 80 | Squelch(value='NOT_FOUND'), 81 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 82 | ], 83 | ret_field='clusterTemplates', clusterTemplateNames=names 84 | ) 85 | 86 | def create_cluster_template(self, name, description, content): 87 | return self.sdk.call( 88 | svc='datahub', func='create_cluster_template', squelch=[ 89 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 90 | ], 91 | ret_field='clusterTemplate', clusterTemplateName=name, 92 | description=description, clusterTemplateContent=content 93 | ) 94 | 95 | def list_cluster_definitions(self): 96 | return self.sdk.call( 97 | svc='datahub', func='list_cluster_definitions', squelch=[ 98 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 99 | ], 100 | ret_field='clusterDefinitions' 101 | ) 102 | 103 | def describe_cluster_definition(self, name): 104 | return self.sdk.call( 105 | svc='datahub', func='describe_cluster_definition', squelch=[ 106 | Squelch(value='NOT_FOUND'), 107 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 108 | ], 109 | ret_field='clusterDefinition', clusterDefinitionName=name 110 | ) 111 | 112 | def start_cluster(self, name): 113 | return self.sdk.call( 114 | svc='datahub', func='start_cluster', squelch=[ 115 | Squelch('NOT_FOUND'), 116 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 117 | ], 118 | clusterName=name 119 | ) 120 | 121 | def stop_cluster(self, name): 122 | return self.sdk.call( 123 | svc='datahub', func='stop_cluster', squelch=[ 124 | Squelch('NOT_FOUND'), 125 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 126 | ], 127 | clusterName=name 128 | ) -------------------------------------------------------------------------------- /src/cdpy/datalake.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from cdpy.common import CdpSdkBase, Squelch 4 | 5 | 6 | class CdpyDatalake(CdpSdkBase): 7 | def __init__(self, *args, **kwargs): 8 | super().__init__(*args, **kwargs) 9 | 10 | def list_datalakes(self, name=None): 11 | return self.sdk.call( 12 | svc='datalake', func='list_datalakes', ret_field='datalakes', squelch=[Squelch('NOT_FOUND')], 13 | environmentName=name 14 | ) 15 | 16 | def is_datalake_running(self, environment_name): 17 | resp = self.list_datalakes(environment_name) 18 | if resp and len(resp) == 1: 19 | if resp[0]['status'] in self.sdk.STARTED_STATES: 20 | return True 21 | return False 22 | 23 | def describe_datalake(self, name): 24 | return self.sdk.call( 25 | svc='datalake', func='describe_datalake', ret_field='datalake', 26 | squelch=[Squelch('NOT_FOUND'), Squelch('UNKNOWN') 27 | ], 28 | datalakeName=name 29 | ) 30 | 31 | def delete_datalake(self, name, force=False): 32 | return self.sdk.call( 33 | svc='datalake', func='delete_datalake', squelch=[Squelch('NOT_FOUND')], 34 | datalakeName=name, force=force 35 | ) 36 | 37 | def describe_all_datalakes(self, environment_name=None): 38 | datalakes_listing = self.list_datalakes(environment_name) 39 | if datalakes_listing: 40 | return [self.describe_datalake(datalake['datalakeName']) for datalake in datalakes_listing] 41 | return datalakes_listing 42 | -------------------------------------------------------------------------------- /src/cdpy/de.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from cdpy.common import CdpSdkBase, Squelch, CdpcliWrapper 4 | 5 | ENTITLEMENT_DISABLED='Data Engineering not enabled on CDP Tenant' 6 | 7 | 8 | class CdpyDe(CdpSdkBase): 9 | def __init__(self, *args, **kwargs): 10 | super().__init__(*args, **kwargs) 11 | 12 | def describe_vc(self, cluster_id, vc_id): 13 | return self.sdk.call( 14 | svc='de', func='describe_vc', ret_field='vc', squelch=[ 15 | Squelch('NOT_FOUND'), Squelch('INVALID_ARGUMENT'), 16 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 17 | ], 18 | clusterId=cluster_id, 19 | vcId=vc_id 20 | ) 21 | 22 | def list_vcs(self, cluster_id): 23 | return self.sdk.call( 24 | svc='de', func='list_vcs', ret_field='vcs', squelch=[ 25 | Squelch(value='NOT_FOUND', default=list()), 26 | Squelch(field='status_code', value='504', default=list(), 27 | warning="No VCS in this Cluster"), 28 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 29 | ], 30 | clusterId=cluster_id 31 | ) 32 | 33 | def create_vc(self, name, cluster_id, cpu_requests, memory_requests, chart_value_overrides=None, 34 | runtime_spot_component=None, spark_version=None, acl_users=None): 35 | return self.sdk.call( 36 | svc='de', func='create_vc', ret_field='Vc', squelch=[ 37 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 38 | ], 39 | name=name, 40 | clusterId=cluster_id, 41 | cpuRequests=cpu_requests, 42 | memoryRequests=memory_requests, 43 | chartValueOverrides=chart_value_overrides, 44 | runtimeSpotComponent=runtime_spot_component, 45 | sparkVersion=spark_version, 46 | aclUsers=acl_users 47 | ) 48 | 49 | def delete_vc(self, cluster_id, vc_id): 50 | return self.sdk.call( 51 | svc='de', func='delete_vc', ret_field='status', squelch=[ 52 | Squelch('NOT_FOUND'), 53 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 54 | ], 55 | clusterId=cluster_id, vcId=vc_id 56 | ) 57 | 58 | def describe_service(self, cluster_id): 59 | return self.sdk.call( 60 | svc='de', func='describe_service', ret_field='service', squelch=[ 61 | Squelch('NOT_FOUND'), Squelch('INVALID_ARGUMENT'), 62 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 63 | ], 64 | clusterId=cluster_id, 65 | ) 66 | 67 | def list_services(self, env=None, remove_deleted=False): 68 | services = self.sdk.call( 69 | svc='de', func='list_services', ret_field='services', squelch=[ 70 | Squelch(value='NOT_FOUND', default=list()), 71 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED, default=list()) 72 | ], removeDeleted=remove_deleted 73 | ) 74 | return [s for s in services if env is None or s['environmentName'] == env] 75 | 76 | def enable_service(self, name, env, instance_type, minimum_instances, maximum_instances, 77 | initial_instances=None, minimum_spot_instances=None, maximum_spot_instances=None, 78 | initial_spot_instances=None, chart_value_overrides=None, enable_public_endpoint=False, 79 | enable_private_network=False, enable_workload_analytics=False, root_volume_size=None, 80 | skip_validation=False, tags=None, use_ssd=None, loadbalancer_allowlist=None, whitelist_ips=None): 81 | return self.sdk.call( 82 | svc='de', func='enable_service', ret_field='service', squelch=[ 83 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 84 | ], 85 | name=name, 86 | env=env, 87 | instanceType=instance_type, 88 | minimumInstances=minimum_instances, 89 | maximumInstances=maximum_instances, 90 | initialInstances=initial_instances, 91 | minimumSpotInstances=minimum_spot_instances, 92 | maximumSpotInstances=maximum_spot_instances, 93 | initialSpotInstances=initial_spot_instances, 94 | chartValueOverrides=chart_value_overrides, 95 | enablePublicEndpoint=enable_public_endpoint, 96 | enablePrivateNetwork=enable_private_network, 97 | enableWorkloadAnalytics=enable_workload_analytics, 98 | rootVolumeSize=root_volume_size, 99 | skipValidation=skip_validation, 100 | tags=tags, 101 | useSsd=use_ssd, 102 | whitelistIps=whitelist_ips, 103 | loadbalancerAllowlist=loadbalancer_allowlist 104 | ) 105 | 106 | def disable_service(self, cluster_id, force=False): 107 | return self.sdk.call( 108 | svc='de', func='disable_service', ret_field='status', squelch=[ 109 | Squelch('NOT_FOUND'), 110 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 111 | ], 112 | clusterId=cluster_id, force=force 113 | ) 114 | 115 | def get_kubeconfig(self, cluster_id): 116 | return self.sdk.call( 117 | svc='de', func='get_kubeconfig', ret_field='kubeconfig', squelch=[ 118 | Squelch('NOT_FOUND'), 119 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 120 | ], 121 | clusterId=cluster_id 122 | ) 123 | 124 | def get_service_id_by_name(self, name, env): 125 | cluster_id = None 126 | for service in self.list_services(env, remove_deleted=True): 127 | if service['name'] == name: 128 | cluster_id = service['clusterId'] 129 | break 130 | return cluster_id 131 | 132 | def get_vc_id_by_name(self, name, cluster_id, remove_deleted=True): 133 | vc_id = None 134 | for vc in self.list_vcs(cluster_id): 135 | vc_stopped = vc['status'] in self.sdk.STOPPED_STATES 136 | if vc['vcName'] == name and (not vc_stopped if remove_deleted else True): 137 | vc_id = vc['vcId'] 138 | break 139 | return vc_id 140 | -------------------------------------------------------------------------------- /src/cdpy/df.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from cdpy.common import CdpSdkBase, Squelch, CdpError, CdpWarning 4 | from cdpcli.extensions.df.createdeployment import CreateDeploymentOperationCaller 5 | 6 | ENTITLEMENT_DISABLED = 'DataFlow not enabled on CDP Tenant' 7 | 8 | 9 | class CdpyDf(CdpSdkBase): 10 | def __init__(self, *args, **kwargs): 11 | self.DEPLOYMENT_SIZES = ['EXTRA_SMALL', 'SMALL', 'MEDIUM', 'LARGE'] 12 | super().__init__(*args, **kwargs) 13 | 14 | def list_services(self, only_enabled=False, env_crn=None, df_crn=None, name=None): 15 | result = self.sdk.call( 16 | svc='df', func='list_services', ret_field='services', squelch=[ 17 | Squelch(value='NOT_FOUND', default=list(), 18 | warning='No DataFlow Services found'), 19 | Squelch(value='PATH_DISABLED', default=list(), 20 | warning=ENTITLEMENT_DISABLED) 21 | ], 22 | pageSize=self.sdk.DEFAULT_PAGE_SIZE 23 | ) 24 | if only_enabled: 25 | result = [x for x in result if x['status']['state'] in self.sdk.STARTED_STATES] 26 | if name is not None: 27 | result = [x for x in result if x['name'] == name] 28 | if df_crn is not None: 29 | result = [x for x in result if x['crn'] == df_crn] 30 | if env_crn is not None: 31 | result = [x for x in result if x['environmentCrn'] == env_crn] 32 | return result 33 | 34 | def describe_service(self, df_crn: str = None, env_crn: str = None): 35 | resolved_df_crn = None 36 | if df_crn is not None: 37 | resolved_df_crn = df_crn 38 | elif env_crn is not None: 39 | services = self.list_services(env_crn=env_crn) 40 | if len(services) == 0: 41 | return None 42 | elif len(services) == 1: 43 | resolved_df_crn = services[0]['crn'] 44 | else: 45 | self.sdk.throw_error( 46 | CdpError('More than one DataFlow service found for env_crn, please try list instead') 47 | ) 48 | else: 49 | self.sdk.throw_error(CdpError("Either df_crn or env_crn must be supplied to df.describe_service")) 50 | if resolved_df_crn is not None: 51 | return self.sdk.call( 52 | svc='df', func='describe_service', ret_field='service', squelch=[ 53 | Squelch(value='NOT_FOUND', 54 | warning='No DataFlow Service with crn %s found' % df_crn), 55 | Squelch(value='PATH_DISABLED', 56 | warning=ENTITLEMENT_DISABLED), 57 | Squelch(value='PERMISSION_DENIED') # DF GRPC sometimes returns 403 when finishing deletion 58 | ], 59 | serviceCrn=resolved_df_crn 60 | ) 61 | else: 62 | return None 63 | 64 | def resolve_service_crn_from_name(self, name, only_enabled=True): 65 | listing = self.list_services(only_enabled=only_enabled, name=name) 66 | # More than one DF Service may exist with a given name if it was previously uncleanly deleted 67 | if len(listing) == 1: 68 | return listing[0]['crn'] 69 | elif len(listing) == 0: 70 | self.sdk.throw_warning(CdpWarning("No DataFlow Service found matching name %s" % name)) 71 | return None 72 | else: 73 | self.sdk.throw_error(CdpError("Multiple DataFlow Services found matching name %s" % name)) 74 | 75 | def enable_service(self, env_crn: str, lb_ips: list = None, min_nodes: int = 3, max_nodes: int = 3, 76 | enable_public_ip: bool = True, private_cluster: bool = False, kube_ips: list = None, 77 | cluster_subnets: list = None, lb_subnets: list = None, tags: dict = None): 78 | self.sdk.validate_crn(env_crn) 79 | return self.sdk.call( 80 | svc='df', func='enable_service', ret_field='service', 81 | environmentCrn=env_crn, minK8sNodeCount=min_nodes, maxK8sNodeCount=max_nodes, 82 | usePublicLoadBalancer=enable_public_ip, privateCluster=private_cluster, 83 | kubeApiAuthorizedIpRanges=kube_ips, loadBalancerAuthorizedIpRanges=lb_ips, 84 | clusterSubnets=cluster_subnets, loadBalancerSubnets=lb_subnets, tags=tags 85 | ) 86 | 87 | def disable_service(self, df_crn: str, persist: bool = False, terminate=False): 88 | self.sdk.validate_crn(df_crn) 89 | return self.sdk.call( 90 | svc='df', func='disable_service', ret_field='status', ret_error=True, 91 | serviceCrn=df_crn, persist=persist, terminateDeployments=terminate 92 | ) 93 | 94 | def reset_service(self, df_crn: str): 95 | self.sdk.validate_crn(df_crn) 96 | return self.sdk.call( 97 | svc='df', func='reset_service', 98 | serviceCrn=df_crn 99 | ) 100 | 101 | def list_deployments(self, env_crn=None, df_crn=None, name=None, dep_crn=None, described=False): 102 | result = self.sdk.call( 103 | svc='df', func='list_deployments', ret_field='deployments', squelch=[ 104 | Squelch(value='NOT_FOUND', default=list(), 105 | warning='No DataFlow Deployments found'), 106 | Squelch(value='PATH_DISABLED', default=list(), 107 | warning=ENTITLEMENT_DISABLED) 108 | ], 109 | pageSize=self.sdk.DEFAULT_PAGE_SIZE 110 | ) 111 | if dep_crn is not None: 112 | result = [x for x in result if x['crn'] == dep_crn] 113 | if name is not None: 114 | result = [x for x in result if x['name'] == name] 115 | if df_crn is not None: 116 | result = [x for x in result if x['service']['crn'] == df_crn] 117 | if env_crn is not None: 118 | result = [x for x in result if x['service']['environmentCrn'] == env_crn] 119 | if described is False: 120 | return result 121 | else: 122 | return [self.describe_deployment(dep_crn=x['crn']) for x in result] 123 | 124 | def describe_deployment(self, dep_crn=None, df_crn=None, name=None): 125 | if dep_crn is not None: 126 | self.sdk.validate_crn(dep_crn, 'deployment') 127 | elif df_crn is not None and name is not None: 128 | deployments = self.list_deployments(df_crn=df_crn, name=name) 129 | if len(deployments) == 0: 130 | return None 131 | elif len(deployments) == 1: 132 | dep_crn = deployments[0]['crn'] 133 | else: 134 | self.sdk.throw_error( 135 | CdpError('More than one DataFlow Deployment found, please try list instead') 136 | ) 137 | else: 138 | self.sdk.throw_error( 139 | CdpError( 140 | "Either dep_crn or both of df_crn and name must be supplied" 141 | ) 142 | ) 143 | return self.sdk.call( 144 | svc='df', func='describe_deployment', ret_field='deployment', squelch=[ 145 | Squelch(value='NOT_FOUND', 146 | warning='No DataFlow Deployment with crn %s found' % dep_crn), 147 | Squelch(value='PATH_DISABLED', 148 | warning=ENTITLEMENT_DISABLED) 149 | ], 150 | deploymentCrn=dep_crn 151 | ) 152 | 153 | def list_readyflows(self, name=None): 154 | # Lists readyflows that can be added to the Catalog for Deployment 155 | result = self.sdk.call( 156 | svc='df', func='list_readyflows', ret_field='readyflows', squelch=[ 157 | Squelch(value='NOT_FOUND', 158 | warning='No ReadyFlows found within your CDP Tenant'), 159 | Squelch(value='PATH_DISABLED', 160 | warning=ENTITLEMENT_DISABLED) 161 | ], 162 | ) 163 | if name is not None: 164 | result = [x for x in result if x['name'] == name] 165 | return result 166 | 167 | def list_flow_definitions(self, name=None): 168 | # Lists definitions in the Catalog. May contain more than one artefactType: flows, readyFlows 169 | result = self.sdk.call( 170 | svc='df', func='list_flow_definitions', ret_field='flows', squelch=[ 171 | Squelch(value='NOT_FOUND', 172 | warning='No Flow Definitions found within your CDP Tenant Catalog'), 173 | Squelch(value='PATH_DISABLED', 174 | warning=ENTITLEMENT_DISABLED) 175 | ], 176 | ) 177 | if name is not None: 178 | result = [x for x in result if x['name'] == name] 179 | return result 180 | 181 | def describe_readyflow(self, def_crn): 182 | # Describes readyFlow not added to the Catalog 183 | self.sdk.validate_crn(def_crn, 'readyflow') 184 | return self.sdk.call( 185 | svc='df', func='describe_readyflow', ret_field='readyflowDetail', squelch=[ 186 | Squelch(value='NOT_FOUND', 187 | warning='No ReadyFlow Definition with crn %s found' % def_crn), 188 | Squelch(value='PATH_DISABLED', 189 | warning=ENTITLEMENT_DISABLED) 190 | ], 191 | readyflowCrn=def_crn 192 | ) 193 | 194 | def import_readyflow(self, def_crn): 195 | # Imports a Readyflow from the Control Plane into the Tenant Flow Catalog 196 | self.sdk.validate_crn(def_crn, 'readyflow') 197 | return self.sdk.call( 198 | svc='df', func='add_readyflow', ret_field='addedReadyflowDetail', squelch=[ 199 | Squelch(value='NOT_FOUND', 200 | warning='No ReadyFlow Definition with crn %s found' % def_crn), 201 | Squelch(value='PATH_DISABLED', 202 | warning=ENTITLEMENT_DISABLED) 203 | ], 204 | readyflowCrn=def_crn 205 | ) 206 | 207 | def delete_added_readyflow(self, def_crn): 208 | # Deletes an added Readyflow from the Tenant Flow Catalog 209 | self.sdk.validate_crn(def_crn, 'readyflow') 210 | return self.sdk.call( 211 | svc='df', func='delete_added_readyflow', ret_field='readyflowDetail', squelch=[ 212 | Squelch(value='NOT_FOUND', 213 | warning='No ReadyFlow Definition with crn %s found' % def_crn), 214 | Squelch(value='PATH_DISABLED', 215 | warning=ENTITLEMENT_DISABLED) 216 | ], 217 | readyflowCrn=def_crn 218 | ) 219 | 220 | def describe_added_readyflow(self, def_crn, sort_versions=True): 221 | # Describes readyFlows added to the Catalog 222 | self.sdk.validate_crn(def_crn, 'readyflow') 223 | result = self.sdk.call( 224 | svc='df', func='describe_added_readyflow', ret_field='addedReadyflowDetail', squelch=[ 225 | Squelch(value='NOT_FOUND', 226 | warning='No ReadyFlow Definition with crn %s found' % def_crn), 227 | Squelch(value='PATH_DISABLED', 228 | warning=ENTITLEMENT_DISABLED) 229 | ], 230 | addedReadyflowCrn=def_crn 231 | ) 232 | out = result 233 | if sort_versions and out: 234 | out['versions'] = sorted(result['versions'], key=lambda d: d['version'], reverse=True) 235 | return out 236 | 237 | def describe_customflow(self, def_crn, sort_versions=True): 238 | self.sdk.validate_crn(def_crn, 'flow') 239 | result = self.sdk.call( 240 | svc='df', func='describe_flow', ret_field='flowDetail', squelch=[ 241 | Squelch(value='NOT_FOUND', 242 | warning='No Flow Definition with crn %s found' % def_crn), 243 | Squelch(value='PATH_DISABLED', 244 | warning=ENTITLEMENT_DISABLED) 245 | ], 246 | flowCrn=def_crn 247 | ) 248 | out = result 249 | if sort_versions and out: 250 | out['versions'] = sorted(result['versions'], key=lambda d: d['version'], reverse=True) 251 | return out 252 | 253 | def import_customflow(self, def_file, name, description=None, comments=None): 254 | # cdpcli/extensions/df/__init__.py: DfExtension._df_upload_flow 255 | 256 | return self.sdk.call( 257 | svc='df', func='import_flow_definition', squelch=[ 258 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED), 259 | Squelch(field='status_code', value='409') 260 | ], 261 | redirect_headers={ 262 | 'Content-Type': 'application/json', 263 | 'Flow-Definition-Name': self.sdk.encode_value(name), 264 | 'Flow-Definition-Description': self.sdk.encode_value(description), 265 | 'Flow-Definition-Comments': self.sdk.encode_value(comments) 266 | }, 267 | name=name, 268 | file=def_file, 269 | description=description, 270 | comments=comments 271 | ) 272 | 273 | def import_customflow_version(self, def_crn, def_file, comments=None): 274 | self.sdk.validate_crn(def_crn, 'flow') 275 | return self.sdk.call( 276 | svc='df', func='import_flow_definition_version', ret_field='.', squelch=[ 277 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED), 278 | Squelch(field='status_code', value='409') 279 | ], 280 | redirect_headers={ 281 | 'Content-Type': 'application/json', 282 | 'Flow-Definition-Comments': self.sdk.encode_value(comments) 283 | }, 284 | flowCrn=def_crn, 285 | file=def_file, 286 | comments=comments 287 | ) 288 | 289 | def delete_customflow(self, def_crn): 290 | self.sdk.validate_crn(def_crn, 'flow') 291 | return self.sdk.call( 292 | svc='df', func='delete_flow', ret_field='flow', squelch=[ 293 | Squelch(value='NOT_FOUND', 294 | warning='No Flow Definition with crn %s found' % def_crn), 295 | Squelch(value='PATH_DISABLED', 296 | warning=ENTITLEMENT_DISABLED) 297 | ], 298 | flowCrn=def_crn 299 | ) 300 | 301 | def get_version_crn_from_flow_definition(self, flow_name, version=None): 302 | summary_list = self.list_flow_definitions(name=flow_name) 303 | if summary_list: 304 | if len(summary_list) == 1: 305 | flow_def = summary_list[0] 306 | kind = flow_def['artifactType'] 307 | if kind == 'flow': 308 | detail = self.describe_customflow(flow_def['crn']) 309 | elif kind == 'readyFlow': 310 | detail = self.describe_added_readyflow(flow_def['crn']) 311 | else: 312 | detail = None 313 | self.sdk.throw_error(CdpError("DataFlow Definition type not supported %s" % kind)) 314 | if version is None: 315 | # versions are sorted descending by default 316 | return detail['versions'][0]['crn'] 317 | else: 318 | out = [x for x in detail['versions'] if x['version'] == version] 319 | if out: 320 | return out[0]['crn'] 321 | else: 322 | self.sdk.throw_error(CdpError( 323 | "Could not find version %d for DataFlow Definition named %s" % (version, flow_name) 324 | )) 325 | else: 326 | self.sdk.throw_error(CdpError("More than one DataFlow Definition found for name %s" % flow_name)) 327 | else: 328 | self.sdk.throw_warning(CdpWarning("DataFlow Definition not found for name %s" % flow_name)) 329 | 330 | def resolve_env_crn_from_df_crn(self, df_crn): 331 | if ':service:' in df_crn: 332 | self.sdk.validate_crn(df_crn, 'df') 333 | df_info = self.describe_service(df_crn=df_crn) 334 | if df_info: 335 | return df_info['environmentCrn'] 336 | elif ':deployment:' in df_crn: 337 | self.sdk.validate_crn(df_crn, 'deployment') 338 | df_info = self.describe_deployment(df_crn) 339 | if df_info: 340 | return df_info['service']['environmentCrn'] 341 | else: 342 | self.sdk.throw_error( 343 | CdpError( 344 | "Could not resolve an Environment CRN from DataFlow CRN %s" % df_crn 345 | ) 346 | ) 347 | 348 | def create_deployment(self, df_crn, flow_ver_crn, deployment_name, size_name=None, static_node_count=None, 349 | autoscale_enabled=None, autoscale_nodes_min=None, autoscale_nodes_max=None, nifi_ver=None, 350 | autostart_flow=None, parameter_groups=None, kpis=None): 351 | # Validations 352 | if size_name is not None and size_name not in self.DEPLOYMENT_SIZES: 353 | self.sdk.throw_error(CdpError("Deployment size_name %s not in supported size list: %s" 354 | % (size_name, str(self.DEPLOYMENT_SIZES)))) 355 | self.sdk.validate_crn(df_crn, 'df') 356 | if self.list_deployments(name=deployment_name): 357 | self.sdk.throw_error(CdpError("Deployment already exists with conflicting name %s" % deployment_name)) 358 | # Setup 359 | config = dict( 360 | autoStartFlow=autostart_flow if autostart_flow is not None else True, 361 | parameterGroups=parameter_groups, 362 | deploymentName=deployment_name, 363 | environmentCrn=self.resolve_env_crn_from_df_crn(df_crn), 364 | clusterSizeName=size_name if size_name is not None else 'EXTRA_SMALL', 365 | cfmNifiVersion=nifi_ver, 366 | kpis=kpis 367 | ) 368 | if autoscale_enabled: 369 | config['autoScalingEnabled'] = True 370 | config['autoScaleMinNodes'] = autoscale_nodes_min if autoscale_nodes_min is not None else 1 371 | config['autoScaleMaxNodes'] = autoscale_nodes_max if autoscale_nodes_max is not None else 3 372 | else: 373 | config['staticNodeCount'] = static_node_count if static_node_count is not None else 1 374 | 375 | # cdpcli/extensions/df/createdeployment.py cdpcli-beta v0.9.48+ 376 | dep_req_crn = self.sdk.call( 377 | svc='df', func='initiate_deployment', ret_field='deploymentRequestCrn', 378 | serviceCrn=df_crn, flowVersionCrn=flow_ver_crn 379 | ) 380 | df_handler = CreateDeploymentOperationCaller() 381 | df_handler._upload_assets( 382 | df_workload_client=self.sdk._client( 383 | service='dfworkload', 384 | parameters=config 385 | ), 386 | deployment_request_crn=dep_req_crn, 387 | parameters=config 388 | ) 389 | resp = df_handler._create_deployment( 390 | df_workload_client=self.sdk._client( 391 | service='dfworkload', 392 | parameters=config 393 | ), 394 | deployment_request_crn=dep_req_crn, 395 | environment_crn=config['environmentCrn'], 396 | parameters=config 397 | ) 398 | return resp 399 | 400 | def terminate_deployment(self, dep_crn, env_crn=None): 401 | if env_crn is None: 402 | env_crn = self.resolve_env_crn_from_df_crn(df_crn=dep_crn) 403 | _ = [self.sdk.validate_crn(x[0], x[1]) for x in [(env_crn, 'env'), (dep_crn, 'deployment')]] 404 | return self.sdk.call( 405 | svc='dfworkload', func='terminate_deployment', ret_field='deployment', 406 | environmentCrn=env_crn, deploymentCrn=dep_crn 407 | ) 408 | -------------------------------------------------------------------------------- /src/cdpy/dw.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from cdpy.common import CdpSdkBase, Squelch, CdpError 4 | 5 | ENTITLEMENT_DISABLED = 'Data Warehousing not enabled on CDP Tenant' 6 | 7 | 8 | class CdpyDw(CdpSdkBase): 9 | def __init__(self, *args, **kwargs): 10 | super().__init__(*args, **kwargs) 11 | 12 | def list_dbcs(self, cluster_id): 13 | return self.sdk.call( 14 | svc='dw', func='list_dbcs', ret_field='dbcs', squelch=[ 15 | Squelch(value='NOT_FOUND', default=list()), 16 | Squelch(field='status_code', value='504', default=list(), 17 | warning="No Data Catalogs found in this Cluster"), 18 | Squelch(value='PATH_DISABLED', 19 | warning=ENTITLEMENT_DISABLED, default=list()) 20 | ], 21 | clusterId=cluster_id 22 | ) 23 | 24 | def list_vws(self, cluster_id): 25 | return self.sdk.call( 26 | svc='dw', func='list_vws', ret_field='vws', squelch=[ 27 | Squelch(value='NOT_FOUND', default=list()), 28 | Squelch(field='status_code', value='504', default=list(), 29 | warning="No Virtual Warehouses found in this Cluster"), 30 | Squelch(value='PATH_DISABLED', 31 | warning=ENTITLEMENT_DISABLED, default=list()) 32 | ], 33 | clusterId=cluster_id 34 | ) 35 | 36 | def describe_cluster(self, cluster_id): 37 | return self.sdk.call( 38 | svc='dw', func='describe_cluster', ret_field='cluster', squelch=[ 39 | Squelch('NOT_FOUND'), 40 | Squelch('INVALID_ARGUMENT'), 41 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 42 | ], 43 | clusterId=cluster_id 44 | ) 45 | 46 | def describe_vw(self, cluster_id, vw_id): 47 | return self.sdk.call( 48 | svc='dw', func='describe_vw', ret_field='vw', squelch=[ 49 | Squelch('NOT_FOUND'), 50 | Squelch('INVALID_ARGUMENT'), 51 | Squelch('UNKNOWN'), 52 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 53 | ], 54 | clusterId=cluster_id, 55 | vwId=vw_id 56 | ) 57 | 58 | def describe_dbc(self, cluster_id, dbc_id): 59 | return self.sdk.call( 60 | svc='dw', func='describe_dbc', ret_field='dbc', squelch=[ 61 | Squelch('NOT_FOUND'), 62 | Squelch('INVALID_ARGUMENT'), 63 | Squelch('UNKNOWN'), 64 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 65 | ], 66 | clusterId=cluster_id, 67 | dbcId=dbc_id 68 | ) 69 | 70 | def describe_data_visualization(self, cluster_id, data_viz_id): 71 | return self.sdk.call( 72 | svc='dw', func='describe_data_visualization', ret_field='dataVisualization', squelch=[ 73 | Squelch('NOT_FOUND'), 74 | Squelch('not found', field='violations'), 75 | Squelch('INVALID_ARGUMENT'), 76 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 77 | ], 78 | clusterId=cluster_id, 79 | dataVisualizationId=data_viz_id, 80 | ) 81 | 82 | def list_clusters(self, env_crn=None): 83 | resp = self.sdk.call( 84 | svc='dw', func='list_clusters', ret_field='clusters', squelch=[ 85 | Squelch(value='NOT_FOUND', default=list()), 86 | Squelch(value='PATH_DISABLED', 87 | warning=ENTITLEMENT_DISABLED, default=list()) 88 | ] 89 | ) 90 | if env_crn: 91 | return [x for x in resp if env_crn == x['environmentCrn']] 92 | return resp 93 | 94 | def list_data_visualizations(self, cluster_id): 95 | return self.sdk.call( 96 | svc='dw', func='list_data_visualizations', ret_field='dataVisualizations', squelch=[ 97 | Squelch(value='NOT_FOUND', default=list()), 98 | Squelch(value='PATH_DISABLED', 99 | warning=ENTITLEMENT_DISABLED, default=list()) 100 | ], 101 | clusterId=cluster_id 102 | ) 103 | 104 | def gather_clusters(self, env_crn=None): 105 | self.sdk.validate_crn(env_crn) 106 | clusters = self.list_clusters(env_crn=env_crn) 107 | out = [] 108 | if clusters: 109 | for base in clusters: 110 | out.append({ 111 | 'dbcs': self.list_dbcs(base['id']), 112 | 'vws': self.list_vws(base['id']), 113 | **base 114 | }) 115 | return out 116 | 117 | def create_cluster(self, env_crn: str, overlay: bool, aws_lb_subnets: list = None, 118 | aws_worker_subnets: list = None, az_subnet: str = None, az_enable_az: bool = None, 119 | az_managed_identity: str = None, az_enable_private_aks: bool = None, az_enable_private_sql: bool = None, 120 | az_enable_spot_instances: bool = None, az_log_analytics_workspace_id: str = None, az_network_outbound_type: str = None, 121 | az_aks_private_dns_zone: str = None, az_compute_instance_types: list = None, private_load_balancer: bool = None, 122 | public_worker_node: bool = None, custom_subdomain: str = None, database_backup_retention_period: int = None, 123 | reserved_compute_nodes: int = None, reserved_shared_services_nodes: int = None, resource_pool: str = None, 124 | lb_ip_ranges: list = None, k8s_ip_ranges: list = None 125 | ): 126 | self.sdk.validate_crn(env_crn) 127 | if all(x is not None for x in [aws_worker_subnets, aws_lb_subnets]): 128 | aws_options = dict(lbSubnetIds=aws_lb_subnets, 129 | workerSubnetIds=aws_worker_subnets) 130 | else: 131 | aws_options = None 132 | if all(x is not None for x in [az_subnet, az_enable_az, az_managed_identity]): 133 | azure_options_all = dict( 134 | subnetId=az_subnet, enableAZ=az_enable_az, userAssignedManagedIdentity=az_managed_identity, 135 | enablePrivateAks=az_enable_private_aks, enablePrivateSQL=az_enable_private_sql, 136 | enableSpotInstances=az_enable_spot_instances, logAnalyticsWorkspaceId=az_log_analytics_workspace_id, 137 | outboundType=az_network_outbound_type, privateDNSZoneAKS=az_aks_private_dns_zone, computeInstanceTypes=az_compute_instance_types) 138 | 139 | azure_options = {k: v for k, v in azure_options_all.items() if v is not None} 140 | else: 141 | azure_options = None 142 | return self.sdk.call( 143 | svc='dw', func='create_cluster', ret_field='clusterId', environmentCrn=env_crn, 144 | useOverlayNetwork=overlay, usePrivateLoadBalancer=private_load_balancer, 145 | usePublicWorkerNode=public_worker_node, awsOptions=aws_options, azureOptions=azure_options, 146 | customSubdomain=custom_subdomain, databaseBackupRetentionPeriod=database_backup_retention_period, 147 | reservedComputeNodes=reserved_compute_nodes, reservedSharedServicesNodes=reserved_shared_services_nodes, 148 | resourcePool=resource_pool, whitelistK8sClusterAccessIpCIDRs=k8s_ip_ranges, whitelistWorkloadAccessIpCIDRs=lb_ip_ranges, 149 | squelch=[ 150 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 151 | ] 152 | ) 153 | 154 | def create_data_visualization(self, cluster_id: str, name: str, config: dict = None, 155 | template_name: str = None, image_version: str = None): 156 | return self.sdk.call( 157 | svc='dw', func='create_data_visualization', ret_field='dataVisualizationId', 158 | squelch=[ 159 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 160 | ], 161 | clusterId=cluster_id, 162 | name=name, 163 | config=config, 164 | templateName = template_name, 165 | imageVersion = image_version 166 | ) 167 | 168 | def delete_cluster(self, cluster_id: str, force: bool = False): 169 | return self.sdk.call( 170 | svc='dw', func='delete_cluster', squelch=[ 171 | Squelch('NOT_FOUND'), 172 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 173 | ], 174 | clusterId=cluster_id, force=force 175 | ) 176 | 177 | def delete_data_visualization(self, cluster_id: str, data_viz_id: str): 178 | return self.sdk.call( 179 | svc='dw', func='delete_data_visualization', squelch=[ 180 | Squelch('NOT_FOUND'), 181 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 182 | ], 183 | clusterId=cluster_id, 184 | dataVisualizationId=data_viz_id, 185 | ) 186 | 187 | def update_data_visualization(self, cluster_id: str, data_viz_id: str, config: dict): 188 | return self.sdk.call( 189 | svc='dw', func='update_data_visualization', squelch=[ 190 | Squelch('NOT_FOUND'), 191 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 192 | ], 193 | clusterId=cluster_id, 194 | dataVisualizationId=data_viz_id, 195 | config=config, 196 | ) 197 | 198 | def create_vw(self, cluster_id: str, dbc_id: str, vw_type: str, name: str, template: str = None, 199 | autoscaling_min_cluster: int = None, autoscaling_max_cluster: int = None, 200 | autoscaling_auto_suspend_timeout_seconds: int = None, autoscaling_disable_auto_suspend: bool = None, 201 | autoscaling_hive_desired_free_capacity: int = None, autoscaling_hive_scale_wait_time_seconds: int = None, 202 | autoscaling_impala_scale_down_delay_seconds: int = None, autoscaling_impala_scale_up_delay_seconds: int = None, 203 | autoscaling_pod_config_name: str = None, common_configs: dict = None, application_configs: dict = None, 204 | ldap_groups: list = None, enable_sso: bool = None, tags: dict = None, 205 | enable_unified_analytics: bool = None, enable_platform_jwt_auth: bool = None, 206 | impala_ha_enable_catalog_high_availability: bool = None, impala_ha_enable_shutdown_of_coordinator: bool = None, 207 | impala_ha_high_availability_mode: str = None, impala_ha_num_of_active_coordinators: int = None, 208 | impala_ha_shutdown_of_coordinator_delay_seconds: int = None 209 | ): 210 | 211 | if any(x is not None for x in [autoscaling_min_cluster, autoscaling_max_cluster, autoscaling_auto_suspend_timeout_seconds, 212 | autoscaling_disable_auto_suspend, 213 | autoscaling_hive_desired_free_capacity, 214 | autoscaling_hive_scale_wait_time_seconds, 215 | autoscaling_impala_scale_down_delay_seconds, 216 | autoscaling_impala_scale_up_delay_seconds, 217 | autoscaling_pod_config_name]): 218 | autoscaling_all = dict(autoSuspendTimeoutSeconds=autoscaling_auto_suspend_timeout_seconds, disableAutoSuspend=autoscaling_disable_auto_suspend, hiveDesiredFreeCapacity=autoscaling_hive_desired_free_capacity, 219 | hiveScaleWaitTimeSeconds=autoscaling_hive_scale_wait_time_seconds, impalaScaleDownDelaySeconds=autoscaling_impala_scale_down_delay_seconds, impalaScaleUpDelaySeconds=autoscaling_impala_scale_up_delay_seconds, podConfigName=autoscaling_pod_config_name) 220 | if autoscaling_min_cluster is not None and autoscaling_min_cluster != 0: 221 | autoscaling_all['minClusters'] = autoscaling_min_cluster 222 | if autoscaling_max_cluster is not None and autoscaling_max_cluster != 0: 223 | autoscaling_all['maxClusters'] = autoscaling_max_cluster 224 | 225 | autoscaling = {k: v for k, 226 | v in autoscaling_all.items() if v is not None} 227 | 228 | else: 229 | autoscaling = None 230 | 231 | if tags is not None: 232 | tag_list = [] 233 | for key, value in tags.items(): 234 | tag_list.append({'key': key, 'value': value}) 235 | else: 236 | tag_list = None 237 | 238 | if any(x is not None for x in [common_configs, application_configs, ldap_groups, enable_sso]): 239 | config = {} 240 | if common_configs is not None and common_configs: 241 | config['commonConfigs'] = common_configs 242 | if application_configs is not None and application_configs: 243 | config['applicationConfigs'] = application_configs 244 | if ldap_groups is not None and ldap_groups: 245 | config['ldapGroups'] = ldap_groups 246 | if enable_sso is not None: 247 | config['enableSSO'] = enable_sso 248 | else: 249 | config = None 250 | 251 | if any(x is not None for x in [impala_ha_enable_catalog_high_availability, impala_ha_enable_shutdown_of_coordinator, impala_ha_high_availability_mode, impala_ha_num_of_active_coordinators, impala_ha_shutdown_of_coordinator_delay_seconds]): 252 | 253 | impala_ha_settings_all = dict(enableCatalogHighAvailability=impala_ha_enable_catalog_high_availability, enableShutdownOfCoordinator=impala_ha_enable_shutdown_of_coordinator, 254 | highAvailabilityMode=impala_ha_high_availability_mode, numOfActiveCoordinators=impala_ha_num_of_active_coordinators, 255 | shutdownOfCoordinatorDelaySeconds=impala_ha_shutdown_of_coordinator_delay_seconds) 256 | 257 | impala_ha_settings = {k: v for k, 258 | v in impala_ha_settings_all.items() if v is not None} 259 | 260 | else: 261 | impala_ha_settings = None 262 | 263 | return self.sdk.call( 264 | svc='dw', func='create_vw', ret_field='vwId', clusterId=cluster_id, dbcId=dbc_id, 265 | vwType=vw_type, name=name, template=template, autoscaling=autoscaling, config=config, 266 | tags=tag_list, enableUnifiedAnalytics=enable_unified_analytics, 267 | platformJwtAuth=enable_platform_jwt_auth, impalaHaSettings=impala_ha_settings, 268 | squelch=[ 269 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 270 | ] 271 | ) 272 | 273 | def delete_vw(self, cluster_id: str, vw_id: str): 274 | return self.sdk.call( 275 | svc='dw', func='delete_vw', squelch=[ 276 | Squelch('NOT_FOUND'), 277 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 278 | ], 279 | clusterId=cluster_id, vwId=vw_id 280 | ) 281 | 282 | def start_vw(self, cluster_id: str, vw_id: str): 283 | return self.sdk.call( 284 | svc='dw', func='start_vw', squelch=[ 285 | Squelch('NOT_FOUND'), 286 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 287 | ], 288 | clusterId=cluster_id, vwId=vw_id 289 | ) 290 | 291 | def pause_vw(self, cluster_id: str, vw_id: str): 292 | return self.sdk.call( 293 | svc='dw', func='pause_vw', squelch=[ 294 | Squelch('NOT_FOUND'), 295 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 296 | ], 297 | clusterId=cluster_id, vwId=vw_id 298 | ) 299 | 300 | def restart_vw(self, cluster_id: str, vw_id: str): 301 | return self.sdk.call( 302 | svc='dw', func='restart_vw', squelch=[ 303 | Squelch('NOT_FOUND'), 304 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 305 | ], 306 | clusterId=cluster_id, vwId=vw_id 307 | ) 308 | 309 | def create_dbc(self, cluster_id: str, name: str, load_demo_data: bool = None): 310 | return self.sdk.call( 311 | svc='dw', func='create_dbc', ret_field='dbcId', clusterId=cluster_id, name=name, 312 | loadDemoData=load_demo_data, squelch=[ 313 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 314 | ] 315 | ) 316 | 317 | def delete_dbc(self, cluster_id: str, dbc_id: str): 318 | return self.sdk.call( 319 | svc='dw', func='delete_dbc', squelch=[ 320 | Squelch('NOT_FOUND'), 321 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 322 | ], 323 | clusterId=cluster_id, dbcId=dbc_id 324 | ) 325 | 326 | def restart_dbc(self, cluster_id: str, dbc_id: str): 327 | return self.sdk.call( 328 | svc='dw', func='restart_dbc', squelch=[ 329 | Squelch('NOT_FOUND'), 330 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 331 | ], 332 | clusterId=cluster_id, dbcId=dbc_id 333 | ) 334 | -------------------------------------------------------------------------------- /src/cdpy/environments.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from typing import Union 4 | from cdpy.common import CdpSdkBase, Squelch, CdpError, CdpWarning 5 | 6 | 7 | class CdpyEnvironments(CdpSdkBase): 8 | def __init__(self, *args, **kwargs): 9 | super().__init__(*args, **kwargs) 10 | 11 | def get_credential_prerequisites(self, cloud: str): 12 | return self.sdk.call(svc='environments', func='get_credential_prerequisites', cloudPlatform=cloud.upper()) 13 | 14 | def describe_proxy_config(self, name): 15 | resp = self.list_proxy_configs(name) 16 | return self.sdk.first_item_if_exists(resp) 17 | 18 | def create_proxy_config(self, proxyConfigName, host=None, port=None, protocol=None, description=None, 19 | noProxyHosts=None, user=None, password=None): 20 | 21 | return self.sdk.call( 22 | svc='environments', func='create_proxy_config', ret_field='proxyConfig', 23 | proxyConfigName=proxyConfigName, host=host, port=port, 24 | protocol=protocol, description=description, noProxyHosts=noProxyHosts, user=user, password=password 25 | ) 26 | 27 | def delete_proxy_config(self, name): 28 | return self.sdk.call( 29 | svc='environments', func='delete_proxy_config', ret_field='credentials', squelch=[Squelch('NOT_FOUND')], 30 | proxyConfigName=name 31 | ) 32 | 33 | def list_proxy_configs(self, name=None): 34 | return self.sdk.call( 35 | svc='environments', func='list_proxy_configs', ret_field='proxyConfigs', squelch=[ 36 | Squelch('NOT_FOUND', default=list()) 37 | ], 38 | proxyConfigName=name 39 | ) 40 | 41 | def get_id_broker_mapping_sync(self, name): 42 | return self.sdk.call( 43 | svc='environments', func='get_id_broker_mappings_sync_status', squelch=[ 44 | Squelch('NOT_FOUND'), Squelch('INVALID_ARGUMENT') 45 | ], 46 | environmentName=name 47 | ) 48 | 49 | def get_id_broker_mappings(self, name): 50 | return self.sdk.call( 51 | svc='environments', func='get_id_broker_mappings', squelch=[Squelch('NOT_FOUND')], 52 | environmentName=name 53 | ) 54 | 55 | def describe_environment(self, name): 56 | resp = self.sdk.call( 57 | svc='environments', func='describe_environment', ret_field='environment', ret_error=True, 58 | environmentName=name 59 | ) 60 | if isinstance(resp, CdpError): 61 | if resp.error_code == 'NOT_FOUND': 62 | # Describe will fail in certain fault scenarios early in Environment creation 63 | # We helpfully provide the summary listing as a backup set of information 64 | # If the environment truly does not exist, then this will give the same response 65 | self.sdk.throw_warning(CdpWarning(str(resp.violations))) 66 | return self.summarize_environment(name) 67 | self.sdk.throw_error(resp) 68 | return resp 69 | 70 | def describe_all_environments(self): 71 | envs_listing = self.list_environments() 72 | if envs_listing is not None: 73 | return [self.describe_environment(env['environmentName']) for env in envs_listing] 74 | else: 75 | return list() 76 | 77 | def summarize_environment(self, name): 78 | result = self.list_environments() 79 | if result: 80 | for env in result: 81 | if env['environmentName'] == name: 82 | return env 83 | return None 84 | 85 | def gather_idbroker_mappings(self, name): 86 | results = dict() 87 | mappings = self.get_id_broker_mappings(name) 88 | if mappings is not None: 89 | results.update(**mappings) 90 | broker_sync_status = self.get_id_broker_mapping_sync(name) 91 | if broker_sync_status is not None: 92 | results.update(syncStatus=broker_sync_status) 93 | return results 94 | 95 | def list_environments(self): 96 | return self.sdk.call( 97 | svc='environments', func='list_environments', ret_field='environments', squelch=[ 98 | Squelch('NOT_FOUND', default=list(), warning='No Environments found in CDP Tenant')] 99 | ) 100 | 101 | def create_aws_environment(self, **kwargs): 102 | # TODO: Rework with named kwargs 103 | resp = self.sdk.call( 104 | svc='environments', func='create_aws_environment', ret_field='environment', ret_error=True, 105 | **kwargs 106 | ) 107 | if isinstance(resp, CdpError): 108 | if resp.error_code == 'INVALID_ARGUMENT': 109 | if 'constraintViolations' not in str(resp.violations): 110 | resp.update(message="Received violation warning:\n%s" % self.sdk.dumps(str(resp.violations))) 111 | self.sdk.throw_warning(CdpWarning(str(resp.violations))) 112 | self.sdk.throw_error(resp) 113 | return resp 114 | 115 | def create_azure_environment(self, **kwargs): 116 | # TODO: Rework with named kwargs 117 | resp = self.sdk.call( 118 | svc='environments', func='create_azure_environment', ret_field='environment', ret_error=True, 119 | **kwargs 120 | ) 121 | if isinstance(resp, CdpError): 122 | if resp.error_code == 'INVALID_ARGUMENT': 123 | if 'constraintViolations' not in str(resp.violations): 124 | resp.update(message="Received violation warning:\n%s" % self.sdk.dumps(str(resp.violations))) 125 | self.sdk.throw_warning(CdpWarning(str(resp.violations))) 126 | self.sdk.throw_error(resp) 127 | return resp 128 | 129 | def create_gcp_environment(self, **kwargs): 130 | # TODO: Rework with named kwargs 131 | resp = self.sdk.call( 132 | svc='environments', func='create_gcp_environment', ret_field='environment', ret_error=True, 133 | **kwargs 134 | ) 135 | if isinstance(resp, CdpError): 136 | if resp.error_code == 'INVALID_ARGUMENT': 137 | if 'constraintViolations' not in str(resp.violations): 138 | resp.update(message="Received violation warning:\n%s" % self.sdk.dumps(str(resp.violations))) 139 | self.sdk.throw_warning(CdpWarning(str(resp.violations))) 140 | self.sdk.throw_error(resp) 141 | return resp 142 | 143 | def stop_environment(self, name): 144 | return self.sdk.call( 145 | svc='environments', func='stop_environment', ret_field='environment', squelch=[ 146 | Squelch(field='error_code', value='CONFLICT', default=self.describe_environment(name), 147 | warning='Environment %s is already scheduled to stop' % name) 148 | ], 149 | environmentName=name 150 | ) 151 | 152 | def delete_environment(self, name, cascade=False, force=False): 153 | return self.sdk.call( 154 | svc='environments', func='delete_environment', ret_field='environment', 155 | environmentName=name, 156 | cascading=cascade, 157 | forced=force 158 | ) 159 | 160 | def start_environment(self, name, datahub_start=True): 161 | return self.sdk.call( 162 | svc='environments', func='start_environment', ret_field='environment', squelch=[ 163 | Squelch(field='error_code', value='CONFLICT', default=self.describe_environment(name), 164 | warning='Environment %s is already scheduled to start' % name) 165 | ], 166 | environmentName=name, 167 | withDatahubStart=datahub_start 168 | ) 169 | 170 | def set_password(self, password, environment_names=None): 171 | payload = dict(password=password) 172 | if environment_names is not None: 173 | if not isinstance(environment_names, list): 174 | environment_names = [environment_names] 175 | environment_crns = [] 176 | for env in environment_names: 177 | r = self.describe_environment(env) 178 | if r is not None: 179 | environment_crns.append(r['crn']) 180 | else: 181 | self.sdk.throw_error(CdpError("Environment not found")) 182 | payload.update(environmentCRNs=environment_crns) 183 | return self.sdk.call( 184 | svc='environments', func='set_password', squelch=[ 185 | Squelch(field='error_code', value='CONFLICT', default=None, 186 | warning='Password Update Conflict') 187 | ], 188 | **payload 189 | ) 190 | 191 | def sync_current_user(self): 192 | return self.sdk.call(svc='environments', func='sync_user') 193 | 194 | def sync_users(self, environments=None): 195 | if isinstance(environments, list): 196 | if len(environments) == 0: 197 | environments = None 198 | elif isinstance(environments, str): 199 | environments = [environments] 200 | else: 201 | self.sdk.throw_error(CdpError("environments must be a list of one or more strings or a string")) 202 | resp = self.sdk.call( 203 | svc='environments', func='sync_all_users', ret_error=True, 204 | environmentNames=environments 205 | ) 206 | if isinstance(resp, CdpError): 207 | if resp.error_code == 'CONFLICT': 208 | operation_match = self.sdk.regex_search(self.sdk.OPERATION_REGEX, str(resp.violations)) 209 | if operation_match is not None: 210 | existing_op_id = operation_match.group(1) 211 | if not self.sdk.strict_errors: 212 | self.sdk.throw_warning( 213 | CdpWarning('Sync All Users Operation already running, tracking existing job {0}' 214 | .format(existing_op_id))) 215 | return self.get_sync_status(existing_op_id) 216 | self.sdk.throw_error(resp) 217 | return resp 218 | 219 | def get_sync_status(self, operation): 220 | return self.sdk.call( 221 | svc='environments', func='sync_status', squelch=[ 222 | Squelch(field='error_code', value='NOT_FOUND', default=None, 223 | warning='No User Sync Operation found matching %s' % operation) 224 | ], 225 | operationId=operation 226 | ) 227 | 228 | def get_keytab(self, actor, environment): 229 | return self.sdk.call( 230 | svc='environments', func='get_keytab', ret_field='contents', 231 | actorCrn=actor, 232 | environmentName=environment 233 | ) 234 | 235 | def list_credentials(self, name=None): 236 | return self.sdk.call( 237 | svc='environments', func='list_credentials', ret_field='credentials', squelch=[ 238 | Squelch('NOT_FOUND', default=list())], 239 | credentialName=name 240 | ) 241 | 242 | def describe_credential(self, name): 243 | resp = self.list_credentials(name) 244 | # Return singular None if credential not found, rather than default list() 245 | return self.sdk.first_item_if_exists(resp) if resp else None 246 | 247 | def delete_credential(self, name): 248 | return self.sdk.call( 249 | svc='environments', func='delete_credential', ret_field='credentials', squelch=[Squelch('NOT_FOUND')], 250 | credentialName=name 251 | ) 252 | 253 | def create_aws_credential(self, name, role, description, retries=3, delay=2): 254 | resp = self.sdk.call( 255 | svc='environments', func='create_aws_credential', ret_error=True, 256 | ret_field='credential', squelch=[Squelch(field='violations', value='Credential already exists with name', 257 | warning='Credential with this name already exists', default=None)], 258 | credentialName=name, 259 | roleArn=role, 260 | description=description 261 | ) 262 | if isinstance(resp, CdpError): 263 | if retries > 0: 264 | consistency_violations = [ 265 | 'Unable to verify credential', 'sts:AssumeRole', 'You are not authorized' 266 | ] 267 | if any(x in str(resp.violations) for x in consistency_violations): 268 | retries = retries - 1 269 | self.sdk.throw_warning( 270 | CdpWarning('Got likely AWS IAM eventual consistency error [%s], %d retries left...' 271 | % (str(resp.violations), retries)) 272 | ) 273 | self.sdk.sleep(delay) 274 | return self.create_aws_credential(name, role, description, retries, delay) 275 | else: 276 | self.sdk.throw_error(resp) 277 | return resp 278 | 279 | def create_azure_credential(self, name, subscription, tenant, application, secret, retries=3, delay=5): 280 | resp = self.sdk.call( 281 | svc='environments', func='create_azure_credential', ret_error=True, squelch=[ 282 | Squelch(field='violations', value='Credential already exists with name', 283 | warning='Credential with this name already exists', default=None)], 284 | credentialName=name, 285 | subscriptionId=subscription, 286 | tenantId=tenant, 287 | appBased={'applicationId': application, 'secretKey': secret} 288 | ) 289 | if isinstance(resp, CdpError): 290 | if retries > 0: 291 | consistency_violations = [ 292 | 'You may have sent your authentication request to the wrong tenant' 293 | ] 294 | if any(x in str(resp.violations) for x in consistency_violations): 295 | retries = retries - 1 296 | self.sdk.throw_warning( 297 | CdpWarning('Got likely Azure eventual consistency error [%s], %d retries left...' 298 | % (str(resp.violations), retries)) 299 | ) 300 | self.sdk.sleep(delay) 301 | return self.create_azure_credential(name, subscription, tenant, application, secret, retries, delay) 302 | else: 303 | self.sdk.throw_error(resp) 304 | return resp 305 | 306 | def create_gcp_credential(self, name, key_file): 307 | return self.sdk.call( 308 | svc='environments', func='create_gcp_credential', squelch=[ 309 | Squelch(field='violations', value='Credential already exists with name', 310 | warning='Credential with this name already exists', default=None)], 311 | credentialName=name, 312 | credentialKey=self.sdk.read_file(key_file) 313 | ) 314 | 315 | def get_root_cert(self, environment): 316 | return self.sdk.call( 317 | svc='environments', func='get_root_certificate', ret_field='contents', 318 | environmentName=environment 319 | ) 320 | 321 | def set_telemetry(self, name, workload_analytics=None, logs_collection=None): 322 | return self.sdk.call( 323 | svc='environments', func='set_telemetry_features', 324 | environmentName=name, 325 | workloadAnalytics=workload_analytics, 326 | reportDeploymentLogs=logs_collection 327 | ) 328 | 329 | def resolve_environment_crn(self, env: Union[str, None]): 330 | """Ensures a given env string is either the environment crn or None""" 331 | if isinstance(env, str): 332 | if env.startswith('crn:'): 333 | return env 334 | else: 335 | env_desc = self.describe_environment(env) 336 | return env_desc['crn'] if env_desc else None 337 | else: 338 | return None 339 | -------------------------------------------------------------------------------- /src/cdpy/iam.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from cdpy.common import CdpSdkBase, Squelch 4 | 5 | 6 | class CdpyIam(CdpSdkBase): 7 | def __init__(self, *args, **kwargs): 8 | super().__init__(*args, **kwargs) 9 | 10 | def get_user(self, name=None): 11 | return self.sdk.call( 12 | # Describe base function calls 13 | svc='iam', # Name of the client service 14 | func='get_user', # Name of the Function within the service to call 15 | ret_field='user', # Optional child field to return, often CDP CLI responses are wrapped like this 16 | squelch=[ # List of below Client Error Handlers 17 | # Describe any Client Error responses using the provided Squelch class 18 | Squelch( 19 | field='error_code', # CdpError Field to test 20 | value='NOT_FOUND', # String to check for in Field 21 | warning='CDP User could not be retrieved', # Warning to throw if encountered 22 | default=None # Value to return instead of Error 23 | ) 24 | ], 25 | # Include any keyword args that may be used in the function call, None/'' args will be ignored 26 | userId=name # As name is None by default, it will be ignored unless provided 27 | ) 28 | 29 | def set_cloudera_sso(self, enabled: bool): 30 | func = 'enable_cloudera_sso_login' if enabled is True else 'disable_cloudera_sso_login' 31 | return self.sdk.call(svc='iam', func=func) 32 | 33 | def set_password_lifetime(self, lifetime: int): 34 | return self.sdk.call(svc='iam', func='set_workload_password_policy', maxPasswordLifetimeDays=lifetime) 35 | 36 | def create_group(self, name: str, sync: bool = True): 37 | return self.sdk.call( 38 | svc='iam', func='create_group', ret_field='group', 39 | groupName=name, 40 | syncMembershipOnUserLogin=sync 41 | ) 42 | 43 | def update_group(self, name, sync=True): 44 | return self.sdk.call( 45 | svc='iam', func='update_group', ret_field='group', 46 | groupName=name, 47 | syncMembershipOnUserLogin=sync 48 | ) 49 | 50 | def delete_group(self, name): 51 | return self.sdk.call( 52 | svc='iam', func='delete_group', ret_field='group', 53 | groupName=name 54 | ) 55 | 56 | def add_group_user(self, group, user): 57 | return self.sdk.call( 58 | svc='iam', func='add_user_to_group', ret_field='group', 59 | groupName=group, userId=user 60 | ) 61 | 62 | def remove_group_user(self, group, user): 63 | return self.sdk.call( 64 | svc='iam', func='remove_user_from_group', ret_field='group', 65 | groupName=group, userId=user 66 | ) 67 | 68 | def assign_group_role(self, group, role): 69 | return self.sdk.call( 70 | svc='iam', func='assign_group_role', ret_field='group', 71 | groupName=group, role=role 72 | ) 73 | 74 | def unassign_group_role(self, group, role): 75 | return self.sdk.call( 76 | svc='iam', func='unassign_group_role', ret_field='group', 77 | groupName=group, role=role 78 | ) 79 | 80 | def assign_group_resource_role(self, group, resource, role): 81 | return self.sdk.call( 82 | svc='iam', func='assign_group_resource_role', ret_field='group', 83 | groupName=group, resourceCrn=resource, resourceRoleCrn=role 84 | ) 85 | 86 | def unassign_group_resource_role(self, group, resource, role): 87 | return self.sdk.call( 88 | svc='iam', func='unassign_group_resource_role', ret_field='group', 89 | groupName=group, resourceCrn=resource, resourceRoleCrn=role 90 | ) 91 | 92 | def gather_groups(self, group_names=None): 93 | # TODO: Needs tests 94 | resp = self.list_groups(group_names=group_names) 95 | if resp: 96 | list(map(lambda grp: grp.update(users=self.list_group_membership(grp['crn'])), resp)) 97 | list(map(lambda grp: grp.update(roles=self.list_group_assigned_roles(grp['crn'])), resp)) 98 | list(map(lambda grp: grp.update(resource_roles=self.list_group_assigned_resource_roles(grp['crn'])), resp)) 99 | return resp 100 | 101 | def list_groups(self, group_names=None): 102 | group_names = group_names if group_names is None or isinstance(group_names, list) else [group_names] 103 | return self.sdk.call( 104 | svc='iam', func='list_groups', ret_field='groups', squelch=[ 105 | Squelch(field='error_code', value='NOT_FOUND', default=list(), 106 | warning='No Groups found for Group Names, %s' % str(group_names)) 107 | ], 108 | groupNames=group_names 109 | ) 110 | 111 | def gather_users(self, users=None): 112 | resp = self.list_users(users=users) 113 | return resp if resp is None else self.sdk.filter_by_key(resp, 'crn') 114 | 115 | def list_users(self, users=None): 116 | users = users if users is None or isinstance(users, list) else [users] 117 | return self.sdk.call( 118 | svc='iam', func='list_users', ret_field='users', squelch=[ 119 | Squelch(field='error_code', value='NOT_FOUND', default=list(), 120 | warning='No Users found for UserIds, %s' % str(users)) 121 | ], 122 | userIds=users 123 | ) 124 | 125 | def list_group_membership(self, group_name): 126 | return self.sdk.call( 127 | svc='iam', func='list_group_members', ret_field='memberCrns', squelch=[ 128 | Squelch(field='error_code', value='NOT_FOUND', default=list(), 129 | warning='No Group Members found for Group, %s' % group_name) 130 | ], 131 | groupName=group_name 132 | ) 133 | 134 | def list_group_assigned_roles(self, group_name): 135 | return self.sdk.call( 136 | svc='iam', func='list_group_assigned_roles', ret_field='roleCrns', squelch=[ 137 | Squelch(field='error_code', value='NOT_FOUND', default=list(), 138 | warning='No Roles found for Group, %s' % group_name) 139 | ], 140 | groupName=group_name 141 | ) 142 | 143 | def list_group_assigned_resource_roles(self, group_name): 144 | return self.sdk.call( 145 | svc='iam', func='list_group_assigned_resource_roles', ret_field='resourceAssignments', squelch=[ 146 | Squelch(field='error_code', value='NOT_FOUND', default=list(), 147 | warning='No Group Assigned Resource Roles found for Group, %s' % group_name) 148 | ], 149 | groupName=group_name 150 | ) 151 | 152 | def list_resource_roles(self, roles=None): 153 | return self.sdk.call( 154 | svc='iam', func='list_resource_roles', ret_field='resourceRoles', squelch=[ 155 | Squelch(field='error_code', value='NOT_FOUND', default=list(), 156 | warning='No Resource Roles found for Names, %s' % str(roles)) 157 | ], 158 | resourceRoleNames=roles 159 | ) 160 | 161 | def get_account(self): 162 | return self.sdk.call( 163 | svc='iam', func='get_account', ret_field='account', squelch=[ 164 | Squelch(field='error_code', value='NOT_FOUND', default=None, 165 | warning='CDP Account could not be retrieved') 166 | ] 167 | ) 168 | -------------------------------------------------------------------------------- /src/cdpy/ml.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from cdpy.common import CdpError, CdpWarning, CdpSdkBase, Squelch 4 | 5 | ENTITLEMENT_DISABLED='Machine Learning not enabled on CDP Tenant' 6 | 7 | 8 | class CdpyMl(CdpSdkBase): 9 | def __init__(self, *args, **kwargs): 10 | super().__init__(*args, **kwargs) 11 | 12 | def describe_workspace(self, name=None, crn=None, env=None): 13 | return self.sdk.call( 14 | svc='ml', func='describe_workspace', ret_field='workspace', squelch=[ 15 | Squelch('NOT_FOUND'), 16 | Squelch('INVALID_ARGUMENT'), 17 | Squelch('UNKNOWN'), 18 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 19 | ], 20 | workspaceName=name, 21 | environmentName=env, 22 | workspaceCrn=crn 23 | ) 24 | 25 | def list_workspaces(self, env=None): 26 | resp = self.sdk.call( 27 | svc='ml', func='list_workspaces', ret_field='workspaces', squelch=[ 28 | Squelch(value='NOT_FOUND', default=list(), 29 | warning='No Workspaces found in Tenant'), 30 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED, 31 | default=list()) 32 | ] 33 | ) 34 | # TODO: Replace with Filters 35 | if env: 36 | return [x for x in resp if env == x['environmentName']] 37 | return resp 38 | 39 | def describe_all_workspaces(self, env=None): 40 | ws_list = self.list_workspaces(env) 41 | resp = [] 42 | for ws in ws_list: 43 | ws_desc = self.describe_workspace(crn=ws['crn']) 44 | if ws_desc is not None: 45 | resp.append(ws_desc) 46 | return resp 47 | 48 | def list_workspace_access(self, name: str = None, crn: str = None, env: str = None): 49 | resp = self.sdk.call( 50 | svc='ml', func='list_workspace_access', ret_field='users', ret_error=True, 51 | squelch=[ 52 | Squelch(value='UNKNOWN', default=list()), 53 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED, default=list()) 54 | ], 55 | workspaceName=name, 56 | environmentName=env, 57 | workspaceCrn=crn 58 | ) 59 | if isinstance(resp, CdpError) and resp.error_code == 'INVALID_ARGUMENT': 60 | resp.update(message=resp.violations) 61 | self.sdk.throw_error(resp) 62 | return resp 63 | 64 | def grant_workspace_access(self, identifier: str, name: str = None, crn: str = None, env: str = None): 65 | resp = self.sdk.call( 66 | svc='ml', func='grant_workspace_access', ret_error=True, 67 | squelch=[ 68 | Squelch(value='UNKNOWN'), 69 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 70 | ], 71 | workspaceName=name, 72 | environmentName=env, 73 | workspaceCrn=crn, 74 | identifier=identifier 75 | ) 76 | if isinstance(resp, CdpError) and resp.error_code == 'INVALID_ARGUMENT': 77 | resp.update(message=resp.violations) 78 | self.sdk.throw_error(resp) 79 | return resp 80 | 81 | def revoke_workspace_access(self, identifier: str, name: str = None, crn: str = None, env: str = None): 82 | resp = self.sdk.call( 83 | svc='ml', func='revoke_workspace_access', ret_error=True, 84 | squelch=[ 85 | Squelch(value='UNKNOWN'), 86 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 87 | ], 88 | workspaceName=name, 89 | environmentName=env, 90 | workspaceCrn=crn, 91 | identifier=identifier 92 | ) 93 | if isinstance(resp, CdpError) and resp.error_code == 'INVALID_ARGUMENT': 94 | resp.update(message=resp.violations) 95 | self.sdk.throw_error(resp) 96 | return resp 97 | -------------------------------------------------------------------------------- /src/cdpy/opdb.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import array 3 | 4 | from cdpy.common import CdpSdkBase, Squelch 5 | 6 | ENTITLEMENT_DISABLED='Operational Database not enabled on CDP Tenant' 7 | 8 | 9 | class CdpyOpdb(CdpSdkBase): 10 | def __init__(self, *args, **kwargs): 11 | super().__init__(*args, **kwargs) 12 | 13 | def describe_database(self, name=None, env=None): 14 | return self.sdk.call( 15 | svc='opdb', func='describe_database', ret_field='databaseDetails', squelch=[ 16 | Squelch('NOT_FOUND'), 17 | Squelch('INVALID_ARGUMENT'), 18 | Squelch('UNKNOWN'), 19 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 20 | ], 21 | databaseName=name, 22 | environmentName=env, 23 | ) 24 | 25 | def list_databases(self, env=None): 26 | return self.sdk.call( 27 | svc='opdb', func='list_databases', ret_field='databases', squelch=[ 28 | Squelch(value='NOT_FOUND', default=list(), 29 | warning='No OpDB Databases found in Tenant'), 30 | Squelch(value='PATH_DISABLED', default=list(), 31 | warning=ENTITLEMENT_DISABLED) 32 | ], 33 | environmentName=env 34 | ) 35 | 36 | def describe_all_databases(self, env=None): 37 | ws_list = self.list_databases(env) 38 | resp = [] 39 | for db in ws_list: 40 | db_desc = self.describe_database(db['databaseName'], db['environmentCrn']) 41 | if db_desc is not None: 42 | resp.append(db_desc) 43 | return resp 44 | 45 | def drop_database(self, name, env): 46 | return self.sdk.call( 47 | svc='opdb', func='drop_database', ret_field='status', squelch=[ 48 | Squelch('NOT_FOUND'), 49 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 50 | ], 51 | databaseName=name, 52 | environmentName=env, 53 | ) 54 | 55 | def create_database(self, name: str, env: str, disable_ephemeral_storage: bool = False, 56 | disable_jwt_auth: bool = False, auto_scaling_params: dict = None, 57 | dns_forward_domain: str = None, dns_forward_ns_ip: str = None, 58 | subnet_id: str = None, use_hdfs: bool = False, disable_multi_az: bool = False, 59 | disable_kerberos: bool = False, num_edge_nodes: int = 0, image: dict = None, 60 | enable_region_canary: bool = False, master_node_type: str = None, 61 | gateway_node_type: str = None, custom_user_tags: array = None, 62 | attached_storage_for_workers: dict = None): 63 | return self.sdk.call( 64 | svc='opdb', func='create_database', ret_field='databaseDetails', 65 | squelch=[ 66 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 67 | ], 68 | databaseName=name, 69 | environmentName=env, 70 | disableEphemeralStorage=disable_ephemeral_storage, 71 | disableJwtAuth=disable_jwt_auth, 72 | autoScalingParameters=auto_scaling_params, 73 | dnsForwardDomain=dns_forward_domain, 74 | dnsForwardNsIp=dns_forward_ns_ip, 75 | enableRegionCanary=enable_region_canary, 76 | useHdfs=use_hdfs, 77 | subnetId=subnet_id, 78 | disableMultiAz=disable_multi_az, 79 | disableKerberos=disable_kerberos, 80 | numEdgeNodes=num_edge_nodes, 81 | image=image, 82 | masterNodeType=master_node_type, 83 | gatewayNodeType=gateway_node_type, 84 | customUserTags=custom_user_tags, 85 | attachedStorageForWorkers=attached_storage_for_workers, 86 | ) 87 | 88 | def start_database(self, name, env): 89 | return self.sdk.call( 90 | svc='opdb', func='start_database', squelch=[ 91 | Squelch('NOT_FOUND'), 92 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 93 | ], 94 | databaseName=name, 95 | environmentName=env, 96 | ) 97 | 98 | def stop_database(self, name, env): 99 | return self.sdk.call( 100 | svc='opdb', func='stop_database', squelch=[ 101 | Squelch('NOT_FOUND'), 102 | Squelch(value='PATH_DISABLED', warning=ENTITLEMENT_DISABLED) 103 | ], 104 | databaseName=name, 105 | environmentName=env, 106 | ) -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloudera-labs/cdpy/0b6b1a5d8a31cb29d3c486c7b415c1a53dd4a683/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Dummy conftest.py for cdpy. 4 | 5 | If you don't know what this is for, just leave it empty. 6 | Read more about conftest.py under: 7 | https://pytest.org/latest/plugins.html 8 | """ 9 | 10 | import pytest 11 | -------------------------------------------------------------------------------- /tests/test_credentials.py: -------------------------------------------------------------------------------- 1 | from cdpy.common import StaticCredentials 2 | from cdpy.cdpy import Cdpy 3 | from cdpy.iam import CdpyIam 4 | 5 | def test_static_credentials(): 6 | ACCESS_KEY_ID = "thekey" 7 | PRIVATE_KEY = "theanswer" 8 | 9 | iam = Cdpy(cdp_credentials=StaticCredentials(ACCESS_KEY_ID, PRIVATE_KEY)).iam 10 | 11 | assert iam.sdk.cdp_credentials.access_key_id == ACCESS_KEY_ID 12 | assert iam.sdk.cdp_credentials.private_key == PRIVATE_KEY 13 | 14 | def test_static_credentials_submodule(): 15 | ACCESS_KEY_ID = "thekey" 16 | PRIVATE_KEY = "theanswer" 17 | 18 | iam = CdpyIam(cdp_credentials=StaticCredentials(ACCESS_KEY_ID, PRIVATE_KEY)) 19 | 20 | assert iam.sdk.cdp_credentials.access_key_id == ACCESS_KEY_ID 21 | assert iam.sdk.cdp_credentials.private_key == PRIVATE_KEY 22 | -------------------------------------------------------------------------------- /tests/test_environments.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from cdpy.environments import CdpyEnvironments 3 | from tests import conftest 4 | 5 | sdk = CdpyEnvironments() 6 | 7 | 8 | def test_get_credential_prerequisites(): 9 | pass 10 | 11 | # Example Test File 12 | -------------------------------------------------------------------------------- /tests/test_iam.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from cdpy.iam import CdpyIam 3 | from tests import conftest 4 | 5 | sdk = CdpyIam() 6 | 7 | 8 | def test_get_user(): 9 | r1 = sdk.get_user() 10 | assert isinstance(r1, dict) 11 | assert isinstance(r1['workloadUsername'], str) 12 | 13 | r2 = sdk.get_user(name=r1['userId']) 14 | assert r2['userId'] == r1['userId'] 15 | 16 | with pytest.warns(UserWarning, match='Removing empty string arg'): 17 | r3 = sdk.get_user(name='') # Invalid submission 18 | assert isinstance(r3['workloadUsername'], str) 19 | 20 | with pytest.warns(UserWarning, match='User could not be retrieved'): 21 | r4 = sdk.get_user(name='obviouslyfakeomgwtfbbq') 22 | assert r4 is None 23 | 24 | # Example test function 25 | --------------------------------------------------------------------------------