├── .github └── workflows │ ├── deploy.yaml │ └── integration.yaml ├── .gitignore ├── .readthedocs.yaml ├── CHANGELOG.rst ├── LICENSE ├── README.rst ├── cloudbridge ├── __init__.py ├── base │ ├── __init__.py │ ├── helpers.py │ ├── middleware.py │ ├── provider.py │ ├── resources.py │ ├── services.py │ └── subservices.py ├── factory.py ├── interfaces │ ├── __init__.py │ ├── exceptions.py │ ├── provider.py │ ├── resources.py │ ├── services.py │ └── subservices.py └── providers │ ├── __init__.py │ ├── aws │ ├── __init__.py │ ├── helpers.py │ ├── provider.py │ ├── resources.py │ ├── services.py │ └── subservices.py │ ├── azure │ ├── __init__.py │ ├── azure_client.py │ ├── helpers.py │ ├── provider.py │ ├── resources.py │ ├── services.py │ └── subservices.py │ ├── gcp │ ├── README.rst │ ├── __init__.py │ ├── helpers.py │ ├── provider.py │ ├── resources.py │ ├── services.py │ └── subservices.py │ ├── mock │ ├── __init__.py │ └── provider.py │ └── openstack │ ├── __init__.py │ ├── helpers.py │ ├── provider.py │ ├── resources.py │ ├── services.py │ └── subservices.py ├── docs ├── .gitignore ├── Makefile ├── api_docs │ ├── cloud │ │ ├── exceptions.rst │ │ ├── providers.rst │ │ ├── resources.rst │ │ └── services.rst │ └── ref.rst ├── concepts.rst ├── conf.py ├── extras │ └── _images │ │ └── object_relationships_detailed.svg ├── getting_started.rst ├── images │ ├── lifecycle_image.svg │ ├── lifecycle_instance.svg │ ├── lifecycle_snapshot.svg │ ├── lifecycle_volume.svg │ └── object_relationships_overview.svg ├── index.rst ├── requirements.txt └── topics │ ├── aws_mapping.rst │ ├── azure_mapping.rst │ ├── block_storage.rst │ ├── captures │ ├── aws-ami-dash.png │ ├── aws-bucket.png │ ├── aws-instance-dash.png │ ├── aws-services-dash.png │ ├── az-app-1.png │ ├── az-app-2.png │ ├── az-app-3.png │ ├── az-app-4.png │ ├── az-app-5.png │ ├── az-app-6.png │ ├── az-app-7.png │ ├── az-dir-1.png │ ├── az-dir-2.png │ ├── az-label-dash.png │ ├── az-net-id.png │ ├── az-net-label.png │ ├── az-role-1.png │ ├── az-role-2.png │ ├── az-role-3.png │ ├── az-storacc.png │ ├── az-sub-1.png │ ├── az-sub-2.png │ ├── az-subnet-label.png │ ├── az-subnet-name.png │ ├── gcp-sa-1.png │ ├── gcp-sa-2.png │ ├── gcp-sa-3.png │ ├── gcp-sa-4.png │ ├── gcp-sa-5.png │ ├── os-instance-dash.png │ └── os-kp-dash.png │ ├── contributor_guide.rst │ ├── design_decisions.rst │ ├── design_goals.rst │ ├── dns.rst │ ├── event_system.rst │ ├── faq.rst │ ├── install.rst │ ├── launch.rst │ ├── networking.rst │ ├── object_lifecycles.rst │ ├── object_storage.rst │ ├── os_mapping.rst │ ├── overview.rst │ ├── paging_and_iteration.rst │ ├── procuring_credentials.rst │ ├── provider_development.rst │ ├── release_process.rst │ ├── resource_types_and_mapping.rst │ ├── setup.rst │ ├── testing.rst │ └── troubleshooting.rst ├── requirements.txt ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── fixtures │ ├── custom_amis.json │ └── logo.jpg ├── helpers │ ├── __init__.py │ └── standard_interface_tests.py ├── test_base_helpers.py ├── test_block_store_service.py ├── test_cloud_factory.py ├── test_cloud_helpers.py ├── test_compute_service.py ├── test_dns_service.py ├── test_image_service.py ├── test_interface.py ├── test_middleware_system.py ├── test_network_service.py ├── test_object_life_cycle.py ├── test_object_store_service.py ├── test_region_service.py ├── test_security_service.py └── test_vm_types_service.py └── tox.ini /.github/workflows/deploy.yaml: -------------------------------------------------------------------------------- 1 | name: Publish cloudbridge to PyPI 2 | on: 3 | release: 4 | types: [published] 5 | push: 6 | tags: 7 | - '*' 8 | jobs: 9 | build-n-publish: 10 | name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@master 14 | - name: Set up Python 3.10.12 15 | uses: actions/setup-python@v1 16 | with: 17 | python-version: 3.10.12 18 | - name: Install dependencies 19 | run: | 20 | python3 -m pip install --upgrade pip setuptools 21 | python3 -m pip install --upgrade twine wheel 22 | - name: Create and check packages 23 | run: | 24 | python3 setup.py sdist bdist_wheel 25 | twine check dist/* 26 | ls -l dist 27 | - name: Publish distribution 📦 to Test PyPI 28 | uses: pypa/gh-action-pypi-publish@master 29 | with: 30 | password: ${{ secrets.TEST_PYPI_API_TOKEN }} 31 | repository_url: https://test.pypi.org/legacy/ 32 | skip_existing: true 33 | - name: Publish distribution 📦 to PyPI 34 | if: github.event_name == 'release' 35 | uses: pypa/gh-action-pypi-publish@master 36 | with: 37 | password: ${{ secrets.PYPI_API_TOKEN }} 38 | -------------------------------------------------------------------------------- /.github/workflows/integration.yaml: -------------------------------------------------------------------------------- 1 | name: Integration tests 2 | 3 | # Run this workflow every time a new commit pushed to your repository 4 | on: 5 | push: 6 | branches: 7 | - main 8 | pull_request_target: 9 | branches: 10 | - main 11 | workflow_dispatch: {} 12 | 13 | jobs: 14 | # Set the job key. The key is displayed as the job name 15 | # when a job name is not provided 16 | lint: 17 | name: Lint code 18 | runs-on: ubuntu-latest 19 | strategy: 20 | matrix: 21 | python-version: [ '3.10' ] 22 | steps: 23 | - name: Checkout code 24 | uses: actions/checkout@v4 25 | with: 26 | ref: ${{ github.event.pull_request.head.sha }} 27 | 28 | - name: Setup Python 29 | uses: actions/setup-python@v5 30 | with: 31 | python-version: ${{ matrix.python-version }} 32 | 33 | - name: Cache pip dir 34 | uses: actions/cache@v4 35 | with: 36 | path: ~/.cache/pip 37 | key: pip-cache-${{ matrix.python-version }}-lint 38 | 39 | - name: Install required packages 40 | run: pip install tox 41 | 42 | - name: Run tox 43 | run: tox -e lint 44 | 45 | integration: 46 | # Name the Job 47 | name: Per-cloud integration tests 48 | needs: lint 49 | # Set the type of machine to run on 50 | runs-on: ubuntu-latest 51 | strategy: 52 | fail-fast: false 53 | matrix: 54 | python-version: ['3.10'] 55 | cloud-provider: ['aws', 'azure', 'gcp', 'mock', 'openstack'] 56 | 57 | steps: 58 | - name: Checkout code 59 | uses: actions/checkout@v4 60 | with: 61 | ref: ${{ github.event.pull_request.head.sha }} 62 | 63 | - name: Setup Python 64 | uses: actions/setup-python@v5 65 | with: 66 | python-version: ${{ matrix.python-version }} 67 | 68 | - name: Cache pip dir 69 | uses: actions/cache@v4 70 | with: 71 | path: ~/.cache/pip 72 | key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('**/setup.py', '**/requirements.txt') }} 73 | 74 | - name: Install required packages 75 | run: pip install tox 76 | 77 | - name: Run tox 78 | id: tox 79 | run: tox -e py${{ matrix.python-version }}-${{ matrix.cloud-provider }} 80 | env: 81 | PYTHONUNBUFFERED: "True" 82 | # aws 83 | AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }} 84 | AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }} 85 | CB_VM_TYPE_AWS: ${{ secrets.CB_VM_TYPE_AWS }} 86 | # azure 87 | AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} 88 | AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} 89 | AZURE_SECRET: ${{ secrets.AZURE_SECRET }} 90 | AZURE_TENANT: ${{ secrets.AZURE_TENANT }} 91 | AZURE_RESOURCE_GROUP: ${{ secrets.AZURE_RESOURCE_GROUP }} 92 | AZURE_STORAGE_ACCOUNT: ${{ secrets.AZURE_STORAGE_ACCOUNT }} 93 | CB_IMAGE_AZURE: ${{ secrets.CB_IMAGE_AZURE }} 94 | CB_VM_TYPE_AZURE: ${{ secrets.CB_VM_TYPE_AZURE }} 95 | # gcp 96 | GCP_SERVICE_CREDS_DICT: ${{ secrets.GCP_SERVICE_CREDS_DICT }} 97 | CB_IMAGE_GCP: ${{ secrets.CB_IMAGE_GCP }} 98 | CB_VM_TYPE_GCP: ${{ secrets.CB_VM_TYPE_GCP }} 99 | # openstack 100 | OS_AUTH_URL: ${{ secrets.OS_AUTH_URL }} 101 | OS_PASSWORD: ${{ secrets.OS_PASSWORD }} 102 | OS_PROJECT_NAME: ${{ secrets.OS_PROJECT_NAME }} 103 | OS_PROJECT_DOMAIN_NAME: ${{ secrets.OS_PROJECT_DOMAIN_NAME }} 104 | OS_TENANT_NAME: ${{ secrets.OS_TENANT_NAME }} 105 | OS_USERNAME: ${{ secrets.OS_USERNAME }} 106 | OS_REGION_NAME: ${{ secrets.OS_REGION_NAME }} 107 | OS_USER_DOMAIN_NAME: ${{ secrets.OS_USER_DOMAIN_NAME }} 108 | OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} 109 | OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} 110 | CB_IMAGE_OS: ${{ secrets.CB_IMAGE_OS }} 111 | CB_VM_TYPE_OS: ${{ secrets.CB_VM_TYPE_OS }} 112 | CB_PLACEMENT_OS: ${{ secrets.CB_PLACEMENT_OS }} 113 | 114 | - name: Create Build Status Badge 115 | if: github.ref == 'refs/heads/master' 116 | uses: schneegans/dynamic-badges-action@v1.1.0 117 | with: 118 | auth: ${{ secrets.BUILD_STATUS_GIST_SECRET }} 119 | gistID: ${{ secrets.BUILD_STATUS_GIST_ID }} 120 | filename: cloudbridge_py${{ matrix.python-version }}_${{ matrix.cloud-provider }}.json 121 | label: ${{ matrix.cloud-provider }} 122 | message: ${{ fromJSON('["passing", "failing"]')[steps.tox.outcome != 'success'] }} 123 | color: ${{ fromJSON('["green", "red"]')[steps.tox.outcome != 'success'] }} 124 | 125 | - name: Coveralls 126 | if: ${{ steps.tox.outcome == 'success' }} 127 | uses: AndreMiras/coveralls-python-action@develop 128 | with: 129 | github-token: ${{ secrets.GITHUB_TOKEN }} 130 | flag-name: run-${{ matrix.python-version }}-${{ matrix.cloud-provider }} 131 | parallel: true 132 | 133 | finish: 134 | needs: integration 135 | runs-on: ubuntu-latest 136 | steps: 137 | - name: Coveralls Finished 138 | uses: AndreMiras/coveralls-python-action@develop 139 | with: 140 | github-token: ${{ secrets.github_token }} 141 | parallel-finished: true 142 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .coverage.* 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | *,cover 45 | 46 | # Translations 47 | *.mo 48 | *.pot 49 | 50 | # Django stuff: 51 | *.log 52 | 53 | # Sphinx documentation 54 | docs/_build/ 55 | 56 | # PyBuilder 57 | target/ 58 | 59 | *.DS_Store 60 | /venv/ 61 | 62 | credentials.tar.gz 63 | bootstrap.py 64 | ISB-* 65 | launch.json 66 | settings.json 67 | run_nose.py 68 | *ipynb* 69 | 70 | # PyCharm 71 | .idea/ 72 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | python: 4 | install: 5 | - requirements: docs/requirements.txt 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 CloudVE 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | CloudBridge provides a consistent layer of abstraction over different 2 | Infrastructure-as-a-Service cloud providers, reducing or eliminating the need 3 | to write conditional code for each cloud. 4 | 5 | Documentation 6 | ~~~~~~~~~~~~~ 7 | Detailed documentation can be found at http://cloudbridge.cloudve.org. 8 | 9 | 10 | Build Status Tests 11 | ~~~~~~~~~~~~~~~~~~ 12 | .. image:: https://github.com/CloudVE/cloudbridge/actions/workflows/integration.yaml/badge.svg 13 | :target: https://github.com/CloudVE/cloudbridge/actions/ 14 | :alt: Integration Tests 15 | 16 | .. image:: https://codecov.io/gh/CloudVE/cloudbridge/graph/badge.svg?token=w0LAfAIVdd 17 | :target: https://codecov.io/gh/CloudVE/cloudbridge 18 | :alt: Code Coverage 19 | 20 | .. image:: https://img.shields.io/pypi/v/cloudbridge.svg 21 | :target: https://pypi.python.org/pypi/cloudbridge/ 22 | :alt: latest version available on PyPI 23 | 24 | .. image:: https://readthedocs.org/projects/cloudbridge/badge/?version=latest 25 | :target: http://cloudbridge.readthedocs.org/en/latest/?badge=latest 26 | :alt: Documentation Status 27 | 28 | .. image:: https://img.shields.io/pypi/dm/cloudbridge 29 | :target: https://pypistats.org/packages/cloudbridge 30 | :alt: Download stats 31 | 32 | .. |aws-py38| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/nuwang/d354f151eb8c9752da13e6dec012fb07/raw/cloudbridge_py3.8_aws.json 33 | :target: https://github.com/CloudVE/cloudbridge/actions/ 34 | 35 | .. |azure-py38| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/nuwang/d354f151eb8c9752da13e6dec012fb07/raw/cloudbridge_py3.8_azure.json 36 | :target: https://github.com/CloudVE/cloudbridge/actions/ 37 | 38 | .. |gcp-py38| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/nuwang/d354f151eb8c9752da13e6dec012fb07/raw/cloudbridge_py3.8_gcp.json 39 | :target: https://github.com/CloudVE/cloudbridge/actions/ 40 | 41 | .. |mock-py38| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/nuwang/d354f151eb8c9752da13e6dec012fb07/raw/cloudbridge_py3.8_mock.json 42 | :target: https://github.com/CloudVE/cloudbridge/actions/ 43 | 44 | .. |os-py38| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/nuwang/d354f151eb8c9752da13e6dec012fb07/raw/cloudbridge_py3.8_openstack.json 45 | :target: https://github.com/CloudVE/cloudbridge/actions/ 46 | 47 | +---------------------------+----------------+ 48 | | **Provider/Environment** | **Python 3.8** | 49 | +---------------------------+----------------+ 50 | | **Amazon Web Services** | |aws-py38| | 51 | +---------------------------+----------------+ 52 | | **Google Cloud Platform** | |gcp-py38| | 53 | +---------------------------+----------------+ 54 | | **Microsoft Azure** | |azure-py38| | 55 | +---------------------------+----------------+ 56 | | **OpenStack** | |os-py38| | 57 | +---------------------------+----------------+ 58 | | **Mock Provider** | |mock-py38| | 59 | +---------------------------+----------------+ 60 | 61 | Installation 62 | ~~~~~~~~~~~~ 63 | Install the latest release from PyPi: 64 | 65 | .. code-block:: shell 66 | 67 | pip install cloudbridge[full] 68 | 69 | For other installation options, see the `installation page`_ in 70 | the documentation. 71 | 72 | 73 | Usage example 74 | ~~~~~~~~~~~~~ 75 | 76 | To `get started`_ with CloudBridge, export your cloud access credentials 77 | (e.g., AWS_ACCESS_KEY and AWS_SECRET_KEY for your AWS credentials) and start 78 | exploring the API: 79 | 80 | .. code-block:: python 81 | 82 | from cloudbridge.factory import CloudProviderFactory, ProviderList 83 | 84 | provider = CloudProviderFactory().create_provider(ProviderList.AWS, {}) 85 | print(provider.security.key_pairs.list()) 86 | 87 | The exact same command (as well as any other CloudBridge method) will run with 88 | any of the supported providers: ``ProviderList.[AWS | AZURE | GCP | OPENSTACK]``! 89 | 90 | 91 | Citation 92 | ~~~~~~~~ 93 | 94 | N. Goonasekera, A. Lonie, J. Taylor, and E. Afgan, 95 | "CloudBridge: a Simple Cross-Cloud Python Library," 96 | presented at the Proceedings of the XSEDE16 Conference on Diversity, Big Data, and Science at Scale, Miami, USA, 2016. 97 | DOI: http://dx.doi.org/10.1145/2949550.2949648 98 | 99 | 100 | Quick Reference 101 | ~~~~~~~~~~~~~~~ 102 | The following object graph shows how to access various provider services, and the resource 103 | that they return. 104 | 105 | .. image:: http://cloudbridge.readthedocs.org/en/latest/_images/object_relationships_detailed.svg 106 | :target: http://cloudbridge.readthedocs.org/en/latest/?badge=latest#quick-reference 107 | :alt: CloudBridge Quick Reference 108 | 109 | 110 | Design Goals 111 | ~~~~~~~~~~~~ 112 | 113 | 1. Create a cloud abstraction layer which minimises or eliminates the need for 114 | cloud specific special casing (i.e., Not require clients to write 115 | ``if EC2 do x else if OPENSTACK do y``.) 116 | 117 | 2. Have a suite of conformance tests which are comprehensive enough that goal 118 | 1 can be achieved. This would also mean that clients need not manually test 119 | against each provider to make sure their application is compatible. 120 | 121 | 3. Opt for a minimum set of features that a cloud provider will support, 122 | instead of a lowest common denominator approach. This means that reasonably 123 | mature clouds like Amazon and OpenStack are used as the benchmark against 124 | which functionality & features are determined. Therefore, there is a 125 | definite expectation that the cloud infrastructure will support a compute 126 | service with support for images and snapshots and various machine sizes. 127 | The cloud infrastructure will very likely support block storage, although 128 | this is currently optional. It may optionally support object storage. 129 | 130 | 4. Make the CloudBridge layer as thin as possible without compromising goal 1. 131 | By wrapping the cloud provider's native SDK and doing the minimal work 132 | necessary to adapt the interface, we can achieve greater development speed 133 | and reliability since the native provider SDK is most likely to have both 134 | properties. 135 | 136 | 137 | Contributing 138 | ~~~~~~~~~~~~ 139 | Community contributions for any part of the project are welcome. If you have 140 | a completely new idea or would like to bounce your idea before moving forward 141 | with the implementation, feel free to create an issue to start a discussion. 142 | 143 | Contributions should come in the form of a pull request. We strive for 100% test 144 | coverage so code will only be accepted if it comes with appropriate tests and it 145 | does not break existing functionality. Further, the code needs to be well 146 | documented and all methods have docstrings. We are largely adhering to the 147 | `PEP8 style guide`_ with 80 character lines, 4-space indentation (spaces 148 | instead of tabs), explicit, one-per-line imports among others. Please keep the 149 | style consistent with the rest of the project. 150 | 151 | Conceptually, the library is laid out such that there is a factory used to 152 | create a reference to a cloud provider. Each provider offers a set of services 153 | and resources. Services typically perform actions while resources offer 154 | information (and can act on itself, when appropriate). The structure of each 155 | object is defined via an abstract interface (see 156 | ``cloudbridge/providers/interfaces``) and any object should implement the 157 | defined interface. If adding a completely new provider, take a look at the 158 | `provider development page`_ in the documentation. 159 | 160 | 161 | .. _`installation page`: http://cloudbridge.readthedocs.org/en/ 162 | latest/topics/install.html 163 | .. _`get started`: http://cloudbridge.readthedocs.org/en/latest/ 164 | getting_started.html 165 | .. _`PEP8 style guide`: https://www.python.org/dev/peps/pep-0008/ 166 | .. _`provider development page`: http://cloudbridge.readthedocs.org/ 167 | en/latest/ 168 | topics/provider_development.html 169 | -------------------------------------------------------------------------------- /cloudbridge/__init__.py: -------------------------------------------------------------------------------- 1 | """Library setup.""" 2 | import logging 3 | 4 | # Current version of the library 5 | __version__ = '3.2.0' 6 | 7 | 8 | def get_version(): 9 | """ 10 | Return a string with the current version of the library. 11 | 12 | :rtype: ``string`` 13 | :return: Library version (e.g., "0.1.0"). 14 | """ 15 | return __version__ 16 | 17 | 18 | def init_logging(): 19 | """ 20 | Initialize logging for testing. 21 | 22 | Temporary workaround for build timeouts by enabling logging to 23 | stdout so that Travis doesn't think the build has hung. 24 | """ 25 | set_stream_logger(__name__, level=logging.DEBUG) 26 | 27 | 28 | class NullHandler(logging.Handler): 29 | """A null handler for the logger.""" 30 | 31 | def emit(self, record): 32 | """Don't emit a log.""" 33 | pass 34 | 35 | 36 | TRACE = 5 # Lower than debug which is 10 37 | 38 | 39 | class CBLogger(logging.Logger): 40 | """ 41 | A custom logger, adds logging level below debug. 42 | 43 | Add a ``trace`` log level, numeric value 5: ``log.trace("Log message")`` 44 | """ 45 | 46 | def trace(self, msg, *args, **kwargs): 47 | """Add ``trace`` log level.""" 48 | self.log(TRACE, msg, *args, **kwargs) 49 | 50 | 51 | # By default, do not force any logging by the library. If you want to see the 52 | # log messages in your scripts, add the following to the top of your script: 53 | # import cloudbridge 54 | # cloudbridge.set_stream_logger(__name__) 55 | # OR 56 | # cloudbridge.set_file_logger(__name__, '/tmp/log') 57 | default_format_string = "%(asctime)s [%(levelname)s] %(name)s: %(message)s" 58 | logging.setLoggerClass(CBLogger) 59 | logging.addLevelName(TRACE, "TRACE") 60 | log = logging.getLogger('cloudbridge') 61 | log.addHandler(NullHandler()) 62 | 63 | # Convenience functions to set logging to a particular file or stream 64 | # To enable either of these by default within CloudBridge, add the following 65 | # at the top of a CloudBridge module: 66 | # import cloudbridge 67 | # cloudbridge.set_stream_logger(__name__) 68 | # OR 69 | # cloudbridge.set_file_logger(__name__, '/tmp/log') 70 | 71 | 72 | def set_stream_logger(name, level=TRACE, format_string=None): 73 | """A convenience method to set the global logger to stream.""" 74 | global log 75 | if not format_string: 76 | format_string = default_format_string 77 | logger = logging.getLogger(name) 78 | logger.setLevel(level) 79 | fh = logging.StreamHandler() 80 | fh.setLevel(level) 81 | formatter = logging.Formatter(format_string) 82 | fh.setFormatter(formatter) 83 | logger.addHandler(fh) 84 | log = logger 85 | 86 | 87 | def set_file_logger(name, filepath, level=logging.INFO, format_string=None): 88 | """A convenience method to set the global logger to a file.""" 89 | global log 90 | if not format_string: 91 | format_string = default_format_string 92 | logger = logging.getLogger(name) 93 | logger.setLevel(level) 94 | fh = logging.FileHandler(filepath) 95 | fh.setLevel(level) 96 | formatter = logging.Formatter(format_string) 97 | fh.setFormatter(formatter) 98 | logger.addHandler(fh) 99 | log = logger 100 | -------------------------------------------------------------------------------- /cloudbridge/base/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Public interface exports 3 | """ 4 | from .provider import BaseCloudProvider # noqa 5 | -------------------------------------------------------------------------------- /cloudbridge/base/helpers.py: -------------------------------------------------------------------------------- 1 | import fnmatch 2 | import functools 3 | import logging 4 | import os 5 | import re 6 | import sys 7 | from contextlib import contextmanager 8 | 9 | from cryptography.hazmat.backends import default_backend 10 | from cryptography.hazmat.primitives import serialization as crypt_serialization 11 | from cryptography.hazmat.primitives.asymmetric import rsa 12 | 13 | from deprecation import deprecated 14 | 15 | import six 16 | 17 | import cloudbridge 18 | 19 | from ..interfaces.exceptions import InvalidParamException 20 | 21 | log = logging.getLogger(__name__) 22 | 23 | 24 | def generate_key_pair(): 25 | """ 26 | This method generates a keypair and returns it as a tuple 27 | of (public, private) keys. 28 | The public key format is OpenSSH and private key format is PEM. 29 | """ 30 | key_pair = rsa.generate_private_key( 31 | backend=default_backend(), 32 | public_exponent=65537, 33 | key_size=2048) 34 | private_key = key_pair.private_bytes( 35 | crypt_serialization.Encoding.PEM, 36 | crypt_serialization.PrivateFormat.PKCS8, 37 | crypt_serialization.NoEncryption()).decode('utf-8') 38 | public_key = key_pair.public_key().public_bytes( 39 | crypt_serialization.Encoding.OpenSSH, 40 | crypt_serialization.PublicFormat.OpenSSH).decode('utf-8') 41 | return public_key, private_key 42 | 43 | 44 | def filter_by(prop_name, kwargs, objs): 45 | """ 46 | Utility method for filtering a list of objects by a property. 47 | If the given property has a non empty value in kwargs, then 48 | the list of objs is filtered by that value. Otherwise, the 49 | list of objs is returned as is. 50 | """ 51 | prop_val = kwargs.pop(prop_name, None) 52 | if prop_val: 53 | if isinstance(prop_val, six.string_types): 54 | regex = fnmatch.translate(prop_val) 55 | results = [o for o in objs 56 | if getattr(o, prop_name) 57 | and re.search(regex, getattr(o, prop_name))] 58 | else: 59 | results = [o for o in objs 60 | if getattr(o, prop_name) == prop_val] 61 | return results 62 | else: 63 | return objs 64 | 65 | 66 | def generic_find(filter_names, kwargs, objs): 67 | """ 68 | Utility method for filtering a list of objects by a list of filters. 69 | """ 70 | matches = objs 71 | for name in filter_names: 72 | matches = filter_by(name, kwargs, matches) 73 | 74 | # All kwargs should have been popped at this time. 75 | if len(kwargs) > 0: 76 | raise InvalidParamException( 77 | "Unrecognised parameters for search: %s. Supported attributes: %s" 78 | % (kwargs, filter_names)) 79 | 80 | return matches 81 | 82 | 83 | @contextmanager 84 | def cleanup_action(cleanup_func): 85 | """ 86 | Context manager to carry out a given 87 | cleanup action after carrying out a set 88 | of tasks, or when an exception occurs. 89 | If any errors occur during the cleanup 90 | action, those are ignored, and the original 91 | traceback is preserved. 92 | 93 | :params func: This function is called if 94 | an exception occurs or at the end of the 95 | context block. If any exceptions raised 96 | by func are ignored. 97 | Usage: 98 | with cleanup_action(lambda e: print("Oops!")): 99 | do_something() 100 | """ 101 | try: 102 | yield 103 | except Exception: 104 | ex_class, ex_val, ex_traceback = sys.exc_info() 105 | try: 106 | cleanup_func() 107 | except Exception: 108 | log.exception("Error during exception cleanup: ") 109 | six.reraise(ex_class, ex_val, ex_traceback) 110 | try: 111 | cleanup_func() 112 | except Exception: 113 | log.exception("Error during exception cleanup: ") 114 | 115 | 116 | def get_env(varname, default_value=None): 117 | """ 118 | Return the value of the environment variable or default_value. 119 | 120 | This is a helper method that wraps ``os.environ.get`` to ensure type 121 | compatibility across py2 and py3. For py2, any value obtained from an 122 | environment variable, ensure ``unicode`` type and ``str`` for py3. The 123 | casting is done only for string variables. 124 | 125 | :type varname: ``str`` 126 | :param varname: Name of the environment variable for which to check. 127 | 128 | :param default_value: Return this value is the env var is not found. 129 | Defaults to ``None``. 130 | 131 | :return: Value of the supplied environment if found; value of 132 | ``default_value`` otherwise. 133 | """ 134 | value = os.environ.get(varname, default_value) 135 | if isinstance(value, six.string_types) and not isinstance( 136 | value, six.text_type): 137 | return six.u(value) 138 | return value 139 | 140 | 141 | # Alias deprecation decorator, following: 142 | # https://stackoverflow.com/questions/49802412/ 143 | # how-to-implement-deprecation-in-python-with-argument-alias 144 | def deprecated_alias(**aliases): 145 | def deco(f): 146 | @functools.wraps(f) 147 | def wrapper(*args, **kwargs): 148 | rename_kwargs(f.__name__, kwargs, aliases) 149 | return f(*args, **kwargs) 150 | return wrapper 151 | return deco 152 | 153 | 154 | def rename_kwargs(func_name, kwargs, aliases): 155 | for alias, new in aliases.items(): 156 | if alias in kwargs: 157 | if new in kwargs: 158 | raise InvalidParamException( 159 | '{} received both {} and {}'.format(func_name, alias, new)) 160 | # Manually invoke the deprecated decorator with an empty lambda 161 | # to signal deprecation 162 | deprecated(deprecated_in='1.1', 163 | removed_in='2.0', 164 | current_version=cloudbridge.__version__, 165 | details='{} is deprecated, use {} instead'.format( 166 | alias, new))(lambda: None)() 167 | kwargs[new] = kwargs.pop(alias) 168 | 169 | 170 | NON_ALPHA_NUM = re.compile(r"[^A-Za-z0-9]+") 171 | 172 | 173 | def to_resource_name(value, replace_with="-"): 174 | """ 175 | Converts a given string to a valid resource name by stripping 176 | all characters that are not alphanumeric. 177 | 178 | :param value: the value to strip 179 | :param replace_with: the value to replace mismatching characters with 180 | :return: a string with all mismatching characters removed. 181 | """ 182 | val = re.sub(NON_ALPHA_NUM, replace_with, value) 183 | return val.strip("-") 184 | -------------------------------------------------------------------------------- /cloudbridge/base/middleware.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | 4 | from pyeventsystem.middleware import dispatch as pyevent_dispatch 5 | from pyeventsystem.middleware import intercept 6 | from pyeventsystem.middleware import observe 7 | 8 | import six 9 | 10 | from ..interfaces.exceptions import CloudBridgeBaseException 11 | 12 | log = logging.getLogger(__name__) 13 | 14 | 15 | dispatch = pyevent_dispatch 16 | 17 | 18 | class EventDebugLoggingMiddleware(object): 19 | """ 20 | Logs all event parameters. This middleware should not be enabled other 21 | than for debugging, as it could log sensitive parameters such as 22 | access keys. 23 | """ 24 | @observe(event_pattern="*", priority=100) 25 | def pre_log_event(self, event_args, *args, **kwargs): 26 | log.debug("Event: {0}, args: {1} kwargs: {2}".format( 27 | event_args.get("event"), args, kwargs)) 28 | 29 | @observe(event_pattern="*", priority=4900) 30 | def post_log_event(self, event_args, *args, **kwargs): 31 | log.debug("Event: {0}, result: {1}".format( 32 | event_args.get("event"), event_args.get("result"))) 33 | 34 | 35 | class ExceptionWrappingMiddleware(object): 36 | """ 37 | Wraps all unhandled exceptions in cloudbridge exceptions. 38 | """ 39 | @intercept(event_pattern="*", priority=1050) 40 | def wrap_exception(self, event_args, *args, **kwargs): 41 | next_handler = event_args.pop("next_handler") 42 | if not next_handler: 43 | return 44 | try: 45 | return next_handler.invoke(event_args, *args, **kwargs) 46 | except Exception as e: 47 | if isinstance(e, CloudBridgeBaseException): 48 | raise 49 | else: 50 | ex_type, ex_value, traceback = sys.exc_info() 51 | cb_ex = CloudBridgeBaseException( 52 | "CloudBridgeBaseException: {0} from exception type: {1}" 53 | .format(ex_value, ex_type)) 54 | if sys.version_info >= (3, 0): 55 | six.raise_from(cb_ex, e) 56 | else: 57 | six.reraise(CloudBridgeBaseException, cb_ex, traceback) 58 | -------------------------------------------------------------------------------- /cloudbridge/base/provider.py: -------------------------------------------------------------------------------- 1 | """Base implementation of a provider interface.""" 2 | import ast 3 | import functools 4 | import logging 5 | import os 6 | from os.path import expanduser 7 | try: 8 | from configparser import ConfigParser 9 | except ImportError: # Python 2 10 | from ConfigParser import SafeConfigParser as ConfigParser 11 | 12 | from pyeventsystem.middleware import SimpleMiddlewareManager 13 | 14 | import six 15 | 16 | from ..base.middleware import ExceptionWrappingMiddleware 17 | from ..interfaces import CloudProvider 18 | from ..interfaces.exceptions import ProviderConnectionException 19 | from ..interfaces.resources import Configuration 20 | 21 | log = logging.getLogger(__name__) 22 | 23 | DEFAULT_RESULT_LIMIT = 50 24 | DEFAULT_WAIT_TIMEOUT = 600 25 | DEFAULT_WAIT_INTERVAL = 5 26 | 27 | # By default, use two locations for CloudBridge configuration 28 | CloudBridgeConfigPath = '/etc/cloudbridge.ini' 29 | CloudBridgeConfigLocations = [CloudBridgeConfigPath] 30 | UserConfigPath = os.path.join(expanduser('~'), '.cloudbridge') 31 | CloudBridgeConfigLocations.append(UserConfigPath) 32 | 33 | 34 | class BaseConfiguration(Configuration): 35 | 36 | def __init__(self, user_config): 37 | self.update(user_config) 38 | 39 | @property 40 | def default_result_limit(self): 41 | """ 42 | Get the maximum number of results to return for a 43 | list method 44 | 45 | :rtype: ``int`` 46 | :return: The maximum number of results to return 47 | """ 48 | log.debug("Maximum number of results for list methods %s", 49 | DEFAULT_RESULT_LIMIT) 50 | return self.get('default_result_limit', DEFAULT_RESULT_LIMIT) 51 | 52 | @property 53 | def default_wait_timeout(self): 54 | """ 55 | Gets the default wait timeout for LifeCycleObjects. 56 | """ 57 | log.debug("Default wait timeout for LifeCycleObjects %s", 58 | DEFAULT_WAIT_TIMEOUT) 59 | return self.get('default_wait_timeout', DEFAULT_WAIT_TIMEOUT) 60 | 61 | @property 62 | def default_wait_interval(self): 63 | """ 64 | Gets the default wait interval for LifeCycleObjects. 65 | """ 66 | log.debug("Default wait interfal for LifeCycleObjects %s", 67 | DEFAULT_WAIT_INTERVAL) 68 | return self.get('default_wait_interval', DEFAULT_WAIT_INTERVAL) 69 | 70 | @property 71 | def debug_mode(self): 72 | """ 73 | A flag indicating whether CloudBridge is in debug mode. Setting 74 | this to True will cause the underlying provider's debug 75 | output to be turned on. 76 | 77 | The flag can be toggled by sending in the cb_debug value via 78 | the config dictionary, or setting the CB_DEBUG environment variable. 79 | 80 | :rtype: ``bool`` 81 | :return: Whether debug mode is on. 82 | """ 83 | return self.get('cb_debug', os.environ.get('CB_DEBUG', False)) 84 | 85 | 86 | class BaseCloudProvider(CloudProvider): 87 | def __init__(self, config): 88 | self._config = BaseConfiguration(config) 89 | self._config_parser = ConfigParser() 90 | self._config_parser.read(CloudBridgeConfigLocations) 91 | self._middleware = SimpleMiddlewareManager() 92 | self.add_required_middleware() 93 | self._region_name = None 94 | self._zone_name = None 95 | 96 | @property 97 | def region_name(self): 98 | return self._region_name 99 | 100 | @property 101 | def zone_name(self): 102 | if not self._zone_name: 103 | region = self.compute.regions.current 104 | zone = region.default_zone 105 | self._zone_name = zone.name if zone else None 106 | return self._zone_name 107 | else: 108 | try: 109 | zone_dict = ast.literal_eval(self._zone_name) 110 | if isinstance(zone_dict, dict): 111 | return zone_dict 112 | except (ValueError, SyntaxError): 113 | pass 114 | return self._zone_name 115 | 116 | @property 117 | def config(self): 118 | return self._config 119 | 120 | @property 121 | def name(self): 122 | return str(self.__class__.__name__) 123 | 124 | @property 125 | def middleware(self): 126 | return self._middleware 127 | 128 | def add_required_middleware(self): 129 | """ 130 | Adds common middleware that is essential for cloudbridge to function. 131 | Any other extra middleware can be added through the provider factory. 132 | """ 133 | self.middleware.add(ExceptionWrappingMiddleware()) 134 | 135 | def authenticate(self): 136 | """ 137 | A basic implementation which simply runs a low impact command to 138 | check whether cloud credentials work. Providers should override with 139 | more efficient implementations. 140 | """ 141 | log.debug("Checking if cloud credential works...") 142 | try: 143 | self.security.key_pairs.list() 144 | return True 145 | except Exception as e: 146 | log.exception("ProviderConnectionException occurred") 147 | raise ProviderConnectionException( 148 | "Authentication with cloud provider failed: %s" % (e,)) 149 | 150 | def clone(self, zone=None): 151 | cloned_config = self.config.copy() 152 | cloned_provider = self.__class__(cloned_config) 153 | if zone: 154 | # pylint:disable=protected-access 155 | cloned_provider._zone_name = zone.name 156 | return cloned_provider 157 | 158 | def _deepgetattr(self, obj, attr): 159 | """Recurses through an attribute chain to get the ultimate value.""" 160 | return functools.reduce(getattr, attr.split('.'), obj) 161 | 162 | def has_service(self, service_type): 163 | """ 164 | Checks whether this provider supports a given service. 165 | 166 | :type service_type: str or :class:``.CloudServiceType`` 167 | :param service_type: Type of service to check support for. 168 | 169 | :rtype: bool 170 | :return: ``True`` if the service type is supported. 171 | """ 172 | log.info("Checking if provider supports %s", service_type) 173 | try: 174 | if self._deepgetattr(self, service_type): 175 | log.info("This provider supports %s", 176 | service_type) 177 | return True 178 | except AttributeError: 179 | pass # Undefined service type 180 | except NotImplementedError: 181 | pass # service not implemented 182 | log.info("This provider doesn't support %s", 183 | service_type) 184 | return False 185 | 186 | def _get_config_value(self, key, default_value=None): 187 | """ 188 | A convenience method to extract a configuration value. 189 | 190 | :type key: str 191 | :param key: a field to look for in the ``self.config`` field 192 | 193 | :type default_value: anything 194 | :param default_value: the default value to return if a value for the 195 | ``key`` is not available 196 | 197 | :return: a configuration value for the supplied ``key`` 198 | """ 199 | log.debug("Getting config key %s, with supplied default value: %s", 200 | key, default_value) 201 | value = default_value 202 | if isinstance(self.config, dict) and self.config.get(key): 203 | value = self.config.get(key, default_value) 204 | elif hasattr(self.config, key) and getattr(self.config, key): 205 | value = getattr(self.config, key) 206 | elif (self._config_parser.has_option(self.PROVIDER_ID, key) and 207 | self._config_parser.get(self.PROVIDER_ID, key)): 208 | value = self._config_parser.get(self.PROVIDER_ID, key) 209 | if isinstance(value, six.string_types) and not isinstance( 210 | value, six.text_type): 211 | return six.u(value) 212 | return value 213 | -------------------------------------------------------------------------------- /cloudbridge/base/subservices.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from cloudbridge.interfaces.subservices import BucketObjectSubService 4 | from cloudbridge.interfaces.subservices import DnsRecordSubService 5 | from cloudbridge.interfaces.subservices import FloatingIPSubService 6 | from cloudbridge.interfaces.subservices import GatewaySubService 7 | from cloudbridge.interfaces.subservices import SubnetSubService 8 | from cloudbridge.interfaces.subservices import VMFirewallRuleSubService 9 | 10 | from .resources import BasePageableObjectMixin 11 | 12 | log = logging.getLogger(__name__) 13 | 14 | 15 | class BaseBucketObjectSubService(BasePageableObjectMixin, 16 | BucketObjectSubService): 17 | 18 | def __init__(self, provider, bucket): 19 | self.__provider = provider 20 | self.bucket = bucket 21 | 22 | @property 23 | def _provider(self): 24 | return self.__provider 25 | 26 | def get(self, name): 27 | return self._provider.storage._bucket_objects.get(self.bucket, name) 28 | 29 | def list(self, limit=None, marker=None, prefix=None): 30 | return self._provider.storage._bucket_objects.list(self.bucket, limit, 31 | marker, prefix) 32 | 33 | def find(self, **kwargs): 34 | return self._provider.storage._bucket_objects.find(self.bucket, 35 | **kwargs) 36 | 37 | def create(self, name): 38 | return self._provider.storage._bucket_objects.create(self.bucket, name) 39 | 40 | 41 | class BaseGatewaySubService(GatewaySubService, BasePageableObjectMixin): 42 | 43 | def __init__(self, provider, network): 44 | self._network = network 45 | self.__provider = provider 46 | 47 | @property 48 | def _provider(self): 49 | return self.__provider 50 | 51 | def get_or_create(self): 52 | return (self._provider.networking 53 | ._gateways 54 | .get_or_create(self._network)) 55 | 56 | def delete(self, gateway): 57 | return (self._provider.networking 58 | ._gateways 59 | .delete(self._network, gateway)) 60 | 61 | def list(self, limit=None, marker=None): 62 | return (self._provider.networking 63 | ._gateways 64 | .list(self._network, limit, marker)) 65 | 66 | 67 | class BaseVMFirewallRuleSubService(BasePageableObjectMixin, 68 | VMFirewallRuleSubService): 69 | 70 | def __init__(self, provider, firewall): 71 | self.__provider = provider 72 | self._firewall = firewall 73 | 74 | @property 75 | def _provider(self): 76 | return self.__provider 77 | 78 | def get(self, rule_id): 79 | return self._provider.security._vm_firewall_rules.get(self._firewall, 80 | rule_id) 81 | 82 | def list(self, limit=None, marker=None): 83 | return self._provider.security._vm_firewall_rules.list(self._firewall, 84 | limit, marker) 85 | 86 | def create(self, direction, protocol=None, from_port=None, 87 | to_port=None, cidr=None, src_dest_fw=None): 88 | return (self._provider 89 | .security 90 | ._vm_firewall_rules 91 | .create(self._firewall, direction, protocol, from_port, 92 | to_port, cidr, src_dest_fw)) 93 | 94 | def find(self, **kwargs): 95 | return self._provider.security._vm_firewall_rules.find(self._firewall, 96 | **kwargs) 97 | 98 | def delete(self, rule_id): 99 | return (self._provider 100 | .security 101 | ._vm_firewall_rules 102 | .delete(self._firewall, rule_id)) 103 | 104 | 105 | class BaseFloatingIPSubService(FloatingIPSubService, BasePageableObjectMixin): 106 | 107 | def __init__(self, provider, gateway): 108 | self.__provider = provider 109 | self.gateway = gateway 110 | 111 | @property 112 | def _provider(self): 113 | return self.__provider 114 | 115 | def get(self, fip_id): 116 | return self._provider.networking._floating_ips.get(self.gateway, 117 | fip_id) 118 | 119 | def list(self, limit=None, marker=None): 120 | return self._provider.networking._floating_ips.list(self.gateway, 121 | limit, marker) 122 | 123 | def find(self, **kwargs): 124 | return self._provider.networking._floating_ips.find(self.gateway, 125 | **kwargs) 126 | 127 | def create(self): 128 | return self._provider.networking._floating_ips.create(self.gateway) 129 | 130 | def delete(self, fip): 131 | return self._provider.networking._floating_ips.delete(self.gateway, 132 | fip) 133 | 134 | 135 | class BaseSubnetSubService(SubnetSubService, BasePageableObjectMixin): 136 | 137 | def __init__(self, provider, network): 138 | self.__provider = provider 139 | self.network = network 140 | 141 | @property 142 | def _provider(self): 143 | return self.__provider 144 | 145 | def get(self, subnet_id): 146 | sn = self._provider.networking.subnets.get(subnet_id) 147 | if sn.network_id != self.network.id: 148 | log.warning("The SubnetSubService nested in the network '{}' " 149 | "returned subnet '{}' which is attached to another " 150 | "network '{}'".format(str(self.network), str(sn), 151 | str(sn.network))) 152 | return sn 153 | 154 | def list(self, limit=None, marker=None): 155 | return self._provider.networking.subnets.list(network=self.network, 156 | limit=limit, 157 | marker=marker) 158 | 159 | def find(self, **kwargs): 160 | return self._provider.networking.subnets.find(network=self.network, 161 | **kwargs) 162 | 163 | def create(self, label, cidr_block): 164 | return self._provider.networking.subnets.create(label, 165 | self.network, 166 | cidr_block) 167 | 168 | def delete(self, subnet): 169 | return self._provider.networking.subnets.delete(subnet) 170 | 171 | 172 | class BaseDnsRecordSubService(DnsRecordSubService, BasePageableObjectMixin): 173 | 174 | def __init__(self, provider, dns_zone): 175 | self.__provider = provider 176 | self.dns_zone = dns_zone 177 | 178 | @property 179 | def _provider(self): 180 | return self.__provider 181 | 182 | def get(self, rec_id): 183 | # pylint:disable=protected-access 184 | return self._provider.dns._records.get(self.dns_zone, rec_id) 185 | 186 | def list(self, limit=None, marker=None): 187 | # pylint:disable=protected-access 188 | return self._provider.dns._records.list( 189 | dns_zone=self.dns_zone, limit=limit, marker=marker) 190 | 191 | def find(self, **kwargs): 192 | # pylint:disable=protected-access 193 | return self._provider.dns._records.find( 194 | dns_zone=self.dns_zone, **kwargs) 195 | 196 | def create(self, name, type, data, ttl=None): 197 | # pylint:disable=protected-access 198 | return self._provider.dns._records.create( 199 | self.dns_zone, name, type, data, ttl) 200 | 201 | def delete(self, rec): 202 | return self._provider.dns._records.delete(self.dns_zone, rec) 203 | -------------------------------------------------------------------------------- /cloudbridge/factory.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import inspect 3 | import logging 4 | import pkgutil 5 | from collections import defaultdict 6 | 7 | from cloudbridge import providers 8 | from cloudbridge.interfaces import CloudProvider 9 | from cloudbridge.interfaces import TestMockHelperMixin 10 | 11 | 12 | log = logging.getLogger(__name__) 13 | 14 | 15 | class ProviderList(object): 16 | AWS = 'aws' 17 | AZURE = 'azure' 18 | GCP = 'gcp' 19 | OPENSTACK = 'openstack' 20 | MOCK = 'mock' 21 | 22 | 23 | class CloudProviderFactory(object): 24 | 25 | """ 26 | Get info and handle on the available cloud provider implementations. 27 | """ 28 | 29 | def __init__(self): 30 | self.provider_list = defaultdict(dict) 31 | log.debug("Providers List: %s", self.provider_list) 32 | 33 | def register_provider_class(self, cls): 34 | """ 35 | Registers a provider class with the factory. The class must 36 | inherit from cloudbridge.interfaces.CloudProvider 37 | and also have a class attribute named PROVIDER_ID. 38 | 39 | The PROVIDER_ID is a user friendly name for the cloud provider, 40 | such as 'aws'. The PROVIDER_ID must also be included in the 41 | cloudbridge.factory.ProviderList. 42 | 43 | :type cls: class 44 | :param cls: A class implementing the CloudProvider interface. 45 | Mock providers must also implement 46 | :py:class:`cloudbridge.base.helpers. 47 | TestMockHelperMixin`. 48 | """ 49 | if isinstance(cls, type) and issubclass(cls, CloudProvider): 50 | if hasattr(cls, "PROVIDER_ID"): 51 | provider_id = getattr(cls, "PROVIDER_ID") 52 | if self.provider_list.get(provider_id, {}).get('class'): 53 | log.warning("Provider with id: %s is already " 54 | "registered. Overriding with class: %s", 55 | provider_id, cls) 56 | self.provider_list[provider_id]['class'] = cls 57 | else: 58 | log.warning("Provider class: %s implements CloudProvider but" 59 | " does not define PROVIDER_ID. Ignoring...", cls) 60 | else: 61 | log.debug("Class: %s does not implement the CloudProvider" 62 | " interface. Ignoring...", cls) 63 | 64 | def discover_providers(self): 65 | """ 66 | Discover all available providers within the 67 | ``cloudbridge.providers`` package. 68 | Note that this methods does not guard against a failed import. 69 | """ 70 | for _, modname, _ in pkgutil.iter_modules(providers.__path__): 71 | log.debug("Importing provider: %s", modname) 72 | try: 73 | self._import_provider(modname) 74 | except Exception as e: 75 | log.debug("Could not import provider: %s", e) 76 | 77 | def _import_provider(self, module_name): 78 | """ 79 | Imports and registers providers from the given module name. 80 | Raises an ImportError if the import does not succeed. 81 | """ 82 | log.debug("Importing providers from %s", module_name) 83 | module = importlib.import_module( 84 | "{0}.{1}".format(providers.__name__, 85 | module_name)) 86 | classes = inspect.getmembers(module, inspect.isclass) 87 | for _, cls in classes: 88 | log.debug("Registering the provider: %s", cls) 89 | self.register_provider_class(cls) 90 | 91 | def list_providers(self): 92 | """ 93 | Get a list of available providers. 94 | 95 | It uses a simple automatic discovery system by iterating through all 96 | submodules in cloudbridge.providers. 97 | 98 | :rtype: dict 99 | :return: A dict of available providers and their implementations in the 100 | following format:: 101 | {'aws': {'class': aws.provider.AWSCloudProvider}, 102 | 'openstack': {'class': openstack.provider.OpenStackCloudProvi 103 | der} 104 | } 105 | """ 106 | if not self.provider_list: 107 | self.discover_providers() 108 | log.debug("List of available providers: %s", self.provider_list) 109 | return self.provider_list 110 | 111 | def create_provider(self, name, config): 112 | """ 113 | Searches all available providers for a CloudProvider interface with the 114 | given name, and instantiates it based on the given config dictionary, 115 | where the config dictionary is a dictionary understood by that 116 | cloud provider. 117 | 118 | :type name: str 119 | :param name: Cloud provider name: one of ``aws``, ``openstack``, 120 | ``azure``. 121 | 122 | :type config: :class:`dict` 123 | :param config: A dictionary or an iterable of key/value pairs (as 124 | tuples or other iterables of length two). See specific 125 | provider implementation for the required fields. 126 | 127 | :return: a concrete provider instance 128 | :rtype: ``object`` of :class:`.CloudProvider` 129 | """ 130 | log.info("Creating '%s' provider", name) 131 | provider_class = self.get_provider_class(name) 132 | if provider_class is None: 133 | log.exception("A provider with the name %s could not " 134 | "be found", name) 135 | raise NotImplementedError( 136 | 'A provider with name {0} could not be' 137 | ' found'.format(name)) 138 | log.debug("Created '%s' provider", name) 139 | return provider_class(config) 140 | 141 | def get_provider_class(self, name): 142 | """ 143 | Return a class for the requested provider. 144 | 145 | :rtype: provider class or ``None`` 146 | :return: A class corresponding to the requested provider or ``None`` 147 | if the provider was not found. 148 | """ 149 | log.debug("Returning a class for the %s provider", name) 150 | impl = self.list_providers().get(name) 151 | if impl: 152 | log.debug("Returning provider class for %s", name) 153 | return impl["class"] 154 | else: 155 | log.debug("Provider with the name: %s not found", name) 156 | return None 157 | 158 | def get_all_provider_classes(self, ignore_mocks=False): 159 | """ 160 | Returns a list of classes for all available provider implementations 161 | 162 | :type ignore_mocks: ``bool`` 163 | :param ignore_mocks: If True, does not return mock providers. Mock 164 | providers are providers which implement the TestMockHelperMixin. 165 | 166 | :rtype: type ``class`` or ``None`` 167 | :return: A list of all available provider classes or an empty list 168 | if none found. 169 | """ 170 | all_providers = [] 171 | for impl in self.list_providers().values(): 172 | if ignore_mocks: 173 | if not issubclass(impl["class"], TestMockHelperMixin): 174 | all_providers.append(impl["class"]) 175 | else: 176 | all_providers.append(impl["class"]) 177 | log.info("List of provider classes: %s", all_providers) 178 | return all_providers 179 | -------------------------------------------------------------------------------- /cloudbridge/interfaces/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Public interface exports 3 | """ 4 | from .provider import CloudProvider # noqa 5 | from .provider import TestMockHelperMixin # noqa 6 | from .resources import CloudServiceType # noqa 7 | from .resources import InstanceState # noqa 8 | from .resources import LaunchConfig # noqa 9 | from .resources import MachineImageState # noqa 10 | from .resources import NetworkState # noqa 11 | from .resources import Region # noqa 12 | from .resources import SnapshotState # noqa 13 | from .resources import VolumeState # noqa 14 | from .exceptions import InvalidConfigurationException # noqa 15 | -------------------------------------------------------------------------------- /cloudbridge/interfaces/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Specification for exceptions raised by a provider 3 | """ 4 | 5 | 6 | class CloudBridgeBaseException(Exception): 7 | """ 8 | Base class for all CloudBridge exceptions 9 | """ 10 | pass 11 | 12 | 13 | class WaitStateException(CloudBridgeBaseException): 14 | """ 15 | Marker interface for object wait exceptions. 16 | Thrown when a timeout or errors occurs waiting for an object does not reach 17 | the expected state within a specified time limit. 18 | """ 19 | pass 20 | 21 | 22 | class InvalidConfigurationException(CloudBridgeBaseException): 23 | """ 24 | Marker interface for invalid launch configurations. 25 | Thrown when a combination of parameters in a LaunchConfig 26 | object results in an illegal state. 27 | """ 28 | pass 29 | 30 | 31 | class ProviderInternalException(CloudBridgeBaseException): 32 | """ 33 | Marker interface for provider specific errors. 34 | Thrown when CloudBridge encounters an error internal to a 35 | provider. 36 | """ 37 | pass 38 | 39 | 40 | class ProviderConnectionException(CloudBridgeBaseException): 41 | """ 42 | Marker interface for connection errors to a cloud provider. 43 | Thrown when CloudBridge is unable to connect with a provider, 44 | for example, when credentials are incorrect, or connection 45 | settings are invalid. 46 | """ 47 | pass 48 | 49 | 50 | class InvalidNameException(CloudBridgeBaseException): 51 | """ 52 | Marker interface for any attempt to set an invalid name on 53 | a CloudBridge resource. An example would be setting uppercase 54 | letters, which are not allowed in a resource name. 55 | """ 56 | 57 | def __init__(self, msg): 58 | super(InvalidNameException, self).__init__(msg) 59 | 60 | 61 | class InvalidLabelException(InvalidNameException): 62 | """ 63 | Marker interface for any attempt to set an invalid label on 64 | a CloudBridge resource. An example would be setting uppercase 65 | letters, which are not allowed in a resource label. 66 | InvalidLabelExceptions inherit from, and are a special case 67 | of InvalidNameExceptions. At present, these restrictions are 68 | identical. 69 | """ 70 | 71 | def __init__(self, msg): 72 | super(InvalidLabelException, self).__init__(msg) 73 | 74 | 75 | class InvalidValueException(CloudBridgeBaseException): 76 | """ 77 | Marker interface for any attempt to set an invalid value on a CloudBridge 78 | resource. An example would be setting an unrecognised value for the 79 | direction of a firewall rule other than TrafficDirection.INBOUND or 80 | TrafficDirection.OUTBOUND. 81 | """ 82 | def __init__(self, param, value): 83 | super(InvalidValueException, self).__init__( 84 | "Param %s has been given an unrecognised value %s" % 85 | (param, value)) 86 | 87 | 88 | class DuplicateResourceException(CloudBridgeBaseException): 89 | """ 90 | Marker interface for any attempt to create a CloudBridge resource that 91 | already exists. For example, creating a KeyPair with the same name will 92 | result in a DuplicateResourceException. 93 | """ 94 | pass 95 | 96 | 97 | class InvalidParamException(InvalidNameException): 98 | """ 99 | Marker interface for an invalid or unexpected parameter, for example, 100 | to a service.find() method. 101 | """ 102 | 103 | def __init__(self, msg): 104 | super(InvalidParamException, self).__init__(msg) 105 | -------------------------------------------------------------------------------- /cloudbridge/providers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/cloudbridge/providers/__init__.py -------------------------------------------------------------------------------- /cloudbridge/providers/aws/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Exports from this provider 3 | """ 4 | 5 | from .provider import AWSCloudProvider # noqa 6 | -------------------------------------------------------------------------------- /cloudbridge/providers/aws/provider.py: -------------------------------------------------------------------------------- 1 | """Provider implementation based on boto library for AWS-compatible clouds.""" 2 | import logging 3 | 4 | import boto3 5 | 6 | from botocore.client import Config 7 | 8 | from cloudbridge.base import BaseCloudProvider 9 | from cloudbridge.base.helpers import get_env 10 | 11 | from .services import AWSComputeService 12 | from .services import AWSDnsService 13 | from .services import AWSNetworkingService 14 | from .services import AWSSecurityService 15 | from .services import AWSStorageService 16 | 17 | 18 | log = logging.getLogger(__name__) 19 | 20 | 21 | class AWSCloudProvider(BaseCloudProvider): 22 | '''AWS cloud provider interface''' 23 | PROVIDER_ID = 'aws' 24 | 25 | def __init__(self, config): 26 | super(AWSCloudProvider, self).__init__(config) 27 | 28 | # Initialize cloud connection fields 29 | # These are passed as-is to Boto 30 | self._region_name = self._get_config_value('aws_region_name', 31 | 'us-east-1') 32 | self._zone_name = self._get_config_value('aws_zone_name') 33 | self.session_cfg = { 34 | 'aws_access_key_id': self._get_config_value( 35 | 'aws_access_key', get_env('AWS_ACCESS_KEY')), 36 | 'aws_secret_access_key': self._get_config_value( 37 | 'aws_secret_key', get_env('AWS_SECRET_KEY')), 38 | 'aws_session_token': self._get_config_value( 39 | 'aws_session_token', None) 40 | } 41 | self.ec2_cfg = { 42 | 'use_ssl': self._get_config_value('ec2_is_secure', True), 43 | 'verify': self._get_config_value('ec2_validate_certs', True), 44 | 'endpoint_url': self._get_config_value('ec2_endpoint_url'), 45 | 'config': Config( 46 | retries={ 47 | 'max_attempts': self._get_config_value('ec2_retries_value', 4), 48 | 'mode': 'standard'}) 49 | } 50 | self.s3_cfg = { 51 | 'use_ssl': self._get_config_value('s3_is_secure', True), 52 | 'verify': self._get_config_value('s3_validate_certs', True), 53 | 'endpoint_url': self._get_config_value('s3_endpoint_url'), 54 | 'config': Config( 55 | signature_version=self._get_config_value( 56 | 's3_signature_version', 's3v4')) 57 | } 58 | 59 | # service connections, lazily initialized 60 | self._session = None 61 | self._ec2_conn = None 62 | self._vpc_conn = None 63 | self._s3_conn = None 64 | 65 | # Initialize provider services 66 | self._compute = AWSComputeService(self) 67 | self._networking = AWSNetworkingService(self) 68 | self._security = AWSSecurityService(self) 69 | self._storage = AWSStorageService(self) 70 | self._dns = AWSDnsService(self) 71 | 72 | @property 73 | def session(self): 74 | '''Get a low-level session object or create one if needed''' 75 | if not self._session: 76 | if self.config.debug_mode: 77 | boto3.set_stream_logger(level=log.DEBUG) 78 | self._session = boto3.session.Session( 79 | region_name=self.region_name, **self.session_cfg) 80 | return self._session 81 | 82 | @property 83 | def ec2_conn(self): 84 | if not self._ec2_conn: 85 | self._ec2_conn = self._connect_ec2() 86 | return self._ec2_conn 87 | 88 | @property 89 | def s3_conn(self): 90 | if not self._s3_conn: 91 | self._s3_conn = self._connect_s3() 92 | return self._s3_conn 93 | 94 | @property 95 | def compute(self): 96 | return self._compute 97 | 98 | @property 99 | def networking(self): 100 | return self._networking 101 | 102 | @property 103 | def security(self): 104 | return self._security 105 | 106 | @property 107 | def storage(self): 108 | return self._storage 109 | 110 | @property 111 | def dns(self): 112 | return self._dns 113 | 114 | def _connect_ec2(self): 115 | """ 116 | Get a boto ec2 connection object. 117 | """ 118 | return self._connect_ec2_region(region_name=self.region_name) 119 | 120 | def _connect_ec2_region(self, region_name=None): 121 | '''Get an EC2 resource object''' 122 | return self.session.resource( 123 | 'ec2', region_name=region_name, **self.ec2_cfg) 124 | 125 | def _connect_s3(self): 126 | '''Get an S3 resource object''' 127 | return self.session.resource( 128 | 's3', region_name=self.region_name, **self.s3_cfg) 129 | -------------------------------------------------------------------------------- /cloudbridge/providers/aws/subservices.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from cloudbridge.base.subservices import BaseBucketObjectSubService 4 | from cloudbridge.base.subservices import BaseDnsRecordSubService 5 | from cloudbridge.base.subservices import BaseFloatingIPSubService 6 | from cloudbridge.base.subservices import BaseGatewaySubService 7 | from cloudbridge.base.subservices import BaseSubnetSubService 8 | from cloudbridge.base.subservices import BaseVMFirewallRuleSubService 9 | 10 | log = logging.getLogger(__name__) 11 | 12 | 13 | class AWSBucketObjectSubService(BaseBucketObjectSubService): 14 | 15 | def __init__(self, provider, bucket): 16 | super(AWSBucketObjectSubService, self).__init__(provider, bucket) 17 | 18 | 19 | class AWSGatewaySubService(BaseGatewaySubService): 20 | 21 | def __init__(self, provider, network): 22 | super(AWSGatewaySubService, self).__init__(provider, network) 23 | 24 | 25 | class AWSVMFirewallRuleSubService(BaseVMFirewallRuleSubService): 26 | 27 | def __init__(self, provider, firewall): 28 | super(AWSVMFirewallRuleSubService, self).__init__(provider, firewall) 29 | 30 | 31 | class AWSFloatingIPSubService(BaseFloatingIPSubService): 32 | 33 | def __init__(self, provider, gateway): 34 | super(AWSFloatingIPSubService, self).__init__(provider, gateway) 35 | 36 | 37 | class AWSSubnetSubService(BaseSubnetSubService): 38 | 39 | def __init__(self, provider, network): 40 | super(AWSSubnetSubService, self).__init__(provider, network) 41 | 42 | 43 | class AWSDnsRecordSubService(BaseDnsRecordSubService): 44 | 45 | def __init__(self, provider, dns_zone): 46 | super(AWSDnsRecordSubService, self).__init__(provider, dns_zone) 47 | -------------------------------------------------------------------------------- /cloudbridge/providers/azure/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Exports from this provider 3 | """ 4 | 5 | from .provider import AzureCloudProvider # noqa 6 | -------------------------------------------------------------------------------- /cloudbridge/providers/azure/helpers.py: -------------------------------------------------------------------------------- 1 | from cloudbridge.interfaces.exceptions import InvalidValueException 2 | 3 | 4 | # def filter_by_tag(list_items, filters): 5 | # """ 6 | # This function filter items on the tags 7 | # :param list_items: 8 | # :param filters: 9 | # :return: 10 | # """ 11 | # filtered_list = [] 12 | # if filters: 13 | # for obj in list_items: 14 | # for key in filters: 15 | # if obj.tags and filters[key] in obj.tags.get(key, ''): 16 | # filtered_list.append(obj) 17 | # 18 | # return filtered_list 19 | # else: 20 | # return list_items 21 | 22 | 23 | def parse_url(template_urls, original_url): 24 | """ 25 | In Azure all the resource IDs are returned as URIs. 26 | ex: '/subscriptions/{subscriptionId}/resourceGroups/' \ 27 | '{resourceGroupName}/providers/Microsoft.Compute/' \ 28 | 'virtualMachines/{vmName}' 29 | This function splits the resource ID based on the template urls passed 30 | and returning the dictionary. 31 | 32 | The only exception to that format are image URN's which are used for 33 | public gallery references: 34 | https://docs.microsoft.com/en-us/azure/virtual-machines/linux/cli-ps-findimage 35 | """ 36 | if not original_url: 37 | raise InvalidValueException(template_urls, original_url) 38 | original_url_parts = original_url.split('/') 39 | if len(original_url_parts) == 1: 40 | original_url_parts = original_url.split(':') 41 | for each_template in template_urls: 42 | template_url_parts = each_template.split('/') 43 | if len(template_url_parts) == 1: 44 | template_url_parts = each_template.split(':') 45 | if len(template_url_parts) == len(original_url_parts): 46 | break 47 | if len(template_url_parts) != len(original_url_parts): 48 | raise InvalidValueException(template_urls, original_url) 49 | resource_param = {} 50 | for key, value in zip(template_url_parts, original_url_parts): 51 | if key.startswith('{') and key.endswith('}'): 52 | resource_param.update({key[1:-1]: value}) 53 | return resource_param 54 | 55 | 56 | def generate_urn(gallery_image): 57 | """ 58 | This function takes an azure gallery image and outputs a corresponding URN 59 | :param gallery_image: a GalleryImageReference object 60 | :return: URN as string 61 | """ 62 | reference_dict = gallery_image.as_dict() 63 | return ':'.join([reference_dict['publisher'], 64 | reference_dict['offer'], 65 | reference_dict['sku'], 66 | reference_dict['version']]) 67 | -------------------------------------------------------------------------------- /cloudbridge/providers/azure/provider.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import uuid 3 | 4 | from deprecation import deprecated 5 | 6 | from msrestazure.azure_exceptions import CloudError 7 | 8 | import tenacity 9 | 10 | import cloudbridge 11 | from cloudbridge.base import BaseCloudProvider 12 | from cloudbridge.base.helpers import get_env 13 | from cloudbridge.interfaces.exceptions import ProviderConnectionException 14 | from cloudbridge.providers.azure.azure_client import AzureClient 15 | 16 | from .services import AzureComputeService 17 | from .services import AzureNetworkingService 18 | from .services import AzureSecurityService 19 | from .services import AzureStorageService 20 | 21 | log = logging.getLogger(__name__) 22 | 23 | 24 | class AzureCloudProvider(BaseCloudProvider): 25 | PROVIDER_ID = 'azure' 26 | 27 | def __init__(self, config): 28 | super(AzureCloudProvider, self).__init__(config) 29 | 30 | # mandatory config values 31 | self.subscription_id = self._get_config_value( 32 | 'azure_subscription_id', get_env('AZURE_SUBSCRIPTION_ID')) 33 | self.client_id = self._get_config_value( 34 | 'azure_client_id', get_env('AZURE_CLIENT_ID')) 35 | self.secret = self._get_config_value( 36 | 'azure_secret', get_env('AZURE_SECRET')) 37 | self.tenant = self._get_config_value( 38 | 'azure_tenant', get_env('AZURE_TENANT')) 39 | 40 | # optional config values 41 | self.access_token = self._get_config_value( 42 | 'azure_access_token', get_env('AZURE_ACCESS_TOKEN')) 43 | self._region_name = self._get_config_value( 44 | 'azure_region_name', get_env('AZURE_REGION_NAME', 'eastus')) 45 | self._zone_name = self._get_config_value( 46 | 'azure_zone_name', get_env('AZURE_ZONE_NAME')) 47 | self.resource_group = self._get_config_value( 48 | 'azure_resource_group', get_env('AZURE_RESOURCE_GROUP', 49 | 'cloudbridge')) 50 | self.networking_resource_group = self._get_config_value( 51 | 'azure_networking_resource_group', get_env('AZURE_NETWORKING_RESOURCE_GROUP', 52 | self.resource_group)) 53 | # Storage account name is limited to a max length of 24 alphanum chars 54 | # and unique across all of Azure. Thus, a uuid is used to generate a 55 | # unique name for the Storage Account based on the resource group, 56 | # while also using the subscription ID to ensure that different users 57 | # having the same resource group name do not have the same SA name. 58 | self.storage_account = self._get_config_value( 59 | 'azure_storage_account', 60 | get_env( 61 | 'AZURE_STORAGE_ACCOUNT', 62 | 'storacc' + self.subscription_id[-6:] + 63 | str(uuid.uuid5(uuid.NAMESPACE_OID, 64 | str(self.resource_group)))[-6:])) 65 | 66 | self.vm_default_user_name = self._get_config_value( 67 | 'azure_vm_default_username', get_env( 68 | 'AZURE_VM_DEFAULT_USERNAME')) \ 69 | or self.__get_deprecated_username('cbuser') 70 | 71 | self.public_key_storage_table_name = self._get_config_value( 72 | 'azure_public_key_storage_table_name', get_env( 73 | 'AZURE_PUBLIC_KEY_STORAGE_TABLE_NAME', 'cbcerts')) 74 | 75 | self._azure_client = None 76 | 77 | self._security = AzureSecurityService(self) 78 | self._storage = AzureStorageService(self) 79 | self._compute = AzureComputeService(self) 80 | self._networking = AzureNetworkingService(self) 81 | 82 | def __get_deprecated_username(self, default): 83 | username = self._get_config_value( 84 | 'azure_vm_default_user_name', get_env( 85 | 'AZURE_VM_DEFAULT_USER_NAME', None)) 86 | if username: 87 | return self.__wrap_deprecated_username(username) 88 | else: 89 | return default 90 | 91 | @deprecated(deprecated_in='1.1', 92 | removed_in='2.0', 93 | current_version=cloudbridge.__version__, 94 | details='AZURE_VM_DEFAULT_USER_NAME was deprecated in favor ' 95 | 'of AZURE_VM_DEFAULT_USERNAME') 96 | def __wrap_deprecated_username(self, username): 97 | return username 98 | 99 | @property 100 | def compute(self): 101 | return self._compute 102 | 103 | @property 104 | def networking(self): 105 | return self._networking 106 | 107 | @property 108 | def security(self): 109 | return self._security 110 | 111 | @property 112 | def storage(self): 113 | return self._storage 114 | 115 | @property 116 | def dns(self): 117 | raise NotImplementedError() 118 | 119 | @property 120 | def azure_client(self): 121 | if not self._azure_client: 122 | 123 | # create a dict with both optional and mandatory configuration 124 | # values to pass to the azureclient class, rather 125 | # than passing the provider object and taking a dependency. 126 | 127 | provider_config = { 128 | 'azure_subscription_id': self.subscription_id, 129 | 'azure_client_id': self.client_id, 130 | 'azure_secret': self.secret, 131 | 'azure_tenant': self.tenant, 132 | 'azure_region_name': self.region_name, 133 | 'azure_resource_group': self.resource_group, 134 | 'azure_networking_resource_group': self.networking_resource_group, 135 | 'azure_storage_account': self.storage_account, 136 | 'azure_public_key_storage_table_name': 137 | self.public_key_storage_table_name, 138 | 'azure_access_token': self.access_token 139 | } 140 | 141 | self._azure_client = AzureClient(provider_config) 142 | self._initialize() 143 | return self._azure_client 144 | 145 | @tenacity.retry(stop=tenacity.stop_after_attempt(2), 146 | retry=tenacity.retry_if_exception_type(CloudError), 147 | reraise=True) 148 | def _initialize(self): 149 | """ 150 | Verifying that resource group and storage account exists 151 | if not create one with the name provided in the 152 | configuration 153 | """ 154 | try: 155 | self._azure_client.get_resource_group(self.resource_group) 156 | 157 | except CloudError as cloud_error: 158 | if cloud_error.error.error == "ResourceGroupNotFound": 159 | resource_group_params = {'location': self.region_name} 160 | try: 161 | self._azure_client.\ 162 | create_resource_group(self.resource_group, 163 | resource_group_params) 164 | except CloudError as cloud_error2: # pragma: no cover 165 | if cloud_error2.error.error == "AuthorizationFailed": 166 | mess = 'The following error was returned by Azure:\n' \ 167 | '%s\n\nThis is likely because the Role' \ 168 | 'associated with the given credentials does ' \ 169 | 'not allow for Resource Group creation.\nA ' \ 170 | 'Resource Group is necessary to manage ' \ 171 | 'resources in Azure. You must either ' \ 172 | 'provide an existing Resource Group as part ' \ 173 | 'of the configuration, or elevate the ' \ 174 | 'associated role.\nFor more information on ' \ 175 | 'roles, see: https://docs.microsoft.com/' \ 176 | 'en-us/azure/role-based-access-control/' \ 177 | 'overview\n' % cloud_error2 178 | raise ProviderConnectionException(mess) 179 | else: 180 | raise cloud_error2 181 | 182 | else: 183 | raise cloud_error 184 | 185 | """ 186 | Verify that resource group used for network exists, 187 | if not, use the self.resource_group 188 | """ 189 | try: 190 | self._azure_client.get_resource_group(self.networking_resource_group) 191 | except CloudError as cloud_error: 192 | if cloud_error.error.error == "ResourceGroupNotFound": 193 | self.networking_resource_group = self.resource_group 194 | else: 195 | raise cloud_error 196 | -------------------------------------------------------------------------------- /cloudbridge/providers/azure/subservices.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from cloudbridge.base.subservices import BaseBucketObjectSubService 4 | from cloudbridge.base.subservices import BaseFloatingIPSubService 5 | from cloudbridge.base.subservices import BaseGatewaySubService 6 | from cloudbridge.base.subservices import BaseSubnetSubService 7 | from cloudbridge.base.subservices import BaseVMFirewallRuleSubService 8 | 9 | log = logging.getLogger(__name__) 10 | 11 | 12 | class AzureBucketObjectSubService(BaseBucketObjectSubService): 13 | 14 | def __init__(self, provider, bucket): 15 | super(AzureBucketObjectSubService, self).__init__(provider, bucket) 16 | 17 | 18 | class AzureGatewaySubService(BaseGatewaySubService): 19 | def __init__(self, provider, network): 20 | super(AzureGatewaySubService, self).__init__(provider, network) 21 | 22 | 23 | class AzureVMFirewallRuleSubService(BaseVMFirewallRuleSubService): 24 | 25 | def __init__(self, provider, firewall): 26 | super(AzureVMFirewallRuleSubService, self).__init__(provider, firewall) 27 | 28 | 29 | class AzureFloatingIPSubService(BaseFloatingIPSubService): 30 | 31 | def __init__(self, provider, gateway): 32 | super(AzureFloatingIPSubService, self).__init__(provider, gateway) 33 | 34 | 35 | class AzureSubnetSubService(BaseSubnetSubService): 36 | 37 | def __init__(self, provider, network): 38 | super(AzureSubnetSubService, self).__init__(provider, network) 39 | -------------------------------------------------------------------------------- /cloudbridge/providers/gcp/README.rst: -------------------------------------------------------------------------------- 1 | CloudBridge support for `Google Cloud Platform`_. Compute is provided by `Google 2 | Compute Engine`_ (GCE). Object storage is provided by `Google Cloud Storage`_ 3 | (GCS). 4 | 5 | Security Groups 6 | ~~~~~~~~~~~~~~~ 7 | CloudBridge API lets you control incoming traffic to VM instances by creating 8 | VM firewalls, adding rules to VM firewalls, and then assigning instances to VM 9 | firewalls. 10 | 11 | GCP does this a little bit differently. GCP lets you assign `tags`_ to VM 12 | instances. Tags, then, can be used for networking purposes. In particular, you 13 | can create `firewall rules`_ to control incoming traffic to instances having a 14 | specific tag. So, to add GCP support to CloudBridge, we simulate VM firewalls by 15 | tags. 16 | 17 | To make this more clear, let us consider the example of adding a rule to a 18 | VM firewall. When you add a VM firewall rule from the CloudBridge API to a VM 19 | firewall ``vmf``, what really happens is that a firewall with one rule is 20 | created whose ``targetTags`` is ``[vmf]``. This makes sure that the rule 21 | applies to all instances that have ``vmf`` as a tag (in CloudBridge language 22 | instances belonging to the VM firewall ``vmf``). 23 | 24 | **Note**: This implementation does not take advantage of the full power of GCP 25 | firewall format and only creates firewalls with one rule and only can find or 26 | list firewalls with one rule. This should be OK as long as all firewalls are 27 | created through the CloudBridge API. 28 | 29 | .. _`Google Cloud Platform`: https://cloud.google.com/ 30 | .. _`Google Compute Engine`: https://cloud.google.com/compute/docs 31 | .. _`Google Cloud Storage`: https://cloud.google.com/storage/docs 32 | .. _`tags`: https://cloud.google.com/compute/docs/reference/latest/instances/ 33 | setTags 34 | .. _`firewall rules`: https://cloud.google.com/compute/docs/ 35 | networking#firewall_rules 36 | -------------------------------------------------------------------------------- /cloudbridge/providers/gcp/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Exports from this provider 3 | """ 4 | 5 | from .provider import GCPCloudProvider # noqa 6 | -------------------------------------------------------------------------------- /cloudbridge/providers/gcp/subservices.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from cloudbridge.base.subservices import BaseBucketObjectSubService 4 | from cloudbridge.base.subservices import BaseDnsRecordSubService 5 | from cloudbridge.base.subservices import BaseFloatingIPSubService 6 | from cloudbridge.base.subservices import BaseGatewaySubService 7 | from cloudbridge.base.subservices import BaseSubnetSubService 8 | from cloudbridge.base.subservices import BaseVMFirewallRuleSubService 9 | 10 | 11 | log = logging.getLogger(__name__) 12 | 13 | 14 | class GCPBucketObjectSubService(BaseBucketObjectSubService): 15 | 16 | def __init__(self, provider, bucket): 17 | super(GCPBucketObjectSubService, self).__init__(provider, bucket) 18 | 19 | 20 | class GCPGatewaySubService(BaseGatewaySubService): 21 | def __init__(self, provider, network): 22 | super(GCPGatewaySubService, self).__init__(provider, network) 23 | 24 | 25 | class GCPVMFirewallRuleSubService(BaseVMFirewallRuleSubService): 26 | 27 | def __init__(self, provider, firewall): 28 | super(GCPVMFirewallRuleSubService, self).__init__(provider, firewall) 29 | 30 | 31 | class GCPFloatingIPSubService(BaseFloatingIPSubService): 32 | 33 | def __init__(self, provider, gateway): 34 | super(GCPFloatingIPSubService, self).__init__(provider, gateway) 35 | 36 | 37 | class GCPSubnetSubService(BaseSubnetSubService): 38 | 39 | def __init__(self, provider, network): 40 | super(GCPSubnetSubService, self).__init__(provider, network) 41 | 42 | 43 | class GCPDnsRecordSubService(BaseDnsRecordSubService): 44 | 45 | def __init__(self, provider, dns_zone): 46 | super(GCPDnsRecordSubService, self).__init__(provider, dns_zone) 47 | -------------------------------------------------------------------------------- /cloudbridge/providers/mock/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Exports from this provider 3 | """ 4 | 5 | from .provider import MockAWSCloudProvider # noqa 6 | -------------------------------------------------------------------------------- /cloudbridge/providers/mock/provider.py: -------------------------------------------------------------------------------- 1 | """ 2 | Provider implementation based on the moto library (mock boto). This mock 3 | provider is useful for running tests against cloudbridge but should not 4 | be used in tandem with other providers, in particular the AWS provider. 5 | This is because instantiating this provider will result in all calls to 6 | boto being hijacked, which will cause AWS to malfunction. 7 | See notes below. 8 | """ 9 | from moto import mock_ec2 10 | from moto import mock_route53 11 | from moto import mock_s3 12 | 13 | from ..aws import AWSCloudProvider 14 | from ...interfaces.provider import TestMockHelperMixin 15 | 16 | 17 | class MockAWSCloudProvider(AWSCloudProvider, TestMockHelperMixin): 18 | """ 19 | Using this mock driver will result in all boto communications being 20 | hijacked. As a result, this mock driver and the AWS driver cannot be used 21 | at the same time. Do not instantiate the mock driver if you plan to use 22 | the AWS provider within the same python process. Alternatively, call 23 | provider.tearDownMock() to stop the hijacking. 24 | """ 25 | PROVIDER_ID = 'mock' 26 | 27 | def __init__(self, config): 28 | self.setUpMock() 29 | super(MockAWSCloudProvider, self).__init__(config) 30 | 31 | def setUpMock(self): 32 | """ 33 | Let Moto take over all socket communications 34 | """ 35 | self.ec2mock = mock_ec2() 36 | self.ec2mock.start() 37 | self.s3mock = mock_s3() 38 | self.s3mock.start() 39 | self.route53mock = mock_route53() 40 | self.route53mock.start() 41 | 42 | def tearDownMock(self): 43 | """ 44 | Stop Moto intercepting all socket communications 45 | """ 46 | self.s3mock.stop() 47 | self.ec2mock.stop() 48 | self.route53mock.stop() 49 | -------------------------------------------------------------------------------- /cloudbridge/providers/openstack/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Exports from this provider 3 | """ 4 | 5 | from .provider import OpenStackCloudProvider # noqa 6 | -------------------------------------------------------------------------------- /cloudbridge/providers/openstack/helpers.py: -------------------------------------------------------------------------------- 1 | """ 2 | Helper functions 3 | """ 4 | import itertools 5 | import logging as log 6 | 7 | from cloudbridge.base.resources import ServerPagedResultList 8 | 9 | 10 | def os_result_limit(provider, requested_limit=None): 11 | """ 12 | Calculates the limit for OpenStack. 13 | """ 14 | limit = requested_limit or provider.config.default_result_limit 15 | # fetch one more than the limit to help with paging. 16 | # i.e. if length(objects) is one more than the limit, 17 | # we know that the object has another page of results, 18 | # so we always request one extra record. 19 | log.debug("Limit of OpenStack: %s Requested Limit: %s", 20 | limit, requested_limit) 21 | return limit + 1 22 | 23 | 24 | def to_server_paged_list(provider, objects, limit=None): 25 | """ 26 | A convenience function for wrapping a list of OpenStack native objects in 27 | a ServerPagedResultList. OpenStack 28 | initial list of objects. Thereafter, the return list is wrapped in a 29 | BaseResultList, enabling extra properties like 30 | `is_truncated` and `marker` to be accessed. 31 | """ 32 | limit = limit or provider.config.default_result_limit 33 | is_truncated = len(objects) > limit 34 | next_token = objects[limit-1].id if is_truncated else None 35 | results = ServerPagedResultList(is_truncated, 36 | next_token, 37 | False) 38 | for obj in itertools.islice(objects, limit): 39 | results.append(obj) 40 | return results 41 | -------------------------------------------------------------------------------- /cloudbridge/providers/openstack/subservices.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from cloudbridge.base.subservices import BaseBucketObjectSubService 4 | from cloudbridge.base.subservices import BaseDnsRecordSubService 5 | from cloudbridge.base.subservices import BaseFloatingIPSubService 6 | from cloudbridge.base.subservices import BaseGatewaySubService 7 | from cloudbridge.base.subservices import BaseSubnetSubService 8 | from cloudbridge.base.subservices import BaseVMFirewallRuleSubService 9 | 10 | 11 | log = logging.getLogger(__name__) 12 | 13 | 14 | class OpenStackBucketObjectSubService(BaseBucketObjectSubService): 15 | 16 | def __init__(self, provider, bucket): 17 | super(OpenStackBucketObjectSubService, self).__init__(provider, bucket) 18 | 19 | 20 | class OpenStackGatewaySubService(BaseGatewaySubService): 21 | 22 | def __init__(self, provider, network): 23 | super(OpenStackGatewaySubService, self).__init__(provider, network) 24 | 25 | 26 | class OpenStackFloatingIPSubService(BaseFloatingIPSubService): 27 | 28 | def __init__(self, provider, gateway): 29 | super(OpenStackFloatingIPSubService, self).__init__(provider, gateway) 30 | 31 | 32 | class OpenStackVMFirewallRuleSubService(BaseVMFirewallRuleSubService): 33 | 34 | def __init__(self, provider, firewall): 35 | super(OpenStackVMFirewallRuleSubService, self).__init__( 36 | provider, firewall) 37 | 38 | 39 | class OpenStackSubnetSubService(BaseSubnetSubService): 40 | 41 | def __init__(self, provider, network): 42 | super(OpenStackSubnetSubService, self).__init__(provider, network) 43 | 44 | 45 | class OpenStackDnsRecordSubService(BaseDnsRecordSubService): 46 | 47 | def __init__(self, provider, dns_zone): 48 | super(OpenStackDnsRecordSubService, self).__init__(provider, dns_zone) 49 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | _build 2 | _static 3 | _templates 4 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/api_docs/cloud/exceptions.rst: -------------------------------------------------------------------------------- 1 | Exceptions 2 | ========== 3 | 4 | .. contents:: :local: 5 | 6 | CloudBridgeBaseException 7 | ------------------------ 8 | .. autoclass:: cloudbridge.interfaces.exceptions.CloudBridgeBaseException 9 | :members: 10 | 11 | WaitStateException 12 | ------------------ 13 | .. autoclass:: cloudbridge.interfaces.exceptions.WaitStateException 14 | :members: 15 | 16 | InvalidConfigurationException 17 | ----------------------------- 18 | .. autoclass:: cloudbridge.interfaces.exceptions.InvalidConfigurationException 19 | :members: 20 | 21 | ProviderConnectionException 22 | ----------------------------- 23 | .. autoclass:: cloudbridge.interfaces.exceptions.ProviderConnectionException 24 | :members: 25 | 26 | InvalidLabelException 27 | ----------------------------- 28 | .. autoclass:: cloudbridge.interfaces.exceptions.InvalidLabelException 29 | :members: 30 | 31 | InvalidValueException 32 | ----------------------------- 33 | .. autoclass:: cloudbridge.interfaces.exceptions.InvalidValueException 34 | :members: 35 | -------------------------------------------------------------------------------- /docs/api_docs/cloud/providers.rst: -------------------------------------------------------------------------------- 1 | Providers 2 | ========= 3 | 4 | CloudProvider 5 | ------------- 6 | .. autoclass:: cloudbridge.interfaces.provider.CloudProvider 7 | :members: 8 | :special-members: __init__ 9 | -------------------------------------------------------------------------------- /docs/api_docs/cloud/resources.rst: -------------------------------------------------------------------------------- 1 | Resources 2 | ========= 3 | 4 | .. contents:: :local: 5 | 6 | CloudServiceType 7 | ------------------------ 8 | .. autoclass:: cloudbridge.interfaces.resources.CloudServiceType 9 | :members: 10 | 11 | CloudResource 12 | ------------- 13 | .. autoclass:: cloudbridge.interfaces.resources.CloudResource 14 | :members: 15 | 16 | Configuration 17 | ------------- 18 | .. autoclass:: cloudbridge.interfaces.resources.Configuration 19 | :members: 20 | 21 | ObjectLifeCycleMixin 22 | -------------------- 23 | .. autoclass:: cloudbridge.interfaces.resources.ObjectLifeCycleMixin 24 | :members: 25 | 26 | PageableObjectMixin 27 | -------------------- 28 | .. autoclass:: cloudbridge.interfaces.resources.PageableObjectMixin 29 | :members: 30 | 31 | ResultList 32 | ---------- 33 | .. autoclass:: cloudbridge.interfaces.resources.ResultList 34 | :members: 35 | 36 | InstanceState 37 | ------------- 38 | .. autoclass:: cloudbridge.interfaces.resources.InstanceState 39 | :members: 40 | 41 | Instance 42 | -------- 43 | .. autoclass:: cloudbridge.interfaces.resources.Instance 44 | :members: 45 | 46 | MachineImageState 47 | ----------------- 48 | .. autoclass:: cloudbridge.interfaces.resources.MachineImageState 49 | :members: 50 | 51 | LaunchConfig 52 | ------------ 53 | .. autoclass:: cloudbridge.interfaces.resources.LaunchConfig 54 | :members: 55 | 56 | MachineImage 57 | ------------ 58 | .. autoclass:: cloudbridge.interfaces.resources.MachineImage 59 | :members: 60 | 61 | NetworkState 62 | ------------ 63 | .. autoclass:: cloudbridge.interfaces.resources.NetworkState 64 | :members: 65 | 66 | Network 67 | ------- 68 | .. autoclass:: cloudbridge.interfaces.resources.Network 69 | :members: 70 | 71 | SubnetState 72 | ------------ 73 | .. autoclass:: cloudbridge.interfaces.resources.SubnetState 74 | :members: 75 | 76 | Subnet 77 | ------ 78 | .. autoclass:: cloudbridge.interfaces.resources.Subnet 79 | :members: 80 | 81 | FloatingIP 82 | ---------- 83 | .. autoclass:: cloudbridge.interfaces.resources.FloatingIP 84 | :members: 85 | 86 | RouterState 87 | ------------ 88 | .. autoclass:: cloudbridge.interfaces.resources.RouterState 89 | :members: 90 | 91 | Router 92 | ------ 93 | .. autoclass:: cloudbridge.interfaces.resources.Router 94 | :members: 95 | 96 | Gateway 97 | -------- 98 | .. autoclass:: cloudbridge.interfaces.resources.Gateway 99 | :members: 100 | 101 | InternetGateway 102 | --------------- 103 | .. autoclass:: cloudbridge.interfaces.resources.InternetGateway 104 | :members: 105 | 106 | VolumeState 107 | ----------- 108 | .. autoclass:: cloudbridge.interfaces.resources.VolumeState 109 | :members: 110 | 111 | Volume 112 | ------ 113 | .. autoclass:: cloudbridge.interfaces.resources.Volume 114 | :members: 115 | 116 | SnapshotState 117 | ------------- 118 | .. autoclass:: cloudbridge.interfaces.resources.SnapshotState 119 | :members: 120 | 121 | Snapshot 122 | -------- 123 | .. autoclass:: cloudbridge.interfaces.resources.Snapshot 124 | :members: 125 | 126 | KeyPair 127 | ------- 128 | .. autoclass:: cloudbridge.interfaces.resources.KeyPair 129 | :members: 130 | 131 | Region 132 | ------ 133 | .. autoclass:: cloudbridge.interfaces.resources.Region 134 | :members: 135 | 136 | PlacementZone 137 | ------------- 138 | .. autoclass:: cloudbridge.interfaces.resources.PlacementZone 139 | :members: 140 | 141 | VMType 142 | ------------ 143 | .. autoclass:: cloudbridge.interfaces.resources.VMType 144 | :members: 145 | 146 | VMFirewall 147 | ------------- 148 | .. autoclass:: cloudbridge.interfaces.resources.VMFirewall 149 | :members: 150 | 151 | VMFirewallRule 152 | ----------------- 153 | .. autoclass:: cloudbridge.interfaces.resources.VMFirewallRule 154 | :members: 155 | :undoc-members: 156 | 157 | TrafficDirection 158 | ----------------- 159 | .. autoclass:: cloudbridge.interfaces.resources.TrafficDirection 160 | :members: 161 | 162 | BucketObject 163 | --------------- 164 | .. autoclass:: cloudbridge.interfaces.resources.BucketObject 165 | :members: 166 | 167 | Bucket 168 | --------- 169 | .. autoclass:: cloudbridge.interfaces.resources.Bucket 170 | :members: 171 | 172 | DnsZone 173 | --------- 174 | .. autoclass:: cloudbridge.interfaces.resources.DnsZone 175 | :members: 176 | 177 | 178 | DnsRecord 179 | --------- 180 | .. autoclass:: cloudbridge.interfaces.resources.DnsRecord 181 | :members: 182 | -------------------------------------------------------------------------------- /docs/api_docs/cloud/services.rst: -------------------------------------------------------------------------------- 1 | Services 2 | ======== 3 | 4 | .. contents:: :local: 5 | 6 | CloudService 7 | ------------ 8 | .. autoclass:: cloudbridge.interfaces.services.CloudService 9 | :members: 10 | 11 | ComputeService 12 | -------------- 13 | .. autoclass:: cloudbridge.interfaces.services.ComputeService 14 | :members: 15 | 16 | InstanceService 17 | --------------- 18 | .. autoclass:: cloudbridge.interfaces.services.InstanceService 19 | :members: 20 | 21 | VolumeService 22 | ------------- 23 | .. autoclass:: cloudbridge.interfaces.services.VolumeService 24 | :members: 25 | 26 | SnapshotService 27 | --------------- 28 | .. autoclass:: cloudbridge.interfaces.services.SnapshotService 29 | :members: 30 | 31 | StorageService 32 | -------------- 33 | .. autoclass:: cloudbridge.interfaces.services.StorageService 34 | :members: 35 | 36 | ImageService 37 | ------------ 38 | .. autoclass:: cloudbridge.interfaces.services.ImageService 39 | :members: 40 | 41 | NetworkingService 42 | ----------------- 43 | .. autoclass:: cloudbridge.interfaces.services.NetworkingService 44 | :members: 45 | 46 | NetworkService 47 | -------------- 48 | .. autoclass:: cloudbridge.interfaces.services.NetworkService 49 | :members: 50 | 51 | SubnetService 52 | ------------- 53 | .. autoclass:: cloudbridge.interfaces.services.SubnetService 54 | :members: 55 | 56 | FloatingIPService 57 | ----------------- 58 | .. autoclass:: cloudbridge.interfaces.subservices.FloatingIPSubService 59 | :members: 60 | 61 | RouterService 62 | ------------- 63 | .. autoclass:: cloudbridge.interfaces.services.RouterService 64 | :members: 65 | 66 | GatewayService 67 | -------------- 68 | .. autoclass:: cloudbridge.interfaces.subservices.GatewaySubService 69 | :members: 70 | 71 | BucketService 72 | ------------- 73 | .. autoclass:: cloudbridge.interfaces.services.BucketService 74 | :members: 75 | 76 | SecurityService 77 | --------------- 78 | .. autoclass:: cloudbridge.interfaces.services.SecurityService 79 | :members: 80 | 81 | KeyPairService 82 | -------------- 83 | .. autoclass:: cloudbridge.interfaces.services.KeyPairService 84 | :members: 85 | 86 | VMFirewallService 87 | ----------------- 88 | .. autoclass:: cloudbridge.interfaces.services.VMFirewallService 89 | :members: 90 | 91 | VMTypeService 92 | -------------------- 93 | .. autoclass:: cloudbridge.interfaces.services.VMTypeService 94 | :members: 95 | 96 | RegionService 97 | ------------- 98 | .. autoclass:: cloudbridge.interfaces.services.RegionService 99 | :members: 100 | 101 | DnsService 102 | ---------- 103 | .. autoclass:: cloudbridge.interfaces.services.DnsService 104 | :members: 105 | 106 | DnsZoneService 107 | -------------- 108 | .. autoclass:: cloudbridge.interfaces.services.DnsZoneService 109 | :members: 110 | 111 | DnsRecordService 112 | ---------------- 113 | .. autoclass:: cloudbridge.interfaces.services.DnsRecordService 114 | :members: -------------------------------------------------------------------------------- /docs/api_docs/ref.rst: -------------------------------------------------------------------------------- 1 | API reference 2 | ============= 3 | 4 | This section includes the API documentation for the reference interface. 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | :glob: 9 | 10 | cloud/providers.rst 11 | cloud/services.rst 12 | cloud/resources.rst 13 | cloud/exceptions.rst 14 | -------------------------------------------------------------------------------- /docs/concepts.rst: -------------------------------------------------------------------------------- 1 | Concepts and Organisation 2 | ========================= 3 | 4 | Object types 5 | ------------ 6 | 7 | Conceptually, CloudBridge consists of the following types of objects. 8 | 9 | 1. Providers - Represents a connection to a cloud provider, and is 10 | the gateway to using its services. 11 | 12 | 2. Services - Represents a service provided by a cloud provider, 13 | such as its compute service, storage service, networking service etc. 14 | Services may in turn be divided into smaller services. Smaller services 15 | tend to have uniform methods, such as create, find and list. For example, 16 | InstanceService.list(), InstanceService.find() etc. which can be used 17 | to access cloud resources. Larger services tend to provide organisational 18 | structure only. For example, the storage service provides access to 19 | the VolumeService, SnapshotService and BucketService. 20 | 21 | 3. Resources - resources are objects returned by a service, 22 | and represent a remote resource. For example, InstanceService.list() 23 | will return a list of Instance objects, which can be used to manipulate 24 | an instance. Similarly, VolumeService.create() will return a Volume object. 25 | 26 | 27 | .. image:: images/object_relationships_overview.svg 28 | 29 | The actual source code structure of CloudBridge also mirrors this organisation. 30 | 31 | Object identification and naming 32 | --------------------------------- 33 | 34 | In order to function uniformly across cloud providers, object identity 35 | and naming must be conceptually consistent. In CloudBridge, there are three 36 | main properties for identifying and naming an object. 37 | 38 | 1.Id - The `id` corresponds to a unique identifier that can be reliably used to 39 | reference a resource. All CloudBridge resources have an id. Most methods in 40 | CloudBridge services, such as `get`, use the `id` property to identify and 41 | retrieve objects. 42 | 43 | 2. Name - The `name` property is a more human-readable identifier for 44 | a particular resource, and is often useful to display to the end user instead 45 | of the `id`. While it is often unique, it is not guaranteed to be so, and 46 | therefore, the `id` property must always be used for uniquely identifying 47 | objects. All CloudBridge resources have a `name` property. The `name` property 48 | is often assigned during resource creation, and is often derived from the 49 | `label` property by appending some unique characters to it. Once assigned 50 | however, it is unchangeable. 51 | 52 | 3. Label - Most resources also support a `label` property, which is a user 53 | changeable value that can be used to describe an object. When creating 54 | resources, cloudbridge often accepts a `label` property as a parameter. 55 | The `name` property is derived from the `label`, by appending some unique 56 | characters to it. However, there are some resources which do not support a 57 | `label` property, such as key pairs and buckets. In the latter case, the 58 | `name` can be specified during resource creation, but cannot be changed 59 | thereafter. 60 | 61 | 62 | Detailed class relationships 63 | ---------------------------- 64 | 65 | The following diagram shows a typical provider object graph and the relationship 66 | between services. 67 | 68 | .. raw:: html 69 | 70 | 71 | 72 | Some services are nested. For example, to access the instance service, you can 73 | use `provider.compute.instances`. Similarly, to get a list of all instances, 74 | you can use the following code. 75 | 76 | .. code-block:: python 77 | 78 | instances = provider.compute.instances.list() 79 | print(instances[0].name) -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | import sys 15 | 16 | sys.path.insert(0, os.path.abspath('../')) 17 | 18 | import sphinx_rtd_theme 19 | import cloudbridge 20 | 21 | # -- Project information ----------------------------------------------------- 22 | 23 | project = 'cloudbridge' 24 | copyright = '2021, GVL and Galaxy Projects' 25 | author = 'GVL and Galaxy Projects' 26 | 27 | # The full version, including alpha/beta/rc tags 28 | release = cloudbridge.get_version() 29 | 30 | 31 | # -- General configuration --------------------------------------------------- 32 | 33 | # Add any Sphinx extension module names here, as strings. They can be 34 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 35 | # ones. 36 | extensions = [ 37 | 'sphinx.ext.autodoc', 38 | 'sphinx.ext.doctest', 39 | 'sphinx.ext.todo', 40 | 'sphinx.ext.coverage', 41 | 'sphinx.ext.viewcode', 42 | ] 43 | 44 | # Add any paths that contain templates here, relative to this directory. 45 | templates_path = ['_templates'] 46 | 47 | # List of patterns, relative to source directory, that match files and 48 | # directories to ignore when looking for source files. 49 | # This pattern also affects html_static_path and html_extra_path. 50 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 51 | 52 | 53 | # -- Options for HTML output ------------------------------------------------- 54 | 55 | # The theme to use for HTML and HTML Help pages. See the documentation for 56 | # a list of builtin themes. 57 | # 58 | html_theme = 'sphinx_rtd_theme' 59 | 60 | # Add any paths that contain custom themes here, relative to this directory. 61 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 62 | 63 | # Add any paths that contain custom static files (such as style sheets) here, 64 | # relative to this directory. They are copied after the builtin static files, 65 | # so a file named "default.css" will overwrite the builtin "default.css". 66 | # html_static_path = ['_static'] 67 | 68 | # Add any extra paths that contain custom files (such as robots.txt or 69 | # .htaccess) here, relative to this directory. These files are copied 70 | # directly to the root of the documentation. 71 | html_extra_path = ['extras'] 72 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. cloudbridge documentation master file, created by 2 | sphinx-quickstart on Sat Oct 10 03:17:52 2015. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to CloudBridge's documentation! 7 | ======================================= 8 | 9 | CloudBridge aims to provide a simple layer of abstraction over 10 | different cloud providers, reducing or eliminating the need to write 11 | conditional code for each cloud. 12 | 13 | Usage example 14 | ------------- 15 | 16 | The simplest possible example for doing something useful with CloudBridge would 17 | look like the following. 18 | 19 | .. code-block:: python 20 | 21 | from cloudbridge.factory import CloudProviderFactory, ProviderList 22 | 23 | provider = CloudProviderFactory().create_provider(ProviderList.AWS, {}) 24 | print(provider.compute.instances.list()) 25 | 26 | In the example above, the AWS_ACCESS_KEY and AWS_SECRET_KEY environment variables 27 | must be set to your cloud credentials. 28 | 29 | Quick Reference 30 | --------------- 31 | 32 | The following object graph shows how to access various provider services, and the resource 33 | that they return. Click on any object to drill down into its details. 34 | 35 | .. raw:: html 36 | 37 | 38 | 39 | Installation 40 | ------------ 41 | 42 | The latest release can always be installed form PyPI. For other installation 43 | options, see the `installation page `_:: 44 | 45 | pip install cloudbridge[full] 46 | 47 | Documentation 48 | ------------- 49 | .. toctree:: 50 | :maxdepth: 2 51 | 52 | concepts.rst 53 | getting_started.rst 54 | topics/overview.rst 55 | topics/contributor_guide.rst 56 | api_docs/ref.rst 57 | 58 | Page index 59 | ---------- 60 | * :ref:`genindex` 61 | 62 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | # https://github.yuuza.net/sphinx-doc/sphinx/issues/9727 2 | sphinx>=4.2.0 3 | sphinx_rtd_theme>=1.0.0 4 | -------------------------------------------------------------------------------- /docs/topics/block_storage.rst: -------------------------------------------------------------------------------- 1 | Working with block storage 2 | ========================== 3 | To add persistent storage to your cloud environments, you would use block 4 | storage devices, namely volumes and volume snapshots. A volume is attached to 5 | an instance and mounted as a file system for use by an application. A volume 6 | snapshot is a point-in-time snapshot of a volume that can be shared with other 7 | cloud users. Before a snapshot can be used, it is necessary to create a volume 8 | from it. 9 | 10 | Volume storage 11 | -------------- 12 | Operations, such as creating a new volume and listing the existing ones, are 13 | performed via the :class:`.VolumeService`. To start, let's create a 1GB volume. 14 | 15 | .. code-block:: python 16 | 17 | vol = provider.storage.volumes.create('cloudbridge-vol', 1) 18 | vol.wait_till_ready() 19 | provider.storage.volumes.list() 20 | 21 | Next, let's attach the volume to a running instance as device ``/dev/sdh``: 22 | 23 | vol.attach('i-dbf37022', '/dev/sdh') 24 | vol.refresh() 25 | vol.state 26 | # 'in-use' 27 | 28 | Once attached, from within the instance, it is necessary to create a file 29 | system on the new volume and mount it. 30 | 31 | Once you wish to detach a volume from an instance, it is necessary to unmount 32 | the file system from within the instance and detach it. The volume can then be 33 | attached to a different instance with all the data on it preserved. 34 | 35 | .. code-block:: python 36 | 37 | vol.detach() 38 | vol.refresh() 39 | vol.state 40 | # 'available' 41 | 42 | Snapshot storage 43 | ---------------- 44 | A volume snapshot it created from an existing volume. Note that it may take a 45 | long time for a snapshot to become ready, particularly on AWS. 46 | 47 | .. code-block:: python 48 | 49 | snap = vol.create_snapshot('cloudbridge-snap', 50 | 'A demo snapshot created via CloudBridge.') 51 | snap.wait_till_ready() 52 | snap.state 53 | # 'available' 54 | 55 | In order to make use of a snapshot, it is necessary to create a volume from it:: 56 | 57 | vol = provider.storage.volumes.create( 58 | 'cloudbridge-snap-vol', 1, 'us-east-1e', snapshot=snap) 59 | 60 | The newly created volume behaves just like any other volume and can be attached 61 | to an instance for use. 62 | -------------------------------------------------------------------------------- /docs/topics/captures/aws-ami-dash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/aws-ami-dash.png -------------------------------------------------------------------------------- /docs/topics/captures/aws-bucket.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/aws-bucket.png -------------------------------------------------------------------------------- /docs/topics/captures/aws-instance-dash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/aws-instance-dash.png -------------------------------------------------------------------------------- /docs/topics/captures/aws-services-dash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/aws-services-dash.png -------------------------------------------------------------------------------- /docs/topics/captures/az-app-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/az-app-1.png -------------------------------------------------------------------------------- /docs/topics/captures/az-app-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/az-app-2.png -------------------------------------------------------------------------------- /docs/topics/captures/az-app-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/az-app-3.png -------------------------------------------------------------------------------- /docs/topics/captures/az-app-4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/az-app-4.png -------------------------------------------------------------------------------- /docs/topics/captures/az-app-5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/az-app-5.png -------------------------------------------------------------------------------- /docs/topics/captures/az-app-6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/az-app-6.png -------------------------------------------------------------------------------- /docs/topics/captures/az-app-7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/az-app-7.png -------------------------------------------------------------------------------- /docs/topics/captures/az-dir-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/az-dir-1.png -------------------------------------------------------------------------------- /docs/topics/captures/az-dir-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/az-dir-2.png -------------------------------------------------------------------------------- /docs/topics/captures/az-label-dash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/az-label-dash.png -------------------------------------------------------------------------------- /docs/topics/captures/az-net-id.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/az-net-id.png -------------------------------------------------------------------------------- /docs/topics/captures/az-net-label.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/az-net-label.png -------------------------------------------------------------------------------- /docs/topics/captures/az-role-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/az-role-1.png -------------------------------------------------------------------------------- /docs/topics/captures/az-role-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/az-role-2.png -------------------------------------------------------------------------------- /docs/topics/captures/az-role-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/az-role-3.png -------------------------------------------------------------------------------- /docs/topics/captures/az-storacc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/az-storacc.png -------------------------------------------------------------------------------- /docs/topics/captures/az-sub-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/az-sub-1.png -------------------------------------------------------------------------------- /docs/topics/captures/az-sub-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/az-sub-2.png -------------------------------------------------------------------------------- /docs/topics/captures/az-subnet-label.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/az-subnet-label.png -------------------------------------------------------------------------------- /docs/topics/captures/az-subnet-name.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/az-subnet-name.png -------------------------------------------------------------------------------- /docs/topics/captures/gcp-sa-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/gcp-sa-1.png -------------------------------------------------------------------------------- /docs/topics/captures/gcp-sa-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/gcp-sa-2.png -------------------------------------------------------------------------------- /docs/topics/captures/gcp-sa-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/gcp-sa-3.png -------------------------------------------------------------------------------- /docs/topics/captures/gcp-sa-4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/gcp-sa-4.png -------------------------------------------------------------------------------- /docs/topics/captures/gcp-sa-5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/gcp-sa-5.png -------------------------------------------------------------------------------- /docs/topics/captures/os-instance-dash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/os-instance-dash.png -------------------------------------------------------------------------------- /docs/topics/captures/os-kp-dash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/docs/topics/captures/os-kp-dash.png -------------------------------------------------------------------------------- /docs/topics/contributor_guide.rst: -------------------------------------------------------------------------------- 1 | Contributor Guide 2 | ================= 3 | This section has information on how to contribute to CloudBridge development, 4 | and a walkthrough of the process of getting started on developing a new 5 | CloudBridge Provider. 6 | 7 | .. toctree:: 8 | :maxdepth: 1 9 | 10 | Design Goals 11 | Design Decisions 12 | Testing 13 | Provider Development Walkthrough 14 | Release Process 15 | 16 | -------------------------------------------------------------------------------- /docs/topics/design_goals.rst: -------------------------------------------------------------------------------- 1 | Design Goals 2 | ~~~~~~~~~~~~ 3 | 4 | 1. Create a cloud abstraction layer which minimises or eliminates the need for 5 | cloud specific special casing (i.e., Not require clients to write 6 | ``if EC2 do x else if OPENSTACK do y``.) 7 | 8 | 2. Have a suite of conformance tests which are comprehensive enough that goal 9 | 1 can be achieved. This would also mean that clients need not manually test 10 | against each provider to make sure their application is compatible. 11 | 12 | 3. Opt for a minimum set of features that a cloud provider will support, 13 | instead of a lowest common denominator approach. This means that reasonably 14 | mature clouds like Amazon and OpenStack are used as the benchmark against 15 | which functionality & features are determined. Therefore, there is a 16 | definite expectation that the cloud infrastructure will support a compute 17 | service with support for images and snapshots and various machine sizes. 18 | The cloud infrastructure will very likely support block storage, although 19 | this is currently optional. It may optionally support object storage. 20 | 21 | 4. Make the CloudBridge layer as thin as possible without compromising goal 1. 22 | By wrapping the cloud provider's native SDK and doing the minimal work 23 | necessary to adapt the interface, we can achieve greater development speed 24 | and reliability since the native provider SDK is most likely to have both 25 | properties. 26 | -------------------------------------------------------------------------------- /docs/topics/dns.rst: -------------------------------------------------------------------------------- 1 | DNS Service 2 | =========== 3 | The DNS service provides a cloud-independent way to create and edit 4 | dns zones and records. 5 | 6 | 1. Creating a DNS zone 7 | ------------------------ 8 | At the top-level, dns records are organized into zones. A zone 9 | is a portion of the dns namespace that's managed by a particular 10 | organization or group. 11 | 12 | .. code-block:: python 13 | 14 | host_zone = provider.dns.host_zones.create("cloudve.org.", "admin@cloudve.org") 15 | 16 | 17 | 2. Create a DNS record 18 | ---------------------- 19 | Once a zone is created, you can create records as required. 20 | 21 | .. code-block:: python 22 | 23 | host_zone = provider.dns.host_zones.find(name="cloudve.org.") 24 | # create an A record 25 | rec1 = host_zone.records.create("mysubdomain.cloudve.org.", DnsRecordType.A, data='10.1.1.1') 26 | # create a wildcard record 27 | rec2 = host_zone.records.create("*.cloudve.org.", DnsRecordType.A, data='10.1.1.2') 28 | # create an MX record 29 | MX_DATA = ['10 mx1.hello.com.', '20 mx2.hello.com.'] 30 | test_rec2 = host_zone.records.create("cloudve.org.", DnsRecordType.MX, data=MX_DATA, ttl=300) 31 | -------------------------------------------------------------------------------- /docs/topics/faq.rst: -------------------------------------------------------------------------------- 1 | FAQ 2 | === 3 | 4 | 1. Using cloudbridge across zones 5 | 6 | Currently, each instance of a cloudbridge provider is designated to work within a 7 | particular zone, for reasons clarified here: :ref:`single-zone-provider`. 8 | 9 | To perform cross-zonal operations, we recommend cloning the provider into a different 10 | zone as in this example: 11 | 12 | .. code-block:: python 13 | 14 | all_instances = [] 15 | for zone in provider.compute.regions.current.zones: 16 | new_provider = provider.clone(zone=zone) 17 | all_instances.append(list(new_provider.compute.instances)) 18 | print(all_instances) 19 | 20 | 21 | 2. Cleaning up resources/left over resources 22 | 23 | The trickiest part about using cloud resources is the orderly cleanup of resources 24 | when they are no longer needed. Cleanup is often complicated, as cloud-providers 25 | may have delays in responding at certain times, and transient errors at other times. 26 | While cloudbridge does not designate a particular strategy to combat this, 27 | the `controller pattern`_ is a recommended mechanism for handling such scenarios: 28 | 29 | 30 | Cloudbridge provides some utilities that can aid in simpler scenarios, such as 31 | `wait_for`, the cleanup helper and retries. 32 | 33 | The following example demonstrates a scenario where an instance and its attached 34 | volume must be deleted. 35 | 36 | .. code-block:: python 37 | 38 | from cloudbridge.base import helpers as cb_helpers 39 | import tenacity 40 | 41 | def does_instance_or_volume_still_exist(inst, vol): 42 | return provider.compute.instances.get(inst.id) or 43 | provider.storage.volumes.get(vol.id) 44 | 45 | def detach_and_delete(inst, vol) 46 | with cb_helpers.cleanup_action(lambda: inst.delete()): 47 | vol.detach() 48 | vol.wait_for( 49 | [VolumeState.AVAILABLE], 50 | terminal_states=[VolumeState.ERROR, VolumeState.DELETED]) 51 | vol.delete() 52 | self.wait_for([VolumeState.UNKNOWN, VolumeState.ERROR]) 53 | 54 | def delete_my_instance_and_attached_volume(provider, instance, vol): 55 | retryer = tenacity.Retrying( 56 | stop=tenacity.stop_after_delay(300), 57 | retry=tenacity.retry_if_result(does_instance_or_volume_still_exist(instance, vol), 58 | wait=tenacity.wait_fixed(5)) 59 | 60 | retryer(detach_and_delete, instance, vol) 61 | 62 | # invoke with the instance and vol you want to delete 63 | delete_my_instance_and_attached_volume(my_inst, my_vol) 64 | 65 | 66 | The code above attempts to first detach and then delete the volume. 67 | If an exception occurs, such as the volume not existing, the `cleanup_action` code 68 | ensures that the `inst.delete()` code runs regardless of the success or failure 69 | of the volume deletion operation. The tenacity.retryer wraps the entire operation 70 | so that the overall process will repeat till both the volume nor the instance no 71 | longer exist. 72 | 73 | 74 | .. _controller pattern: https://kubernetes.io/docs/concepts/architecture/controller/#controller-pattern 75 | -------------------------------------------------------------------------------- /docs/topics/install.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | 4 | **Prerequisites**: CloudBridge runs on Python 2.7 and higher. Python 3 is 5 | recommended. 6 | 7 | We highly recommend installing CloudBridge in a 8 | `virtualenv `_. Creating a new virtualenv 9 | is simple: 10 | 11 | .. code-block:: shell 12 | 13 | pip install virtualenv 14 | virtualenv .venv 15 | source .venv/bin/activate 16 | 17 | Latest stable release 18 | --------------------- 19 | The latest release of CloudBridge can be installed from PyPI:: 20 | 21 | pip install cloudbridge[full] 22 | 23 | Latest unreleased dev version 24 | ----------------------------- 25 | The development version of the library can be installed directly from the 26 | `GitHub repo `_:: 27 | 28 | $ pip install --upgrade git+https://github.com/CloudVE/cloudbridge.git 29 | 30 | Single Provider Installation 31 | ----------------------------- 32 | If you only require to integrate with one to two providers, you can install 33 | the particular providers only as the following. 34 | 35 | $ pip install cloudbridge[aws,gcp] 36 | 37 | The available options are aws, azure, gcp and openstack. 38 | 39 | Developer installation 40 | ---------------------- 41 | To install additional libraries required by CloudBridge contributors, such as 42 | `tox `_, clone the source code 43 | repository and run the following command from the repository root directory:: 44 | 45 | $ git clone https://github.com/CloudVE/cloudbridge.git 46 | $ cd cloudbridge 47 | $ pip install --upgrade --editable .[dev] 48 | 49 | Checking installation 50 | --------------------- 51 | To check what version of the library you have installed, do the following:: 52 | 53 | import cloudbridge 54 | cloudbridge.get_version() 55 | -------------------------------------------------------------------------------- /docs/topics/launch.rst: -------------------------------------------------------------------------------- 1 | Launching instances 2 | =================== 3 | Before being able to run below commands, you will need a ``provider`` object 4 | (see `this page `_). 5 | 6 | Common launch data 7 | ------------------ 8 | Before launching an instance, you need to decide what image to launch 9 | as well as what type of instance. We will create those objects here. The 10 | specified image ID is a base Ubuntu image on AWS so feel free to change it as 11 | desired. For instance type, we're going to let CloudBridge figure out what's 12 | the appropriate name on a given provider for an instance with at least 2 CPUs 13 | and 4 GB RAM. 14 | 15 | .. code-block:: python 16 | 17 | img = provider.compute.images.get('ami-759bc50a') # Ubuntu 16.04 on AWS 18 | vm_type = sorted([t for t in provider.compute.vm_types 19 | if t.vcpus >= 2 and t.ram >= 4], 20 | key=lambda x: x.vcpus*x.ram)[0] 21 | 22 | In addition, CloudBridge instances must be launched into a private subnet. 23 | While it is possible to create complex network configurations as shown in the 24 | `Private networking`_ section, if you don't particularly care in which subnet 25 | the instance is launched, CloudBridge provides a convenience function to 26 | quickly obtain a default subnet for use. 27 | 28 | .. code-block:: python 29 | 30 | subnet = provider.networking.subnets.get_or_create_default() 31 | 32 | When launching an instance, you can also specify several optional arguments 33 | such as the firewall (aka security group), a key pair, or instance user data. 34 | To allow you to connect to the launched instances, we will also supply those 35 | parameters (note that we're making an assumption here these resources exist; 36 | if you don't have those resources under your account, take a look at the 37 | `Getting Started <../getting_started.html>`_ guide). 38 | 39 | .. code-block:: python 40 | 41 | kp = provider.security.key_pairs.find(name='cloudbridge-intro')[0] 42 | fw = provider.security.vm_firewalls.list()[0] 43 | 44 | Launch an instance 45 | ------------------ 46 | Once we have all the desired pieces, we'll use them to launch an instance. 47 | Note that the instance is launched in the provider's default region and zone, 48 | and can be overridden by changing the provider config. 49 | 50 | .. code-block:: python 51 | 52 | inst = provider.compute.instances.create( 53 | label='cloudbridge-vpc', image=img, vm_type=vm_type, 54 | subnet=subnet, key_pair=kp, vm_firewalls=[fw]) 55 | 56 | Private networking 57 | ~~~~~~~~~~~~~~~~~~ 58 | Private networking gives you control over the networking setup for your 59 | instance(s) and is considered the preferred method for launching instances. To 60 | launch an instance with an explicit private network, you can create a custom 61 | network and make sure it has internet connectivity. You can then launch into 62 | that subnet. 63 | 64 | .. code-block:: python 65 | 66 | net = self.provider.networking.networks.create( 67 | label='my-network', cidr_block='10.0.0.0/16') 68 | sn = net.subnets.create(label='my-subnet', cidr_block='10.0.0.0/28') 69 | # make sure subnet has internet access 70 | router = self.provider.networking.routers.create(label='my-router', network=net) 71 | router.attach_subnet(sn) 72 | gateway = net.gateways.get_or_create() 73 | router.attach_gateway(gateway) 74 | 75 | inst = provider.compute.instances.create( 76 | label='cloudbridge-vpc', image=img, vm_type=vm_type, 77 | subnet=sn, key_pair=kp, vm_firewalls=[fw]) 78 | 79 | For more information on how to create and setup a private network, take a look 80 | at `Networking <./networking.html>`_. 81 | 82 | Block device mapping 83 | ~~~~~~~~~~~~~~~~~~~~ 84 | Optionally, you may want to provide a block device mapping at launch, 85 | specifying volume or ephemeral storage mappings for the instance. While volumes 86 | can also be attached and mapped after instance boot using the volume service, 87 | specifying block device mappings at launch time is especially useful when it is 88 | necessary to resize the root volume. 89 | 90 | The code below demonstrates how to resize the root volume. For more information, 91 | refer to :class:`.LaunchConfig`. 92 | 93 | .. code-block:: python 94 | 95 | lc = provider.compute.instances.create_launch_config() 96 | lc.add_volume_device(source=img, size=11, is_root=True) 97 | inst = provider.compute.instances.create( 98 | label='cloudbridge-bdm', image=img, vm_type=vm_type, 99 | launch_config=lc, key_pair=kp, vm_firewalls=[fw], 100 | subnet=subnet) 101 | 102 | where ``img`` is the :class:`.Image` object to use for the root volume. 103 | 104 | After launch 105 | ------------ 106 | After an instance has launched, you can access its properties: 107 | 108 | .. code-block:: python 109 | 110 | # Wait until ready 111 | inst.wait_till_ready() # This is a blocking call 112 | inst.state 113 | # 'running' 114 | 115 | Depending on the provider's networking setup, it may be necessary to explicitly 116 | assign a floating IP address to your instance. This can be done as follows: 117 | 118 | .. code-block:: python 119 | 120 | # Create a new floating IP address 121 | fip = provider.networking.floating_ips.create() 122 | # Assign the desired IP to the instance 123 | inst.add_floating_ip(fip) 124 | inst.refresh() 125 | inst.public_ips 126 | # [u'149.165.168.143'] 127 | -------------------------------------------------------------------------------- /docs/topics/networking.rst: -------------------------------------------------------------------------------- 1 | Private networking 2 | ================== 3 | Private networking gives you control over the networking setup for your 4 | instance(s) and is considered the preferred method for launching instances. 5 | Also, providers these days are increasingly requiring use of private networks. 6 | All CloudBridge deployed VMs must be deployed into a particular subnet. 7 | 8 | If you do not explicitly specify a private network to use when launching an 9 | instance, CloudBridge will attempt to use a default one. A 'default' network is 10 | one tagged as such by the native API. If such tag or functionality does not 11 | exist, CloudBridge will look for one with a predefined label (by default, 12 | called 'cloudbridge-net', which can be overridden with environment variable 13 | ``CB_DEFAULT_NETWORK_LABEL``). 14 | 15 | Once a VM is deployed, CloudBridge's networking capabilities must address 16 | several common scenarios. 17 | 18 | 1. Allowing internet access from a launched VM 19 | 20 | In the simplest scenario, a user may simply want to launch an instance and 21 | allow the instance to access the internet. 22 | 23 | 24 | 2. Allowing internet access to a launched VM 25 | 26 | Alternatively, the user may want to allow the instance to be contactable 27 | from the internet. In a more complex scenario, a user may want to deploy 28 | VMs into several subnets, and deploy a gateway, jump host, or bastion host 29 | to access other VMs which are not directly connected to the internet. In 30 | the latter scenario, the gateway/jump host/bastion host will need to be 31 | contactable over the internet. 32 | 33 | 34 | 3. Secure access between subnets for n-tier applications 35 | 36 | In this third scenario, a multi-tier app may be deployed into several 37 | subnets depending on their tier. For example, consider the following 38 | scenario: 39 | 40 | - Tier 1/Subnet 1 - Web Server needs to be externally accessible over the 41 | internet. However, in this particular scenario, the web server itself does 42 | not need access to the internet. 43 | 44 | - Tier 2/Subnet 2 - Application Server must only be able to communicate with 45 | the database server in Subnet 3, and receive communication from the Web 46 | Server in Subnet 1. However, we assume a special case here where the 47 | application server needs to access the internet. 48 | 49 | - Tier 3/Subnet 3 - Database Server must only be able to receive incoming 50 | traffic from Tier 2, but must not be able to make outgoing traffic outside 51 | of its subnet. 52 | 53 | At present, CloudBridge does not provide support for this scenario, 54 | primarily because OpenStack's FwaaS (Firewall-as-a-Service) is not widely 55 | available. 56 | 57 | 1. Allowing internet access from a launched VM 58 | ---------------------------------------------- 59 | Creating a private network is a simple, one-line command but appropriately 60 | connecting it so that it has uniform internet access across all providers 61 | is a multi-step process: 62 | (1) create a network; (2) create a subnet within this network; (3) create a 63 | router; (4) attach the router to the subnet; and (5) attach the router to the 64 | internet gateway. 65 | 66 | When creating a network, we need to set an address pool. Any subsequent 67 | subnets you create must have a CIDR block that falls within the parent 68 | network's CIDR block. CloudBridge also defines a default IPv4 network range in 69 | ``BaseNetwork.CB_DEFAULT_IPV4RANGE``. Below, we'll create a subnet starting 70 | from the beginning of the block and allow up to 16 IP addresses within a 71 | subnet (``/28``). 72 | 73 | .. code-block:: python 74 | 75 | net = provider.networking.networks.create( 76 | label='my-network', cidr_block='10.0.0.0/16') 77 | sn = net.subnets.create(label='my-subnet', 78 | cidr_block='10.0.0.0/28') 79 | router = provider.networking.routers.create(label='my-router', network=net) 80 | router.attach_subnet(sn) 81 | gateway = net.gateways.get_or_create() 82 | router.attach_gateway(gateway) 83 | 84 | 85 | 2. Allowing internet access to a launched VM 86 | -------------------------------------------- 87 | The additional step that's required here is to assign a floating IP to the VM: 88 | 89 | .. code-block:: python 90 | 91 | net = provider.networking.networks.create( 92 | label='my-network', cidr_block='10.0.0.0/16') 93 | sn = net.subnets.create(label='my-subnet', cidr_block='10.0.0.0/28') 94 | 95 | vm = provider.compute.instances.create(label='my-inst', subnet=sn, ...) 96 | 97 | router = provider.networking.routers.create(label='my-router', network=net) 98 | router.attach_subnet(sn) 99 | gateway = net.gateways.get_or_create() 100 | router.attach_gateway(gateway) 101 | 102 | fip = provider.networking.floating_ips.create() 103 | vm.add_floating_ip(fip) 104 | 105 | 106 | Retrieve an existing private network 107 | ------------------------------------ 108 | If you already have existing networks, we can query for it: 109 | 110 | .. code-block:: python 111 | 112 | provider.networking.networks.list() # Find a desired network ID 113 | net = provider.networking.networks.get('desired network ID') 114 | -------------------------------------------------------------------------------- /docs/topics/object_storage.rst: -------------------------------------------------------------------------------- 1 | Working with object storage 2 | =========================== 3 | Object storage provides a simple way to store and retrieve large amounts of 4 | unstructured data over HTTP. Object Storage is also referred to as Blob (Binary 5 | Large OBject) Storage by Azure, and Simple Storage Service (S3) by Amazon. 6 | 7 | Typically, you would store your objects within a Bucket, as it is known in 8 | AWS and GCP. A Bucket is also called a Container in OpenStack and Azure. In 9 | CloudBridge, we use the term Bucket. 10 | 11 | Storing objects in a bucket 12 | --------------------------- 13 | To store an object within a bucket, we need to first create a bucket or 14 | retrieve an existing bucket. 15 | 16 | .. code-block:: python 17 | 18 | bucket = provider.storage.buckets.create('my-bucket') 19 | bucket.objects.list() 20 | 21 | Next, let's upload some data to this bucket. To efficiently upload a file, 22 | simple use the upload_from_file method. 23 | 24 | .. code-block:: python 25 | 26 | obj = bucket.objects.create('my-data.txt') 27 | obj.upload_from_file('/path/to/myfile.txt') 28 | 29 | You can also use the upload() function to upload from an in memory stream. 30 | Note that, an object you create with objects.create() doesn't actually get 31 | persisted until you upload some content. 32 | 33 | To locate and download this uploaded file again, you can do the following: 34 | 35 | .. code-block:: python 36 | 37 | bucket = provider.storage.buckets.find(name='my-bucket')[0] 38 | obj = bucket.objects.find(name='my-data.txt')[0] 39 | print("Size: {0}, Modified: {1}".format(obj.size, obj.last_modified)) 40 | with open('/tmp/myfile.txt', 'wb') as f: 41 | obj.save_content(f) 42 | 43 | 44 | Using tokens for authentication 45 | ------------------------------- 46 | Some providers may support using temporary credentials with a session token, 47 | in which case you will be able to access a particular bucket by using that 48 | session token. 49 | 50 | .. code-block:: python 51 | 52 | provider = CloudProviderFactory().create_provider( 53 | ProviderList.AWS, 54 | {'aws_access_key': 'ACCESS_KEY', 55 | 'aws_secret_key': 'SECRET_KEY', 56 | 'aws_session_token': 'MY_SESSION_TOKEN'}) 57 | .. code-block:: python 58 | 59 | provider = CloudProviderFactory().create_provider( 60 | ProviderList.OPENSTACK, 61 | {'os_storage_url': 'SWIFT_STORAGE_URL', 62 | 'os_auth_token': 'MY_SESSION_TOKEN'}) 63 | 64 | Once a provider is obtained, you can access the container as usual: 65 | 66 | .. code-block:: python 67 | 68 | bucket = provider.storage.buckets.get(container) 69 | obj = bucket.objects.create('my_object.txt') 70 | obj.upload_from_file(source) 71 | 72 | 73 | Generating signed URLs 74 | ---------------------- 75 | 76 | Signed URLs are a great way to allow users who do not have credentials for 77 | the cloud provider of your choice, to interact with an object within a 78 | storage bucket. 79 | 80 | You can generate signed URLs with ``GET`` permissions to allow a user to 81 | get an object. 82 | 83 | .. code-block:: python 84 | 85 | provider = CloudProviderFactory().create_provider( 86 | ProviderList.AWS, 87 | {'aws_access_key': 'ACCESS_KEY', 88 | 'aws_secret_key': 'SECRET_KEY', 89 | 'aws_session_token': 'MY_SESSION_TOKEN'}) 90 | 91 | bucket = provider.storage.buckets.get("my-bucket") 92 | obj = bucket.objects.get("my-file.txt") 93 | 94 | url = obj.generate_url(expires_in=7200) 95 | 96 | You can also generate a signed URL with `PUT` permissions to allow users 97 | to upload files to your storage bucket. 98 | 99 | .. code-block:: python 100 | 101 | provider = CloudProviderFactory().create_provider( 102 | ProviderList.AWS, 103 | {'aws_access_key': 'ACCESS_KEY', 104 | 'aws_secret_key': 'SECRET_KEY', 105 | 'aws_session_token': 'MY_SESSION_TOKEN'}) 106 | 107 | bucket = provider.storage.buckets.get("my-bucket") 108 | obj = bucket.objects.create("my-file.txt") 109 | url = obj.generate_url(expires_in=7200, writable=True) 110 | 111 | 112 | With your signed URL, you or someone on your team can upload a file like this 113 | 114 | .. code-block:: python 115 | 116 | import requests 117 | 118 | content = b"Hello world!" 119 | # Only Azure requires the x-ms-blob-type header to be present, but there's no harm 120 | # in sending this in for all providers. 121 | headers = {'x-ms-blob-type': 'BlockBlob'} 122 | requests.put(url, data=content) 123 | -------------------------------------------------------------------------------- /docs/topics/os_mapping.rst: -------------------------------------------------------------------------------- 1 | Detailed OpenStack Type and Resource Mappings 2 | ============================================= 3 | 4 | OpenStack - Labeled Resources 5 | ----------------------------- 6 | +------------------------+------------------------+-----------+----------------+----------+ 7 | | Labeled Resource | OS Resource Type | CB ID | CB Name | CB Label | 8 | +========================+========================+===========+================+==========+ 9 | | OpenStackInstance | Instance | ID | ID | Name | 10 | +------------------------+------------------------+-----------+----------------+----------+ 11 | | OpenStackMachineImage | Image | ID | ID | Name | 12 | +------------------------+------------------------+-----------+----------------+----------+ 13 | | OpenStackNetwork | Network | ID | ID | Name | 14 | +------------------------+------------------------+-----------+----------------+----------+ 15 | | OpenStackSubnet | Subnet | ID | ID | Name | 16 | +------------------------+------------------------+-----------+----------------+----------+ 17 | | OpenStackRouter | Router | ID | ID | Name | 18 | +------------------------+------------------------+-----------+----------------+----------+ 19 | | OpenStackVolume | Volume | ID | ID | Name | 20 | +------------------------+------------------------+-----------+----------------+----------+ 21 | | OpenStackSnapshot | Snapshot | ID | ID | Name | 22 | +------------------------+------------------------+-----------+----------------+----------+ 23 | | OpenStackVMFirewall | Security Group | ID | ID | Name | 24 | +------------------------+------------------------+-----------+----------------+----------+ 25 | 26 | The resources listed above are labeled, they thus have both the `name` and 27 | `label` properties in CloudBridge. These resources require a mandatory `label` 28 | parameter at creation. For all labeled resources, the `label` property in 29 | OpenStack maps to the Name attribute. However, unlike in Azure or AWS, no 30 | resource has an unchangeable name by which to identify it in our OpenStack 31 | implementation. The `name` property will therefore map to the ID, preserving 32 | its role as an unchangeable identifier even though not easily readable in this 33 | context. Finally, labeled resources support a `label` parameter for the `find` 34 | method in their corresponding services. The below screenshots will help map 35 | these properties to OpenStack objects in the web portal. Additionally, although 36 | OpenStack Security Groups are not associated with a specific network, such an 37 | association is done in CloudBridge, due to its necessity in AWS. As such, the 38 | VMFirewall creation method requires a `network` parameter and the association 39 | is accomplished in OpenStack through the description, by appending the 40 | following string to the user-provided description (if any) at creation: 41 | "[CB-AUTO-associated-network-id: associated_net_id]" 42 | 43 | .. figure:: captures/os-instance-dash.png 44 | :alt: name, ID, and label properties for OS Instances 45 | 46 | The CloudBridge `name` and `ID` properties map to the unchangeable 47 | resource ID in OpenStack as resources do not allow for an unchangeable 48 | name. The `label` property maps to the 'Name' for all resources in 49 | OpenStack. By default, this label will appear in the first column. 50 | 51 | 52 | OpenStack - Unlabeled Resources 53 | ------------------------------- 54 | +-----------------------+------------------------+-------+---------+----------+ 55 | | Unlabeled Resource | OS Resource Type | CB ID | CB Name | CB Label | 56 | +=======================+========================+=======+=========+==========+ 57 | | OpenStackKeyPair | Key Pair | Name | Name | - | 58 | +-----------------------+------------------------+-------+---------+----------+ 59 | | OpenStackBucket | Object Store Container | Name | Name | - | 60 | +-----------------------+------------------------+-------+---------+----------+ 61 | | OpenStackBucketObject | Object | Name | Name | - | 62 | +-----------------------+------------------------+-------+---------+----------+ 63 | 64 | The resources listed above are unlabeled. They thus only have the `name` 65 | property in CloudBridge. These resources require a mandatory `name` 66 | parameter at creation, which will directly map to the unchangeable `name` 67 | property. Additionally, for these resources, the `ID` property also maps to 68 | the `name` in OpenStack, as these resources don't have an `ID` in the 69 | traditional sense and can be identified by name. Finally, unlabeled resources 70 | support a `name` parameter for the `find` method in their corresponding 71 | services. 72 | 73 | .. figure:: captures/os-kp-dash.png 74 | :alt: KeyPair details on OS dashboard 75 | 76 | KeyPairs and other unlabeled resources in OpenStack have `name` that is 77 | unique and unmodifiable. The `ID` will thus map to the `name` property when 78 | no other `ID` exists for that OpenStack resource. 79 | 80 | 81 | OpenStack - Special Unlabeled Resources 82 | --------------------------------------- 83 | +--------------------------+------------------------+-------+------------------------------------------------------------------------+----------+ 84 | | Unlabeled Resource | OS Resource Type | CB ID | CB Name | CB Label | 85 | +==========================+========================+=======+========================================================================+==========+ 86 | | OpenStackFloatingIP | Floating IP | ID | [public_ip] | - | 87 | +--------------------------+------------------------+-------+------------------------------------------------------------------------+----------+ 88 | | OpenStackInternetGateway | Network `public` | ID | 'public' | - | 89 | +--------------------------+------------------------+-------+------------------------------------------------------------------------+----------+ 90 | | OpenStackVMFirewallRule | Security Group Rule | ID | Generated: [direction]-[protocol]-[from_port]-[to_port]-[cidr]-[fw_id] | - | 91 | +--------------------------+------------------------+-------+------------------------------------------------------------------------+----------+ 92 | 93 | While these resources are similarly unlabeled, they do not follow the same 94 | general rules as the ones listed before. Firstly, they differ by the fact 95 | that they take neither a `name` nor a `label` parameter at creation. 96 | Moreover, each of them has other special properties. 97 | 98 | The FloatingIP resource has a traditional resource ID, but instead of a 99 | traditional name, its `name` property maps to its Public IP. 100 | Moreover, the corresponding `find` method for Floating IPs can thus help 101 | find a resource by `Public IP Address`. 102 | 103 | In terms of the gateway in OpenStack, it maps to the network named 'public.' 104 | Thus, the internet gateway create method does not take a name parameter, and 105 | the `name` property will be 'public'. 106 | 107 | Finally, Firewall Rules in OpenStack differ from traditional unlabeled resources 108 | by the fact that they do not take a `name` parameter at creation, and the 109 | `name` property is automatically generated from the rule's properties, as 110 | shown above. These rules can be found within each Firewall (i.e. Security 111 | Group) in the web portal, and will not have any name in the OpenStack dashboard. 112 | -------------------------------------------------------------------------------- /docs/topics/overview.rst: -------------------------------------------------------------------------------- 1 | Using CloudBridge 2 | ================= 3 | Introductions to all the key parts of CloudBridge you'll need to know: 4 | 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | How to install CloudBridge 9 | Procuring access credentials 10 | Connection and authentication setup 11 | Launching instances 12 | Networking 13 | DNS 14 | Object states and lifecycles 15 | Paging and iteration 16 | Using block storage 17 | Using object storage 18 | Resource types and mapping 19 | Using the event system 20 | Troubleshooting 21 | FAQ 22 | -------------------------------------------------------------------------------- /docs/topics/paging_and_iteration.rst: -------------------------------------------------------------------------------- 1 | Paging and iteration 2 | ==================== 3 | 4 | Overview 5 | -------- 6 | Most provider services have list() methods, and all list methods accept a limit 7 | parameter which specifies the maximum number of results to return. If a limit 8 | is not specified, CloudBridge will default to the global configuration variable 9 | `default_result_limit`, which can be modified through the provider config. 10 | 11 | Since the returned result list may have more records available, CloudBridge 12 | will always return a :py:class:`ResultList` object to assist with paging through 13 | additional results. A ResultList extends the standard :py:class:`list` and 14 | the following example illustrates how to fetch additional records. 15 | 16 | Example: 17 | 18 | .. code-block:: python 19 | 20 | # get first page of results 21 | rl = provider.compute.instances.list(limit=50) 22 | for result in rl: 23 | print("Instance Data: {0}", result) 24 | if rl.supports_total: 25 | print("Total results: {0}".format(rl.total_results)) 26 | else: 27 | print("Total records unknown," 28 | "but has more data?: {0}."format(rl.is_truncated)) 29 | 30 | # Page to next set of results 31 | if (rl.is_truncated) 32 | rl = provider.compute.instances.list(limit=100, 33 | marker=rl.marker) 34 | 35 | 36 | To ease development, CloudBridge also provides standard Python iterators that 37 | will page the results in for you automatically. Therefore, when you need to 38 | iterate through all available objects, the following shorthand is recommended: 39 | 40 | Example: 41 | 42 | .. code-block:: python 43 | 44 | # Iterate through all results 45 | for instance in provider.compute.instances: 46 | print("Instance Data: {0}", instance) 47 | -------------------------------------------------------------------------------- /docs/topics/provider_development.rst: -------------------------------------------------------------------------------- 1 | Provider Development Walkthrough 2 | ================================ 3 | This guide will walk you through the basic process of developing a new provider 4 | for CloudBridge. 5 | 6 | 7 | 1. We start off by creating a new folder for the provider within the 8 | ``cloudbridge/cloud/providers`` folder. In this case: ``gcp``. Further, install 9 | the native cloud provider Python library, here 10 | ``pip install google-api-python-client==1.4.2`` and a couple of its requirements 11 | ``oauth2client==1.5.2`` and ``pycrypto==2.6.1``. 12 | 13 | 2. Add a ``provider.py`` file. This file will contain the main implementation 14 | of the cloud provider and will be the entry point that CloudBridge uses for all 15 | provider related services. You will need to subclass ``BaseCloudProvider`` and 16 | add a class variable named ``PROVIDER_ID``. 17 | 18 | .. code-block:: python 19 | 20 | from cloudbridge.base import BaseCloudProvider 21 | 22 | 23 | class GCPCloudProvider(BaseCloudProvider): 24 | 25 | PROVIDER_ID = 'gcp' 26 | 27 | def __init__(self, config): 28 | super(GCPCloudProvider, self).__init__(config) 29 | 30 | 31 | 32 | 3. Add an ``__init__.py`` to the ``cloudbridge/cloud/providers/gcp`` folder 33 | and export the provider. 34 | 35 | .. code-block:: python 36 | 37 | from .provider import GCPCloudProvider # noqa 38 | 39 | .. tip :: 40 | 41 | You can view the code so far here: `commit 1`_ 42 | 43 | 4. Next, we need to register the provider with the factory. 44 | This only requires that you register the provider's ID in the ``ProviderList``. 45 | Add GCP to the ``ProviderList`` class in ``cloudbridge/cloud/factory.py``. 46 | 47 | 48 | 5. Run the test suite. We will get the tests passing on py27 first. 49 | 50 | .. code-block:: bash 51 | 52 | export CB_TEST_PROVIDER=gcp 53 | tox -e py27 54 | 55 | You should see the tests fail with the following message: 56 | 57 | .. code-block:: bash 58 | 59 | "TypeError: Can't instantiate abstract class GCPCloudProvider with abstract 60 | methods storage, compute, security, network." 61 | 62 | 6. Therefore, our next step is to implement these methods. We can start off by 63 | implementing these methods in ``provider.py`` and raising a 64 | ``NotImplementedError``. 65 | 66 | .. code-block:: python 67 | 68 | @property 69 | def compute(self): 70 | raise NotImplementedError( 71 | "GCPCloudProvider does not implement this service") 72 | 73 | @property 74 | def network(self): 75 | raise NotImplementedError( 76 | "GCPCloudProvider does not implement this service") 77 | 78 | @property 79 | def security(self): 80 | raise NotImplementedError( 81 | "GCPCloudProvider does not implement this service") 82 | 83 | @property 84 | def storage(self): 85 | raise NotImplementedError( 86 | "GCPCloudProvider does not implement this service") 87 | 88 | 89 | Running the tests now will complain as much. We will next implement each 90 | Service in turn. 91 | 92 | 93 | 7. We will start with the compute service. Add a ``services.py`` file. 94 | 95 | .. code-block:: python 96 | 97 | from cloudbridge.base.services import BaseSecurityService 98 | 99 | 100 | class GCPSecurityService(BaseSecurityService): 101 | 102 | def __init__(self, provider): 103 | super(GCPSecurityService, self).__init__(provider) 104 | 105 | 106 | 8. We can now return this new service from the security property in 107 | ``provider.py`` as follows: 108 | 109 | .. code-block:: python 110 | 111 | def __init__(self, config): 112 | super(GCPCloudProvider, self).__init__(config) 113 | self._security = GCPSecurityService(self) 114 | 115 | @property 116 | def security(self): 117 | return self._security 118 | 119 | .. tip :: 120 | 121 | You can view the code so far here: `commit 2`_ 122 | 123 | 9. Run the tests, and the following message will cause all security service 124 | tests to fail: 125 | 126 | .. code-block:: bash 127 | 128 | "TypeError: Can't instantiate abstract class GCPSecurityService with abstract 129 | methods key_pairs, security_groups." 130 | 131 | The Abstract Base Classes are doing their job and flagging all methods that 132 | need to be implemented. 133 | 134 | 10. Since the security service simply provides organisational structure, and is 135 | a container for the ``key_pairs`` and ``security_groups`` services, we must 136 | next implement these services. 137 | 138 | .. code-block:: python 139 | 140 | from cloudbridge.base.services import BaseKeyPairService 141 | from cloudbridge.base.services import BaseSecurityGroupService 142 | from cloudbridge.base.services import BaseSecurityService 143 | 144 | 145 | class GCPSecurityService(BaseSecurityService): 146 | 147 | def __init__(self, provider): 148 | super(GCPSecurityService, self).__init__(provider) 149 | 150 | # Initialize provider services 151 | self._key_pairs = GCPKeyPairService(provider) 152 | self._security_groups = GCPSecurityGroupService(provider) 153 | 154 | @property 155 | def key_pairs(self): 156 | return self._key_pairs 157 | 158 | @property 159 | def security_groups(self): 160 | return self._security_groups 161 | 162 | 163 | class GCPKeyPairService(BaseKeyPairService): 164 | 165 | def __init__(self, provider): 166 | super(GCPKeyPairService, self).__init__(provider) 167 | 168 | 169 | class GCPSecurityGroupService(BaseSecurityGroupService): 170 | 171 | def __init__(self, provider): 172 | super(GCPSecurityGroupService, self).__init__(provider) 173 | 174 | .. tip :: 175 | 176 | You can view the code so far here: `commit 3`_ 177 | 178 | 179 | Once again, running the tests will complain of missing methods: 180 | 181 | .. code-block:: bash 182 | 183 | "TypeError: Can't instantiate abstract class GCPKeyPairService with abstract 184 | methods create, find, get, list." 185 | 186 | 11. Keep implementing the methods till the security service works, and the 187 | tests pass. 188 | 189 | .. note :: 190 | 191 | We start off by implementing the list keypairs method. Therefore, to obtain 192 | the keypair, we need to have a connection to the cloud provider. For this, 193 | we need to install the Google sdk, and thereafter, to obtain the desired 194 | connection via the sdk. While the design and structure of that connection 195 | is up to the implementor, a general design we have followed is to have the 196 | cloud connection globally available within the provider. 197 | 198 | To add the sdk, we edit CloudBridge's main ``setup.py`` and list the 199 | dependencies. 200 | 201 | .. code-block:: python 202 | 203 | gcp_reqs = ['google-api-python-client==1.4.2'] 204 | full_reqs = base_reqs + aws_reqs + openstack_reqs + gcp_reqs 205 | 206 | We will also register the provider in ``cloudbridge/cloud/factory.py``'s 207 | provider list. 208 | 209 | .. code-block:: python 210 | 211 | class ProviderList(object): 212 | AWS = 'aws' 213 | OPENSTACK = 'openstack' 214 | ... 215 | GCP = 'gcp' 216 | 217 | .. tip :: 218 | 219 | You can view the code so far here: `commit 4`_ 220 | 221 | 222 | 12. Thereafter, we create the actual connection through the sdk. In the case of 223 | GCP, we need a Compute API client object. We will make this connection 224 | available as a public property named ``gcp_compute`` in the provider. We will 225 | then lazily initialize this connection. 226 | 227 | A full implementation of the KeyPair service can now be made in a provider 228 | specific manner. 229 | 230 | .. tip :: 231 | 232 | You can view the code so far here: `commit 5`_ 233 | 234 | 235 | 236 | .. _commit 1: https://github.com/CloudVE/cloudbridge/commit/54c67e93a3cd9d51e7d2b1195ebf4e257d165297 237 | .. _commit 2: https://github.com/CloudVE/cloudbridge/commit/82c0244aa4229ae0aecfe40d769eb93b06470dc7 238 | .. _commit 3: https://github.com/CloudVE/cloudbridge/commit/e90a7f6885814a3477cd0b38398d62af64f91093 239 | .. _commit 4: https://github.com/CloudVE/cloudbridge/commit/2d5c14166a538d320e54eed5bc3fa04997828715 240 | .. _commit 5: https://github.com/CloudVE/cloudbridge/commit/98c9cf578b672867ee503027295f9d901411e496 241 | -------------------------------------------------------------------------------- /docs/topics/release_process.rst: -------------------------------------------------------------------------------- 1 | Release Process 2 | ~~~~~~~~~~~~~~~ 3 | 4 | 1. Make sure `all tests pass `_. 5 | 6 | 2. Increment version number in ``cloudbridge/__init__.py`` as per 7 | `semver rules `_. 8 | 9 | 3. Freeze all library dependencies in ``setup.py`` and commit. 10 | The version numbers can be a range with the upper limit being the latest 11 | known working version, and the lowest being the last known working version. 12 | 13 | In general, our strategy is to make provider sdk libraries fixed within 14 | relatively known compatibility ranges, so that we reduce the chances of 15 | breakage. If someone uses CloudBridge, presumably, they do not use the SDKs 16 | directly. For all other libraries, especially, general purpose libraries 17 | (e.g. ``six``), our strategy is to make compatibility as broad and 18 | unrestricted as possible. 19 | 20 | 4. Add release notes to ``CHANGELOG.rst``. Also add last commit hash to 21 | changelog. List of commits can be obtained using 22 | ``git shortlog ..HEAD`` 23 | 24 | 5. Release to PyPi. 25 | (make sure you have run `pip install wheel twine`) 26 | First, test release with PyPI staging server as described in: 27 | https://hynek.me/articles/sharing-your-labor-of-love-pypi-quick-and-dirty/ 28 | 29 | Once tested, run: 30 | 31 | .. code-block:: bash 32 | 33 | # remove stale files or wheel might package them 34 | rm -r build dist 35 | python setup.py sdist bdist_wheel 36 | twine upload -r pypi dist/cloudbridge-3.0.0* 37 | 38 | 6. Tag release and make a GitHub release. 39 | 40 | .. code-block:: bash 41 | 42 | git tag -a v3.0.0 -m "Release 3.0.0" 43 | git push 44 | git push --tags 45 | 46 | 7. Increment version number in ``cloudbridge/__init__.py`` to ``version-dev`` 47 | to indicate the development cycle, commit, and push the changes. 48 | -------------------------------------------------------------------------------- /docs/topics/resource_types_and_mapping.rst: -------------------------------------------------------------------------------- 1 | Resource Types and Dashboard Mapping 2 | ==================================== 3 | 4 | Cross-Platform Concepts 5 | ----------------------- 6 | 7 | Given CloudBridge's goal to work uniformly across cloud providers, some 8 | compromises were necessary in order to bridge the many differences between 9 | providers' resources and features. Notably, in order to create a robust and 10 | conceptually consistent cross-cloud library, resources were separated into 11 | `labeled` and `unlabeled resources,` and were given three main properties: 12 | `ID`, `name`, and `label`. 13 | 14 | The `ID` corresponds to a unique identifier that can be reliably used to 15 | reference a resource. Users can safely use an ID knowing that it will always 16 | point to the same resource. All resources have an `ID` property, thus making 17 | it the recommended property for reliably identifying a resource. 18 | 19 | The `label` property, conversely, is a modifiable value that does not need 20 | to be unique. Unlike the `name` property, it is not used to identify a 21 | particular resource, but rather label a resource for easier distinction. 22 | Only labeled resources have the `label` property, and these resources require 23 | a `label` parameter be set at creation time. 24 | 25 | The `name` property corresponds to an unchangeable and unique designation for a 26 | particular resource. This property is meant to be, in some ways, a more 27 | human-readable identifier. Thus, when no conceptually comparable property 28 | exists for a given resource in a particular provider, the `ID` is returned 29 | instead, as is the case for all OpenStack and some AWS resources. Given the 30 | discrepancy between providers, using the `name` property is not advisable for 31 | cross-cloud usage of the library. Labeled resources will use the label given at 32 | creation as a prefix to the set `name`, when this property is separable from 33 | the `ID` as is the case in Azure and some AWS resources. Finally, unlabeled 34 | resources will always support a `name`, and some unlabeled resources will 35 | require a `name` parameter at creation. Below is a list of all resources 36 | classified by whether they support a `label` property. 37 | 38 | +-------------------+---------------------+ 39 | | Labeled Resources | Unlabeled Resources | 40 | +===================+=====================+ 41 | | Instance | Key Pair | 42 | +-------------------+---------------------+ 43 | | MachineImage | Bucket | 44 | +-------------------+---------------------+ 45 | | Network | Bucket Object | 46 | +-------------------+---------------------+ 47 | | Subnet | FloatingIP | 48 | +-------------------+---------------------+ 49 | | Router | Internet Gateway | 50 | +-------------------+---------------------+ 51 | | Volume | VMFirewall Rule | 52 | +-------------------+---------------------+ 53 | | Snapshot | | 54 | +-------------------+---------------------+ 55 | | VMFirewall | | 56 | +-------------------+---------------------+ 57 | 58 | 59 | Properties per Resource per Provider 60 | ------------------------------------ 61 | For each provider, we documented the mapping of CloudBridge resources and 62 | properties to provider objects, as well as some useful dashboard navigation. 63 | These sections will thus present summary tables delineating the different types of 64 | CloudBridge resources, as well as present some design decisions made to 65 | preserve consistency across providers: 66 | 67 | .. toctree:: 68 | :maxdepth: 1 69 | 70 | Detailed AWS Mappings 71 | Detailed Azure Mappings 72 | Detailed OpenStack Mappings 73 | 74 | .. - `Detailed Azure Mappings `_ 75 | .. - `Detailed AWS Mappings `_ 76 | .. - `Detailed OpenStack Mappings `_ 77 | -------------------------------------------------------------------------------- /docs/topics/testing.rst: -------------------------------------------------------------------------------- 1 | Running tests 2 | ============= 3 | In the spirit of the library's :doc:`design_goals`, the aim is to have thorough 4 | tests for the entire library. This page explains the testing philosophy and 5 | shows how to run the tests locally. 6 | 7 | Testing philosophy 8 | ------------------ 9 | Our testing goals are to: 10 | 11 | 1. Write one set of tests that all provider implementations must pass. 12 | 13 | 2. Make that set of tests a 'conformance' test suite, which validates that each 14 | implementation correctly implements the CloudBridge specification. 15 | 16 | 3. Make the test suite comprehensive enough that a provider which passes all 17 | the tests can be used safely by an application with no additional testing. 18 | In other words, the CloudBridge specification and accompanying test suite 19 | must be comprehensive enough that no provider specific workarounds, code or 20 | testing is required. 21 | 22 | 4. For development, mock providers may be used to speed up the feedback cycle, 23 | but providers must also pass the full suite of tests when run against actual 24 | cloud infrastructure to ensure that we are not testing against an idealised 25 | or imagined environment. 26 | 27 | 5. Aim for 100% code coverage. 28 | 29 | 30 | Running tests 31 | ------------- 32 | To run the test suite locally: 33 | 1. Install `tox`_ with :code:`pip install tox` 34 | 2. Export all environment variables listed in ``tox.ini`` (under ``passenv``) 35 | 3. Run ``tox`` command 36 | 37 | This will run all the tests for all the environments defined in file 38 | ``tox.ini``. 39 | 40 | 41 | Specific environment and infrastructure 42 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 43 | If you’d like to run the tests on a specific environment only, say Python 2.7, 44 | against a specific infrastructure, say aws, use a command like this: 45 | ``tox -e py27-aws``. The available provider names are listed in the 46 | `ProviderList`_ class (e.g., ``aws`` or ``openstack``). 47 | 48 | Specific test cases 49 | ~~~~~~~~~~~~~~~~~~~~ 50 | You can run a specific test case, as follows: 51 | ``tox -- tests/test_image_service.py:CloudImageServiceTestCase.test_create_and_list_imag`` 52 | 53 | It can also be restricted to a particular environment as follows: 54 | ``tox -e "py27-aws" -- tests/test_cloud_factory.py:CloudFactoryTestCase`` 55 | 56 | See nosetest documentation for other parameters that can be passed in. 57 | 58 | Using unittest directly 59 | ~~~~~~~~~~~~~~~~~~~~~~~ 60 | You can also run the tests against your active virtual environment directly 61 | with ``python setup.py test``. You will need to set the ``CB_TEST_PROVIDER`` 62 | environment variable prior to running the tests, or they will default to 63 | ``CB_TEST_PROVIDER=aws``. 64 | 65 | You can also run a specific test case, as follows: 66 | ``python setup.py test -s tests.test_cloud_factory.CloudFactoryTestCase`` 67 | 68 | Using a mock provider 69 | ~~~~~~~~~~~~~~~~~~~~~ 70 | 71 | Note that running the tests may create various cloud resources, for which you 72 | may incur costs. For the AWS cloud, there is also a mock provider (`moto`_) that 73 | will simulate AWS resources. You can use ``CB_TEST_PROVIDER=mock`` to run tests 74 | against the mock provider only, which will provide faster feedback times. 75 | 76 | Alternatively you can run the mock tests through tox. 77 | ``tox -e "py27-mock"`` 78 | 79 | .. _design goals: https://github.com/CloudVE/cloudbridge/ 80 | blob/main/README.rst 81 | .. _tox: https://tox.readthedocs.org/en/latest/ 82 | .. _ProviderList: https://github.com/CloudVE/cloudbridge/blob/main/ 83 | cloudbridge/cloud/factory.py#L15 84 | .. _moto: https://github.com/spulec/moto 85 | -------------------------------------------------------------------------------- /docs/topics/troubleshooting.rst: -------------------------------------------------------------------------------- 1 | Common Setup Issues 2 | =================== 3 | 4 | macOS Issues 5 | ------------ 6 | 7 | * If you are getting an error message like so: ``Authentication with cloud provider failed: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:749)`` 8 | then this indicates that you are probably using a newer version of Python on 9 | macOS. Starting with Python 3.6, the Python installer includes its own version 10 | of OpenSSL and it no longer uses the system trusted certificate keychains. 11 | 12 | Python 3.6 includes a script that can install a bundle of root certificates 13 | from ``certifi``. To install this bundle execute the following: 14 | 15 | .. code-block:: bash 16 | 17 | cd /Applications/Python\ 3.6/ 18 | sudo ./Install\ Certificates.command 19 | 20 | For more information see `this StackOverflow 21 | answer `_ and the `Python 3.6 22 | Release Notes `_. 23 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # needed by moto 2 | sshpubkeys 3 | -e ".[dev]" 4 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [coverage:run] 2 | branch = True 3 | source = cloudbridge 4 | omit = 5 | cloudbridge/interfaces/* 6 | cloudbridge/__init__.py 7 | parallel = True 8 | 9 | [bdist_wheel] 10 | universal = 1 11 | 12 | [flake8] 13 | application_import_names = cloudbridge, tests 14 | max-line-length = 120 15 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | CloudBridge provides a uniform interface to multiple IaaS cloud providers. 3 | """ 4 | 5 | import ast 6 | import os 7 | import re 8 | 9 | from setuptools import find_packages, setup 10 | 11 | # Cannot use "from cloudbridge import get_version" because that would try to 12 | # import the six package which may not be installed yet. 13 | reg = re.compile(r'__version__\s*=\s*(.+)') 14 | with open(os.path.join('cloudbridge', '__init__.py')) as f: 15 | for line in f: 16 | m = reg.match(line) 17 | if m: 18 | version = ast.literal_eval(m.group(1)) 19 | break 20 | 21 | REQS_BASE = [ 22 | 'six>=1.11', 23 | 'tenacity>=6.0', 24 | 'deprecation>=2.0.7', 25 | 'pyeventsystem<2' 26 | ] 27 | REQS_AWS = [ 28 | 'boto3>=1.9.86,<2.0.0' 29 | ] 30 | # Install azure>=3.0.0 package to find which of the azure libraries listed 31 | # below are compatible with each other. List individual libraries instead 32 | # of using the azure umbrella package to speed up installation. 33 | REQS_AZURE = [ 34 | 'msrestazure<1.0.0', 35 | 'azure-identity<2.0.0', 36 | 'azure-common<2.0.0', 37 | 'azure-mgmt-devtestlabs<10.0.0', 38 | 'azure-mgmt-resource<22.0.0', 39 | 'azure-mgmt-compute>=27.2.0,<28.0.0', 40 | 'azure-mgmt-network<22.0.0', 41 | 'azure-mgmt-storage<21.0.0', 42 | 'azure-storage-blob<13.0.0', 43 | 'azure-cosmosdb-table<2.0.0', 44 | 'pysftp<1.0.0' 45 | ] 46 | REQS_GCP = [ 47 | 'google-api-python-client>=2.0,<3.0.0' 48 | ] 49 | REQS_OPENSTACK = [ 50 | 'openstacksdk>=0.12.0,<1.0.0', 51 | 'python-novaclient>=7.0.0,<19.0', 52 | 'python-swiftclient>=3.2.0,<5.0', 53 | 'python-neutronclient>=6.0.0,<9.0', 54 | 'python-keystoneclient>=3.13.0,<6.0' 55 | ] 56 | REQS_FULL = REQS_AWS + REQS_GCP + REQS_OPENSTACK + REQS_AZURE 57 | # httpretty is required with/for moto 1.0.0 or AWS tests fail 58 | REQS_DEV = ([ 59 | 'tox>=4.0.0', 60 | 'pytest', 61 | 'moto>=3.1.18', 62 | 'sphinx>=1.3.1', 63 | 'pydevd', 64 | 'flake8>=3.3.0', 65 | 'flake8-import-order>=0.12'] + REQS_FULL 66 | ) 67 | 68 | setup( 69 | name='cloudbridge', 70 | version=version, 71 | description='A simple layer of abstraction over multiple cloud providers.', 72 | long_description=__doc__, 73 | author='Galaxy and GVL Projects', 74 | author_email='help@genome.edu.au', 75 | url='http://cloudbridge.cloudve.org/', 76 | install_requires=REQS_BASE, 77 | extras_require={ 78 | ':python_version<"3.3"': ['ipaddress'], 79 | 'azure': REQS_AZURE, 80 | 'gcp': REQS_GCP, 81 | 'aws': REQS_AWS, 82 | 'openstack': REQS_OPENSTACK, 83 | 'full': REQS_FULL, 84 | 'dev': REQS_DEV 85 | }, 86 | packages=find_packages(), 87 | license='MIT', 88 | classifiers=[ 89 | 'Development Status :: 5 - Production/Stable', 90 | 'Environment :: Console', 91 | 'Intended Audience :: Developers', 92 | 'Intended Audience :: System Administrators', 93 | 'License :: OSI Approved :: MIT License', 94 | 'Operating System :: OS Independent', 95 | 'Programming Language :: Python', 96 | 'Topic :: Software Development :: Libraries :: Python Modules', 97 | 'Programming Language :: Python :: 2.7', 98 | 'Programming Language :: Python :: 3', 99 | 'Programming Language :: Python :: 3.4', 100 | 'Programming Language :: Python :: 3.5', 101 | 'Programming Language :: Python :: 3.6', 102 | 'Programming Language :: Python :: Implementation :: CPython'], 103 | test_suite="tests" 104 | ) 105 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Use ``python setup.py test`` to run these unit tests (alternatively, use 3 | ``python -m unittest test``). 4 | """ 5 | -------------------------------------------------------------------------------- /tests/fixtures/custom_amis.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "ami_id": "ami-aa2ea6d0", 4 | "state": "available", 5 | "public": true, 6 | "owner_id": "099720109477", 7 | "image_location": "amazon/getting-started", 8 | "sriov": "simple", 9 | "root_device_type": "ebs", 10 | "root_device_name": "/dev/sda1", 11 | "description": "Canonical, Ubuntu, 16.04 LTS, amd64 xenial image build on 2017-11-21", 12 | "image_type": "machine", 13 | "platform": null, 14 | "architecture": "x86_64", 15 | "name": "ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-20171121.1", 16 | "virtualization_type": "hvm", 17 | "hypervisor": "xen" 18 | } 19 | ] 20 | -------------------------------------------------------------------------------- /tests/fixtures/logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloudVE/cloudbridge/dfcc6c10830ba1855b225b90f523fd3091b05045/tests/fixtures/logo.jpg -------------------------------------------------------------------------------- /tests/test_base_helpers.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from cloudbridge.base import helpers as cb_helpers 4 | from cloudbridge.interfaces.exceptions import InvalidParamException 5 | 6 | 7 | class BaseHelpersTestCase(unittest.TestCase): 8 | 9 | _multiprocess_can_split_ = True 10 | 11 | def test_cleanup_action_body_has_no_exception(self): 12 | invoke_order = [""] 13 | 14 | def cleanup_func(): 15 | invoke_order[0] += "cleanup" 16 | 17 | with cb_helpers.cleanup_action(lambda: cleanup_func()): 18 | invoke_order[0] += "body_" 19 | self.assertEqual(invoke_order[0], "body_cleanup") 20 | 21 | def test_cleanup_action_body_has_exception(self): 22 | invoke_order = [""] 23 | 24 | def cleanup_func(): 25 | invoke_order[0] += "cleanup" 26 | 27 | class CustomException(Exception): 28 | pass 29 | 30 | with self.assertRaises(CustomException): 31 | with cb_helpers.cleanup_action(lambda: cleanup_func()): 32 | invoke_order[0] += "body_" 33 | raise CustomException() 34 | self.assertEqual(invoke_order[0], "body_cleanup") 35 | 36 | def test_cleanup_action_cleanup_has_exception(self): 37 | invoke_order = [""] 38 | 39 | def cleanup_func(): 40 | invoke_order[0] += "cleanup" 41 | raise Exception("test") 42 | 43 | with cb_helpers.cleanup_action(lambda: cleanup_func()): 44 | invoke_order[0] += "body_" 45 | self.assertEqual(invoke_order[0], "body_cleanup") 46 | 47 | def test_cleanup_action_body_and_cleanup_has_exception(self): 48 | invoke_order = [""] 49 | 50 | def cleanup_func(): 51 | invoke_order[0] += "cleanup" 52 | raise Exception("test") 53 | 54 | class CustomException(Exception): 55 | pass 56 | 57 | with self.assertRaises(CustomException): 58 | with cb_helpers.cleanup_action(lambda: cleanup_func()): 59 | invoke_order[0] += "body_" 60 | raise CustomException() 61 | self.assertEqual(invoke_order[0], "body_cleanup") 62 | 63 | def test_deprecated_alias_no_rename(self): 64 | param_values = {} 65 | 66 | @cb_helpers.deprecated_alias(old_param='new_param') 67 | def custom_func(new_param=None, old_param=None): 68 | param_values['new_param'] = new_param 69 | param_values['old_param'] = old_param 70 | 71 | custom_func(new_param="hello") 72 | self.assertDictEqual(param_values, 73 | { 74 | 'new_param': "hello", 75 | 'old_param': None 76 | }) 77 | 78 | def test_deprecated_alias_force_rename(self): 79 | param_values = {} 80 | 81 | @cb_helpers.deprecated_alias(old_param='new_param') 82 | def custom_func(new_param=None, old_param=None): 83 | param_values['new_param'] = new_param 84 | param_values['old_param'] = old_param 85 | 86 | custom_func(old_param="hello") 87 | self.assertDictEqual(param_values, 88 | { 89 | 'new_param': "hello", 90 | 'old_param': None 91 | }) 92 | 93 | def test_deprecated_alias_force_conflict(self): 94 | param_values = {} 95 | 96 | @cb_helpers.deprecated_alias(old_param='new_param') 97 | def custom_func(new_param=None, old_param=None): 98 | param_values['new_param'] = new_param 99 | param_values['old_param'] = old_param 100 | 101 | with self.assertRaises(InvalidParamException): 102 | custom_func(new_param="world", old_param="hello") 103 | -------------------------------------------------------------------------------- /tests/test_cloud_factory.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from cloudbridge import factory, interfaces 4 | from cloudbridge.factory import CloudProviderFactory 5 | from cloudbridge.interfaces import TestMockHelperMixin 6 | from cloudbridge.interfaces.provider import CloudProvider 7 | from cloudbridge.providers.aws import AWSCloudProvider 8 | 9 | 10 | class CloudFactoryTestCase(unittest.TestCase): 11 | 12 | _multiprocess_can_split_ = True 13 | 14 | def test_create_provider_valid(self): 15 | # Creating a provider with a known name should return 16 | # a valid implementation 17 | self.assertIsInstance(CloudProviderFactory().create_provider( 18 | factory.ProviderList.AWS, {}), 19 | interfaces.CloudProvider, 20 | "create_provider did not return a valid VM type") 21 | 22 | def test_create_provider_invalid(self): 23 | # Creating a provider with an invalid name should raise a 24 | # NotImplementedError 25 | with self.assertRaises(NotImplementedError): 26 | CloudProviderFactory().create_provider("ec23", {}) 27 | 28 | def test_get_provider_class_valid(self): 29 | # Searching for a provider class with a known name should return a 30 | # valid class 31 | self.assertEqual(CloudProviderFactory().get_provider_class( 32 | factory.ProviderList.AWS), AWSCloudProvider) 33 | 34 | def test_get_provider_class_invalid(self): 35 | # Searching for a provider class with an invalid name should 36 | # return None 37 | self.assertIsNone(CloudProviderFactory().get_provider_class("aws1")) 38 | 39 | def test_find_provider_include_mocks(self): 40 | self.assertTrue( 41 | any(cls for cls 42 | in CloudProviderFactory().get_all_provider_classes() 43 | if issubclass(cls, TestMockHelperMixin)), 44 | "expected to find at least one mock provider") 45 | 46 | def test_find_provider_exclude_mocks(self): 47 | for cls in CloudProviderFactory().get_all_provider_classes( 48 | ignore_mocks=True): 49 | self.assertTrue( 50 | not issubclass(cls, TestMockHelperMixin), 51 | "Did not expect mock but %s implements mock provider" % cls) 52 | 53 | def test_register_provider_class_invalid(self): 54 | # Attempting to register an invalid test class should be ignored 55 | class DummyClass(object): 56 | PROVIDER_ID = 'aws' 57 | 58 | factory = CloudProviderFactory() 59 | factory.register_provider_class(DummyClass) 60 | self.assertTrue(DummyClass not in 61 | factory.get_all_provider_classes()) 62 | 63 | def test_register_provider_class_double(self): 64 | # Attempting to register the same class twice should register second 65 | # instance 66 | class DummyClass(CloudProvider): 67 | PROVIDER_ID = 'aws' 68 | 69 | factory = CloudProviderFactory() 70 | factory.list_providers() 71 | factory.register_provider_class(DummyClass) 72 | self.assertTrue(DummyClass in 73 | factory.get_all_provider_classes()) 74 | self.assertTrue(AWSCloudProvider not in 75 | factory.get_all_provider_classes()) 76 | 77 | def test_register_provider_class_without_id(self): 78 | # Attempting to register a class without a PROVIDER_ID attribute 79 | # should be ignored. 80 | class DummyClass(CloudProvider): 81 | pass 82 | 83 | factory = CloudProviderFactory() 84 | factory.register_provider_class(DummyClass) 85 | self.assertTrue(DummyClass not in 86 | factory.get_all_provider_classes()) 87 | -------------------------------------------------------------------------------- /tests/test_cloud_helpers.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | 3 | import six 4 | 5 | from cloudbridge.base.helpers import get_env 6 | from cloudbridge.base.resources import ClientPagedResultList 7 | from cloudbridge.base.resources import ServerPagedResultList 8 | 9 | from tests.helpers import ProviderTestBase 10 | 11 | 12 | class DummyResult(object): 13 | 14 | def __init__(self, objid, name): 15 | self.id = objid 16 | self.name = name 17 | 18 | def __repr__(self): 19 | return "%s (%s)" % (self.id, self.name) 20 | 21 | 22 | class CloudHelpersTestCase(ProviderTestBase): 23 | 24 | _multiprocess_can_split_ = True 25 | 26 | def setUp(self): 27 | super(CloudHelpersTestCase, self).setUp() 28 | self.objects = [DummyResult(1, "One"), 29 | DummyResult(2, "Two"), 30 | DummyResult(3, "Three"), 31 | DummyResult(4, "Four"), 32 | ] 33 | 34 | def test_client_paged_result_list(self): 35 | objects = self.objects 36 | 37 | # A list with limit=2 and marker=None 38 | results = ClientPagedResultList(self.provider, objects, 2, None) 39 | self.assertListEqual(results, list(itertools.islice(objects, 2))) 40 | self.assertEqual(results.marker, objects[1].id) 41 | self.assertTrue(results.is_truncated) 42 | self.assertTrue(results.supports_total) 43 | self.assertEqual(results.total_results, 4) 44 | self.assertEqual(results.data, objects) 45 | 46 | # A list with limit=2 and marker=2 47 | results = ClientPagedResultList(self.provider, objects, 2, 2) 48 | self.assertListEqual(results, list(itertools.islice(objects, 2, 4))) 49 | self.assertEqual(results.marker, None) 50 | self.assertFalse(results.is_truncated) 51 | self.assertTrue(results.supports_total) 52 | self.assertEqual(results.total_results, 4) 53 | self.assertEqual(results.data, objects) 54 | 55 | # A list with limit=2 and marker=3 56 | results = ClientPagedResultList(self.provider, objects, 2, 3) 57 | self.assertListEqual(results, list(itertools.islice(objects, 3, 4))) 58 | self.assertFalse(results.is_truncated) 59 | self.assertEqual(results.marker, None) 60 | self.assertEqual(results.data, objects) 61 | 62 | self.assertFalse(results.supports_server_paging, "Client paged result" 63 | " lists should return False for server paging.") 64 | 65 | def test_server_paged_result_list(self): 66 | 67 | objects = list(itertools.islice(self.objects, 2)) 68 | results = ServerPagedResultList(is_truncated=True, 69 | marker=objects[-1].id, 70 | supports_total=True, 71 | total=2, data=objects) 72 | self.assertTrue(results.is_truncated) 73 | self.assertListEqual(results, objects) 74 | self.assertEqual(results.marker, objects[-1].id) 75 | self.assertTrue(results.supports_total) 76 | self.assertEqual(results.total_results, 2) 77 | self.assertTrue(results.supports_server_paging, "Server paged result" 78 | " lists should return True for server paging.") 79 | with self.assertRaises(NotImplementedError): 80 | results.data 81 | 82 | def test_type_validation(self): 83 | # Make sure internal type checking implementation properly sets types. 84 | self.provider.config['text_type_check'] = 'test-text' 85 | # pylint:disable=protected-access 86 | config_value = self.provider._get_config_value('text_type_check', None) 87 | self.assertIsInstance(config_value, six.string_types) 88 | 89 | # pylint:disable=protected-access 90 | none_value = self.provider._get_config_value( 91 | 'some_config_value', get_env('MISSING_ENV', None)) 92 | self.assertIsNone(none_value) 93 | 94 | # pylint:disable=protected-access 95 | bool_value = self.provider._get_config_value( 96 | 'some_config_value', get_env('MISSING_ENV', True)) 97 | self.assertIsInstance(bool_value, bool) 98 | 99 | # pylint:disable=protected-access 100 | int_value = self.provider._get_config_value( 101 | 'default_result_limit', None) 102 | self.assertIsInstance(int_value, int) 103 | -------------------------------------------------------------------------------- /tests/test_dns_service.py: -------------------------------------------------------------------------------- 1 | from cloudbridge.base import helpers as cb_helpers 2 | from cloudbridge.interfaces.resources import DnsRecord 3 | from cloudbridge.interfaces.resources import DnsRecordType 4 | from cloudbridge.interfaces.resources import DnsZone 5 | 6 | from tests import helpers 7 | from tests.helpers import ProviderTestBase 8 | from tests.helpers import standard_interface_tests as sit 9 | 10 | 11 | class CloudDnsServiceTestCase(ProviderTestBase): 12 | 13 | _multiprocess_can_split_ = True 14 | 15 | @helpers.skipIfNoService(['dns.host_zones']) 16 | def test_crud_dns_zones(self): 17 | 18 | def create_dns_zone(name): 19 | if name: 20 | name = name + ".com." 21 | return self.provider.dns.host_zones.create( 22 | name, "admin@cloudve.org") 23 | 24 | def cleanup_dns_zone(dns_zone): 25 | if dns_zone: 26 | dns_zone.delete() 27 | 28 | def test_zone_props(dns_zone): 29 | self.assertEqual(dns_zone.admin_email, "admin@cloudve.org") 30 | 31 | sit.check_crud(self, self.provider.dns.host_zones, DnsZone, 32 | "cb-crudzone", create_dns_zone, cleanup_dns_zone, 33 | skip_name_check=True, extra_test_func=test_zone_props) 34 | 35 | @helpers.skipIfNoService(['dns.host_zones']) 36 | def test_create_dns_zones_not_fully_qualified(self): 37 | zone_name = "cb-dnszonenfq-{0}.com".format(helpers.get_uuid()) 38 | test_zone = None 39 | with cb_helpers.cleanup_action(lambda: test_zone.delete()): 40 | # If zone name is not fully qualified, it should automatically be 41 | # handled 42 | test_zone = self.provider.dns.host_zones.create( 43 | zone_name, "admin@cloudve.org") 44 | 45 | @helpers.skipIfNoService(['dns.host_zones']) 46 | def test_crud_dns_record(self): 47 | test_zone = None 48 | zone_name = "cb-dnsrec-{0}.com.".format(helpers.get_uuid()) 49 | 50 | def create_dns_rec(name): 51 | if name: 52 | name = name + "." + zone_name 53 | else: 54 | name = zone_name 55 | return test_zone.records.create( 56 | name, DnsRecordType.A, data='10.1.1.1') 57 | 58 | def cleanup_dns_rec(dns_rec): 59 | if dns_rec: 60 | dns_rec.delete() 61 | 62 | with cb_helpers.cleanup_action(lambda: test_zone.delete()): 63 | test_zone = self.provider.dns.host_zones.create( 64 | zone_name, "admin@cloudve.org") 65 | sit.check_crud(self, test_zone.records, DnsRecord, 66 | "cb-dnsrec", create_dns_rec, 67 | cleanup_dns_rec, skip_name_check=True) 68 | 69 | @helpers.skipIfNoService(['dns.host_zones']) 70 | def test_create_wildcard_dns_record(self): 71 | test_zone = None 72 | zone_name = "cb-dnswild-{0}.com.".format(helpers.get_uuid()) 73 | 74 | with cb_helpers.cleanup_action(lambda: test_zone.delete()): 75 | test_zone = self.provider.dns.host_zones.create( 76 | zone_name, "admin@cloudve.org") 77 | test_rec = None 78 | with cb_helpers.cleanup_action(lambda: test_rec.delete()): 79 | test_rec = test_zone.records.create( 80 | "*.cb-wildcard." + zone_name, DnsRecordType.A, 81 | data='10.1.1.1') 82 | 83 | @helpers.skipIfNoService(['dns.host_zones']) 84 | def test_dns_record_properties(self): 85 | test_zone = None 86 | zone_name = "cb-recprop-{0}.com.".format(helpers.get_uuid()) 87 | 88 | with cb_helpers.cleanup_action(lambda: test_zone.delete()): 89 | test_zone = self.provider.dns.host_zones.create( 90 | zone_name, "admin@cloudve.org") 91 | test_rec = None 92 | 93 | with cb_helpers.cleanup_action(lambda: test_rec.delete()): 94 | zone_name = "subdomain." + zone_name 95 | test_rec = test_zone.records.create( 96 | zone_name, DnsRecordType.CNAME, data='hello.com.', ttl=500) 97 | self.assertEqual(test_rec.zone_id, test_zone.id) 98 | self.assertEqual(test_rec.type, DnsRecordType.CNAME) 99 | self.assertEqual(test_rec.data, ['hello.com.']) 100 | self.assertEqual(test_rec.ttl, 500) 101 | 102 | # Check setting data array 103 | test_rec2 = None 104 | with cb_helpers.cleanup_action(lambda: test_rec2.delete()): 105 | MX_DATA = ['10 mx1.hello.com.', '20 mx2.hello.com.'] 106 | test_rec2 = test_zone.records.create( 107 | zone_name, DnsRecordType.MX, data=MX_DATA, ttl=300) 108 | self.assertEqual(test_rec2.zone_id, test_zone.id) 109 | self.assertEqual(test_rec2.type, DnsRecordType.MX) 110 | self.assertSetEqual(set(test_rec2.data), set(MX_DATA)) 111 | self.assertEqual(test_rec2.ttl, 300) 112 | 113 | @helpers.skipIfNoService(['dns.host_zones']) 114 | def test_create_dns_rec_not_fully_qualified(self): 115 | test_zone = None 116 | root_zone_name = "cb-recprop-{0}.com.".format(helpers.get_uuid()) 117 | 118 | with cb_helpers.cleanup_action(lambda: test_zone.delete()): 119 | test_zone = self.provider.dns.host_zones.create( 120 | root_zone_name, "admin@cloudve.org") 121 | test_rec = None 122 | 123 | with cb_helpers.cleanup_action(lambda: test_rec.delete()): 124 | zone_name = "subdomain." + root_zone_name 125 | test_rec = test_zone.records.create( 126 | zone_name, DnsRecordType.CNAME, data='hello.com', ttl=500) 127 | 128 | with cb_helpers.cleanup_action(lambda: test_rec.delete()): 129 | test_rec = test_zone.records.create( 130 | root_zone_name, DnsRecordType.MX, 131 | data=['10 mx1.hello.com', '20 mx2.hello.com'], ttl=500) 132 | -------------------------------------------------------------------------------- /tests/test_image_service.py: -------------------------------------------------------------------------------- 1 | from cloudbridge.base import helpers as cb_helpers 2 | from cloudbridge.interfaces import MachineImageState 3 | from cloudbridge.interfaces.resources import Instance 4 | from cloudbridge.interfaces.resources import MachineImage 5 | 6 | from tests import helpers 7 | from tests.helpers import ProviderTestBase 8 | from tests.helpers import standard_interface_tests as sit 9 | 10 | 11 | class CloudImageServiceTestCase(ProviderTestBase): 12 | 13 | _multiprocess_can_split_ = True 14 | 15 | @helpers.skipIfNoService(['compute.images']) 16 | def test_storage_services_event_pattern(self): 17 | self.assertEqual(self.provider.compute.images._service_event_pattern, 18 | "provider.compute.images", 19 | "Event pattern for {} service should be '{}', " 20 | "but found '{}'.".format("images", 21 | "provider.compute.images", 22 | self.provider.compute.images. 23 | _service_event_pattern)) 24 | 25 | @helpers.skipIfNoService(['compute.images', 'networking.networks', 26 | 'compute.instances']) 27 | def test_create_and_list_image(self): 28 | instance_label = "cb-crudimage-{0}".format(helpers.get_uuid()) 29 | img_inst_label = "cb-crudimage-{0}".format(helpers.get_uuid()) 30 | 31 | # Declare these variables and late binding will allow 32 | # the cleanup method access to the most current values 33 | test_instance = None 34 | subnet = None 35 | 36 | def create_img(label): 37 | return test_instance.create_image(label=label) 38 | 39 | def cleanup_img(img): 40 | if img: 41 | img.delete() 42 | img.wait_for( 43 | [MachineImageState.UNKNOWN, MachineImageState.ERROR]) 44 | img.refresh() 45 | self.assertTrue( 46 | img.state == MachineImageState.UNKNOWN, 47 | "MachineImage.state must be unknown when refreshing after " 48 | "a delete but got %s" 49 | % img.state) 50 | 51 | def extra_tests(img): 52 | # check image size 53 | img.refresh() 54 | self.assertGreater(img.min_disk, 0, "Minimum disk" 55 | " size required by image is invalid") 56 | create_instance_from_image(img) 57 | 58 | def create_instance_from_image(img): 59 | img_instance = None 60 | with cb_helpers.cleanup_action( 61 | lambda: helpers.cleanup_test_resources(img_instance)): 62 | img_instance = self.provider.compute.instances.create( 63 | img_inst_label, img, 64 | helpers.get_provider_test_data(self.provider, 'vm_type'), 65 | subnet=subnet) 66 | img_instance.wait_till_ready() 67 | self.assertIsInstance(img_instance, Instance) 68 | self.assertEqual( 69 | img_instance.label, img_inst_label, 70 | "Instance label {0} is not equal to the expected label" 71 | " {1}".format(img_instance.label, img_inst_label)) 72 | image_id = img.id 73 | self.assertEqual(img_instance.image_id, image_id, 74 | "Image id {0} is not equal to the expected id" 75 | " {1}".format(img_instance.image_id, 76 | image_id)) 77 | self.assertIsInstance(img_instance.public_ips, list) 78 | if img_instance.public_ips: 79 | self.assertTrue( 80 | img_instance.public_ips[0], 81 | "public ip should contain a" 82 | " valid value if a list of public_ips exist") 83 | self.assertIsInstance(img_instance.private_ips, list) 84 | self.assertTrue(img_instance.private_ips[0], 85 | "private ip should" 86 | " contain a valid value") 87 | 88 | with cb_helpers.cleanup_action(lambda: helpers.cleanup_test_resources( 89 | test_instance)): 90 | subnet = helpers.get_or_create_default_subnet( 91 | self.provider) 92 | test_instance = helpers.get_test_instance( 93 | self.provider, instance_label, subnet=subnet) 94 | sit.check_crud(self, self.provider.compute.images, MachineImage, 95 | "cb-listimg", create_img, cleanup_img, 96 | extra_test_func=extra_tests) 97 | -------------------------------------------------------------------------------- /tests/test_interface.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import cloudbridge 4 | from cloudbridge import interfaces 5 | from cloudbridge.base import helpers as cb_helpers 6 | from cloudbridge.factory import CloudProviderFactory 7 | from cloudbridge.interfaces import TestMockHelperMixin 8 | from cloudbridge.interfaces.exceptions import ProviderConnectionException 9 | 10 | from tests import helpers 11 | from tests.helpers import ProviderTestBase 12 | 13 | 14 | class CloudInterfaceTestCase(ProviderTestBase): 15 | 16 | _multiprocess_can_split_ = True 17 | 18 | def test_name_property(self): 19 | # Name should always return a value and should not raise an exception 20 | assert self.provider.name 21 | 22 | def test_has_service_valid_service_type(self): 23 | # has_service with a valid service type should return 24 | # a boolean and raise no exceptions 25 | for key, value in interfaces.CloudServiceType.__dict__.items(): 26 | if not key.startswith("__"): 27 | self.provider.has_service(value) 28 | 29 | def test_has_service_invalid_service_type(self): 30 | # has_service with an invalid service type should return False 31 | self.assertFalse( 32 | self.provider.has_service("NON_EXISTENT_SERVICE"), 33 | "has_service should not return True for a non-existent service") 34 | 35 | def test_library_version(self): 36 | # Check that the library version can be retrieved. 37 | self.assertIsNotNone(cloudbridge.get_version(), 38 | "Did not get library version.") 39 | 40 | def test_authenticate_success(self): 41 | self.assertTrue(self.provider.authenticate()) 42 | 43 | def test_authenticate_failure(self): 44 | if isinstance(self.provider, TestMockHelperMixin): 45 | raise unittest.SkipTest( 46 | "Mock providers are not expected to" 47 | " authenticate correctly") 48 | 49 | # Mock up test by clearing credentials on a per provider basis 50 | cloned_config = self.provider.config.copy() 51 | if self.provider.PROVIDER_ID == 'aws': 52 | cloned_config['aws_access_key'] = "dummy_a_key" 53 | cloned_config['aws_secret_key'] = "dummy_s_key" 54 | elif self.provider.PROVIDER_ID == 'openstack': 55 | cloned_config['os_username'] = "cb_dummy" 56 | cloned_config['os_password'] = "cb_dummy" 57 | elif self.provider.PROVIDER_ID == 'azure': 58 | cloned_config['azure_subscription_id'] = "cb_dummy" 59 | elif self.provider.PROVIDER_ID == 'gcp': 60 | cloned_config['gcp_service_creds_dict'] = {'dummy': 'dict'} 61 | 62 | with self.assertRaises(ProviderConnectionException): 63 | cloned_provider = CloudProviderFactory().create_provider( 64 | self.provider.PROVIDER_ID, cloned_config) 65 | cloned_provider.authenticate() 66 | 67 | def test_provider_zone_in_region(self): 68 | cloned_config = self.provider.config.copy() 69 | # Just a simpler way set zone to null for any provider 70 | # instead of doing it individually for each provider 71 | cloned_config['aws_zone_name'] = None 72 | cloned_config['azure_zone_name'] = None 73 | cloned_config['gcp_zone_name'] = None 74 | cloned_config['os_zone_name'] = None 75 | cloned_provider = CloudProviderFactory().create_provider( 76 | self.provider.PROVIDER_ID, cloned_config) 77 | region = cloned_provider.compute.regions.get( 78 | cloned_provider.region_name) 79 | matches = [zone.name for zone in region.zones 80 | if zone.name == cloned_provider.zone_name] 81 | # FIXME: GCP always requires a zone, so skip for now 82 | if self.provider.PROVIDER_ID != 'gcp': 83 | self.assertListEqual([cloned_provider.zone_name], matches) 84 | 85 | def test_provider_always_has_zone(self): 86 | cloned_config = self.provider.config.copy() 87 | # Just a simpler way set zone to null for any provider 88 | # instead of doing it individually for each provider 89 | cloned_config['aws_zone_name'] = None 90 | cloned_config['azure_zone_name'] = None 91 | cloned_config['gcp_zone_name'] = None 92 | cloned_config['os_zone_name'] = None 93 | cloned_provider = CloudProviderFactory().create_provider( 94 | self.provider.PROVIDER_ID, cloned_config) 95 | # FIXME: GCP always requires a zone, so skip for now 96 | if self.provider.PROVIDER_ID != 'gcp': 97 | self.assertIsNotNone(cloned_provider.zone_name) 98 | 99 | def test_clone_provider_zone(self): 100 | for zone in list(self.provider.compute.regions.current.zones)[:2]: 101 | cloned_provider = self.provider.clone(zone=zone) 102 | test_vol = None 103 | # Currently, volumes are the cheapest object that's actually 104 | # cross-zonal for all providers 105 | with cb_helpers.cleanup_action(lambda: test_vol.delete()): 106 | label = "cb-attachvol-{0}".format(helpers.get_uuid()) 107 | test_vol = cloned_provider.storage.volumes.create(label, 1) 108 | self.assertEqual(test_vol.zone_id, zone.id) 109 | -------------------------------------------------------------------------------- /tests/test_middleware_system.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from pyeventsystem.events import SimpleEventDispatcher 4 | from pyeventsystem.middleware import SimpleMiddlewareManager 5 | from pyeventsystem.middleware import implement 6 | 7 | from cloudbridge.base.middleware import EventDebugLoggingMiddleware 8 | from cloudbridge.base.middleware import ExceptionWrappingMiddleware 9 | from cloudbridge.interfaces.exceptions import CloudBridgeBaseException 10 | from cloudbridge.interfaces.exceptions import \ 11 | InvalidConfigurationException 12 | 13 | from .helpers import skipIfPython 14 | 15 | 16 | class ExceptionWrappingMiddlewareTestCase(unittest.TestCase): 17 | 18 | _multiprocess_can_split_ = True 19 | 20 | def test_unknown_exception_is_wrapped(self): 21 | EVENT_NAME = "an.exceptional.event" 22 | 23 | class SomeDummyClass(object): 24 | 25 | @implement(event_pattern=EVENT_NAME, priority=2500) 26 | def raise_a_non_cloudbridge_exception(self, *args, **kwargs): 27 | raise Exception("Some unhandled exception") 28 | 29 | dispatcher = SimpleEventDispatcher() 30 | manager = SimpleMiddlewareManager(dispatcher) 31 | middleware = ExceptionWrappingMiddleware() 32 | manager.add(middleware) 33 | 34 | # no exception should be raised when there's no next handler 35 | dispatcher.dispatch(self, EVENT_NAME) 36 | 37 | some_obj = SomeDummyClass() 38 | manager.add(some_obj) 39 | 40 | with self.assertRaises(CloudBridgeBaseException): 41 | dispatcher.dispatch(self, EVENT_NAME) 42 | 43 | def test_cloudbridge_exception_is_passed_through(self): 44 | EVENT_NAME = "an.exceptional.event" 45 | 46 | class SomeDummyClass(object): 47 | 48 | @implement(event_pattern=EVENT_NAME, priority=2500) 49 | def raise_a_cloudbridge_exception(self, *args, **kwargs): 50 | raise InvalidConfigurationException() 51 | 52 | dispatcher = SimpleEventDispatcher() 53 | manager = SimpleMiddlewareManager(dispatcher) 54 | some_obj = SomeDummyClass() 55 | manager.add(some_obj) 56 | middleware = ExceptionWrappingMiddleware() 57 | manager.add(middleware) 58 | 59 | with self.assertRaises(InvalidConfigurationException): 60 | dispatcher.dispatch(self, EVENT_NAME) 61 | 62 | 63 | class EventDebugLoggingMiddlewareTestCase(unittest.TestCase): 64 | 65 | _multiprocess_can_split_ = True 66 | 67 | # Only python 3 has assertLogs support 68 | @skipIfPython("<", 3, 0) 69 | def test_messages_logged(self): 70 | EVENT_NAME = "an.exceptional.event" 71 | 72 | class SomeDummyClass(object): 73 | 74 | @implement(event_pattern=EVENT_NAME, priority=2500) 75 | def return_some_value(self, *args, **kwargs): 76 | return "hello world" 77 | 78 | dispatcher = SimpleEventDispatcher() 79 | manager = SimpleMiddlewareManager(dispatcher) 80 | middleware = EventDebugLoggingMiddleware() 81 | manager.add(middleware) 82 | some_obj = SomeDummyClass() 83 | manager.add(some_obj) 84 | 85 | with self.assertLogs('cloudbridge.base.middleware', 86 | level='DEBUG') as cm: 87 | dispatcher.dispatch(self, EVENT_NAME, 88 | "named_param", keyword_param="hello") 89 | self.assertTrue( 90 | "named_param" in cm.output[0] 91 | and "keyword_param" in cm.output[0] and "hello" in cm.output[0], 92 | "Log output {0} not as expected".format(cm.output[0])) 93 | self.assertTrue( 94 | "hello world" in cm.output[1], 95 | "Log output {0} does not contain result".format(cm.output[1])) 96 | -------------------------------------------------------------------------------- /tests/test_object_life_cycle.py: -------------------------------------------------------------------------------- 1 | from cloudbridge.base import helpers as cb_helpers 2 | from cloudbridge.interfaces import VolumeState 3 | from cloudbridge.interfaces.exceptions import WaitStateException 4 | 5 | from tests import helpers 6 | from tests.helpers import ProviderTestBase 7 | 8 | 9 | class CloudObjectLifeCycleTestCase(ProviderTestBase): 10 | 11 | _multiprocess_can_split_ = True 12 | 13 | @helpers.skipIfNoService(['storage.volumes']) 14 | def test_object_life_cycle(self): 15 | # Test object life cycle methods by using a volume. 16 | label = "cb-objlifecycle-{0}".format(helpers.get_uuid()) 17 | test_vol = None 18 | with cb_helpers.cleanup_action(lambda: test_vol.delete()): 19 | test_vol = self.provider.storage.volumes.create( 20 | label, 1) 21 | 22 | # Waiting for an invalid timeout should raise an exception 23 | with self.assertRaises(AssertionError): 24 | test_vol.wait_for([VolumeState.ERROR], timeout=-1, interval=1) 25 | with self.assertRaises(AssertionError): 26 | test_vol.wait_for([VolumeState.ERROR], timeout=1, interval=-1) 27 | 28 | # If interval < timeout, an exception should be raised 29 | with self.assertRaises(AssertionError): 30 | test_vol.wait_for([VolumeState.ERROR], timeout=10, interval=20) 31 | 32 | test_vol.wait_till_ready() 33 | # Hitting a terminal state should raise an exception 34 | with self.assertRaises(WaitStateException): 35 | test_vol.wait_for([VolumeState.ERROR], 36 | terminal_states=[VolumeState.AVAILABLE]) 37 | 38 | # Hitting the timeout should raise an exception 39 | with self.assertRaises(WaitStateException): 40 | test_vol.wait_for([VolumeState.ERROR], timeout=0, interval=0) 41 | -------------------------------------------------------------------------------- /tests/test_region_service.py: -------------------------------------------------------------------------------- 1 | import six 2 | 3 | from cloudbridge.interfaces import Region 4 | 5 | from tests import helpers 6 | from tests.helpers import ProviderTestBase 7 | from tests.helpers import standard_interface_tests as sit 8 | 9 | 10 | class CloudRegionServiceTestCase(ProviderTestBase): 11 | 12 | _multiprocess_can_split_ = True 13 | 14 | @helpers.skipIfNoService(['compute.regions']) 15 | def test_storage_services_event_pattern(self): 16 | # pylint:disable=protected-access 17 | self.assertEqual( 18 | self.provider.compute.regions._service_event_pattern, 19 | "provider.compute.regions", 20 | "Event pattern for {} service should be '{}', " 21 | "but found '{}'.".format("regions", 22 | "provider.compute.regions", 23 | self.provider.compute.regions. 24 | _service_event_pattern)) 25 | 26 | @helpers.skipIfNoService(['compute.regions']) 27 | def test_get_and_list_regions(self): 28 | regions = list(self.provider.compute.regions) 29 | sit.check_standard_behaviour( 30 | self, self.provider.compute.regions, regions[-1]) 31 | 32 | for region in regions: 33 | self.assertIsInstance( 34 | region, 35 | Region, 36 | "regions.list() should return a cloudbridge Region") 37 | self.assertTrue( 38 | region.name, 39 | "Region name should be a non-empty string") 40 | 41 | @helpers.skipIfNoService(['compute.regions']) 42 | def test_regions_unique(self): 43 | regions = self.provider.compute.regions.list() 44 | unique_regions = set([region.id for region in regions]) 45 | self.assertTrue(len(regions) == len(list(unique_regions))) 46 | 47 | @helpers.skipIfNoService(['compute.regions']) 48 | def test_current_region(self): 49 | current_region = self.provider.compute.regions.current 50 | self.assertIsInstance(current_region, Region) 51 | self.assertTrue(current_region in self.provider.compute.regions) 52 | 53 | @helpers.skipIfNoService(['compute.regions']) 54 | def test_zones(self): 55 | zone_find_count = 0 56 | test_zone = helpers.get_provider_test_data(self.provider, "placement") 57 | for region in self.provider.compute.regions: 58 | self.assertTrue(region.name) 59 | for zone in region.zones: 60 | self.assertTrue(zone.id) 61 | self.assertTrue(zone.name) 62 | self.assertTrue(zone.region_name is None or 63 | isinstance(zone.region_name, 64 | six.string_types)) 65 | if test_zone == zone.name: 66 | zone_find_count += 1 67 | # zone info cannot be repeated between regions 68 | self.assertEqual(zone_find_count, 1) 69 | -------------------------------------------------------------------------------- /tests/test_vm_types_service.py: -------------------------------------------------------------------------------- 1 | import six 2 | 3 | from tests import helpers 4 | from tests.helpers import ProviderTestBase 5 | from tests.helpers import standard_interface_tests as sit 6 | 7 | 8 | class CloudVMTypeServiceTestCase(ProviderTestBase): 9 | 10 | _multiprocess_can_split_ = True 11 | 12 | @helpers.skipIfNoService(['compute.vm_types']) 13 | def test_storage_services_event_pattern(self): 14 | self.assertEqual(self.provider.compute.vm_types._service_event_pattern, 15 | "provider.compute.vm_types", 16 | "Event pattern for {} service should be '{}', " 17 | "but found '{}'.".format("vm_types", 18 | "provider.compute.vm_types", 19 | self.provider.compute. 20 | vm_types. 21 | _service_event_pattern)) 22 | 23 | @helpers.skipIfNoService(['compute.vm_types']) 24 | def test_vm_type_properties(self): 25 | 26 | for vm_type in self.provider.compute.vm_types: 27 | sit.check_repr(self, vm_type) 28 | self.assertIsNotNone( 29 | vm_type.id, 30 | "VMType id must have a value") 31 | self.assertIsNotNone( 32 | vm_type.name, 33 | "VMType name must have a value") 34 | self.assertTrue( 35 | vm_type.family is None or isinstance( 36 | vm_type.family, 37 | six.string_types), 38 | "VMType family must be None or a" 39 | " string but is: {0}".format(vm_type.family)) 40 | self.assertTrue( 41 | vm_type.vcpus is None or ( 42 | isinstance(vm_type.vcpus, six.integer_types) and 43 | vm_type.vcpus >= 0), 44 | "VMType vcpus must be None or a positive integer but is: {0}" 45 | .format(vm_type.vcpus)) 46 | self.assertTrue( 47 | vm_type.ram is None or vm_type.ram >= 0, 48 | "VMType ram must be None or a positive number") 49 | self.assertTrue( 50 | vm_type.size_root_disk is None or 51 | vm_type.size_root_disk >= 0, 52 | "VMType size_root_disk must be None or a positive number" 53 | " but is: {0}".format(vm_type.size_root_disk)) 54 | self.assertTrue( 55 | vm_type.size_ephemeral_disks is None or 56 | vm_type.size_ephemeral_disks >= 0, 57 | "VMType size_ephemeral_disk must be None or a positive" 58 | " number") 59 | self.assertTrue( 60 | isinstance(vm_type.num_ephemeral_disks, 61 | six.integer_types) and 62 | vm_type.num_ephemeral_disks >= 0, 63 | "VMType num_ephemeral_disks must be None or a positive" 64 | " number") 65 | self.assertTrue( 66 | vm_type.size_total_disk is None or 67 | vm_type.size_total_disk >= 0, 68 | "VMType size_total_disk must be None or a positive" 69 | " number") 70 | self.assertTrue( 71 | vm_type.extra_data is None or isinstance( 72 | vm_type.extra_data, dict), 73 | "VMType extra_data must be None or a dict") 74 | 75 | @helpers.skipIfNoService(['compute.vm_types']) 76 | def test_vm_types_standard(self): 77 | # Searching for an instance by name should return an 78 | # VMType object and searching for a non-existent 79 | # object should return an empty iterator 80 | vm_type_name = helpers.get_provider_test_data( 81 | self.provider, 82 | "vm_type") 83 | vm_type = self.provider.compute.vm_types.find( 84 | name=vm_type_name)[0] 85 | 86 | sit.check_standard_behaviour( 87 | self, self.provider.compute.vm_types, vm_type) 88 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Tox (http://tox.testrun.org/) is a tool for running tests 2 | # in multiple virtualenvs. This configuration file will run the 3 | # test suite on all supported python versions and providers. 4 | # To use it, "pip install tox" and then run "tox" from this directory. 5 | # You will have to set all required environment variables (below) before 6 | # running the tests. 7 | 8 | [tox] 9 | envlist = {py3.10,pypy}-{aws,azure,gcp,openstack,mock},lint 10 | 11 | [testenv] 12 | commands = # see setup.cfg for options sent to pytest and coverage 13 | coverage run --source=cloudbridge -m pytest -n 5 tests/ -v {posargs} 14 | setenv = 15 | # Fix for moto import issue: https://github.com/travis-ci/travis-ci/issues/7940 16 | BOTO_CONFIG=/dev/null 17 | aws: CB_TEST_PROVIDER=aws 18 | azure: CB_TEST_PROVIDER=azure 19 | gcp: CB_TEST_PROVIDER=gcp 20 | openstack: CB_TEST_PROVIDER=openstack 21 | mock: CB_TEST_PROVIDER=mock 22 | # https://github.com/nedbat/coveragepy/issues/883#issuecomment-650562896 23 | COVERAGE_FILE=.coverage.{envname} 24 | passenv = 25 | PYTHONUNBUFFERED 26 | aws: CB_IMAGE_AWS 27 | aws: CB_VM_TYPE_AWS 28 | aws: CB_PLACEMENT_AWS 29 | aws: AWS_ACCESS_KEY 30 | aws: AWS_SECRET_KEY 31 | azure: CB_IMAGE_AZURE 32 | azure: CB_VM_TYPE_AZURE 33 | azure: AZURE_SUBSCRIPTION_ID 34 | azure: AZURE_CLIENT_ID 35 | azure: AZURE_SECRET 36 | azure: AZURE_TENANT 37 | azure: AZURE_REGION_NAME 38 | azure: AZURE_RESOURCE_GROUP 39 | azure: AZURE_STORAGE_ACCOUNT 40 | azure: AZURE_VM_DEFAULT_USER_NAME 41 | azure: AZURE_PUBLIC_KEY_STORAGE_TABLE_NAME 42 | gcp: CB_IMAGE_GCP 43 | gcp: CB_VM_TYPE_GCP 44 | gcp: CB_PLACEMENT_GCP 45 | gcp: GCP_DEFAULT_REGION 46 | gcp: GCP_DEFAULT_ZONE 47 | gcp: GCP_PROJECT_NAME 48 | gcp: GCP_SERVICE_CREDS_FILE 49 | gcp: GCP_SERVICE_CREDS_DICT 50 | openstack: CB_IMAGE_OS 51 | openstack: CB_VM_TYPE_OS 52 | openstack: CB_PLACEMENT_OS 53 | openstack: OS_AUTH_URL 54 | openstack: OS_PASSWORD 55 | openstack: OS_PROJECT_NAME 56 | openstack: OS_TENANT_NAME 57 | openstack: OS_USERNAME 58 | openstack: OS_REGION_NAME 59 | openstack: OS_USER_DOMAIN_NAME 60 | openstack: OS_PROJECT_DOMAIN_NAME 61 | openstack: NOVA_SERVICE_NAME 62 | openstack: OS_APPLICATION_CREDENTIAL_ID 63 | openstack: OS_APPLICATION_CREDENTIAL_SECRET 64 | mock: CB_IMAGE_AWS 65 | mock: CB_VM_TYPE_AWS 66 | mock: CB_PLACEMENT_AWS 67 | mock: AWS_ACCESS_KEY 68 | mock: AWS_SECRET_KEY 69 | deps = 70 | -rrequirements.txt 71 | coverage 72 | pytest-xdist 73 | 74 | [testenv:lint] 75 | commands = flake8 cloudbridge tests setup.py 76 | deps = flake8 77 | --------------------------------------------------------------------------------