├── docs ├── source │ ├── _static │ │ └── readme.md │ ├── index.rst │ └── conf.py ├── Makefile └── make.bat ├── tests ├── __init__.py ├── unittest_base.py ├── test_rule_factory.py ├── test_table_storage.py ├── test_config_generator.py ├── test_scheduler.py ├── test_resource_service_factory.py ├── test_account_service_factory.py ├── test_resource_storage_factory.py ├── test_storage_container_factory.py ├── test_example_rules.py ├── test_resource_scanner.py ├── test_resource_storage.py ├── test_resource_tagger.py └── test_integration.py ├── cloud_scanner ├── settings.py ├── config │ ├── __init__.py │ ├── configuration.py │ └── process_config.py ├── helpers │ ├── __init__.py │ ├── functions.py │ └── entry_storage.py ├── rules │ ├── __init__.py │ └── example_rules.py ├── __init__.py ├── services │ ├── __init__.py │ ├── resource_storage.py │ ├── task_scheduler.py │ ├── resource_scanner.py │ └── resource_tagger.py ├── version.py ├── simulators │ ├── __init__.py │ ├── account_service_simulator.py │ ├── queue_simulator.py │ ├── table_storage_simulator.py │ ├── resource_service_simulator.py │ └── container_storage_simulator.py └── contracts │ ├── account_service.py │ ├── storage_container.py │ ├── __init__.py │ ├── queue.py │ ├── rule.py │ ├── table_storage.py │ ├── resource_service.py │ ├── cloud_config_reader.py │ ├── account_service_factory.py │ ├── rule_factory.py │ ├── resource_storage_factory.py │ ├── resource_service_factory.py │ ├── storage_container_factory.py │ ├── tag_update_rule.py │ ├── queue_factory.py │ ├── cloud_config_generator.py │ └── resource.py ├── requirements.txt ├── Makefile ├── setup.py ├── azure-pipelines.yml ├── LICENSE ├── .gitignore └── README.md /docs/source/_static/readme.md: -------------------------------------------------------------------------------- 1 | This file is required for sphinx build -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | from .unittest_base import TestCase, FakeQueueMessage -------------------------------------------------------------------------------- /cloud_scanner/settings.py: -------------------------------------------------------------------------------- 1 | from dotenv import load_dotenv 2 | load_dotenv() 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | python-dotenv 2 | pytest 3 | pytest-cov 4 | flake8 5 | click 6 | Sphinx -------------------------------------------------------------------------------- /cloud_scanner/config/__init__.py: -------------------------------------------------------------------------------- 1 | from .configuration import Config 2 | from .process_config import ProcessConfig 3 | -------------------------------------------------------------------------------- /cloud_scanner/helpers/__init__.py: -------------------------------------------------------------------------------- 1 | from .entry_storage import EntryOperations 2 | from .functions import batch_list 3 | -------------------------------------------------------------------------------- /cloud_scanner/rules/__init__.py: -------------------------------------------------------------------------------- 1 | from .example_rules import ( 2 | ExampleRule1, ExampleRule2, ExampleRule3, ExampleRule4 3 | ) 4 | -------------------------------------------------------------------------------- /cloud_scanner/__init__.py: -------------------------------------------------------------------------------- 1 | from . import config, contracts, helpers, rules, services, settings 2 | from .version import __version__ 3 | -------------------------------------------------------------------------------- /cloud_scanner/services/__init__.py: -------------------------------------------------------------------------------- 1 | from .resource_tagger import ResourceTagger 2 | from .resource_scanner import ResourceScanner 3 | from .resource_storage import ResourceStorage 4 | from .task_scheduler import TaskScheduler 5 | -------------------------------------------------------------------------------- /cloud_scanner/version.py: -------------------------------------------------------------------------------- 1 | # Store the version here so: 2 | # 1) we don't load dependencies by storing it in __init__.py 3 | # 2) we can import it in setup.py for the same reason 4 | # 3) we can import it into your module module 5 | __version__ = '0.1.3' 6 | -------------------------------------------------------------------------------- /cloud_scanner/simulators/__init__.py: -------------------------------------------------------------------------------- 1 | from .account_service_simulator import AccountServiceSimulator 2 | from .container_storage_simulator import MockBlobStorageSimulator 3 | from .queue_simulator import QueueSimulator 4 | from .resource_service_simulator import ResourceServiceSimulator 5 | from .table_storage_simulator import TableStorageSimulator 6 | -------------------------------------------------------------------------------- /cloud_scanner/helpers/functions.py: -------------------------------------------------------------------------------- 1 | 2 | def batch_list(items, batch_size): 3 | """Create batches from list of elements. 4 | 5 | :param items: List of all elements 6 | :param batch_size: Desired size of batches 7 | :return: Batches of list 8 | """ 9 | for i in range(0, len(items), batch_size): 10 | yield items[i:i + batch_size] 11 | -------------------------------------------------------------------------------- /cloud_scanner/contracts/account_service.py: -------------------------------------------------------------------------------- 1 | from abc import ABC 2 | 3 | 4 | class AccountService(ABC): 5 | """Service to retrieve account information for cloud provider.""" 6 | 7 | def get_accounts(self): 8 | """ 9 | :return: list of accounts from cloud provider 10 | """ 11 | raise NotImplementedError("accounts is not implemented") 12 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | package = ../cloud_scanner 2 | 3 | sphinx: 4 | cd docs && \ 5 | make clean && \ 6 | sphinx-apidoc -f -o source/generated $(package) && \ 7 | make html 8 | 9 | ghpages: 10 | -git checkout gh-pages && \ 11 | mv docs/build/html new-docs && \ 12 | rm -rf docs && \ 13 | mv new-docs docs && \ 14 | cp -r docs/* . && \ 15 | rm -rf docs && \ 16 | touch .nojekyll && \ 17 | git add . && \ 18 | git commit -m "Updated generated Sphinx documentation" 19 | -------------------------------------------------------------------------------- /tests/unittest_base.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import json 3 | 4 | # Helper to fake an Azure Queue Message 5 | class FakeQueueMessage: 6 | def __init__(self, msg): 7 | self._msg = msg 8 | 9 | def get_body(self): 10 | return self._msg 11 | 12 | def get_json(self): 13 | return json.loads(self.get_body().decode("utf-8")) 14 | 15 | class TestCase(unittest.TestCase): 16 | 17 | def _use_adapters(self): 18 | return False 19 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SOURCEDIR = source 8 | BUILDDIR = build 9 | 10 | # Put it first so that "make" without argument is like "make help". 11 | help: 12 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 13 | 14 | .PHONY: help Makefile 15 | 16 | # Catch-all target: route all unknown targets to Sphinx using the new 17 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 18 | %: Makefile 19 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | with open("README.md", "r") as fh: 4 | long_description = fh.read() 5 | 6 | exec(open('cloud_scanner/version.py').read()) 7 | setup(name='cloud_scanner', 8 | version=__version__, 9 | description='Core package for scanning cloud resources across providers', 10 | url='https://microsoft.github.io/cloud-scanner', 11 | author='Microsoft', 12 | author_email='cloudscanner@microsoft.com', 13 | license='MIT', 14 | packages=find_packages(), 15 | install_requires=[ 16 | 'python-dotenv', 17 | 'pytest', 18 | 'pytest-cov', 19 | 'click' 20 | ]) 21 | -------------------------------------------------------------------------------- /cloud_scanner/contracts/storage_container.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | 4 | class StorageContainer(ABC): 5 | """Base class for storage container.""" 6 | 7 | @abstractmethod 8 | def upload_text(self, filename, text): 9 | """Upload text to file in storage container Not implemented in this 10 | class.""" 11 | raise NotImplementedError("Should have implemented upload_text") 12 | 13 | @abstractmethod 14 | def list_blobs(self): 15 | """Get list of files in storage container Not implemented in this 16 | class.""" 17 | raise NotImplementedError("Should have implemented push") 18 | 19 | @abstractmethod 20 | def get_blob_to_text(self, file): 21 | """Get text content from file in storage container Not implemented in 22 | this class.""" 23 | raise NotImplementedError("Should have implemented pop") 24 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /azure-pipelines.yml: -------------------------------------------------------------------------------- 1 | # Python package 2 | # Create and test a Python package on multiple Python versions. 3 | # Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: 4 | # https://docs.microsoft.com/azure/devops/pipelines/languages/python 5 | 6 | trigger: 7 | - master 8 | 9 | pool: 10 | vmImage: 'ubuntu-latest' 11 | strategy: 12 | matrix: 13 | 14 | Python36: 15 | python.version: '3.6' 16 | Python37: 17 | python.version: '3.7' 18 | 19 | steps: 20 | - task: UsePythonVersion@0 21 | inputs: 22 | versionSpec: '$(python.version)' 23 | displayName: 'Use Python $(python.version)' 24 | 25 | - script: | 26 | python -m pip install --upgrade pip 27 | pip install -r requirements.txt 28 | displayName: 'Install dependencies' 29 | 30 | - script: | 31 | pip install pytest pytest-azurepipelines 32 | pytest 33 | displayName: 'pytest' 34 | -------------------------------------------------------------------------------- /cloud_scanner/simulators/account_service_simulator.py: -------------------------------------------------------------------------------- 1 | from cloud_scanner.contracts import AccountService, register_account_service 2 | 3 | 4 | @register_account_service("simulator", lambda: AccountServiceSimulator()) 5 | class AccountServiceSimulator(AccountService): 6 | """Simulator of AccoutService.""" 7 | 8 | def get_accounts(self): 9 | """Get fake accounts. 10 | 11 | :return: List of fake accounts 12 | [ 13 | { 14 | 'subscriptionId': '...', 15 | 'displayName': '...' 16 | }, 17 | ... 18 | ] 19 | """ 20 | return [ 21 | { 22 | "subscriptionId": "00000000-0000-0000-0000-000000000000", 23 | "displayName": "Sub1" 24 | }, 25 | { 26 | "subscriptionId": "00000000-0000-0000-0000-000000000001", 27 | "displayName": "Sub2" 28 | } 29 | ] 30 | -------------------------------------------------------------------------------- /tests/test_rule_factory.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from cloud_scanner.contracts import RuleFactory, register_rule 4 | from .unittest_base import TestCase 5 | from unittest.mock import patch 6 | 7 | def create_func(): 8 | return MockRule() 9 | 10 | class MockRule: 11 | pass 12 | 13 | class TestRuleFactory(TestCase): 14 | def setUp(self): 15 | os.environ["TAG_UPDATES_QUEUE_NAME"] = "resource-tag-updates" 16 | os.environ["QUEUE_TYPE"] = "simulator" 17 | 18 | def test_get_all_rules(self): 19 | rules = RuleFactory.get_rules() 20 | 21 | self.assertEqual(4, len(rules)) 22 | 23 | @patch.object(RuleFactory, 'register_rule') 24 | def test_register_rule_decorator(self, register_rule_mock): 25 | decorator = register_rule(create_func) 26 | decorator(RuleFactory) 27 | 28 | register_rule_mock.assert_called_once() 29 | 30 | def test_get_rules(self): 31 | rules = RuleFactory.get_rules() 32 | self.assertGreater(len(rules), 0) 33 | 34 | -------------------------------------------------------------------------------- /cloud_scanner/helpers/entry_storage.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | 4 | class EntryOperations: 5 | @staticmethod 6 | def prepare_entry_for_insert(resource): 7 | """Optimize resource for table entry insertion. 8 | 9 | :param resource: Resource to be inserted in Storage 10 | :return: Prepared resource 11 | """ 12 | # using location as the partition key. This will keep all the data from 13 | # the same location on the same node for fastest access 14 | location = resource['location'] 15 | resource['PartitionKey'] = location 16 | resource['RowKey'] = str(uuid.uuid3( 17 | uuid.NAMESPACE_DNS, resource['id'])) 18 | 19 | # cosmos does not allow for an entry with key 'id' 20 | modified_data = {} 21 | for key, value in resource.items(): 22 | if key == 'id': 23 | modified_data['resourceid'] = str(value) 24 | else: 25 | modified_data[key] = str(value) 26 | 27 | return modified_data 28 | -------------------------------------------------------------------------------- /cloud_scanner/config/configuration.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | 4 | 5 | def get_enviroment_value(key, default): 6 | """Gets an enviornment variable value. 7 | 8 | :param key: Name of the environment variable 9 | :return: The environmeant variable's value or None if it doesn't exist. 10 | """ 11 | value = os.environ.get(key, default) 12 | if value is None: 13 | error_message = f"Env variable {key} is not set" 14 | logging.error(error_message) 15 | return value 16 | 17 | 18 | class Config: 19 | """Base configuration class. 20 | 21 | Only exposes direct access to get config properties. 22 | """ 23 | 24 | def get_property(self, property_name, default=None): 25 | """Gets a configuration property. 26 | 27 | :param property_name: The name of the configuration property 28 | :param default: Default value of property 29 | :return: The property as a string, or None. 30 | """ 31 | 32 | return get_enviroment_value(property_name, default) 33 | -------------------------------------------------------------------------------- /cloud_scanner/contracts/__init__.py: -------------------------------------------------------------------------------- 1 | from .account_service import AccountService 2 | from .account_service_factory import ( 3 | AccountServiceFactory, register_account_service 4 | ) 5 | from .cloud_config_generator import CloudConfigGenerator 6 | from .cloud_config_reader import CloudConfigReader 7 | from .queue import Queue 8 | from .queue_factory import QueueFactory, register_queue_service 9 | from .resource import Resource 10 | from .resource_service import ResourceService, ResourceFilter 11 | from .resource_service_factory import ( 12 | ResourceServiceFactory, register_resource_service 13 | ) 14 | from .resource_storage_factory import ( 15 | ResourceStorageFactory, register_resource_storage 16 | ) 17 | from .rule import Rule 18 | from .rule_factory import RuleFactory, register_rule 19 | from .storage_container import StorageContainer 20 | from .storage_container_factory import ( 21 | StorageContainerFactory, register_storage_container 22 | ) 23 | from .table_storage import TableStorage 24 | from .tag_update_rule import TagUpdateRule 25 | -------------------------------------------------------------------------------- /cloud_scanner/contracts/queue.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | 4 | class Queue(ABC): 5 | """Generic Queue interface. 6 | 7 | Any queue implementation must expose the methods detailed in this 8 | interface. 9 | """ 10 | 11 | @abstractmethod 12 | def push(self, message): 13 | """Pushes a message onto the queue. 14 | 15 | :param message: The message that will be pushed onto the queue 16 | """ 17 | raise NotImplementedError("Should have implemented push") 18 | 19 | @abstractmethod 20 | def pop(self): 21 | """Pops the first message fom the queue and returns it. 22 | 23 | :return: The first message in the queue 24 | """ 25 | raise NotImplementedError("Should have implemented pop") 26 | 27 | @abstractmethod 28 | def peek(self): 29 | """Returns the first message flom the queue, leaving the message in the 30 | queue. 31 | 32 | :return: First message in the queue 33 | """ 34 | raise NotImplementedError("Should have implemented peek") 35 | -------------------------------------------------------------------------------- /cloud_scanner/simulators/queue_simulator.py: -------------------------------------------------------------------------------- 1 | import collections 2 | 3 | from cloud_scanner.contracts import Queue, register_queue_service 4 | 5 | 6 | @register_queue_service("simulator", 7 | lambda queue_name: QueueSimulator(queue_name)) 8 | class QueueSimulator(Queue): 9 | """Simulator of queue service.""" 10 | 11 | def __init__(self, queue_name, config=None): 12 | self._queue_name = queue_name 13 | self._queue = collections.deque() 14 | 15 | def __len__(self): 16 | return len(self._queue) 17 | 18 | def push(self, message): 19 | """Push new message to queue. 20 | 21 | :param message: message to push 22 | :return: None 23 | """ 24 | self._queue.append(message) 25 | 26 | def pop(self): 27 | """Pop message off of queue. 28 | 29 | :return: Message popped 30 | """ 31 | return self._queue.popleft() 32 | 33 | def peek(self): 34 | """Peek at message, but don't pop. 35 | 36 | :return: Message peeked 37 | """ 38 | return self._queue[0] 39 | -------------------------------------------------------------------------------- /cloud_scanner/contracts/rule.py: -------------------------------------------------------------------------------- 1 | from abc import ABC 2 | 3 | from .resource import Resource 4 | 5 | 6 | class Rule(ABC): 7 | """Interface of a rule. 8 | 9 | Any implemented rule must define each method described in this 10 | interface. 11 | """ 12 | 13 | def check_condition(self, resource: Resource) -> bool: 14 | """Returns True/False whether the rule should be performed on the input 15 | resource. 16 | 17 | :param resource: The resource to check if the rule should be ran upon. 18 | :return: Boolean if the resource should be processed with the rule. 19 | """ 20 | raise NotImplementedError( 21 | "check_condition is not implemented in the abstract base class") 22 | 23 | def process(self, resource: Resource) -> bool: 24 | """Processes the resource with the rule. 25 | 26 | :param resource: The resource to be processed with the rule. 27 | :return: Boolean if the rule had any effect. 28 | """ 29 | raise NotImplementedError( 30 | "check_condition is not implemented in the abstract base class") 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Eric Maino 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. Cloud Scanner documentation master file, created by 2 | sphinx-quickstart on Mon Oct 15 14:47:06 2018. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to Cloud Scanner's documentation! 7 | ========================================= 8 | 9 | This is the core library for the Cloud Scanner project, which is a workflow 10 | for discovering and documenting cloud resources across multiple accounts or 11 | subscriptions with the intent to store, update tags, and/or perform other 12 | generic resource related operations. 13 | 14 | `GitHub Repo `_ 15 | 16 | Related Projects 17 | ---------------- 18 | * `cloud-scanner-generic `_ 19 | * `cloud-scanner-azure `_ 20 | * `cloud-scanner-functions-example `_ 21 | 22 | .. toctree:: 23 | :maxdepth: 1 24 | :caption: API Reference 25 | 26 | generated/modules 27 | 28 | Indices and tables 29 | ================== 30 | 31 | * :ref:`genindex` 32 | * :ref:`modindex` 33 | * :ref:`search` 34 | -------------------------------------------------------------------------------- /cloud_scanner/services/resource_storage.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from cloud_scanner.contracts import Resource, ResourceStorageFactory 4 | 5 | 6 | class ResourceStorage: 7 | """Store resources from scanning.""" 8 | @staticmethod 9 | def process_queue_message(message): 10 | """Receives resources from queue and stores in registered service. 11 | 12 | :param message: Payload of resources to store 13 | :return: Number of resources stored in service 14 | """ 15 | resource_storage = ResourceStorageFactory.create() 16 | resources = _parse_resources(message) 17 | 18 | resource_storage.write_entries(resources) 19 | return len(resources) 20 | 21 | 22 | def _parse_resources(message): 23 | """Parse message from queue as JSON of resources. 24 | 25 | :param message: JSON of resources 26 | :return: Deserialized list of Resource objects 27 | """ 28 | resource_list = message.get_json() 29 | 30 | # Convert message into a list if it isn"t already 31 | if not isinstance(resource_list, list): 32 | resource_list = [resource_list] 33 | 34 | logging.info(f"Found {len(resource_list)} resources to process") 35 | 36 | resource_list = [Resource(resource) for resource in resource_list] 37 | 38 | return resource_list 39 | -------------------------------------------------------------------------------- /cloud_scanner/contracts/table_storage.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | 4 | class TableStorage(ABC): 5 | """Base class for Table Storage.""" 6 | 7 | @abstractmethod 8 | def write(self, entry): 9 | """Write entry to Table Storage Not implemented in this class.""" 10 | raise NotImplementedError("Should have implemented write_entry") 11 | 12 | def write_entries(self, entries): 13 | """Write collection of entries to Table Storage Not implemented in this 14 | class.""" 15 | for entry in entries: 16 | self.write(entry) 17 | 18 | @abstractmethod 19 | def query_list(self) -> list: 20 | """Get list of all entries in table storage Not implemented in this 21 | class.""" 22 | raise NotImplementedError("Should have implemented query_list") 23 | 24 | @abstractmethod 25 | def query(self, partition_key, row_key): 26 | """Query Table Storage for specific entry Not implemented in this 27 | class.""" 28 | raise NotImplementedError("Should have implemented query") 29 | 30 | @abstractmethod 31 | def delete(self, partition_key, row_key): 32 | """Delete specific entry in Table Storage Not implemented in this 33 | class.""" 34 | raise NotImplementedError("Should have implemented delete") 35 | -------------------------------------------------------------------------------- /cloud_scanner/contracts/resource_service.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | 4 | class ResourceFilter(ABC): 5 | """Base class for a resource filter.""" 6 | @abstractmethod 7 | def normalized_filter(self): 8 | """Not implemented in this class.""" 9 | raise NotImplementedError("normalized_filter is not implemented") 10 | 11 | 12 | class ResourceService(ABC): 13 | """Base class for resource service.""" 14 | @property 15 | @abstractmethod 16 | def name(self): 17 | """Name of resource service Not implemented in this class.""" 18 | raise NotImplementedError("name is not implemented") 19 | 20 | @abstractmethod 21 | def get_resources(self, filter: ResourceFilter = None): 22 | """Get resources based on filter Not implemented in this class.""" 23 | raise NotImplementedError("get_resources is not implemented") 24 | 25 | @abstractmethod 26 | def get_filter(self, payload) -> ResourceFilter: 27 | """Get filter object based on payload Not implemented in this class.""" 28 | raise NotImplementedError('get_filter is not implemented') 29 | 30 | @abstractmethod 31 | def update_resource(self, resource): 32 | """Update resource within cloud service provider Not implemented in 33 | this class.""" 34 | raise NotImplementedError("update_resource is not implemented") 35 | -------------------------------------------------------------------------------- /cloud_scanner/contracts/cloud_config_reader.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | 4 | from .storage_container import StorageContainer 5 | 6 | 7 | class CloudConfigReader: 8 | """Helper to read cloud configuration file.""" 9 | 10 | def __init__(self, container_service: StorageContainer): 11 | self._container_service = container_service 12 | 13 | def read_config(self): 14 | """Read cloud configuration file from storage container. 15 | 16 | :return: json payload of cloud config 17 | """ 18 | # get a list of files in the blob container 19 | config_list = self._container_service.list_blobs() 20 | 21 | # find the most recent config file 22 | latest_config = "config-" 23 | for config_filename in config_list: 24 | 25 | if config_filename.name > latest_config: 26 | latest_config = config_filename.name 27 | 28 | if latest_config == "config-": 29 | logging.error("Could not find any config files in blob container") 30 | return None 31 | 32 | # read the contents of the latest config 33 | json_data = self._container_service.get_blob_to_text( 34 | latest_config).content 35 | if json_data == '{}': 36 | logging.error("Empty JSON returned!") 37 | return None 38 | 39 | return json.loads(json_data) 40 | -------------------------------------------------------------------------------- /tests/test_table_storage.py: -------------------------------------------------------------------------------- 1 | import json 2 | import uuid 3 | 4 | from cloud_scanner.simulators import TableStorageSimulator, ResourceServiceSimulator 5 | from .unittest_base import TestCase 6 | 7 | 8 | class TestTableStorage(TestCase): 9 | 10 | def test_entry_is_inserted(self): 11 | 12 | storage_table = TableStorageSimulator() 13 | resource_service = ResourceServiceSimulator() 14 | resources = resource_service.get_resources() 15 | 16 | for resource in resources: 17 | resource_raw = resource.to_str() 18 | data = json.loads(resource_raw) 19 | 20 | # insert the entry onto Cosmos 21 | storage_table.write(resource) 22 | 23 | # for data in datastore: 24 | location = data['location'] 25 | 26 | # save partition and row key for access later 27 | partition_key = location 28 | row_key = str(uuid.uuid3(uuid.NAMESPACE_DNS, data['id'])) 29 | 30 | # verify the entry was written to the table 31 | retrieved_entry = None 32 | retrieved_entry = storage_table.query(partition_key, row_key) 33 | 34 | # verify the entry was inserted on Cosmos 35 | self.assertIsNotNone(retrieved_entry, "No entry was inserted on Cosmos") 36 | 37 | # delete the entry for good practice 38 | storage_table.delete(partition_key, row_key) 39 | -------------------------------------------------------------------------------- /cloud_scanner/contracts/account_service_factory.py: -------------------------------------------------------------------------------- 1 | from .account_service import AccountService 2 | 3 | 4 | def register_account_service(service_name, service_factory): 5 | """Registers an account service for a cloud provider. 6 | 7 | :param service_name: name of cloud provider ('aws' or 'azure') 8 | :param service_factory: 9 | :return: 10 | """ 11 | def decorator(cls): 12 | AccountServiceFactory.register_factory( 13 | service_name, 14 | service_factory) 15 | 16 | return cls 17 | return decorator 18 | 19 | 20 | class AccountServiceFactory: 21 | """Factory to instantiate account services for cloud providers.""" 22 | _factories = {} 23 | 24 | @classmethod 25 | def create(cls, service_type: str) -> AccountService: 26 | """Create an account service based on service type. 27 | 28 | :param service_type: str 29 | :return: 30 | """ 31 | try: 32 | return cls._factories[service_type]() 33 | except KeyError: 34 | raise KeyError( 35 | f"Service type {service_type} is " 36 | "not registered for Account Service") 37 | 38 | @classmethod 39 | def get_providers(cls): 40 | return [cls.create(key) for key in cls._factories] 41 | 42 | @classmethod 43 | def register_factory(cls, service_type: str, factory_func): 44 | cls._factories[service_type] = factory_func 45 | -------------------------------------------------------------------------------- /tests/test_config_generator.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from cloud_scanner.simulators import MockBlobStorageSimulator 4 | from cloud_scanner.contracts import CloudConfigGenerator 5 | from .unittest_base import TestCase 6 | 7 | 8 | class CloudConfigGeneratorTest(TestCase): 9 | 10 | types = ['vm', 'storage'] 11 | provider_types = ['simulator'] 12 | 13 | expected_config = { 14 | "providers": [{ 15 | "type": "simulator", 16 | "subscriptions": [ 17 | { 18 | "subscriptionId": "00000000-0000-0000-0000-000000000000", 19 | "displayName": "Sub1"}, 20 | { 21 | "subscriptionId": "00000000-0000-0000-0000-000000000001", 22 | "displayName": "Sub2" 23 | } 24 | ], 25 | "resourceTypes": [ 26 | {"typeName": "vm"}, 27 | {"typeName": "storage"} 28 | ] 29 | }] 30 | } 31 | 32 | def test_cloud_generator(self): 33 | container = MockBlobStorageSimulator() 34 | config_generator = CloudConfigGenerator(container) 35 | config = config_generator.generate_config( 36 | self.provider_types, self.types) 37 | 38 | # Asserting both string comparison and dictionary comparison 39 | # just to show serialization/deserialization isn't a problem 40 | self.assertEqual(config, json.dumps(self.expected_config)) 41 | self.assertDictEqual(json.loads(config), self.expected_config) 42 | -------------------------------------------------------------------------------- /cloud_scanner/contracts/rule_factory.py: -------------------------------------------------------------------------------- 1 | def register_rule(factory_func=None): 2 | """Decorator for registering a rule with the rule factory. 3 | 4 | :param factory_func: Optional lambda/function that will 5 | create and return an instance of the rule. Required if 6 | the rule has an __init__ function that takes any parameter 7 | other than self. 8 | """ 9 | def decorator(cls): 10 | if factory_func is None: 11 | RuleFactory.register_rule(cls) 12 | else: 13 | RuleFactory.register_rule(lambda: factory_func(cls)) 14 | 15 | return cls 16 | return decorator 17 | 18 | 19 | class RuleFactory: 20 | """Rule factory responsible for maintaining a list of rule 21 | definitions and returning instances of all registered rules. 22 | 23 | Attributes: 24 | _rules_factories: A list of lambda/functions that will instantiate 25 | an instance of each unique rule. 26 | """ 27 | _rules_factories = [] 28 | 29 | @classmethod 30 | def get_rules(cls) -> list: 31 | """Returns an instantiated list of each rule that has been registered. 32 | 33 | :return: list[Rule] a list of instantiated rules. 34 | """ 35 | return [factory() for factory in cls._rules_factories] 36 | 37 | @classmethod 38 | def register_rule(cls, rule_func): 39 | """Utility function used by the register_rule decorator to register a 40 | lambda/function to instantiate a rule.""" 41 | cls._rules_factories.append(rule_func) 42 | -------------------------------------------------------------------------------- /cloud_scanner/contracts/resource_storage_factory.py: -------------------------------------------------------------------------------- 1 | from cloud_scanner.config.process_config import ProcessConfig 2 | from .table_storage import TableStorage 3 | 4 | 5 | def register_resource_storage(service_name, service_factory): 6 | """Register resource storage service. 7 | 8 | :param service_name: Name of service 9 | :param service_factory: Function to instantiate service 10 | :return: None 11 | """ 12 | def decorator(cls): 13 | ResourceStorageFactory.register_factory( 14 | service_name, 15 | service_factory) 16 | 17 | return cls 18 | return decorator 19 | 20 | 21 | class ResourceStorageFactory: 22 | """Instantiate resource storage services.""" 23 | _factories = {} 24 | 25 | @classmethod 26 | def create(cls) -> TableStorage: 27 | """Create resource storage service. 28 | 29 | :return: Resource storage service object 30 | """ 31 | service_type = ProcessConfig().resource_storage_type 32 | try: 33 | return cls._factories[service_type]() 34 | except KeyError: 35 | raise KeyError( 36 | f"Service type {service_type} is not " 37 | "registered for Resource Storage Service") 38 | 39 | @classmethod 40 | def register_factory(cls, service_type: str, factory_func): 41 | """Register factory. 42 | 43 | :param service_type: type of service of factory 44 | :param factory_func: Function to intantiate service 45 | :return: None 46 | """ 47 | cls._factories[service_type] = factory_func 48 | -------------------------------------------------------------------------------- /cloud_scanner/contracts/resource_service_factory.py: -------------------------------------------------------------------------------- 1 | from .resource_service import ResourceService 2 | 3 | 4 | def register_resource_service(service_name, service_factory): 5 | """Register resource service. 6 | 7 | :param service_name: Name of service 8 | :param service_factory: Function to instantiate service 9 | :return: None 10 | """ 11 | def decorator(cls): 12 | ResourceServiceFactory.register_factory( 13 | service_name, 14 | service_factory) 15 | 16 | return cls 17 | return decorator 18 | 19 | 20 | class ResourceServiceFactory: 21 | """Instantiate resource services.""" 22 | _factories = {} 23 | 24 | @classmethod 25 | def create(cls, service_type: str, subscription_id) -> ResourceService: 26 | """Create resource service. 27 | 28 | :param service_type: type of service 29 | :param subscription_id: cloud service subscription or account ID 30 | :return: Resource service object 31 | """ 32 | try: 33 | return cls._factories[service_type](subscription_id) 34 | except KeyError: 35 | raise KeyError( 36 | f"Service type {service_type} is not " 37 | "registered for Resource Service") 38 | 39 | @classmethod 40 | def register_factory(cls, service_type: str, factory_func): 41 | """Register factory. 42 | 43 | :param service_type: type of service of factory 44 | :param factory_func: Function to intantiate service 45 | :return: None 46 | """ 47 | cls._factories[service_type] = factory_func 48 | -------------------------------------------------------------------------------- /cloud_scanner/contracts/storage_container_factory.py: -------------------------------------------------------------------------------- 1 | from cloud_scanner.config.process_config import ProcessConfig 2 | from .storage_container import StorageContainer 3 | 4 | 5 | def register_storage_container(service_name, service_factory): 6 | """Register storage container service. 7 | 8 | :param service_name: Name of service 9 | :param service_factory: Function to instantiate service 10 | :return: None 11 | """ 12 | def decorator(cls): 13 | StorageContainerFactory.register_factory( 14 | service_name, 15 | service_factory) 16 | 17 | return cls 18 | return decorator 19 | 20 | 21 | class StorageContainerFactory: 22 | """Instantiate storage container services.""" 23 | _factories = {} 24 | 25 | @classmethod 26 | def create(cls) -> StorageContainer: 27 | """Create storage container service. 28 | 29 | :return: Storage container service object 30 | """ 31 | service_type = ProcessConfig().storage_container_type 32 | try: 33 | return cls._factories[service_type]() 34 | except KeyError: 35 | raise KeyError( 36 | f"Service type {service_type} is not " 37 | f"registered for Storage Container Service") 38 | 39 | @classmethod 40 | def register_factory(cls, service_type: str, factory_func): 41 | """Register factory. 42 | 43 | :param service_type: type of service of factory 44 | :param factory_func: Function to intantiate service 45 | :return: None 46 | """ 47 | cls._factories[service_type] = factory_func 48 | -------------------------------------------------------------------------------- /cloud_scanner/rules/example_rules.py: -------------------------------------------------------------------------------- 1 | from cloud_scanner.config.process_config import ProcessConfig 2 | from ..contracts.queue_factory import QueueFactory 3 | from ..contracts.resource import Resource 4 | from ..contracts.rule_factory import register_rule 5 | from ..contracts.tag_update_rule import TagUpdateRule 6 | 7 | 8 | def create_rule(cls): 9 | queue_name = ProcessConfig().tag_updates_queue_name 10 | queue = QueueFactory.create(queue_name) 11 | return cls(queue) 12 | 13 | 14 | @register_rule(create_rule) 15 | class ExampleRule1(TagUpdateRule): 16 | def check_condition(self, resource: Resource) -> bool: 17 | return True if resource.type.startswith("Microsoft.Web") else False 18 | 19 | def get_tags(self, resource: Resource): 20 | return {"Category": "Azure Web"} 21 | 22 | 23 | @register_rule(create_rule) 24 | class ExampleRule2(TagUpdateRule): 25 | def check_condition(self, resource: Resource) -> bool: 26 | return True if resource.type.startswith("Microsoft.Storage") else False 27 | 28 | def get_tags(self, resource: Resource): 29 | return {"Category": "Storage"} 30 | 31 | 32 | @register_rule(create_rule) 33 | class ExampleRule3(TagUpdateRule): 34 | def check_condition(self, resource: Resource) -> bool: 35 | return True if "Microsoft" in resource.name else False 36 | 37 | def get_tags(self, resource: Resource): 38 | return {"Company": "Microsoft"} 39 | 40 | 41 | @register_rule(create_rule) 42 | class ExampleRule4(TagUpdateRule): 43 | def check_condition(self, resource: Resource) -> bool: 44 | return True 45 | 46 | def get_tags(self, resource: Resource): 47 | return {"Location": resource.location} 48 | -------------------------------------------------------------------------------- /tests/test_scheduler.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from cloud_scanner.simulators import MockBlobStorageSimulator 4 | from cloud_scanner.services.task_scheduler import TaskScheduler 5 | from cloud_scanner.contracts import CloudConfigReader 6 | from .unittest_base import TestCase 7 | 8 | 9 | class TestScheduler(TestCase): 10 | def setUp(self): 11 | os.environ["STORAGE_CONTAINER_TYPE"] = "simulator" 12 | os.environ["QUEUE_TYPE"] = "simulator" 13 | os.environ["TASK_QUEUE_NAME"] = "test-queue" 14 | 15 | def read_config(self): 16 | container = MockBlobStorageSimulator() 17 | config_reader = CloudConfigReader(container) 18 | return config_reader.read_config() 19 | 20 | def test_latest_config_is_picked(self): 21 | 22 | result = self.read_config() 23 | self.assertFalse(result is None) 24 | 25 | providers = result['providers'] 26 | for provider in providers: 27 | self.assertFalse(provider is None) 28 | 29 | def test_tasks_are_created(self): 30 | 31 | result = self.read_config() 32 | tasks = TaskScheduler._create_tasks('simulator', result["providers"][0]) 33 | for task in tasks: 34 | id = task['subscriptionId'] 35 | type = task['typeName'] 36 | self.assertFalse(id is None) 37 | self.assertFalse(type is None) 38 | 39 | def test_push_tasks_to_queue(self): 40 | # Get expected number of tasks for simulator provider 41 | result = self.read_config() 42 | tasks = TaskScheduler._create_tasks('simulator', result["providers"][0]) 43 | 44 | task_count = TaskScheduler.execute() 45 | self.assertEqual(len(tasks), task_count) 46 | -------------------------------------------------------------------------------- /tests/test_resource_service_factory.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from cloud_scanner.contracts import ResourceServiceFactory, register_resource_service, ResourceService 4 | from .unittest_base import TestCase 5 | from unittest.mock import patch 6 | 7 | 8 | def create_func(subscription_id): 9 | return ResourceServiceMock() 10 | 11 | 12 | @register_resource_service('test', create_func) 13 | class ResourceServiceMock(): 14 | pass 15 | 16 | 17 | class TestResourceServiceFactory(TestCase): 18 | def test_simulator_provider(self): 19 | service = ResourceServiceFactory.create("simulator", "00000000-0000-0000-0000-000000000000") 20 | self.assertFalse(service is None) 21 | self.assertEqual("ResourceServiceSimulator", type(service).__name__) 22 | 23 | @patch.object(ResourceServiceFactory, 'register_factory') 24 | def test_register_resource_service_decorator(self, mock_register_factory): 25 | decorator = register_resource_service('test', create_func) 26 | decorator(ResourceServiceFactory) 27 | 28 | mock_register_factory.assert_called_once_with('test', create_func) 29 | 30 | resource_service = ResourceServiceFactory.create("test", "00000000-0000-0000-0000-000000000000") 31 | self.assertIsNotNone(resource_service) 32 | self.assertEqual(type(resource_service).__name__, "ResourceServiceMock") 33 | 34 | def test_unknown_provider(self): 35 | test_error = None 36 | 37 | try: 38 | ResourceServiceFactory.create("unknown", "00000000-0000-0000-0000-000000000000") 39 | except Exception as create_error: 40 | test_error = create_error 41 | 42 | self.assertFalse(test_error is None, "Expected error to be thrown for invalid provider") 43 | -------------------------------------------------------------------------------- /cloud_scanner/contracts/tag_update_rule.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from .rule import Rule 4 | from .queue import Queue 5 | from .resource import Resource 6 | 7 | 8 | class TagUpdateRule(Rule): 9 | """Utility base class for a rule that will update the tags on a given 10 | resource. Any tag update will be pushed onto a queue with a message 11 | containing the resource and a dictionary of tags to append. 12 | 13 | Attributes: 14 | _queue: An instance of the queue to push the tag update message to. 15 | """ 16 | 17 | def __init__(self, queue: Queue): 18 | """Initializes the rule with a specified queue to push the tag changes 19 | to.""" 20 | self._queue = queue 21 | 22 | def process(self, resource: Resource): 23 | """Processes the resource with the rule. The resource will first be 24 | checked to see the rule should be run using 'check_condition'. 25 | 26 | :param resource: The resource to be processed with the rule. 27 | :return: Boolean if the rule was run. 28 | """ 29 | if self.check_condition(resource): 30 | tags = self.get_tags(resource) 31 | payload = { 32 | "resource": resource.to_dict(), 33 | "tags": tags 34 | } 35 | 36 | self._queue.push(json.dumps(payload)) 37 | return True 38 | 39 | return False 40 | 41 | def get_tags(self, resource: Resource) -> dict: 42 | """The dictionary of tags to update the resource with. 43 | 44 | :param resource: The resource to update tags on. 45 | :return: dict of tags as key value pairs. 46 | """ 47 | raise NotImplementedError( 48 | "get_tags is not implemented in the abstract base class") 49 | -------------------------------------------------------------------------------- /tests/test_account_service_factory.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from cloud_scanner.contracts import AccountServiceFactory, AccountService, register_account_service 4 | from unittest.mock import patch 5 | from .unittest_base import TestCase 6 | 7 | 8 | def create_func(): 9 | return AccountServiceMock() 10 | 11 | @register_account_service('test', create_func) 12 | class AccountServiceMock(AccountService): 13 | pass 14 | 15 | class TestAccountServiceFactory(TestCase): 16 | def test_simulator_account_service(self): 17 | account_service = AccountServiceFactory.create("simulator") 18 | self.assertIsNotNone(account_service) 19 | self.assertEqual(type(account_service).__name__, "AccountServiceSimulator") 20 | 21 | @patch.object(AccountServiceFactory, 'register_factory') 22 | def test_register_account_service_decorator(self, mock_register_factory): 23 | decorator = register_account_service('test', create_func) 24 | decorator(AccountServiceFactory) 25 | 26 | mock_register_factory.assert_called_once_with('test', create_func) 27 | 28 | account_service = AccountServiceFactory.create("test") 29 | self.assertIsNotNone(account_service) 30 | self.assertEqual(type(account_service).__name__, "AccountServiceMock") 31 | 32 | def test_unknown_provider(self): 33 | test_error = None 34 | 35 | try: 36 | AccountServiceFactory.create("unknown") 37 | except Exception as create_error: 38 | test_error = create_error 39 | 40 | self.assertFalse(test_error is None, "Expected error to be thrown for invalid provider") 41 | 42 | def test_get_providers(self): 43 | providers = AccountServiceFactory.get_providers() 44 | self.assertGreater(len(providers), 0) 45 | 46 | 47 | -------------------------------------------------------------------------------- /tests/test_resource_storage_factory.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from cloud_scanner.contracts.resource_storage_factory import ResourceStorageFactory, register_resource_storage 4 | from .unittest_base import TestCase 5 | from unittest.mock import patch 6 | 7 | def create_func(): 8 | return ResourceStorageMock() 9 | 10 | 11 | @register_resource_storage('test', create_func) 12 | class ResourceStorageMock(): 13 | pass 14 | 15 | 16 | class TestResourceStorageFactory(TestCase): 17 | def test_simulator_resource_storage(self): 18 | os.environ["RESOURCE_STORAGE_TYPE"] = "simulator" 19 | 20 | resource_storage = ResourceStorageFactory.create() 21 | self.assertIsNotNone(resource_storage) 22 | self.assertEqual(type(resource_storage).__name__, "TableStorageSimulator") 23 | 24 | @patch.object(ResourceStorageFactory, 'register_factory') 25 | def test_register_resource_storage_decorator(self, mock_register_factory): 26 | os.environ["RESOURCE_STORAGE_TYPE"] = "test" 27 | 28 | decorator = register_resource_storage('test', create_func) 29 | decorator(ResourceStorageFactory) 30 | 31 | mock_register_factory.assert_called_once_with('test', create_func) 32 | 33 | resource_service = ResourceStorageFactory.create() 34 | self.assertIsNotNone(resource_service) 35 | self.assertEqual(type(resource_service).__name__, "ResourceStorageMock") 36 | 37 | def test_unknown_provider(self): 38 | os.environ["RESOURCE_STORAGE_TYPE"] = "unknown" 39 | test_error = None 40 | 41 | try: 42 | ResourceStorageFactory.create() 43 | except Exception as create_error: 44 | test_error = create_error 45 | 46 | self.assertFalse(test_error is None, "Expected error to be thrown for invalid provider") 47 | -------------------------------------------------------------------------------- /tests/test_storage_container_factory.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from cloud_scanner.contracts.storage_container_factory import StorageContainerFactory, register_storage_container 4 | from .unittest_base import TestCase 5 | from unittest.mock import patch 6 | 7 | def create_func(): 8 | return StorageContainerMock() 9 | 10 | 11 | @register_storage_container('test', create_func) 12 | class StorageContainerMock: 13 | pass 14 | 15 | 16 | class TestStorageContainerFactory(TestCase): 17 | def setUp(self): 18 | os.environ["CONFIG_CONTAINER"] = "test-container" 19 | 20 | def test_create_container_simulator(self): 21 | os.environ["STORAGE_CONTAINER_TYPE"] = "simulator" 22 | 23 | container = StorageContainerFactory.create() 24 | self.assertIsNotNone(container) 25 | self.assertEqual(type(container).__name__, "MockBlobStorageSimulator") 26 | 27 | @patch.object(StorageContainerFactory, 'register_factory') 28 | def test_register_storage_container_decorator(self, mock_register_factory): 29 | os.environ["STORAGE_CONTAINER_TYPE"] = "test" 30 | 31 | decorator = register_storage_container('test', create_func) 32 | decorator(StorageContainerFactory) 33 | 34 | mock_register_factory.assert_called_once_with('test', create_func) 35 | 36 | container = StorageContainerFactory.create() 37 | self.assertIsNotNone(container) 38 | self.assertEqual(type(container).__name__, "StorageContainerMock") 39 | 40 | def test_unknown_provider(self): 41 | os.environ["STORAGE_CONTAINER_TYPE"] = "unknown" 42 | test_error = None 43 | 44 | try: 45 | StorageContainerFactory.create() 46 | except Exception as create_error: 47 | test_error = create_error 48 | 49 | self.assertFalse(test_error is None, "Expected error to be thrown for invalid provider") 50 | 51 | 52 | 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | docs/source/generated 69 | 70 | # PyBuilder 71 | target/ 72 | 73 | # Jupyter Notebook 74 | .ipynb_checkpoints 75 | 76 | # pyenv 77 | .python-version 78 | 79 | # celery beat schedule file 80 | celerybeat-schedule 81 | 82 | # SageMath parsed files 83 | *.sage.py 84 | 85 | # Environments 86 | .env 87 | .venv 88 | env/ 89 | venv/ 90 | ENV/ 91 | env.bak/ 92 | venv.bak/ 93 | 94 | # Spyder project settings 95 | .spyderproject 96 | .spyproject 97 | 98 | # Rope project settings 99 | .ropeproject 100 | 101 | # mkdocs documentation 102 | /site 103 | 104 | # mypy 105 | .mypy_cache/ 106 | 107 | .env 108 | .idea/ 109 | .vscode/ 110 | 111 | docs/.buildinfo 112 | docs/.nojekyll 113 | docs/_sources/ 114 | docs/_static/ 115 | docs/generated/ -------------------------------------------------------------------------------- /tests/test_example_rules.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from cloud_scanner.contracts import QueueFactory, Resource 4 | from .unittest_base import TestCase 5 | 6 | 7 | class TestExampleRules(TestCase): 8 | def setUp(self): 9 | os.environ["QUEUE_TYPE"] = "simulator" 10 | os.environ["TAG_UPDATES_QUEUE_NAME"] = "test-queue-name" 11 | self._queue = QueueFactory.create("test-queue-name") 12 | 13 | resource_json = { "id": '/resources/type1/resource1', "accountId": "account1", "type": "Microsoft.Storage/virtualMachine", "name": "resource1", "providerType": "simulator", "location": "location1"} 14 | self._resource = Resource(resource_json) 15 | 16 | def test_rule_1(self): 17 | from cloud_scanner.rules import ExampleRule1 18 | rule1 = ExampleRule1(self._queue) 19 | result = rule1.check_condition(self._resource) 20 | count = rule1.process(self._resource) 21 | 22 | self.assertFalse(result) 23 | self.assertEqual(count, 0) 24 | 25 | def test_rule_2(self): 26 | from cloud_scanner.rules import ExampleRule2 27 | rule2 = ExampleRule2(self._queue) 28 | result = rule2.check_condition(self._resource) 29 | count = rule2.process(self._resource) 30 | 31 | self.assertTrue(result) 32 | self.assertEqual(count, 1) 33 | 34 | def test_rule_3(self): 35 | from cloud_scanner.rules import ExampleRule3 36 | rule3 = ExampleRule3(self._queue) 37 | result = rule3.check_condition(self._resource) 38 | count = rule3.process(self._resource) 39 | 40 | self.assertFalse(result) 41 | self.assertEqual(count, 0) 42 | 43 | def test_rule_4(self): 44 | from cloud_scanner.rules import ExampleRule4 45 | rule4 = ExampleRule4(self._queue) 46 | result = rule4.check_condition(self._resource) 47 | count = rule4.process(self._resource) 48 | 49 | self.assertTrue(result) 50 | self.assertEqual(count, 1) 51 | 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /cloud_scanner/contracts/queue_factory.py: -------------------------------------------------------------------------------- 1 | from cloud_scanner.config.process_config import ProcessConfig 2 | from .queue import Queue 3 | 4 | 5 | def register_queue_service(service_name, service_factory): 6 | """Decorator used to register an implementation of a queue with the queue 7 | factory. 8 | 9 | :param service_name: The name to register this type of queue as. 10 | :param service_factory: A function or lambda that takes a queue_name 11 | (as a string) and will return an instance of the queue implementation. 12 | """ 13 | 14 | def decorator(cls): 15 | QueueFactory.register_factory( 16 | service_name, 17 | service_factory) 18 | return cls 19 | return decorator 20 | 21 | 22 | class QueueFactory: 23 | """Singleton factory responsible for creating queues.""" 24 | 25 | _factories = {} 26 | 27 | @classmethod 28 | def create(cls, queue_name: str) -> Queue: 29 | """Returns a queue with 'queue_name' of type specified in the config 30 | "QUEUE_TYPE" property. 31 | 32 | :param queue_name: Name of the queue 33 | :return: Implemented instance of the Queue contract 34 | """ 35 | # @TODO: No way of using more than one type of queue 36 | # type since service_type is being read from config 37 | # instead of being passed in/dynamic. 38 | service_type = ProcessConfig().queue_type 39 | try: 40 | return cls._factories[service_type](queue_name) 41 | except KeyError: 42 | raise KeyError( 43 | f"Service type {service_type} is not " 44 | "registered for Queue Service") 45 | 46 | @classmethod 47 | def register_factory(cls, service_type: str, factory_func): 48 | """Utility function used to register a type of queue with a string 49 | name. 50 | 51 | Primarily used by the 'register_queue_service' decorator. 52 | """ 53 | cls._factories[service_type] = factory_func 54 | -------------------------------------------------------------------------------- /cloud_scanner/contracts/cloud_config_generator.py: -------------------------------------------------------------------------------- 1 | import json 2 | from datetime import datetime 3 | 4 | from .account_service_factory import AccountServiceFactory 5 | from .storage_container import StorageContainer 6 | 7 | 8 | class CloudConfigGenerator: 9 | """Generate cloud configuration file for process workflow.""" 10 | 11 | def __init__(self, storage_container: StorageContainer): 12 | self._container = storage_container 13 | 14 | def generate_config(self, providers_types: list, resource_types: list): 15 | """Generate cloud configuration payload. 16 | 17 | :param providers_types: 18 | comma-separated list of cloud providers (azure, aws, gcp) 19 | :param resource_types: comma-separated list of cloud resource types 20 | :return: str of Json payload 21 | """ 22 | providers = [] 23 | 24 | for provider_type in providers_types: 25 | account_service = AccountServiceFactory.create(provider_type) 26 | accounts = [] 27 | for account in account_service.get_accounts(): 28 | accounts.append({ 29 | "subscriptionId": account["subscriptionId"], 30 | "displayName": account["displayName"] 31 | }) 32 | 33 | types = [] 34 | for resource_type in resource_types: 35 | types.append({ 36 | "typeName": resource_type 37 | }) 38 | 39 | providers.append({ 40 | "type": provider_type, 41 | "subscriptions": accounts, 42 | "resourceTypes": types 43 | }) 44 | 45 | return json.dumps({ 46 | "providers": providers 47 | }) 48 | 49 | def output_config(self, config): 50 | """Upload config payload to Storage container. 51 | 52 | :param config: json payload of config 53 | :return: None 54 | """ 55 | blob_name = 'config-{date:%Y-%m-%d-%H-%M-%S}.json'.format( 56 | date=datetime.now()) 57 | self._container.upload_text(blob_name, config) 58 | -------------------------------------------------------------------------------- /tests/test_resource_scanner.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | from cloud_scanner.simulators import ResourceServiceSimulator, QueueSimulator 5 | from cloud_scanner.services.resource_scanner import ResourceScanner, ResourceTaskProcessor 6 | from .unittest_base import TestCase, FakeQueueMessage 7 | from cloud_scanner.config import ProcessConfig 8 | 9 | 10 | class TestScanResources(TestCase): 11 | def setUp(self): 12 | os.environ["STORAGE_CONTAINER_TYPE"] = "simulator" 13 | os.environ["RESOURCE_STORAGE_TYPE"] = "simulator" 14 | os.environ["QUEUE_TYPE"] = "simulator" 15 | os.environ["PAYLOAD_QUEUE_NAME"] = "test-queue" 16 | 17 | def test_simulator_task(self): 18 | json_data = ''' 19 | { 20 | "providerType": "simulator", 21 | "subscriptionId": "00000000-0000-0000-0000-000000000001", 22 | "typeName": "Microsoft.Compute/virtualMachines" 23 | } 24 | ''' 25 | message = FakeQueueMessage(bytes(json_data, 'utf-8')) 26 | count = ResourceScanner.process_queue_message(message) 27 | 28 | self.assertEqual(count, 1) 29 | 30 | def test_task_processor(self): 31 | data = { 32 | "subscriptionId" : "12345678-0000-0000-0000-123412341234", 33 | "typeName" : "storage" 34 | } 35 | resource_service = ResourceServiceSimulator() 36 | queue = QueueSimulator("test_queue") 37 | 38 | # Get the expected resources from the provider 39 | resources = resource_service.get_resources(data["subscriptionId"]) 40 | 41 | task_processor = ResourceTaskProcessor(resource_service, queue) 42 | task_processor.execute(data) 43 | 44 | item = queue.pop() 45 | 46 | self.assertEqual(len(json.loads(item)), len(resources)) 47 | 48 | def test_batching(self): 49 | key = 'RESOURCE_BATCH_SIZE' 50 | if key in os.environ: 51 | del os.environ[key] 52 | process_config = ProcessConfig() 53 | self.assertEqual(16, process_config.batch_size) 54 | 55 | os.environ[key] = '100' 56 | self.assertEqual(100, process_config.batch_size) 57 | 58 | 59 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Cloud Scanner 2 | 3 | [![Build Status](https://dev.azure.com/ethomson/cloud-scanner/_apis/build/status/ethomson.cloud-scanner-build?branchName=master)](https://dev.azure.com/ethomson/cloud-scanner/_build/latest?definitionId=72&branchName=master) 4 | [![PyPI](https://img.shields.io/pypi/v/cloud-scanner.svg)](https://pypi.org/project/cloud-scanner/) 5 | 6 | **Note:** this is a fork of the [Microsoft Cloud Scanner](https://github.com/Microsoft/cloud-scanner), originally built by the Microsoft Commercial Software Engineering team. This project was forked for demonstration purposes only; please contribute changes back to the original project. 7 | 8 | ## Developer Documentation 9 | [Read the API docs](https://microsoft.github.io/cloud-scanner/) 10 | 11 | ## Introduction 12 | 13 | Core library for Cloud Scanner project, which is a workflow for discovering and documenting cloud resources across multiple accounts or subscriptions with the intent to store, update tags, and/or perform other generic resource related operations. 14 | 15 | Dependent upon adapter projects such as [cloud-scanner-azure](https://github.com/Microsoft/cloud-scanner-azure) and [cloud-scanner-generic](https://github.com/Microsoft/cloud-scanner-generic) for the registry of service providers. 16 | 17 | For an example of usage within an Azure Functions environment, see [cloud-scanner-azure-functions-sample](https://github.com/Microsoft/cloud-scanner-azure-functions-sample). 18 | 19 | 20 | ## Contributing 21 | 22 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 23 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 24 | the rights to use your contribution. For details, visit https://cla.microsoft.com. 25 | 26 | When you submit a pull request, a CLA-bot will automatically determine whether you need to provide 27 | a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions 28 | provided by the bot. You will only need to do this once across all repos using our CLA. 29 | 30 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 31 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 32 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 33 | -------------------------------------------------------------------------------- /tests/test_resource_storage.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from cloud_scanner.services.resource_storage import ResourceStorage 4 | from .unittest_base import TestCase, FakeQueueMessage 5 | 6 | 7 | class ResourceStorageTest(TestCase): 8 | def test_process_queue_message_with_zero_resources(self): 9 | os.environ["PROVIDER_LIST"] = "simulator" 10 | 11 | json_data = '[]' 12 | message = FakeQueueMessage(bytes(json_data, 'utf-8')) 13 | resource_written = ResourceStorage.process_queue_message(message) 14 | 15 | self.assertEqual(0, resource_written) 16 | 17 | def test_process_queue_message_with_single_resource(self): 18 | os.environ["PROVIDER_LIST"] = "simulator" 19 | 20 | json_data = ''' 21 | [{ 22 | "id": "00000000-0000-0000-0000-000000000123", 23 | "accountId": "00000000-0000-0000-0000-000000000001", 24 | "providerType": "simulator", 25 | "type": "vm", 26 | "name": "MyResource", 27 | "group": "MyGroup", 28 | "location": "WestUS" 29 | }] 30 | ''' 31 | 32 | message = FakeQueueMessage(bytes(json_data, 'utf-8')) 33 | resource_written = ResourceStorage.process_queue_message(message) 34 | 35 | self.assertEqual(1, resource_written) 36 | 37 | def test_process_queue_message_with_multiple_resources(self): 38 | os.environ["PROVIDER_LIST"] = "simulator" 39 | 40 | json_data = ''' 41 | [{ 42 | "id": "00000000-0000-0000-0000-000000000123", 43 | "accountId": "00000000-0000-0000-0000-000000000001", 44 | "providerType": "simulator", 45 | "type": "vm", 46 | "name": "MyResource", 47 | "group": "MyGroup", 48 | "location": "WestUS" 49 | }, 50 | { 51 | "id": "00000000-0000-0000-0000-000000000124", 52 | "accountId": "00000000-0000-0000-0000-000000000002", 53 | "providerType": "simulator", 54 | "type": "vm", 55 | "name": "MyResource2", 56 | "group": "MyGroup2", 57 | "location": "EastUS" 58 | }] 59 | ''' 60 | message = FakeQueueMessage(bytes(json_data, 'utf-8')) 61 | resource_written = ResourceStorage.process_queue_message(message) 62 | 63 | self.assertEqual(2, resource_written) 64 | -------------------------------------------------------------------------------- /cloud_scanner/services/task_scheduler.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | 4 | from cloud_scanner.contracts import ( 5 | CloudConfigReader, QueueFactory, StorageContainerFactory 6 | ) 7 | from cloud_scanner.config.process_config import ProcessConfig 8 | 9 | 10 | class TaskScheduler: 11 | """Schedule tasks for resource scanning.""" 12 | 13 | @staticmethod 14 | def execute(): 15 | """Execute scheduling of tasks. 16 | 17 | :return: int number of tasks scheduled 18 | """ 19 | 20 | queue_name = ProcessConfig().task_queue_name 21 | 22 | task_queue = QueueFactory.create(queue_name) 23 | storage_container = StorageContainerFactory.create() 24 | config_reader = CloudConfigReader(storage_container) 25 | cloud_config = config_reader.read_config() 26 | 27 | task_count = 0 28 | 29 | for provider in cloud_config["providers"]: 30 | tasks = TaskScheduler._create_tasks(provider["type"], provider) 31 | 32 | for task in tasks: 33 | logging.info(f"Pushing task {task} to queue {queue_name}") 34 | task_queue.push(json.dumps(task)) 35 | task_count += 1 36 | 37 | return task_count 38 | 39 | @staticmethod 40 | def _create_tasks(provider_type: str, config): 41 | """Create tasks for scanning. 42 | 43 | :param provider_type: Cloud provider (azure or aws) 44 | :param config: Pulled from existing configuration file 45 | :return: Tasks for scanning 46 | """ 47 | tasks = [] 48 | # put the tasks in the queue 49 | for subscription in config['subscriptions']: 50 | # handle the scenario where no resource type is specified 51 | if config['resourceTypes'] is None: 52 | sub_id = subscription['subscriptionId'] 53 | 54 | data = { 55 | 'providerType': provider_type, 56 | 'subscriptionId': sub_id 57 | } 58 | message = json.dumps(data) 59 | tasks.append(message) 60 | 61 | for resource_type in config['resourceTypes']: 62 | sub_id = subscription['subscriptionId'] 63 | r_type = resource_type['typeName'] 64 | 65 | data = { 66 | 'providerType': provider_type, 67 | 'subscriptionId': sub_id, 68 | 'typeName': r_type 69 | } 70 | 71 | tasks.append(data) 72 | 73 | return tasks 74 | -------------------------------------------------------------------------------- /cloud_scanner/config/process_config.py: -------------------------------------------------------------------------------- 1 | from .configuration import Config 2 | 3 | 4 | class ProcessConfig(Config): 5 | """Configuration of workflow names, types and other process-specific 6 | info.""" 7 | 8 | @property 9 | def task_queue_name(self): 10 | """Gets the name of the task queue Specified by the TASK_QUEUE_NAME 11 | property.""" 12 | return self.get_property('TASK_QUEUE_NAME') 13 | 14 | @property 15 | def payload_queue_name(self): 16 | """Gets the name of the payload queue Specified by the 17 | PAYLOAD_QUEUE_NAME property.""" 18 | 19 | return self.get_property('PAYLOAD_QUEUE_NAME') 20 | 21 | @property 22 | def config_container_name(self): 23 | """Gets the name of the config container Specified by the 24 | CONFIG_CONTAINER property.""" 25 | return self.get_property('CONFIG_CONTAINER') 26 | 27 | @property 28 | def tag_updates_queue_name(self): 29 | """Gets the name of queue used for tag updates Specified by the 30 | TAG_UPDATES_QUEUE_NAME property.""" 31 | return self.get_property('TAG_UPDATES_QUEUE_NAME') 32 | 33 | @property 34 | def queue_type(self): 35 | """Gets the type of queue currently being used Specified by the 36 | QUEUE_TYPE property. 37 | 38 | Acceptable values: 'azure_storage_queue' 39 | """ 40 | return self.get_property('QUEUE_TYPE') 41 | 42 | @property 43 | def storage_container_type(self): 44 | """Gets the type of storage container currently being used Specified by 45 | the STORAGE_CONTAINER_TYPE property. 46 | 47 | Acceptable values: 'azure_storage' 48 | """ 49 | return self.get_property('STORAGE_CONTAINER_TYPE') 50 | 51 | @property 52 | def resource_storage_type(self): 53 | """Gets the type of resource storage currently being used Specified by 54 | the RESOURCE_STORAGE_TYPE property. 55 | 56 | Acceptable values: 'elastic_search' 'azure_cosmos_table' 57 | 'mysql' 'rest_storage_service' (REST_STORAGE_URL must also 58 | be specified) 59 | """ 60 | return self.get_property('RESOURCE_STORAGE_TYPE') 61 | 62 | @property 63 | def rest_storage_url(self): 64 | """Gets the URL for the Rest storage service if being used Specified by 65 | the REST_STORAGE_URL property.""" 66 | return self.get_property("REST_STORAGE_URL") 67 | 68 | @property 69 | def batch_size(self): 70 | """Gets the batch size of resources to send to the storage service, 71 | specified by the RESOURCE_BATCH_SIZE property Defaults to 16.""" 72 | return int(self.get_property('RESOURCE_BATCH_SIZE', '16')) 73 | -------------------------------------------------------------------------------- /cloud_scanner/services/resource_scanner.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | 4 | from cloud_scanner.helpers import batch_list 5 | from cloud_scanner.config.process_config import ProcessConfig 6 | from cloud_scanner.contracts import ( 7 | Queue, ResourceServiceFactory, QueueFactory, ResourceService 8 | ) 9 | 10 | 11 | def _read_as_json(msg): 12 | """Decode message (UTF-8) and read into dictionary. 13 | 14 | :param msg: str to decode and deserialize 15 | :return: Dictionary from message 16 | """ 17 | msg_body = msg.get_body().decode("utf-8") 18 | return json.loads(msg_body) 19 | 20 | 21 | class ResourceScanner: 22 | """Scan cloud service for resources.""" 23 | 24 | @staticmethod 25 | def process_queue_message(message): 26 | """Receives message from queue, which tells it which resources to scan 27 | from cloud provider. 28 | 29 | :param message: Task of which resources to scan 30 | :return: List of resources scanned from cloud provider 31 | """ 32 | task = _read_as_json(message) 33 | 34 | queue_name = ProcessConfig().payload_queue_name 35 | 36 | resource_service = ResourceServiceFactory.create( 37 | task["providerType"], task["subscriptionId"]) 38 | output_queue = QueueFactory.create(queue_name) 39 | 40 | task_processor = ResourceTaskProcessor(resource_service, output_queue) 41 | return task_processor.execute(task) 42 | 43 | 44 | class ResourceTaskProcessor: 45 | """Process resource scanning tasks and return as dictionaries.""" 46 | 47 | def __init__(self, resource_service: ResourceService, output_queue: Queue): 48 | self._resource_service = resource_service 49 | self._queue = output_queue 50 | 51 | def execute(self, task): 52 | """Execute scanning of resources. 53 | 54 | :param task: Defines which resources to scan 55 | :return: List of resources as dictionaries 56 | """ 57 | subscription_id = task["subscriptionId"] 58 | if subscription_id is None: 59 | raise Exception( 60 | "Couldn't find a subscriptionId for the " 61 | "task: " + json.dumps(task)) 62 | 63 | resource_type = task.get("typeName", None) 64 | logging.info( 65 | f"Received task for subscription {subscription_id}" 66 | f" and resource type {resource_type}") 67 | 68 | resource_filter = self._resource_service.get_filter(resource_type) 69 | resources = self._resource_service.get_resources(resource_filter) 70 | 71 | resource_count = 0 72 | 73 | # Transform resources to resource dictionaries 74 | resources = [resource.to_dict() for resource in resources] 75 | 76 | for batch in batch_list(resources, 77 | batch_size=ProcessConfig().batch_size): 78 | self._queue.push(json.dumps(batch)) 79 | resource_count += 1 80 | 81 | return resource_count 82 | -------------------------------------------------------------------------------- /cloud_scanner/simulators/table_storage_simulator.py: -------------------------------------------------------------------------------- 1 | from cloud_scanner.contracts import ( 2 | TableStorage, Resource, register_resource_storage) 3 | from cloud_scanner.helpers import entry_storage 4 | 5 | 6 | @register_resource_storage("simulator", 7 | lambda: TableStorageSimulator()) 8 | class TableStorageSimulator(TableStorage): 9 | """Simulator of TableStorage service.""" 10 | 11 | def __init__(self): 12 | self._data = dict() 13 | 14 | self._resources = [ 15 | {"id": '/resources/type1/resource1', 16 | "accountId": "account1", 17 | "type": "Microsoft.Storage/virtualMachine", 18 | "name": "resource1", 19 | "providerType": "simulator", 20 | "location": "location1"}, 21 | {"id": '/resources/type1/resource2', 22 | "accountId": "account2", 23 | "type": "Microsoft.Storage/virtualMachine", 24 | "name": "resource2", 25 | "providerType": "simulator", 26 | "location": "location2"}, 27 | {"id": '/resources/type1/resource3', 28 | "accountId": "account2", 29 | "type": "Microsoft.Storage/virtualMachine", 30 | "name": "resource3", 31 | "providerType": "simulator", 32 | "location": "location3"}, 33 | {"id": '/resources/type1/resource4', 34 | "accountId": "account3", 35 | "type": "Microsoft.Storage/virtualMachine", 36 | "name": "resource4", 37 | "providerType": "simulator", 38 | "location": "location4"}, 39 | {"id": '/resources/type1/resource5', 40 | "accountId": "account4", 41 | "type": "Microsoft.Storage/virtualMachine", 42 | "name": "resource5", 43 | "providerType": "simulator", 44 | "location": "location5"}, 45 | ] 46 | 47 | # entry is of json type 48 | def write(self, resource): 49 | """Write resource to storage. 50 | 51 | :param resource: Resource to write 52 | :return: None 53 | """ 54 | entry = resource.to_dict() 55 | prepared = entry_storage.EntryOperations.prepare_entry_for_insert( 56 | entry) 57 | key = entry['PartitionKey'] + '-' + entry['RowKey'] 58 | self._data[key] = prepared 59 | 60 | def query(self, partition_key, row_key): 61 | """Get element from table storage. 62 | 63 | :param partition_key: Partition key of resource 64 | :param row_key: Row key of resource 65 | :return: Resource if found 66 | """ 67 | task = self._data[partition_key + '-' + row_key] 68 | return task 69 | 70 | def query_list(self): 71 | """Get all resources in storage. 72 | 73 | :return: List of Resource objects 74 | """ 75 | return [Resource(resource) for resource in self._resources] 76 | 77 | def delete(self, partition_key, row_key): 78 | """Delete resource from storage. 79 | 80 | :param partition_key: Partition key of resource 81 | :param row_key: Row key of resource 82 | :return: None 83 | """ 84 | del self._data[partition_key + '-' + row_key] 85 | -------------------------------------------------------------------------------- /tests/test_resource_tagger.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from cloud_scanner.simulators import ResourceServiceSimulator 4 | from cloud_scanner.services.resource_tagger import ResourceTagger, ResourceTagProcessor 5 | from .unittest_base import TestCase, FakeQueueMessage 6 | 7 | 8 | class ResourceTaggerTest(TestCase): 9 | def setUp(self): 10 | os.environ["RESOURCE_STORAGE_TYPE"] = "simulator" 11 | 12 | def test_process_queue_message(self): 13 | os.environ["PROVIDER_LIST"] = "simulator" 14 | 15 | json_data = ''' 16 | { 17 | "resource": { 18 | "id": "00000000-0000-0000-0000-000000000123", 19 | "type": "vm", 20 | "group": "MyGroup", 21 | "name": "MyResource", 22 | "location": "WestUS", 23 | "providerType": "simulator", 24 | "accountId": "00000000-0000-0000-0000-000000000001" 25 | }, 26 | "tags": { 27 | "a": "1", 28 | "b": "2" 29 | } 30 | } 31 | ''' 32 | message = FakeQueueMessage(bytes(json_data, 'utf-8')) 33 | result = ResourceTagger.process_queue_message(message) 34 | 35 | self.assertEqual(2, result[0]) 36 | self.assertEqual(0, result[1]) 37 | 38 | def test_scanner_multiple_write(self): 39 | target_tags = { 40 | 'tag1': 'value', 41 | 'tag2': 'value' 42 | } 43 | 44 | resource_service = ResourceServiceSimulator() 45 | tag_processor = ResourceTagProcessor(resource_service) 46 | 47 | resource = resource_service.get_resources()[0] 48 | 49 | tag_processor.execute(resource, target_tags, True) 50 | 51 | assert(tag_processor.tags_written == 2) 52 | assert(tag_processor.tags_skipped == 0) 53 | 54 | def test_scanner_overwrite(self): 55 | test_tag_name = 'testTag1' 56 | test_tag_value = 'testTag1Value' 57 | test_tag_default_value = 'default' 58 | 59 | target_tags = dict() 60 | target_tags[test_tag_name] = test_tag_value 61 | 62 | resource_service = ResourceServiceSimulator() 63 | tag_processor = ResourceTagProcessor(resource_service) 64 | 65 | resource = resource_service.get_resources()[0] 66 | 67 | # Test does overwrite 68 | 69 | resource.tags = target_tags.copy() 70 | resource.tags[test_tag_name] = test_tag_default_value 71 | 72 | tag_processor.execute(resource, target_tags, True) 73 | 74 | assert(tag_processor.tags_written == 1) 75 | assert(tag_processor.tags_skipped == 0) 76 | assert(resource.tags[test_tag_name] == test_tag_value) 77 | 78 | # Test does not overwrite 79 | 80 | resource.tags = target_tags.copy() 81 | resource.tags[test_tag_name] = test_tag_default_value 82 | 83 | tag_processor.reset() 84 | tag_processor.execute(resource, target_tags, False) 85 | 86 | assert(tag_processor.tags_written == 0) 87 | assert(tag_processor.tags_skipped == 1) 88 | assert(resource.tags[test_tag_name] == test_tag_default_value) 89 | 90 | def test_process_rules(self): 91 | os.environ["TAG_UPDATES_QUEUE_NAME"] = "resource-tag-updates" 92 | os.environ["QUEUE_TYPE"] = "simulator" 93 | 94 | 95 | processed = ResourceTagger.process_tag_rules() 96 | 97 | self.assertEqual(10, processed) 98 | -------------------------------------------------------------------------------- /cloud_scanner/simulators/resource_service_simulator.py: -------------------------------------------------------------------------------- 1 | from cloud_scanner.contracts import Resource 2 | from cloud_scanner.contracts import ( 3 | ResourceService, ResourceFilter, register_resource_service) 4 | 5 | 6 | @register_resource_service("simulator", 7 | lambda subscription_id: ResourceServiceSimulator()) 8 | class ResourceServiceSimulator(ResourceService): 9 | """Simulator of resource service.""" 10 | @property 11 | def name(self): 12 | """ 13 | :return: 'simulator' 14 | """ 15 | return "simulator" 16 | 17 | resources = [{ 18 | 'id': '/subscriptions/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/' 19 | 'resourceGroups/yyyyyyyyyyyy/providers/' 20 | 'microsoft.insights/components/wwwwwwwwwwww', 21 | 'name': 'wwwwwwwwwwww', 22 | 'type': 'microsoft.insights/components', 23 | 'location': 'southcentralus', 24 | 'tags': { 25 | 'hidden-link:/subscriptions/xxxxxxxx-xxxx-xxxx' 26 | '-xxxx-xxxxxxxxxxxx/resourceGroups/yyyyyyyyyyyy/' 27 | 'providers/Microsoft.Web/sites/wwwwwwwwwwww': 'Resource' 28 | }, 29 | 'kind': 'web' 30 | }, { 31 | 'id': '/subscriptions/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/' 32 | 'resourceGroups/yyyyyyyyyyyy/providers/' 33 | 'Microsoft.ServiceBus/namespaces/wwwwwwwwwwww', 34 | 'name': 'wwwwwwwwwwww', 35 | 'type': 'Microsoft.ServiceBus/namespaces', 36 | 'location': 'southcentralus', 37 | 'tags': {}, 38 | 'sku': { 39 | 'name': 'Standard', 40 | 'tier': 'Standard' 41 | } 42 | }, { 43 | 'id': '/subscriptions/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/' 44 | 'resourceGroups/yyyyyyyyyyyy/providers/' 45 | 'Microsoft.Storage/storageAccounts/wwwwwwwwwwww', 46 | 'name': 'wwwwwwwwwwww', 47 | 'type': 'Microsoft.Storage/storageAccounts', 48 | 'location': 'southcentralus', 49 | 'tags': {}, 50 | 'kind': 'Storage', 51 | 'sku': { 52 | 'name': 'Standard_LRS', 53 | 'tier': 'Standard' 54 | } 55 | }, { 56 | 'id': '/subscriptions/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/' 57 | 'resourceGroups/yyyyyyyyyyyy/providers/' 58 | 'Microsoft.Web/serverFarms/wwwwwwwwwwww', 59 | 'name': 'wwwwwwwwwwww', 60 | 'type': 'Microsoft.Web/serverFarms', 61 | 'location': 'southcentralus', 62 | 'kind': 'functionapp' 63 | }, { 64 | 'id': '/subscriptions/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/' 65 | 'resourceGroups/yyyyyyyyyyyy/providers/' 66 | 'Microsoft.Web/sites/wwwwwwwwwwww', 67 | 'name': 'wwwwwwwwwwww', 68 | 'type': 'Microsoft.Web/sites', 69 | 'location': 'southcentralus', 70 | 'kind': 'functionapp' 71 | }] 72 | 73 | def get_resources(self, filter: ResourceFilter = None): 74 | """Get list of AzureResources from service. 75 | 76 | :param filter: Filter object to filter resources 77 | :return: List of AzureResource objects 78 | """ 79 | return [Resource(resource) for resource in self.resources] 80 | 81 | def update_resource(self, resource): 82 | return None 83 | 84 | def get_filter(self, payload) -> ResourceFilter: 85 | pass 86 | -------------------------------------------------------------------------------- /cloud_scanner/services/resource_tagger.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from cloud_scanner.contracts import ( 4 | Resource, ResourceService, ResourceServiceFactory, 5 | ResourceStorageFactory, RuleFactory 6 | ) 7 | 8 | 9 | class ResourceTagger: 10 | """Tag resources within cloud provider.""" 11 | 12 | @staticmethod 13 | def process_queue_message(message): 14 | """Apply tags to resources specified by message. 15 | 16 | :param message: Payload of resources 17 | :return: Tags written, tags skipped 18 | """ 19 | msg_json = message.get_json() 20 | table_storage = ResourceStorageFactory.create() 21 | 22 | resource = Resource(msg_json["resource"]) 23 | resource_service = ResourceServiceFactory.create( 24 | resource.provider_type, resource.account_id) 25 | 26 | tag_processor = ResourceTagProcessor(resource_service) 27 | 28 | tag_processor.execute(resource, msg_json["tags"]) 29 | table_storage.write(resource) 30 | 31 | return tag_processor.tags_written, tag_processor.tags_skipped 32 | 33 | @staticmethod 34 | def process_tag_rules(): 35 | """Get rules from rules factory and run them. 36 | 37 | :return: Number of matches found and applied tags to 38 | """ 39 | resource_storage = ResourceStorageFactory.create() 40 | resources = resource_storage.query_list() 41 | rules = RuleFactory.get_rules() 42 | 43 | processed_matches = 0 44 | 45 | for resource in resources: 46 | for rule in rules: 47 | if rule.process(resource): 48 | processed_matches += 1 49 | 50 | return processed_matches 51 | 52 | 53 | class ResourceTagProcessor: 54 | """Writes tags to cloud resources.""" 55 | 56 | def __init__(self, resource_service: ResourceService): 57 | self._resource_service = resource_service 58 | self._tags_written = 0 59 | self._tags_skipped = 0 60 | 61 | @property 62 | def tags_written(self): 63 | """ 64 | :return: int number of tags written 65 | """ 66 | return self._tags_written 67 | 68 | @property 69 | def tags_skipped(self): 70 | """ 71 | :return: int number of tags skipped 72 | """ 73 | return self._tags_skipped 74 | 75 | def execute(self, resource: Resource, tags: dict, overwrite=False): 76 | """Execute tagging of resource. 77 | 78 | :param resource: Resource to tag 79 | :param tags: tags to apply 80 | :param overwrite: True if overwrite of existing tags is desired, 81 | default False 82 | """ 83 | # Store tags written during this single execution 84 | local_written = 0 85 | 86 | for tag_key, tag_value in tags.items(): 87 | if not overwrite and tag_key in resource.tags: 88 | logging.info( 89 | f"Skipped tagging {resource.id} with tag {tag_key}" 90 | " since it already exists.") 91 | self._tags_skipped += 1 92 | continue 93 | 94 | resource.tags[tag_key] = tag_value 95 | local_written += 1 96 | self._tags_written += 1 97 | 98 | # Only save if needed 99 | if local_written > 0: 100 | self._resource_service.update_resource(resource) 101 | logging.info(f"Wrote {self._tags_written} tags to {resource.id}.") 102 | 103 | def reset(self): 104 | """Reset the tags written and tags skipped to 0.""" 105 | self._tags_written = 0 106 | self._tags_skipped = 0 107 | -------------------------------------------------------------------------------- /cloud_scanner/simulators/container_storage_simulator.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from cloud_scanner.contracts import ( 4 | StorageContainer, register_storage_container) 5 | 6 | 7 | class MockBlobStorageOutput: 8 | """Simulator of blob storage output.""" 9 | 10 | def __init__(self, name, content): 11 | self._name = name 12 | self._content = str(content) 13 | 14 | @property 15 | def name(self): 16 | """ 17 | :return: str Name of blob file 18 | """ 19 | return self._name 20 | 21 | @property 22 | def content(self): 23 | """ 24 | :return: str Content of blob file 25 | """ 26 | return self._content 27 | 28 | 29 | @register_storage_container("simulator", 30 | lambda: MockBlobStorageSimulator()) 31 | class MockBlobStorageSimulator(StorageContainer): 32 | """Simulator of BlobStorage.""" 33 | 34 | def __init__(self): 35 | 36 | config_content = '''{ 37 | "providers":[ 38 | { 39 | "type":"simulator", 40 | "resourceTypes": [{"typeName": 41 | "Microsoft.Compute/virtualMachines"}], 42 | "subscriptions": [ 43 | { 44 | "subscriptionId": 45 | "00000000-0000-0000-0000-000000000001", 46 | "displayName": "Simulator Sub 1" 47 | }, 48 | { 49 | "subscriptionId": 50 | "00000000-0000-0000-0000-000000000002", 51 | "displayName": "Simulator Sub 2" 52 | } 53 | ] 54 | }] 55 | }''' 56 | 57 | list_of_entries = [] 58 | latest = MockBlobStorageOutput( 59 | 'config-2018-08-29-10-20-49.json ', config_content) 60 | entry1 = MockBlobStorageOutput( 61 | 'config-2018-08-20-12-33-48.json ', '{}') 62 | entry2 = MockBlobStorageOutput( 63 | 'config-2018-08-21-09-41-05.json ', '{}') 64 | entry3 = MockBlobStorageOutput( 65 | 'config-2018-08-21-09-42-12.json ', '{}') 66 | entry4 = MockBlobStorageOutput( 67 | 'config-2018-08-22-11-41-49.json ', '{}') 68 | entry5 = MockBlobStorageOutput( 69 | 'config-2018-08-22-11-50-38.json ', '{}') 70 | 71 | list_of_entries.append(latest) 72 | list_of_entries.append(entry1) 73 | list_of_entries.append(entry2) 74 | list_of_entries.append(entry3) 75 | list_of_entries.append(entry4) 76 | list_of_entries.append(entry5) 77 | 78 | self._entries = list_of_entries 79 | self._latest_entry = latest 80 | 81 | def get_blob_to_text(self, config): 82 | """ 83 | :param config: Config file to get text from 84 | :return: Text contained in config 85 | """ 86 | # ensure the latest config was picked 87 | if config is not self._latest_entry.name: 88 | logging.error("The picked config is not the latest. " 89 | "Returned: %s, latest: %s", 90 | config, self._latest_entry.name) 91 | return None 92 | return self._latest_entry 93 | 94 | def list_blobs(self): 95 | """ 96 | :return: List of blob files 97 | """ 98 | return self._entries 99 | 100 | def get_latest_config(self): 101 | """ 102 | :return: Latest config file 103 | """ 104 | return self._latest_entry 105 | 106 | def upload_text(self, filename, text): 107 | """Fake call to upload text. 108 | 109 | :param filename: name of new config file 110 | :param text: text to put in config file 111 | :return: None 112 | """ 113 | logging.info("upload_text was called.") 114 | -------------------------------------------------------------------------------- /tests/test_integration.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import json 3 | import pytest 4 | import unittest 5 | 6 | from .unittest_base import FakeQueueMessage 7 | from cloud_scanner.services.resource_scanner import ResourceScanner 8 | from cloud_scanner.services.task_scheduler import TaskScheduler 9 | from cloud_scanner.services.resource_storage import ResourceStorage 10 | from cloud_scanner.contracts import ResourceStorageFactory, QueueFactory 11 | 12 | from unittest import mock 13 | 14 | from cloud_scanner.simulators import QueueSimulator 15 | 16 | @pytest.mark.skip(reason="Integration test") 17 | class IntegrationTest(unittest.TestCase): 18 | 19 | # Used when explicitly testing a task 20 | _test_subscription = '12341234-0000-0000-0000-123412341234' 21 | 22 | # 23 | _write_to_storage = True 24 | 25 | def schedule_tasks(self): 26 | result_messages = [] 27 | 28 | def push_intercept(msg): 29 | queue_msg = FakeQueueMessage(bytes(msg, 'utf-8')) 30 | result_messages.append(queue_msg) 31 | 32 | with mock.patch.object(QueueSimulator, "push", side_effect=push_intercept): 33 | logging.info("Running Task Scheduler") 34 | TaskScheduler.execute() 35 | 36 | # Will fail if no tasks were configured 37 | assert(result_messages) 38 | 39 | test_msg = result_messages[0].get_json() 40 | logging.info(f"Inspecting first message: {test_msg}") 41 | assert(test_msg['providerType']) 42 | assert(test_msg['subscriptionId']) 43 | assert(test_msg['typeName']) 44 | 45 | return result_messages 46 | 47 | @staticmethod 48 | def _create_dummy_task(sub_id, resource_type, provider='azure'): 49 | dummy_task = { 50 | 'providerType': provider, 51 | 'subscriptionId': sub_id, 52 | 'typeName': resource_type 53 | } 54 | task_string = json.dumps(dummy_task) 55 | msg = FakeQueueMessage(bytes(task_string, 'utf-8')) 56 | return msg 57 | 58 | def scan_resource_test(self, task_msg): 59 | result_messages = [] 60 | 61 | def push_intercept(msg): 62 | queue_msg = FakeQueueMessage(bytes(msg, 'utf-8')) 63 | result_messages.append(queue_msg) 64 | 65 | with mock.patch.object(QueueFactory, "create") as cm: 66 | fake_push = mock.MagicMock(side_effect=push_intercept) 67 | fake_queue = mock.MagicMock(push=fake_push) 68 | cm.return_value = fake_queue 69 | 70 | logging.info(f"Running Resource Scanner") 71 | ResourceScanner.process_queue_message(task_msg) 72 | 73 | # Will fail if no resources were found 74 | assert(result_messages) 75 | 76 | test_msg = result_messages[0].get_json() 77 | logging.info(f"Inspecting first message: {test_msg}") 78 | assert(test_msg["id"]) 79 | assert(test_msg["name"]) 80 | assert(test_msg["type"]) 81 | assert(test_msg["location"]) 82 | 83 | return result_messages 84 | 85 | def store_resource_test(self, resource_msg): 86 | if self._write_to_storage: 87 | ResourceStorage.process_queue_message(resource_msg) 88 | return 89 | 90 | written_resources = [] 91 | def write_intercept(resource): 92 | logging.info(f"Intercepted storage write {resource.to_normalized_dict()}") 93 | written_resources.append(resource) 94 | 95 | with mock.patch.object(ResourceStorageFactory, "create") as cm: 96 | fake_write = mock.MagicMock(side_effect=write_intercept) 97 | fake_storage = mock.MagicMock(write=fake_write) 98 | cm.return_value = fake_storage 99 | 100 | logging.info(f"Running Resource Storage") 101 | ResourceStorage.process_queue_message(resource_msg) 102 | 103 | assert(written_resources) 104 | normalized_resource = written_resources[0].to_normalized_dict() 105 | 106 | assert(normalized_resource["ARN"]) 107 | assert(normalized_resource["ResourceId"]) 108 | assert(normalized_resource["ResourceType"]) 109 | assert(normalized_resource["Region"]) 110 | 111 | def _run_task(self, task): 112 | resources = self.scan_resource_test(task) 113 | 114 | for resource_msg in resources: 115 | self.store_resource_test(resource_msg) 116 | 117 | # Full integration test of everything defined by configuration 118 | def test_integration(self): 119 | tasks = self.schedule_tasks() 120 | 121 | for task in tasks: 122 | self._run_task(task) 123 | 124 | # Only test storage accounts 125 | def test_storage_account(self): 126 | storage_task = self._create_dummy_task(self._test_subscription, 'Microsoft.Storage/storageAccounts') 127 | self._run_task(storage_task) 128 | 129 | # Only test virtual Machines 130 | def test_virtual_machines(self): 131 | storage_task = self._create_dummy_task(self._test_subscription, 'Microsoft.Compute/virtualMachines') 132 | 133 | self._run_task(storage_task) 134 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Configuration file for the Sphinx documentation builder. 4 | # 5 | # This file does only contain a selection of the most common options. For a 6 | # full list see the documentation: 7 | # http://www.sphinx-doc.org/en/master/config 8 | 9 | # -- Path setup -------------------------------------------------------------- 10 | 11 | # If extensions (or modules to document with autodoc) are in another directory, 12 | # add these directories to sys.path here. If the directory is relative to the 13 | # documentation root, use os.path.abspath to make it absolute, like shown here. 14 | # 15 | import os 16 | import sys 17 | sys.path.insert(0, os.path.abspath('../..')) 18 | 19 | 20 | # -- Project information ----------------------------------------------------- 21 | 22 | project = 'Cloud Scanner' 23 | copyright = '2018, Microsoft' 24 | author = 'Microsoft' 25 | 26 | # The short X.Y version 27 | exec(open('../../cloud_scanner/version.py').read()) 28 | version = '__version__' 29 | # The full version, including alpha/beta/rc tags 30 | release = __version__ 31 | 32 | # -- General configuration --------------------------------------------------- 33 | 34 | # If your documentation needs a minimal Sphinx version, state it here. 35 | # 36 | # needs_sphinx = '1.0' 37 | 38 | # Add any Sphinx extension module names here, as strings. They can be 39 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 40 | # ones. 41 | extensions = [ 42 | 'sphinx.ext.autodoc', 43 | 'sphinx.ext.githubpages', 44 | ] 45 | 46 | # Add any paths that contain templates here, relative to this directory. 47 | templates_path = ['_templates'] 48 | 49 | # The suffix(es) of source filenames. 50 | # You can specify multiple suffix as a list of string: 51 | # 52 | # source_suffix = ['.rst', '.md'] 53 | source_suffix = '.rst' 54 | 55 | # The master toctree document. 56 | master_doc = 'index' 57 | 58 | # The language for content autogenerated by Sphinx. Refer to documentation 59 | # for a list of supported languages. 60 | # 61 | # This is also used if you do content translation via gettext catalogs. 62 | # Usually you set "language" from the command line for these cases. 63 | language = None 64 | 65 | # List of patterns, relative to source directory, that match files and 66 | # directories to ignore when looking for source files. 67 | # This pattern also affects html_static_path and html_extra_path. 68 | exclude_patterns = [] 69 | 70 | # The name of the Pygments (syntax highlighting) style to use. 71 | pygments_style = None 72 | 73 | 74 | # -- Options for HTML output ------------------------------------------------- 75 | 76 | # The theme to use for HTML and HTML Help pages. See the documentation for 77 | # a list of builtin themes. 78 | # 79 | html_theme = 'classic' 80 | 81 | # Theme options are theme-specific and customize the look and feel of a theme 82 | # further. For a list of options available for each theme, see the 83 | # documentation. 84 | # 85 | # html_theme_options = {} 86 | 87 | # Add any paths that contain custom static files (such as style sheets) here, 88 | # relative to this directory. They are copied after the builtin static files, 89 | # so a file named "default.css" will overwrite the builtin "default.css". 90 | html_static_path = ['_static'] 91 | 92 | # Custom sidebar templates, must be a dictionary that maps document names 93 | # to template names. 94 | # 95 | # The default sidebars (for documents that don't match any pattern) are 96 | # defined by theme itself. Builtin themes are using these templates by 97 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', 98 | # 'searchbox.html']``. 99 | # 100 | # html_sidebars = {} 101 | 102 | 103 | # -- Options for HTMLHelp output --------------------------------------------- 104 | 105 | # Output file base name for HTML help builder. 106 | htmlhelp_basename = 'CloudScannerdoc' 107 | 108 | 109 | # -- Options for LaTeX output ------------------------------------------------ 110 | 111 | latex_elements = { 112 | # The paper size ('letterpaper' or 'a4paper'). 113 | # 114 | # 'papersize': 'letterpaper', 115 | 116 | # The font size ('10pt', '11pt' or '12pt'). 117 | # 118 | # 'pointsize': '10pt', 119 | 120 | # Additional stuff for the LaTeX preamble. 121 | # 122 | # 'preamble': '', 123 | 124 | # Latex figure (float) alignment 125 | # 126 | # 'figure_align': 'htbp', 127 | } 128 | 129 | # Grouping the document tree into LaTeX files. List of tuples 130 | # (source start file, target name, title, 131 | # author, documentclass [howto, manual, or own class]). 132 | latex_documents = [ 133 | (master_doc, 'CloudScanner.tex', 'Cloud Scanner Documentation', 134 | 'Microsoft', 'manual'), 135 | ] 136 | 137 | 138 | # -- Options for manual page output ------------------------------------------ 139 | 140 | # One entry per manual page. List of tuples 141 | # (source start file, name, description, authors, manual section). 142 | man_pages = [ 143 | (master_doc, 'cloudscanner', 'Cloud Scanner Documentation', 144 | [author], 1) 145 | ] 146 | 147 | 148 | # -- Options for Texinfo output ---------------------------------------------- 149 | 150 | # Grouping the document tree into Texinfo files. List of tuples 151 | # (source start file, target name, title, author, 152 | # dir menu entry, description, category) 153 | texinfo_documents = [ 154 | (master_doc, 'CloudScanner', 'Cloud Scanner Documentation', 155 | author, 'CloudScanner', 'One line description of project.', 156 | 'Miscellaneous'), 157 | ] 158 | 159 | 160 | # -- Options for Epub output ------------------------------------------------- 161 | 162 | # Bibliographic Dublin Core info. 163 | epub_title = project 164 | 165 | # The unique identifier of the text. This can be a ISBN number 166 | # or the project homepage. 167 | # 168 | # epub_identifier = '' 169 | 170 | # A unique identification for the text. 171 | # 172 | # epub_uid = '' 173 | 174 | # A list of files that should not be packed into the epub file. 175 | epub_exclude_files = ['search.html'] 176 | 177 | 178 | # -- Extension configuration ------------------------------------------------- -------------------------------------------------------------------------------- /cloud_scanner/contracts/resource.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import datetime 3 | import hashlib 4 | import json 5 | from abc import ABC 6 | 7 | 8 | class Resource(ABC): 9 | """Base class for cloud resource object.""" 10 | 11 | def __init__(self, d: dict): 12 | self._raw = d 13 | 14 | self._provider_type = d.get("provider_type", "simulator") 15 | 16 | self._id = d.get("id", None) 17 | self._account_id = d.get("accountId", None) 18 | self._name = d.get("name", None) 19 | self._type = d.get("type", None) 20 | self._location = d.get("location", None) 21 | self._tags = d.get("tags", {}) 22 | 23 | self._environment = self._tags.get('Environment', None) 24 | self._app_name = self._tags.get('AppName', None) 25 | self._tag_name = self._tags.get('Name', None) 26 | self._tag_guid = self._get_tag_guid() 27 | 28 | @staticmethod 29 | def _jsonify(o): 30 | """Handle any non-json serializable types. 31 | 32 | :param o: object 33 | :return: str of json 34 | """ 35 | if isinstance(o, datetime.datetime): 36 | return o.__str__() 37 | 38 | def _generate_hash(self, data): 39 | """Generate a hash of data provided. 40 | 41 | :param data: Data to use for hash 42 | :return: Sha-1 hash of data 43 | """ 44 | stringified = json.dumps(data, ensure_ascii=True, 45 | sort_keys=True, 46 | default=self._jsonify).encode() 47 | 48 | return hashlib.sha1(stringified).hexdigest() 49 | 50 | def _get_tag_guid(self): 51 | """Get GUID for tag. 52 | 53 | :return: GUID 54 | """ 55 | tag_guid = self._tags.get('TagGuid', None) 56 | if tag_guid is None: 57 | tag_guid = self._tags.get('AppDefined02', None) 58 | return tag_guid 59 | 60 | @property 61 | def raw(self): 62 | """ 63 | :return: raw resource data 64 | """ 65 | return self._raw 66 | 67 | @property 68 | def id(self): 69 | """ 70 | :return: resource ID 71 | """ 72 | return self._id 73 | 74 | @property 75 | def environment(self): 76 | """ 77 | :return: environment of resource 78 | """ 79 | return self._environment 80 | 81 | @property 82 | def app_name(self): 83 | """ 84 | :return: app name 85 | """ 86 | return self._app_name 87 | 88 | @property 89 | def tag_guid(self): 90 | """ 91 | :return: guid of tag 92 | """ 93 | return self._tag_guid 94 | 95 | @property 96 | def tag_name(self): 97 | """ 98 | :return: name of tag 99 | """ 100 | return self._tag_name 101 | 102 | @property 103 | def account_id(self): 104 | """ 105 | :return: account ID for account resource lives in 106 | """ 107 | return self._account_id 108 | 109 | @property 110 | def name(self): 111 | """ 112 | :return: name of resource 113 | """ 114 | return self._name 115 | 116 | @name.setter 117 | def name(self, value): 118 | """Set name of resource. 119 | 120 | :param value: new name 121 | :return: None 122 | """ 123 | self._name = value 124 | 125 | @property 126 | def type(self): 127 | """ 128 | :return: resource type 129 | """ 130 | return self._type 131 | 132 | @property 133 | def location(self): 134 | """ 135 | :return: location of resource 136 | """ 137 | return self._location 138 | 139 | @location.setter 140 | def location(self, value): 141 | """Set location of resource. 142 | 143 | :param value: location 144 | :return: None 145 | """ 146 | self._location = value 147 | 148 | @property 149 | def tags(self): 150 | """ 151 | :return: Dictionary of tags 152 | """ 153 | return self._tags 154 | 155 | @tags.setter 156 | def tags(self, value): 157 | """Set tags for resource. 158 | 159 | :param value: tags dictionary 160 | :return: None 161 | """ 162 | self._tags = value 163 | 164 | @property 165 | def provider_type(self): 166 | """ 167 | :return: Resource provider type 168 | """ 169 | return self._provider_type 170 | 171 | @provider_type.setter 172 | def provider_type(self, provider_type): 173 | """Set resource provider type. 174 | 175 | :param provider_type: new provider type for resource 176 | :return: None 177 | """ 178 | self._provider_type = provider_type 179 | 180 | def to_normalized_dict(self): 181 | """Create normalized dictionary for resource across cloud providers. 182 | 183 | :return: Normalized dictionary 184 | """ 185 | 186 | out_dict = copy.deepcopy(self.raw) # Populate with full meta-data? 187 | 188 | out_dict.update({'AppDefined02': self.tag_guid}) 189 | out_dict.update({"Environment": self.environment}) 190 | out_dict.update( 191 | {"ResourceType": self.type.replace("/", "_").replace(".", "_")}) 192 | out_dict.update({"ResourceId": self.name}) 193 | out_dict.update({"ARN": self.id}) 194 | out_dict.update({"Name": self.tag_name}) 195 | out_dict.update({"Type": self.type}) 196 | out_dict.update({"Region": self.location}) 197 | out_dict.update({"OwnerId": self.account_id}) 198 | out_dict.update({"AppName": self.app_name}) 199 | out_dict.update({"Tags": self.tags}) 200 | 201 | out_dict.update({"Hash": self._generate_hash(out_dict)}) 202 | 203 | return out_dict 204 | 205 | def to_dict(self): 206 | """ 207 | :return: Dictionary with resource data 208 | """ 209 | return { 210 | 'id': self.id, 211 | 'accountId': self._account_id, 212 | 'name': self.name, 213 | 'type': self.type, 214 | 'location': self.location, 215 | 'tags': self.tags, 216 | 'providerType': self.provider_type 217 | } 218 | 219 | def to_str(self): 220 | """ 221 | :return: JSON str of resource dictionary 222 | """ 223 | return json.dumps(self.to_dict()) 224 | --------------------------------------------------------------------------------