├── .github └── workflows │ ├── linter-and-tests.yml │ ├── publish-to-pypi-rc.yml │ ├── publish-to-pypi.yml │ └── publish-to-test-pypi.yml ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── cloudimized-example.png ├── cloudimized ├── __init__.py ├── azurecore │ ├── __init__.py │ ├── aksclustersquery.py │ ├── azurecredential.py │ ├── azurequery.py │ ├── networksecuritygroupsquery.py │ ├── resourcegroupsquery.py │ ├── subscriptionsquery.py │ ├── virtualnetworksquery.py │ └── vnetgatewaysquery.py ├── core │ ├── __init__.py │ ├── changeprocessor.py │ ├── core.py │ ├── jiranotifier.py │ ├── result.py │ ├── run.py │ └── slacknotifier.py ├── gcpcore │ ├── __init__.py │ ├── gcpchangelog.py │ ├── gcpexternaltoken.py │ ├── gcpquery.py │ └── gcpservicequery.py ├── gitcore │ ├── __init__.py │ ├── gitchange.py │ └── repo.py ├── singlerunconfigs │ ├── azure │ │ ├── aksClusters.yaml │ │ ├── networkSecurityGroups.yaml │ │ ├── virtualNetworks.yaml │ │ └── vnetGateways.yaml │ └── gcp │ │ ├── addresses.yaml │ │ ├── forwardingRules.yaml │ │ ├── globalAddresses.yaml │ │ ├── globalPublicDelegatedPrefixes.yaml │ │ ├── networks.yaml │ │ ├── publicAdvertisedPrefixes.yaml │ │ ├── publicDelegatedPrefixes.yaml │ │ ├── subnetworks.yaml │ │ └── vpnTunnels.yaml └── tfcore │ ├── __init__.py │ ├── query.py │ └── run.py ├── config-example.yaml ├── requirements.txt ├── setup.cfg ├── setup.py └── tests ├── test_azurevnetgatewaysquery.py ├── test_azurevnetsquery.py ├── test_changeprocessor.py ├── test_gcpchangelog.py ├── test_gcpquery.py ├── test_gcpservicequery.py ├── test_gitchange.py ├── test_gitrepo.py ├── test_jiranotifier.py ├── test_result.py ├── test_slacknotifier.py ├── test_tfquery.py └── test_tfrun.py /.github/workflows/linter-and-tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | # trigger only on pushes to main OR PRs to main 4 | # to prevent double tests on our own 5 | # PRs within egnyte/cloudimized repo from a feature 6 | # branch to main 7 | on: 8 | push: 9 | branches: 10 | - main 11 | - develop 12 | pull_request: 13 | branches: 14 | - main 15 | - develop 16 | jobs: 17 | unit-tests: 18 | runs-on: ubuntu-latest 19 | strategy: 20 | max-parallel: 5 21 | matrix: 22 | python-version: [3.9, "3.10"] 23 | fail-fast : false 24 | steps: 25 | - uses: actions/checkout@v2 26 | - name: Set up Python ${{ matrix.python-version }} 27 | uses: actions/setup-python@v2 28 | with: 29 | python-version: ${{ matrix.python-version }} 30 | - name: Install dependencies & test dependencies 31 | run: | 32 | pip install -e .[test] 33 | - name: Run unit tests 34 | run: | 35 | python -m unittest discover tests 36 | -------------------------------------------------------------------------------- /.github/workflows/publish-to-pypi-rc.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Publish 🐍 to PyPI - RC only 3 | 4 | on: 5 | push: 6 | tags: 7 | - 'v*rc*' 8 | 9 | jobs: 10 | build-n-publish: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v2 14 | - name: Set up Python 3.9 15 | uses: actions/setup-python@v2 16 | with: 17 | python-version: 3.9 18 | - name: Build 19 | run: | 20 | pip install wheel 21 | python setup.py build sdist bdist_wheel 22 | - name: Publish distribution 📦 to PyPI 23 | uses: pypa/gh-action-pypi-publish@v1.4.1 24 | with: 25 | password: ${{ secrets.PYPI_API_TOKEN }} 26 | -------------------------------------------------------------------------------- /.github/workflows/publish-to-pypi.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Publish 🐍 to PyPI 3 | 4 | on: 5 | push: 6 | tags: 7 | - 'v*' 8 | - '!v*rc*' 9 | 10 | jobs: 11 | build-n-publish: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v2 15 | - name: Set up Python 3.9 16 | uses: actions/setup-python@v2 17 | with: 18 | python-version: 3.9 19 | - name: Build 20 | run: | 21 | pip install wheel 22 | python setup.py build sdist bdist_wheel 23 | - name: Publish distribution 📦 to PyPI 24 | uses: pypa/gh-action-pypi-publish@v1.4.1 25 | with: 26 | password: ${{ secrets.PYPI_API_TOKEN }} 27 | - name: Create release in GitHub 28 | id: create_release 29 | uses: actions/create-release@v1 30 | env: 31 | # This token is provided by Actions, you do not need to create your own token 32 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 33 | with: 34 | tag_name: ${{ github.ref }} 35 | release_name: ${{ github.ref }} 36 | draft: false 37 | prerelease: false 38 | -------------------------------------------------------------------------------- /.github/workflows/publish-to-test-pypi.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Publish 🐍 to Test PyPI 3 | 4 | on: 5 | push: 6 | tags: 7 | - 'testv*' 8 | 9 | jobs: 10 | build-n-publish: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v2 14 | - name: Set up Python 3.9 15 | uses: actions/setup-python@v2 16 | with: 17 | python-version: 3.9 18 | - name: Build 19 | run: | 20 | pip install wheel 21 | python setup.py build sdist bdist_wheel 22 | - name: Publish distribution 📦 to Test PyPI 23 | uses: pypa/gh-action-pypi-publish@v1.4.1 24 | with: 25 | password: ${{ secrets.TEST_PYPI_API_TOKEN }} 26 | repository_url: https://test.pypi.org/legacy/ 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/* 2 | __pycache__ 3 | .pytest_cache 4 | cloudimized.egg-info 5 | venv 6 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 2.0.1 4 | 5 | * Add support for Azure calls: networkSecurityGroups & aksClusters 6 | 7 | ## 2.0.0 8 | 9 | * Add support for Azure cloud 10 | 11 | ## 1.3.11 12 | 13 | * Update Jira notifier with token authentication 14 | 15 | ## 1.3.10 16 | 17 | * Update Slack API file upload call 18 | * Upgrade dependencies 19 | 20 | ## 1.3.9 21 | 22 | * Fix paging limitation for GCP API call 23 | * Upgrade dependencies 24 | 25 | ## 1.3.8 26 | 27 | * Upgrade dependencies 28 | 29 | ## 1.3.7 30 | 31 | * Improve result filter include functionality 32 | 33 | ## 1.3.6 34 | 35 | * Upgrade dependencies 36 | 37 | ## 1.3.5 38 | 39 | * Upgrade dependencies 40 | 41 | ## 1.3.4 42 | 43 | * Upgrade dependencies 44 | 45 | ## 1.3.2 46 | 47 | * Upgrade dependencies 48 | 49 | ## 1.3.1 50 | 51 | * Add result sorting configuration option 52 | 53 | ## 1.3.0 54 | 55 | * Add multi-threading for resource scanning queries 56 | 57 | ## 1.2.1 58 | 59 | * Add support for nested results for CSV output 60 | 61 | ## 1.2.0 62 | 63 | * Change code structure - for package installation 64 | * Added single run mode 65 | 66 | ## 1.1.0 67 | 68 | * Added JiraNotifier feature 69 | 70 | ## 1.0.0 71 | 72 | * Initial version of Cloudimized 73 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Egnyte, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Cloudimized 2 | 3 | Cloudimized is a Cloud Provider configuration scanning tool. It allows 4 | monitoring changes of selected resources. Supports Google Cloud Platform (GCP) and 5 | Microsoft Azure. 6 | 7 | Cloudimized performs similar function as Oxidized, but for Cloud environment. 8 | 9 | ![Example Slack notification when using all optional features, including a git diff of the change](https://raw.githubusercontent.com/egnyte/cloudimized/main/cloudimized-example.png) 10 | 11 | ## Overview 12 | 13 | Cloudimized periodical scans of resources via API calls and dumps them into yaml files. Files are tracked in Git, 14 | so every configuration change is being tracked. 15 | It gathers additional information for each change: changer's identity, related Terraform runs 16 | (optionally), identify change ticket number (optionally). 17 | Optionally it sends Slack notifications for each change. 18 | 19 | ## Features 20 | 21 | * Project discovery across GCP organization / Subscription & ResourceGroups discovery in Azure Tenant 22 | * Identifying changer identity (only GCP) 23 | * Manual changes detection (only GCP) 24 | * Identifying Terraform runs 25 | * Identifying change tickets 26 | * Slack notifications 27 | 28 | ## Feature details 29 | 30 | ## Workflow 31 | 32 | On each execution Cloudimized performs following actions: 33 | 34 | 1. Gets configuration from previous execution from Git remote 35 | 2. Clears all configuration files from local repo 36 | 3. Performs resource reading 37 | 1. For all projects for all resources executes API call to get resource configuration 38 | 2. (optional) Performs configured field and item filtering for each resource 39 | 4. Dumps all results in yaml files format in local Git repo 40 | 5. Checks local Git repo state and detects all changed resource 41 | 6. For each detected change performs additional information gathering: 42 | 1. Get GCP Logs for change to identify changer's identity (only GCP) 43 | 2. Identifies manual changes (only GCP) 44 | 1. *changes performed directly by individual users as opposed to service accounts i.e. changes done outside of Terraform* 45 | 3. (optional) If change performed via Terraform Service Account, identify related Terraform runs. 46 | 1. Get Terraform Runs URL 47 | 2. (optional) Get ticket/issue marker from Terraform Run message. Generate ticket URL. 48 | 7. Commit each individual change to Git repo 49 | 1. Contains configuration Git diff 50 | 2. Contains all additional gathered information 51 | 8. (optional) Send Slack notification for each change 52 | 1. Containing same information as Git commit 53 | 9. Push new commits to remote repo 54 | 55 | ## Installation 56 | 57 | 1. Install Cloudimized with [pipx](https://github.com/pypa/pipx) (recommended) or plain pip. 58 | 59 | ``` 60 | pipx install cloudimized 61 | ``` 62 | 63 | 2. Cloudimized for operation requires Git repo for storing cloud's configuration files. 64 | 1. Set-up empty Git repo in remote location i.e. GitHub or GitLab 65 | 66 | ## Running 67 | 68 | After installation: 69 | 1. Perform necessary [configuration](#Configuration) 70 | 2. Schedule periodic, regular script execution 71 | 1. This can be achieved via number of ways i.e. 72 | 1. via cron 73 | 2. via automation server 74 | 2. Execute with `cloudimized -c /config.yaml` 75 | 76 | ## Configuration 77 | 78 | ### Service accounts 79 | 80 | #### GCP Service Account 81 | 82 | GCP Service Account is used to perform resources scanning, log reading and project discovery. Below steps show setting up account with organization wide 83 | reading permissions. 84 | 1. Create dedicated Service Account in selected GCP project 85 | 2. On organization level create custom role with permissions: 86 | 1. *logging.logEntries.list* //required to identify changer 87 | 2. *\.list* //for each resource to be scanned 88 | 1. i.e. **compute.subnetworks.list** for scanning of VPC subnetwork resource 89 | 2. Refer to official Google docs for [method](https://cloud.google.com/compute/docs/reference/rest/v1/subnetworks/list) and its [permission](https://cloud.google.com/compute/docs/reference/rest/v1/subnetworks/list#iam-permissions) mapping 90 | 3. On organization level create IAM policy that binds Service Account with Roles: Browser and custom role you've created 91 | 92 | This will grant Service Account permission to discover all projects in Organzation, perform scanning of selected 93 | resources in all projects and perform Logs scanning in all projects. If needed it is possible to limit permissions to 94 | selected Folders and/or Projects. 95 | 96 | See: 97 | * [Google Cloud listing permissions](https://cloud.google.com/resource-manager/docs/listing-all-resources) 98 | 99 | #### Azure servie accounts 100 | 101 | Authentication to Azure is done using [Azure DefaultAzureCredentials class](https://learn.microsoft.com/en-us/python/api/azure-identity/azure.identity.defaultazurecredential?view=azure-python), 102 | for exact capabilities how authentication can be done view [official docs](https://learn.microsoft.com/en-us/azure/developer/python/sdk/authentication/credential-chains?tabs=dac#defaultazurecredential-overview). 103 | 104 | Additionally, authentication using GCP Service Account via [Workfload Identity Federation is supported](#azure-secrets) 105 | 106 | #### GIT Service Account 107 | 108 | GIT Service Account is used to get configuration repository from remote and pushing to that repo detected configuration changes. 109 | In your GIT remote you need to configure account/access token with proper permissions. Script allows communication via 110 | both HTTPS and SSH. 111 | 112 | #### Terraform Service Account 113 | 114 | Terraform Account is used to read Terraform Runs. Concept of Service Account doesn't exist in Terraform, so this is 115 | performed in following way: 116 | 1. Create Terraform Team with "Runs: Read Runs" permissions 117 | 2. For each Terraform organization generate Team API token for your Team 118 | 119 | See: 120 | * [Terraform teams](https://www.terraform.io/cloud-docs/users-teams-organizations/teams) 121 | * [Terraform permissions](https://www.terraform.io/cloud-docs/users-teams-organizations/permissions) 122 | * [Terraform API tokens](https://www.terraform.io/cloud-docs/users-teams-organizations/api-tokens) 123 | 124 | #### Slack Service Account 125 | 126 | Slack Service Account is used for sending detected change notification to Slack channel. This is done via Slack App. 127 | 1. Create Slack app on Slack API page 128 | 1. Select *From scratch* 129 | 2. App Name - enter *Cloudimized* 130 | 3. Workspace - select your workspace 131 | 2. In *Add features and funcionality* select *Permissions* 132 | 3. In *Scopes* -> *Bot Token Scopes*, click *Add an OAuth Scopes* 133 | 1. Select *files:write* 134 | 4. In *OAuth Tokens for Your Woskrpace*, click *Install to Workspace* 135 | 5. Click *Allow access* 136 | 6. *Bot User OAuth Token* has been generated, take a note 137 | 7. Select/create channel to which notifications will be sent 138 | 1. Add App bot user by @mentioning App name in selected channel 139 | 140 | See: 141 | * [Slack app setup](https://api.slack.com/authentication/basics) 142 | 143 | #### Jira Service Account 144 | 145 | Jira Service Account is used for creating issues(tickets) for selected changes. Depending on your Jira deployement 146 | (Cloud vs Server), you need to setup an account with proper permissions to create Issues of selected type in given 147 | project. 148 | 149 | See: 150 | * [Jira Cloud API token](https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/) 151 | * [Jira Software docs](https://confluence.atlassian.com/jirasoftware/jira-software-documentation-774242447.html) 152 | 153 | ### Secrets 154 | 155 | All secrets required to run script need to be passed via env variables. 156 | 157 | #### GIT secrets 158 | 159 | Communication with GIT remote can be both SSH and HTTPS based. If using SSH you need to take care of your SSH keys 160 | configuration. If using HTTPS you need to pass credentials via following env variables: 161 | 162 | * env var `GIT_USR` - username 163 | * env var `GIT_PSW` - password/token 164 | 165 | #### Google Service Account secrets 166 | 167 | Cloudimized authenticates to GCP using GCP Service Account. After creating account you need to download JSON key file. 168 | Passing credentials to script can be achieved either by setting env var or using Google's ADC mechanism 169 | 170 | * env var `GOOGLE_APPLICATION_CREDENTIALS` set to file path of service account's JSON key token 171 | 172 | or 173 | 174 | * Authenticate via `gcloud auth application-default login` by providing service account's JSON key token 175 | 176 | See: 177 | * [Best practices to securely authenticate applications in Google Cloud ](https://cloud.google.com/docs/authentication/best-practices-applications#overview_of_application_default_credentials) 178 | * [Authenticating as a service account](https://cloud.google.com/docs/authentication/production) 179 | * [gcloud auth application-default login](https://cloud.google.com/sdk/gcloud/reference/auth/application-default/login) 180 | 181 | #### Terraform secrets 182 | 183 | Cloudimized authenticates to Terraform using Team token for each Terraform organization. Tokens are stored in JSON file 184 | that only contains mapping between organization and team token value. File is passed to script by providing JSON file 185 | location. This can be done by: 186 | 187 | * env var `TERRAFORM_READ_TOKENS` set to file path of terraform's JSON mapping key 188 | 189 | or 190 | 191 | * option `workspace_token_file` in configuration file set to file path of terraform's JSON mapping key 192 | 193 | #### Slack secrets 194 | 195 | Cloudimized authenticates to Slack using Slack's Applications Bot User token. Token is passed to script via env var 196 | 197 | * env var `SLACK_TOKEN` set to token value 198 | 199 | #### Jira secrets 200 | 201 | Cloudimized authenticates to Jira using Username and Password/Token combination. Credentials are passed to script via 202 | env var 203 | 204 | * env var `JIRA_USR` - Jira's username - (note: not set when using Token auth) 205 | * env var `JIRA_PSW` - Jira's password/Token 206 | 207 | #### Azure secrets 208 | 209 | For a scenario when authentication to Azure is done via 210 | [Workload Identity Federation](https://learn.microsoft.com/en-us/entra/workload-id/workload-identity-federation) and 211 | using [Azure app to authenticate via external Identity Provider](https://learn.microsoft.com/en-us/entra/workload-id/workload-identity-federation-create-trust?pivots=identity-wif-apps-methods-azp) 212 | following env var need to be set ([see docs](https://learn.microsoft.com/en-us/entra/workload-id/workload-identity-federation-create-trust?pivots=identity-wif-apps-methods-azp)) 213 | 214 | * `AZURE_CLIENT_ID` - Azure's Application's (Client) ID 215 | * `AZURE_TENANT_ID` - Azure's Application's Directory (Tenant) ID 216 | 217 | ### Configuration file 218 | 219 | Example configuration file: 220 | ```YAML 221 | # Configuration queries for Azure resources 222 | azure_queries: 223 | # Type of Azure's resource to query - registration name of AzureQuery class 224 | # Currently supported queries: resourceGroups, subscriptions, virtualNetworks, vnetGateways 225 | - resource: virtualNetworks 226 | # List of fields to exclude from configuration i.e. for status related fields 227 | field_exclude_filter: 228 | - etag 229 | - subnets: 230 | - etag 231 | # Configuration queries for GCP resources 232 | gcp_services: 233 | # GCP API Service name - https://cloud.google.com/compute/docs/reference/rest/v1#service:-compute.googleapis.com 234 | - serviceName: compute 235 | # GCP API version - https://cloud.google.com/compute/docs/reference/rest/v1#service:-compute.googleapis.com 236 | version: v1 237 | # List of all queries to send for this API Service 238 | queries: 239 | # User defined name of resource 240 | - resource: networks 241 | # GCP API method name - https://cloud.google.com/compute/docs/reference/rest/v1/networks/list 242 | gcp_api_call: networks.list 243 | # GCP API method parameters - https://cloud.google.com/compute/docs/reference/rest/v1/networks/list#path-parameters 244 | gcp_function_args: 245 | # Special Cloudimized value for setting ProjectID - script performs dynamic replacement with real Project ID 246 | project: 247 | # resource.type value in GCP Logs entries generated for this resource changes - https://cloud.google.com/logging/docs/api/v2/resource-list#tag_gce_network 248 | gcp_log_resource_type: gce_network 249 | # Field name in response where resources are stored, (DEFAULT: items) - https://cloud.google.com/compute/docs/reference/rest/v1/networks/list#response-body 250 | # items: items 251 | # List of fields to exclude from configuration i.e. for status related fields 252 | field_exclude_filter: 253 | # From each resource remove field "stateDetails" nested under "peerings" 254 | - peerings: 255 | # https://cloud.google.com/compute/docs/reference/rest/v1/networks/list#response-body 256 | - stateDetails 257 | # List of conditions to filter out whole individual resources 258 | item_exclude_filter: 259 | # From results remove each resource that "name" under "peerings" matches "servicenetwork-googleapis-com" 260 | - peerings: 261 | name: "servicenetworking-googleapis-com" 262 | sortFields: 263 | # Define fields used to perform sorting of results 264 | ## Sort items in results using 'name' field as key 265 | - name 266 | ## Sort inner list (under key 'secondaryIpRanges') in each item using 'name' field as key 267 | - secondaryIpRanges: name 268 | 269 | git: 270 | # Git repo's URL for GCP configuration storing 271 | remote_url: https://github.com// 272 | # Local directory for storing GCP configuration 273 | local_directory: gcp_config 274 | 275 | # Perform dynamic discovery of all projects in GCP organization 276 | discover_projects: True 277 | 278 | # Static list of project IDs (alternative to the above approach) 279 | #project_list: 280 | # # GCP project ID 281 | # - my-project-ID 282 | 283 | # List of project IDs to exclude from scanning - for use with dynamic discoery 284 | excluded_projects: 285 | - excluded-project-ID 286 | 287 | # Number of threads for scanning resources 288 | thread_count: 4 # default - 3 289 | 290 | # Change handling configuration 291 | change_processor: 292 | # Interval (in minutes) between each scan - has to match script execution interval 293 | scan_interval: 30 294 | # Regex to identify service account username - match meaning non-manual change 295 | service_account_regex: '^(my-terraform-sa-|\d+@|service-\d+).*' 296 | # Regex to identify ticket/issue from Terraform Run message i.e. ADR-1234 297 | ticket_regex: "^.*?([a-zA-z]{2,3}[-_][0-9]+).*" 298 | # Ticket/Issue URL base in ticketing system. Used to create ticket link 299 | ticket_sys_url: "https://my-tickets.com/list" 300 | # Slack notifications config 301 | slack: 302 | # Slack channel ID for Cloudimized notifications 303 | channelID: "C123456789A" 304 | # Commit URL base in Git system. Used to create commit link 305 | repoCommitURL: "https://github.com///commit" 306 | # Jira issue creator config 307 | jira: 308 | # Jira's URL 309 | url: "https://my.jira.com" 310 | # Jira's Project Key - Project in which create issue 311 | projectKey: "KEY" 312 | # Jira's Issue Type - Issue's type to be created (optional) 313 | issueType: "Task" #default 314 | # Jira's Issue fields - set values on fields in Issue (optional) 315 | fields: 316 | field_name: "field_value" 317 | # Regex filter for selecting projects for which create issues (optional) 318 | filterSet: 319 | projectId: ".*production-only.*" 320 | # Flag to indicate token use for auth instead of Username/Password 321 | isToken: True 322 | # Terraform Runs configuration 323 | terraform: 324 | # Terraform URL 325 | url: "https://app.terraform.io" 326 | # Path to JSON file containing Organization to team token mapping 327 | #workspace_token_file: "" 328 | # Mapping between GCP Service Account performing changes and Terraform Org/Workspace 329 | service_workspace_map: 330 | # GCP Service Account name 331 | terraform-service-account-no1: 332 | # Terraform Organization 333 | org: my-organization 334 | # Terraform workspaces list 335 | workspace: ["my-workspace-no1"] 336 | ``` 337 | 338 | ## Single run mode 339 | 340 | Allows to run cloudimized only to scan given resource and dump them into text files, without performing and additional 341 | functions (no Git, Terraform, Slack, Jira interaction and no GCP logs lookup). 342 | 343 | ### Running 344 | 345 | ``` 346 | cloudimized --singlerun/-s --output/-o {yaml, csv} 347 | 348 | i.e 349 | cloudimized -s -v gcp addresses -o csv 350 | ``` 351 | 352 | ### Configuration 353 | 354 | Resource configurations for single run (**** parameter) to be scanned are stored in **singlerunconfigs** 355 | directory and are selected based on filename. Resource configuration is the same as in main config file. Additional 356 | singe run mode configs can be added to folder as needed. 357 | 358 | Get info available configs or what will be run with with: 359 | 360 | ``` 361 | cloudimized -s --list 362 | cloudimized -s --describe --name 363 | ``` 364 | 365 | ### Output 366 | 367 | By default script will dump results in YAML format same as in main mode. If chosen it can dump results in CSV file 368 | format (single file per resource). 369 | -------------------------------------------------------------------------------- /cloudimized-example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/egnyte/cloudimized/b319f4fc8b49008a2b06b30d9122829679218c4c/cloudimized-example.png -------------------------------------------------------------------------------- /cloudimized/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/egnyte/cloudimized/b319f4fc8b49008a2b06b30d9122829679218c4c/cloudimized/__init__.py -------------------------------------------------------------------------------- /cloudimized/azurecore/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/egnyte/cloudimized/b319f4fc8b49008a2b06b30d9122829679218c4c/cloudimized/azurecore/__init__.py -------------------------------------------------------------------------------- /cloudimized/azurecore/aksclustersquery.py: -------------------------------------------------------------------------------- 1 | """ 2 | Azure query for AKS clusters 3 | """ 4 | from azure.identity import DefaultAzureCredential 5 | from azure.mgmt.containerservice import ContainerServiceClient 6 | from cloudimized.azurecore.azurequery import AzureQuery 7 | from typing import Dict, List 8 | 9 | 10 | @AzureQuery.register_class("aksClusters") 11 | class AksClustersQuery(AzureQuery): 12 | """ 13 | Azure query for AKS clusters 14 | """ 15 | def _AzureQuery__send_query(self, 16 | credential: DefaultAzureCredential, 17 | subscription_id: str, 18 | resource_groups) -> List[Dict]: 19 | """ 20 | Sends Azure query that lists AKS clusters in subscription in project. 21 | See: https://learn.microsoft.com/en-us/rest/api/compute/container-services/list?view=rest-compute-2020-09-30&tabs=HTTP 22 | :param credential: Azure credential object 23 | :param subscription_id: Azure subscription ID to query 24 | :param resource_groups: irrelevant for this implementation, needed due to inheritance 25 | :return: List of resources that were queried 26 | """ 27 | client = ContainerServiceClient(credential=credential, subscription_id=subscription_id) 28 | result = client.managed_clusters.list() 29 | return result 30 | -------------------------------------------------------------------------------- /cloudimized/azurecore/azurecredential.py: -------------------------------------------------------------------------------- 1 | from cloudimized.gcpcore.gcpexternaltoken import get_idtoken 2 | import logging 3 | from os import getenv 4 | import time 5 | from typing import Any, Optional 6 | from azure.core.credentials import AccessToken, TokenCredential 7 | from azure.identity import DefaultAzureCredential 8 | from msal import ConfidentialClientApplication 9 | 10 | AZURE_AUTHORITY = "https://login.microsoftonline.com" 11 | AZURE_ACCESS_TOKEN_AUDIENCE = "api://AzureADTokenExchange" 12 | 13 | ENV_AZURE_CLIENT_ID = "AZURE_CLIENT_ID" 14 | ENV_AZURE_TENANT_ID = "AZURE_TENANT_ID" 15 | 16 | logger = logging.getLogger(__name__) 17 | #### 18 | # Create as potential wrapper - for compatibility with gcpservicequery logic 19 | #### 20 | 21 | class WorkloadIdentityCredential(TokenCredential): 22 | """Custom credential class for Azure SDK""" 23 | def __init__(self, azure_client_id: str, azure_tenant_id: str, ext_token_id: str): 24 | # create a confidential client application 25 | self.app = ConfidentialClientApplication( 26 | azure_client_id, 27 | client_credential={ 28 | 'client_assertion': ext_token_id 29 | }, 30 | authority=f"{AZURE_AUTHORITY}/{azure_tenant_id}" 31 | ) 32 | 33 | def get_token(self, *scopes: str, claims: Optional[str] = None, tenant_id: Optional[str] = None, **kwargs: Any) -> AccessToken: 34 | token = self.app.acquire_token_for_client(list(scopes)) 35 | if "error" in token: 36 | raise Exception(token["error_description"]) 37 | expires_on = time.time() + token["expires_in"] 38 | return AccessToken(token["access_token"], int(expires_on)) 39 | 40 | def get_azure_credential() -> DefaultAzureCredential: 41 | """ 42 | Returns Azure credential object 43 | :param azure_client_id: Azure application's client ID 44 | :param azure_tenant_id: Azure application's tenenat ID 45 | :param ext_token_id: external token ID when used with Federated Workload identity 46 | """ 47 | logger.info(f"Generating Azure credential object") 48 | azure_client_id = getenv(ENV_AZURE_CLIENT_ID) 49 | azure_tenant_id = getenv(ENV_AZURE_TENANT_ID) 50 | if azure_client_id and azure_tenant_id: 51 | logger.info(f"Authenticating via Workload Identity Federation. Getting GCP ID token") 52 | ext_token_id = get_idtoken(AZURE_ACCESS_TOKEN_AUDIENCE) 53 | logger.info(f"GCP ID token retrived. Getting Azure access token") 54 | return WorkloadIdentityCredential(azure_client_id, azure_tenant_id, ext_token_id) 55 | else: 56 | logger.info(f"Logging to Azure using DefaultAzureCredential method") 57 | return DefaultAzureCredential() 58 | -------------------------------------------------------------------------------- /cloudimized/azurecore/azurequery.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import re 4 | import importlib.util 5 | import sys 6 | from typing import Any, Dict, List, Union 7 | from abc import ABC, abstractmethod 8 | from itertools import filterfalse 9 | from operator import itemgetter 10 | from azure.identity import DefaultAzureCredential 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | AZURE_QUERIES_SECTION = "azure_queries" 15 | RESOURCE = "resource" 16 | ITEM_EXCLUDE_FILTER = "item_exclude_filter" 17 | NUM_RETRIES = "num_retries" 18 | SORT_FIELDS = "sortFields" 19 | 20 | DEFAULT_NUM_RETRIES = 3 21 | 22 | AGGREGATED_LIST = "aggregatedList" 23 | DEFAULT_SORT_FIELDS = ["name"] 24 | 25 | class AzureQuery(ABC): 26 | """A class for sending queries to Azure""" 27 | 28 | #Registry of implementation classes 29 | _registry = {} 30 | 31 | def __init__(self, resource_name: str, 32 | field_exclude_filter: List = None, 33 | field_include_filter: List = None, 34 | item_exclude_filter: List[Dict[str, str]] = None, 35 | num_retries: int = 3, 36 | sort_fields: List = DEFAULT_SORT_FIELDS, 37 | **kwargs): 38 | """ 39 | :param resource_name: user-friendly name to describe queried 40 | :param field_exclude_filter: fields to be excluded from each item 41 | :param field_include_filter: fields to keep for each item 42 | :param item_exclude_filter: regex rules to use for filtering whole items 43 | :param num_retries: number of retry attempts for API calls 44 | :param sort_fields: results sorting fields 45 | :param kwargs: kwargs to pass into azure function 46 | """ 47 | if field_include_filter and field_exclude_filter: 48 | raise AzureQueryArgumentError(f"Issue for resource_name {resource_name} field_include_filter and " 49 | f"field_exclude_filter are mutually exclusive") 50 | self.resource_name = resource_name 51 | self.result_exclude_filter = field_exclude_filter 52 | self.result_include_filter = field_include_filter 53 | self.result_item_filter = item_exclude_filter 54 | self.num_retries = num_retries 55 | self.sort_fields = sort_fields 56 | self.kwargs = kwargs 57 | 58 | @classmethod 59 | def register_class(cls, resource_name): 60 | def decorator(subclass): 61 | cls._registry[resource_name] = subclass 62 | return subclass 63 | return decorator 64 | 65 | @classmethod 66 | def create(cls, resource_name, *args, **kwargs): 67 | if resource_name not in cls._registry: 68 | raise ValueError(f"Class '{resource_name}' is not registered") 69 | return cls._registry[resource_name](resource_name, *args, **kwargs) 70 | 71 | def execute(self, 72 | credentials: DefaultAzureCredential, 73 | subscription_id: str, 74 | resource_groups: List[str]) -> List[Dict]: 75 | """ 76 | Sends Azure query that lists virtualProjects in subscription in project. 77 | :param credentials: Azure credential object 78 | :param subscription_id: Azure subscription ID to query 79 | :param resource_groups: list of Rescource Group names 80 | :return: List of resources that were queried 81 | """ 82 | logger.info(f"Running query for '{self.resource_name}' in subscription '{subscription_id}'") 83 | try: 84 | raw_result = self.__send_query(credentials, subscription_id, resource_groups) 85 | except Exception as e: 86 | raise AzureQueryError(f"Issue executing call '{self.resource_name}'") from e 87 | try: 88 | #Most responses from Azure will implement as_dict() to serialize those objects 89 | result = [item.as_dict() for item in raw_result] 90 | except Exception as e: 91 | raise AzureQueryError(f"Issue serializing response from call '{self.resource_name} ") from e 92 | # Sort result list to get predictable results 93 | # Perform sorting based on "name" key if present 94 | self.__sort_result(result, subscription_id) 95 | if self.result_item_filter: 96 | for filter_condition_set in self.result_item_filter: 97 | result[:] = [i for i in result if self.__filter_item(i, filter_condition_set)] 98 | if self.result_exclude_filter: 99 | return self.__filter_field_exclude(self.result_exclude_filter, result) 100 | elif self.result_include_filter: 101 | return self._filter_field_include(self.result_include_filter, result) 102 | else: 103 | return result 104 | 105 | 106 | @abstractmethod 107 | def __send_query(self, subscription_id: str): 108 | pass 109 | 110 | 111 | def __filter_field_exclude(self, fields: List[Union[str, Dict]], result: List[Dict]) -> List[Dict]: 112 | filtered_result = result[:] 113 | for field in fields: 114 | if isinstance(field, str): 115 | for item in filtered_result: 116 | item.pop(field, None) 117 | elif isinstance(field, dict): 118 | for item in filtered_result: 119 | for nested_key, nested_fields in field.items(): 120 | nested_result = item.get(nested_key, {}) 121 | if isinstance(nested_result, dict): 122 | self.__filter_field_exclude(fields=nested_fields, result=[nested_result]) 123 | if isinstance(nested_result, list): 124 | self.__filter_field_exclude(fields=nested_fields, result=nested_result) 125 | return filtered_result 126 | 127 | def _filter_field_include(self, fields: List[Union[str, Dict]], result: List[Dict[str, Any]]) -> List[Dict[str, Any]]: 128 | """ 129 | Filter out all fields except ones specified from each item in results 130 | """ 131 | def filter_dict(entry, filters): 132 | filtered_entry = {} 133 | for key, value in entry.items(): 134 | if isinstance(value, dict): 135 | if key in filters: 136 | if not filters[key]: # Include the entire nested dictionary 137 | filtered_entry[key] = filter_dict(value, {}) 138 | else: 139 | filtered_entry[key] = filter_dict(value, filters[key]) 140 | elif key in filters or not filters: # Include the field if it's in filters or filters are empty 141 | filtered_entry[key] = value 142 | return filtered_entry 143 | 144 | filtered_data = [] 145 | for entry in result: 146 | filtered_entry = filter_dict(entry, fields) 147 | filtered_data.append(filtered_entry) 148 | 149 | return filtered_data 150 | 151 | def __filter_item(self, item: Dict[str, Any], filter_condition_set) -> bool: 152 | for filter_field, filter_condition in filter_condition_set.items(): 153 | if isinstance(filter_condition, str): 154 | field_value = item.get(filter_field, "") 155 | if isinstance(field_value, list): 156 | field_value[:] = [i for i in field_value if not re.match(rf"{filter_condition}", i)] 157 | break 158 | elif isinstance(field_value, str): 159 | if not re.match(rf"{filter_condition}", field_value): 160 | break 161 | elif isinstance(filter_condition, dict): 162 | nested_result = item.get(filter_field, None) 163 | if nested_result is None: 164 | break 165 | if isinstance(nested_result, dict): 166 | return self.__filter_item(nested_result, filter_condition) 167 | elif isinstance(nested_result, list): 168 | nested_result[:] = [i for i in nested_result if self.__filter_item(i, filter_condition)] 169 | break 170 | else: 171 | # If all condition match filter item out 172 | return False 173 | # If there was a break don't filter item out 174 | return True 175 | 176 | def __sort_result(self, result: List[Dict], subscription_id: str) -> None: 177 | """ 178 | Performs sorting of given result list 179 | :param result: results to be sorted 180 | :param subscription_id: Subscription ID for logging purposes 181 | """ 182 | for sort_field in self.sort_fields: 183 | try: 184 | if isinstance(sort_field, str): 185 | result.sort(key=itemgetter(sort_field)) 186 | elif isinstance(sort_field, dict): 187 | for inner_key, inner_field in sort_field.items(): 188 | for outer_result_item in result: 189 | try: 190 | inner_result = outer_result_item[inner_key] 191 | inner_result.sort(key=itemgetter(inner_field)) 192 | except Exception as e: 193 | logger.warning(f"Unable to sort inner list for {sort_field} fields for project " 194 | f"{subscription_id}") 195 | except Exception as e: 196 | logger.warning( 197 | f"Issue sorting results for call:'{self.resource_name}' for subscription:'{subscription_id}' " 198 | f"for sorting field {sort_field}") 199 | logger.debug(f"Reason: {e}") 200 | 201 | 202 | def configure_azure_queries(queries: List[Dict[str, Any]]) -> Dict[str, AzureQuery]: 203 | """ 204 | Configures Azure queries objects from configuration 205 | :param queries: per service query configuration 206 | :returns resource name to AzureQuery object mapping 207 | """ 208 | if not isinstance(queries, list): 209 | raise AzureQueryArgumentError(f"Incorrect Azure queries configuration. Should be list, is {type(queries)}") 210 | #TODO better configuraiton file verification 211 | result = {} 212 | for query in queries: 213 | if RESOURCE not in query: 214 | raise AzureQueryArgumentError(f"Missing required key in query: '{query}'") 215 | #TODO Add Azure logging parsing 216 | if ITEM_EXCLUDE_FILTER in query: 217 | item_exclude_filter = query[ITEM_EXCLUDE_FILTER] 218 | if not isinstance(item_exclude_filter, list): 219 | raise AzureQueryArgumentError(f"Incorrect Azure query configuration. Item exclude filter should be list, " 220 | f"is {type(item_exclude_filter)}") 221 | num_retries = query.get(NUM_RETRIES, DEFAULT_NUM_RETRIES) 222 | sort_fields = query.get(SORT_FIELDS, DEFAULT_SORT_FIELDS) 223 | # Create kwargs from only keyword arguments 224 | ## Skip gcp query kwargs and pass everything else to api call 225 | kwargs = dict(filterfalse(lambda x: x[0] not in set([ 226 | "field_exclude_filter", "field_include_filter", "item_exclude_filter", "result_items_field"]), 227 | query.items())) 228 | try: 229 | result[query[RESOURCE]] = AzureQuery.create(resource_name=query[RESOURCE], 230 | num_retries=num_retries, 231 | sort_fields=sort_fields, 232 | **kwargs) 233 | except Exception as e: 234 | raise AzureQueryArgumentError(f"Issue parsing query config {query}") from e 235 | return result 236 | 237 | #Needed to load all implementation of AzureQuery Base Class 238 | try: 239 | impl_classes_dir = os.path.dirname(__file__) 240 | for filename in os.listdir(impl_classes_dir): 241 | if filename.endswith(".py") and filename not in ("azurequery.py", "__init__.py"): 242 | module_name = __name__.split(".")[-1] 243 | spec = importlib.util.spec_from_file_location(module_name, f"{impl_classes_dir}/{filename}") 244 | module = importlib.util.module_from_spec(spec) 245 | sys.modules[module_name] = module 246 | spec.loader.exec_module(module) 247 | except Exception as e: 248 | logger.error(f"Unable to load AzureQuery classes from {impl_classes_dir}") 249 | raise e 250 | 251 | 252 | class AzureQueryError(Exception): 253 | pass 254 | 255 | 256 | class AzureQueryArgumentError(AzureQueryError): 257 | pass 258 | -------------------------------------------------------------------------------- /cloudimized/azurecore/networksecuritygroupsquery.py: -------------------------------------------------------------------------------- 1 | """ 2 | Azure query for Network Security Group 3 | """ 4 | from azure.identity import DefaultAzureCredential 5 | from azure.mgmt.network import NetworkManagementClient 6 | from cloudimized.azurecore.azurequery import AzureQuery 7 | from typing import Dict, List 8 | 9 | 10 | @AzureQuery.register_class("networkSecurityGroups") 11 | class NetworkSecurityGroupQuery(AzureQuery): 12 | """ 13 | Azure query for Network Security Groups 14 | """ 15 | def _AzureQuery__send_query(self, 16 | credential: DefaultAzureCredential, 17 | subscription_id: str, 18 | resource_groups) -> List[Dict]: 19 | """ 20 | Sends Azure query that lists Network Security Groups in subscription in project. 21 | See: https://learn.microsoft.com/en-us/rest/api/virtualnetwork/network-security-groups/list-all?view=rest-virtualnetwork-2024-05-01&tabs=HTTP 22 | :param credential: Azure credential object 23 | :param subscription_id: Azure subscription ID to query 24 | :param resource_groups: irrelevant for this implementation, needed due to inheritance 25 | :return: List of resources that were queried 26 | """ 27 | client = NetworkManagementClient(credential=credential, subscription_id=subscription_id) 28 | result = client.network_security_groups.list_all() 29 | return result 30 | -------------------------------------------------------------------------------- /cloudimized/azurecore/resourcegroupsquery.py: -------------------------------------------------------------------------------- 1 | """ 2 | Azure query for resource groups 3 | """ 4 | from azure.identity import DefaultAzureCredential 5 | from azure.mgmt.resource import ResourceManagementClient 6 | from cloudimized.azurecore.azurequery import AzureQuery 7 | from typing import Dict, List 8 | 9 | RESOURCE_GROUPS_RESOURCE_NAME = "resourceGroups" 10 | 11 | @AzureQuery.register_class(RESOURCE_GROUPS_RESOURCE_NAME) 12 | class ResourceGropusQuery(AzureQuery): 13 | """ 14 | Azure query for virtual networks 15 | """ 16 | def _AzureQuery__send_query(self, 17 | credential: DefaultAzureCredential, 18 | subscription_id, 19 | resource_groups) -> List[Dict]: 20 | """ 21 | Sends Azure query that lists Resource Groups. 22 | :param credential: Azure credential object 23 | :param subscription_id: Azure subscription id 24 | :param resource_groups: irrelevant for this implementation, needed due to inheritance 25 | :return: List of resources that were queried 26 | """ 27 | client = ResourceManagementClient(credential=credential,subscription_id=subscription_id) 28 | result = client.resource_groups.list() 29 | return result 30 | -------------------------------------------------------------------------------- /cloudimized/azurecore/subscriptionsquery.py: -------------------------------------------------------------------------------- 1 | """ 2 | Azure query for subscriptions 3 | """ 4 | from azure.identity import DefaultAzureCredential 5 | from azure.mgmt.subscription import SubscriptionClient 6 | from cloudimized.azurecore.azurequery import AzureQuery 7 | from typing import Dict, List 8 | 9 | SUBSCRIPTIONS_RESOURCE_NAME = "subscriptions" 10 | 11 | @AzureQuery.register_class(SUBSCRIPTIONS_RESOURCE_NAME) 12 | class SubscriptionsQuery(AzureQuery): 13 | """ 14 | Azure query for virtual networks 15 | """ 16 | def _AzureQuery__send_query(self, 17 | credential: DefaultAzureCredential, 18 | subscription_id, 19 | resource_groups) -> List[Dict]: 20 | """ 21 | Sends Azure query that lists all subscriptions. 22 | See: https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/subscription/azure-mgmt-subscription/generated_samples/list_subscriptions.py 23 | :param credential: Azure credential object 24 | :param subscription_id: irrelevant for this implementation, needed due to inheritance 25 | :param resource_groups: irrelevant for this implementation, needed due to inheritance 26 | :return: List of resources that were queried 27 | """ 28 | client = SubscriptionClient(credential=credential) 29 | result = client.subscriptions.list() 30 | return result 31 | -------------------------------------------------------------------------------- /cloudimized/azurecore/virtualnetworksquery.py: -------------------------------------------------------------------------------- 1 | """ 2 | Azure query for virtual networks 3 | """ 4 | from azure.identity import DefaultAzureCredential 5 | from azure.mgmt.network import NetworkManagementClient 6 | from cloudimized.azurecore.azurequery import AzureQuery 7 | from typing import Dict, List 8 | 9 | 10 | @AzureQuery.register_class("virtualNetworks") 11 | class VirtualNetworksQuery(AzureQuery): 12 | """ 13 | Azure query for virtual networks 14 | """ 15 | def _AzureQuery__send_query(self, 16 | credential: DefaultAzureCredential, 17 | subscription_id: str, 18 | resource_groups) -> List[Dict]: 19 | """ 20 | Sends Azure query that lists Virtual Networks in subscription in project. 21 | See: https://learn.microsoft.com/en-us/rest/api/virtualnetwork/virtual-networks/list-all?view=rest-virtualnetwork-2024-05-01&tabs=HTTP 22 | :param credential: Azure credential object 23 | :param subscription_id: Azure subscription ID to query 24 | :param resource_groups: irrelevant for this implementation, needed due to inheritance 25 | :return: List of resources that were queried 26 | """ 27 | client = NetworkManagementClient(credential=credential, subscription_id=subscription_id) 28 | result = client.virtual_networks.list_all() 29 | return result 30 | -------------------------------------------------------------------------------- /cloudimized/azurecore/vnetgatewaysquery.py: -------------------------------------------------------------------------------- 1 | """ 2 | Azure query for virtual networks 3 | """ 4 | from azure.identity import DefaultAzureCredential 5 | from azure.mgmt.network import NetworkManagementClient 6 | from cloudimized.azurecore.azurequery import AzureQuery 7 | from typing import Dict, List 8 | 9 | @AzureQuery.register_class("vnetGateways") 10 | class VnetGatewaysQuery(AzureQuery): 11 | """ 12 | Azure query for virtual network gateways 13 | """ 14 | def _AzureQuery__send_query(self, 15 | credential: DefaultAzureCredential, 16 | subscription_id: str, 17 | resource_groups: List[str]) -> List[Dict]: 18 | """ 19 | Sends Azure query that lists Virtual Networks in subscription in project. 20 | See: https://learn.microsoft.com/en-us/cli/azure/network/vnet-gateway?view=azure-cli-latest#az-network-vnet-gateway-list 21 | :param credential: Azure credential object 22 | :param subscription_id: Azure subscription ID to query 23 | :param resource_groups: list of Resource Group names to query 24 | :return: List of resources that were queried 25 | """ 26 | client = NetworkManagementClient(credential=credential, subscription_id=subscription_id) 27 | result = [] 28 | for rg in resource_groups: 29 | result.extend(client.virtual_network_gateways.list(resource_group_name=rg)) 30 | return result 31 | -------------------------------------------------------------------------------- /cloudimized/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/egnyte/cloudimized/b319f4fc8b49008a2b06b30d9122829679218c4c/cloudimized/core/__init__.py -------------------------------------------------------------------------------- /cloudimized/core/changeprocessor.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import re 3 | from datetime import datetime 4 | from typing import Dict, List, Any 5 | 6 | from cloudimized.core.jiranotifier import JiraNotifier, JiraNotifierError, configure_jiranotifier 7 | from cloudimized.core.slacknotifier import SlackNotifier, SlackNotifierError, configure_slack_notifier 8 | from cloudimized.gcpcore.gcpchangelog import getChangeLogs 9 | from cloudimized.gcpcore.gcpquery import GcpQuery 10 | from cloudimized.gitcore.gitchange import GitChange 11 | from cloudimized.gitcore.repo import GitRepo 12 | from cloudimized.tfcore.query import TFQuery, TFQueryError, configure_tfquery 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | CHANGE_TF_RUN_STATE = ['applied', 'errored'] 17 | CHANGE_PROCESSOR = "change_processor" 18 | SCAN_INTERVAL = "scan_interval" 19 | SERVICE_ACCOUNT_REGEX = "service_account_regex" 20 | TICKET_REGEX = "ticket_regex" 21 | TICKET_SYS_URL = "ticket_sys_url" 22 | TERRAFORM_SECTION = "terraform" 23 | SLACK_SECTION = "slack" 24 | JIRA_SECTION = "jira" 25 | 26 | 27 | class ChangeProcessor: 28 | """ 29 | Handles processing detected Git changes 30 | """ 31 | 32 | def __init__(self, repo: GitRepo, 33 | gcp_type_queries_map: Dict[str, GcpQuery], 34 | scan_interval: int, 35 | service_account_regex: str, 36 | tf_query: TFQuery = None, 37 | ticket_regex: str = None, 38 | ticket_sys_url: str = None, 39 | slack_notifier: SlackNotifier = None, 40 | jira_notifier: JiraNotifier = None): 41 | """ 42 | :param repo: GitRepo holding configuration 43 | :param gcp_type_queries_map: resource name to GcpQuery mapping 44 | :param scan_interval: interval between each scan in minutes 45 | :param service_account_regex: regex used to identify Service Account changer name 46 | :param tf_query: Terraform connector 47 | :param ticket_regex: Regex to find ticket information in Terraform Run message 48 | :param ticket_sys_url: URL to ticketing system 49 | :param slack_notifier: optional posting to Slack 50 | """ 51 | self.repo = repo 52 | self.gcp_type_queries_map = gcp_type_queries_map 53 | self.scan_interval = scan_interval 54 | self.tf_query = tf_query 55 | self.service_account_regex = service_account_regex 56 | self.ticket_regex = ticket_regex 57 | self.ticket_sys_url = ticket_sys_url 58 | self.slack_notifier = slack_notifier 59 | self.jira_notifier = jira_notifier 60 | 61 | def process_change(self, git_change: GitChange, change_time: datetime = None): 62 | """ 63 | Retrieves GCP logs 64 | :param git_change: change detected by Git 65 | :param change_time: time of scan start/finish 66 | """ 67 | # TODO: implement better time selection 68 | message = f"{git_change.resource_type.title()} updated in {git_change.provider.upper()}: {git_change.project}" 69 | manual_change = False 70 | skip_process_ticket = False 71 | try: 72 | self.repo.repo.git.add(git_change.get_filename()) 73 | except Exception as e: 74 | raise ChangeProcessorError(f"Issue adding file '{git_change.get_filename()}' in Git") from e 75 | try: 76 | if not self.repo.repo.index.diff("HEAD"): 77 | logger.info(f"Skipping non-change '{git_change.get_filename()}'") 78 | return 79 | except Exception as e: 80 | logger.warning(f"Issue checking branch repo HEAD. Empty repo without commit?\n{e}\n{e.__cause__}") 81 | # Log analysis and TF runs supported only for GCP 82 | if git_change.provider == "gcp": 83 | if not change_time: 84 | change_time = datetime.utcnow() 85 | try: 86 | logging.info(f"Retrieving GCP change logs for '{git_change.get_filename()}'") 87 | gcp_change_logs = getChangeLogs(project=git_change.project, 88 | gcp_query=self.gcp_type_queries_map[git_change.resource_type], 89 | change_time=change_time, 90 | time_window=self.scan_interval) 91 | except Exception as e: 92 | logger.warning(f"Issue getting GCP logs for change '{git_change.get_filename()}'\n{e}\n{e.__cause__}") 93 | gcp_change_logs = [] 94 | if len(gcp_change_logs) == 1: 95 | logger.info(f"Found Gcp change log for resource '{git_change.resource_type}' for " 96 | f"project '{git_change.project}'") 97 | elif len(gcp_change_logs) > 1: 98 | # Log multiple Gcp Log entries for analyzing and improvment in future 99 | logger.info(f"Multiple Gcp change logs found for resource '{git_change.resource_type}' for " 100 | f"project '{git_change.project}'. Log count {len(gcp_change_logs)}\n{gcp_change_logs}") 101 | changers = [] 102 | for gcp_change_log in gcp_change_logs: 103 | if not gcp_change_log.changer: 104 | logger.info(f"Missing changer in GCP log for change {git_change.get_filename()}\n{gcp_change_log}") 105 | continue 106 | try: 107 | changer_login = gcp_change_log.changer.split("@")[0] 108 | except Exception as e: 109 | logger.warning(f"Issue retrieving changer login from {gcp_change_log.changer}") 110 | if gcp_change_log.changer not in changers: 111 | changers.append(gcp_change_log.changer) 112 | message += f"\n Change done by unknown user '{gcp_change_log.changer}'" 113 | continue 114 | 115 | if changer_login in changers: 116 | # Skip lookup for same changer 117 | logger.info(f"Skipping lookup for changer '{changer_login}'") 118 | continue 119 | else: 120 | changers.append(changer_login) 121 | if not re.match(rf"{self.service_account_regex}", gcp_change_log.changer): 122 | # Manual change 123 | git_change.manual = True 124 | logger.info(f"Manual change performed by '{changer_login}' detected") 125 | message += f"\n MANUAL change done by {changer_login}" 126 | else: 127 | message += f"\n Terraform change done by {changer_login}" 128 | # Process only if tf_query is set 129 | if self.tf_query: 130 | logger.info(f"Retrieving Terraform Runs for service account '{changer_login}'") 131 | try: 132 | tf_runs = self.tf_query.get_runs(gcp_sa=changer_login) 133 | except TFQueryError as e: 134 | logger.warning(f"Issue getting terraform runs for GCP log {gcp_change_log}\n{e}\n{e.__cause__}") 135 | continue 136 | if not (self.ticket_regex and self.ticket_sys_url): 137 | logger.info(f"Skipping ticket processing - ticket regex and/or ticketing URL not set") 138 | skip_process_ticket = True 139 | for tf_run in tf_runs: 140 | if tf_run.status not in CHANGE_TF_RUN_STATE: 141 | logger.info(f"Skipping processing non-change Terraform Run '{tf_run}") 142 | continue 143 | logger.info(f"Processing Terraform run: {tf_run}") 144 | run_url = (f"{self.tf_query.tf_url}/app/{tf_run.org}/workspaces/{tf_run.workspace}/runs/" 145 | f"{tf_run.run_id}") 146 | message += f"\n Related TF run {run_url}" 147 | if not skip_process_ticket: 148 | ticket_match = re.search(rf"{self.ticket_regex}", tf_run.message) 149 | if ticket_match: 150 | try: 151 | ticket = ticket_match.group(1) 152 | except Exception as e: 153 | logger.warning(f"Issue retrieving ticket number from " 154 | f"run '{tf_run}'\n{e}\n{e.__cause__}") 155 | continue 156 | # TODO Parametrize string replacement 157 | message += f"\n Related ticket {self.ticket_sys_url}/{ticket.replace('_', '-')}" 158 | # Add least one changer has been identified 159 | if changers: 160 | git_change.message = message 161 | git_change.changers = changers 162 | else: 163 | message += f"\n Unable to identify changer" 164 | git_change.message = message 165 | # All other providers. Only Azure for now 166 | else: 167 | git_change.message = message 168 | logger.info(f"Committing change '{git_change.get_filename()}'") 169 | self.repo.repo.git.commit(m=message) 170 | 171 | git_change.commit = self.repo.repo.heads.master.commit 172 | git_change.diff = self.repo.repo.git.diff("HEAD~1..HEAD") 173 | if self.slack_notifier: 174 | try: 175 | logger.info(f"Posting to Slack channel ID: {self.slack_notifier.channelID}") 176 | self.slack_notifier.post(git_change) 177 | except SlackNotifierError as e: 178 | logger.warning(f"Issue sending message to Slack\n{e}\n{e.__cause__}") 179 | #Only supported for GCP changes 180 | if git_change.provider == "gcp" and self.jira_notifier: 181 | try: 182 | self.jira_notifier.post(git_change) 183 | except JiraNotifierError as e: 184 | logger.warning(f"Issue creating ticket in Jira\n{e}\n{e.__cause__}") 185 | 186 | def process(self, git_changes: List[GitChange], change_time: datetime = None): 187 | """ 188 | Process repo for changes 189 | :param git_changes: list of changes detected by Git 190 | :param change_time: time of scan start/finish 191 | """ 192 | for git_change in git_changes: 193 | self.process_change(git_change, change_time) 194 | try: 195 | commits_ahead = self.repo.repo.iter_commits('origin/master..master') 196 | commit_count = sum(1 for c in commits_ahead) 197 | except Exception as e: 198 | logger.warning(f"Issue checking commit number diff in remote. " 199 | f"Empty repo without commit?\n{e}\n{e.__cause__}") 200 | try: 201 | commit_count = self.repo.repo.git.rev_list("--count", "HEAD") 202 | except Exception as sub_e: 203 | logger.warning(f"Unexpected error when counting commit number\n{e}\n{e.__cause__}") 204 | commit_count = 1 205 | if commit_count: 206 | logger.info(f"Pushing {commit_count} commit(s) to remote") 207 | try: 208 | self.repo.repo.remotes.origin.push() 209 | except Exception as e: 210 | raise ChangeProcessorError("Issue pushing changes to remote") from e 211 | 212 | 213 | def configure_change_processor(config: Dict[str, Any], 214 | gcp_type_queries_map: Dict[str, GcpQuery], 215 | repo: GitRepo, 216 | slack_token: str, 217 | jira_user: str, 218 | jira_token: str) -> ChangeProcessor: 219 | """ 220 | Builds Change from configuraiton file 221 | :param config: dictionary containing configuraiton 222 | :param gcp_type_queries_map: resource_name to GcpQuery map 223 | :param repo: repo where config is stored 224 | :param slack_token: Slack's Bot API token 225 | :param jira_user: Jira username for ticket creation 226 | :param jira_token: Jira token/pass for ticket creation 227 | :return: ChangeProcessor with valid configuration 228 | """ 229 | if not isinstance(config, dict): 230 | raise ChangeProcessorError(f"Incorrect type of config element {CHANGE_PROCESSOR}. " 231 | f"Should be dict, is {type(config)}") 232 | if SCAN_INTERVAL not in config: 233 | raise ChangeProcessorError(f"Missing required parameter '{SCAN_INTERVAL}' in '{CHANGE_PROCESSOR}' section.") 234 | if SERVICE_ACCOUNT_REGEX not in config: 235 | raise ChangeProcessorError(f"Missing required parameter '{SERVICE_ACCOUNT_REGEX}'" 236 | f" in '{CHANGE_PROCESSOR}' section.") 237 | scan_interval = config.get(SCAN_INTERVAL) 238 | service_account_regex = config.get(SERVICE_ACCOUNT_REGEX) 239 | ticket_regex = config.get(TICKET_REGEX, None) 240 | ticket_sys_url = config.get(TICKET_SYS_URL, None) 241 | tf_query_config = config.get(TERRAFORM_SECTION, None) 242 | tf_query = configure_tfquery(tf_query_config) 243 | try: 244 | slack_notifier = configure_slack_notifier(config.get(SLACK_SECTION, None), slack_token) 245 | except SlackNotifierError as e: 246 | raise ChangeProcessorError(f"Issue with SlackNotifier\n{e}\n{e.__cause__}") from e 247 | try: 248 | jira_notifier = configure_jiranotifier(config.get(JIRA_SECTION, None), username=jira_user, password=jira_token) 249 | except JiraNotifierError as e: 250 | raise ChangeProcessorError(f"Issue with JiraNotifier\n{e}\n{e.__cause__}") from e 251 | if not isinstance(scan_interval, int): 252 | raise ChangeProcessorError(f"Incorrect type of config element {SCAN_INTERVAL}. " 253 | f"Should be int, is {type(scan_interval)}") 254 | if not isinstance(service_account_regex, str): 255 | raise ChangeProcessorError(f"Incorrect type of config element {SERVICE_ACCOUNT_REGEX}. " 256 | f"Should be str, is {type(service_account_regex)}") 257 | if not isinstance(ticket_regex, str): 258 | raise ChangeProcessorError(f"Incorrect type of config element {TICKET_REGEX}. " 259 | f"Should be str, is {type(ticket_regex)}") 260 | if not isinstance(ticket_sys_url, str): 261 | raise ChangeProcessorError(f"Incorrect type of config element {TICKET_SYS_URL}. " 262 | f"Should be str, is {type(ticket_sys_url)}") 263 | if not isinstance(gcp_type_queries_map, dict): 264 | raise ChangeProcessorError(f"Incorrect type of config element gcp_type_queries_map. " 265 | f"Should be dict, is {type(gcp_type_queries_map)}") 266 | if not isinstance(repo, GitRepo): 267 | raise ChangeProcessorError(f"Incorrect type of repo parameter. " 268 | f"Should be GitRepo, is {type(repo)}") 269 | if tf_query is not None: 270 | if not isinstance(tf_query, TFQuery): 271 | raise ChangeProcessorError(f"Incorrect type of tf_query parameter. " 272 | f"Should be TFQuery, is {type(tf_query)}") 273 | return ChangeProcessor(repo=repo, 274 | gcp_type_queries_map=gcp_type_queries_map, 275 | scan_interval=scan_interval, 276 | service_account_regex=service_account_regex, 277 | tf_query=tf_query, 278 | ticket_regex=ticket_regex, 279 | ticket_sys_url=ticket_sys_url, 280 | slack_notifier=slack_notifier, 281 | jira_notifier=jira_notifier) 282 | 283 | 284 | class ChangeProcessorError(Exception): 285 | pass 286 | -------------------------------------------------------------------------------- /cloudimized/core/jiranotifier.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import re 3 | from typing import Dict, Any 4 | 5 | from jira import JIRA 6 | 7 | from cloudimized.gitcore.gitchange import GitChange 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | JIRA_USR = "JIRA_USR" 12 | JIRA_PSW = "JIRA_PSW" 13 | JIRA_IS_TOKEN = "isToken" 14 | 15 | 16 | DEFAULT_ISSUE_TYPE = "Task" 17 | URL = "url" 18 | PROJECTKEY = "projectKey" 19 | ISSUETYPE = "issueType" 20 | FIELDS = "fields" 21 | FILTERSET = "filterSet" 22 | PROJECTIDFILTER = "projectId" 23 | 24 | 25 | class JiraNotifier: 26 | """ 27 | Creates ticket in Jira for configuration change 28 | """ 29 | 30 | def __init__(self, 31 | jira_url: str, 32 | projectkey: str, 33 | username: str, 34 | password: str, 35 | issuetype: str, 36 | filter_set: Dict, 37 | **kwargs): 38 | """ 39 | :param jira_url: Jira instance URL address 40 | :param username: Jira's service account user 41 | :param password: Jira's service account password 42 | :param kwargs: additional Jira issue fields 43 | """ 44 | self.jira_url = jira_url 45 | self.username = username 46 | self.password = password 47 | self.projectkey = projectkey 48 | self.issuetype = issuetype 49 | self.filter_set = filter_set 50 | self.kwargs = kwargs 51 | 52 | def post(self, change: GitChange): 53 | """ 54 | Creates Jira issue for Git Change 55 | """ 56 | # Skip for non-manual changes 57 | if not change.manual: 58 | logger.info("Skiping ticket creation for non-manual change") 59 | return 60 | # Skip for non-matching projects 61 | if self.filter_set: 62 | if not re.match(rf"{self.filter_set.get(PROJECTIDFILTER)}", change.project): 63 | logger.info("Skipping ticket creation for non-matching project id") 64 | return 65 | try: 66 | if self.username: 67 | logger.info(f"Connecting to Jira '{self.jira_url}' as '{self.username}'") 68 | jira = JIRA(options={'server': self.jira_url}, basic_auth=(self.username, self.password)) 69 | else: 70 | logger.info(f"Connecting to Jira '{self.jira_url}' using token auth") 71 | jira = JIRA(options={'server': self.jira_url}, token_auth=self.password) 72 | summary = f"GCP manual change detected - project: {change.project}, resource: {change.resource_type}" 73 | # In case multiple changers were identified 74 | if len(change.changers) == 0: 75 | changer = "Unknown changer" 76 | elif len(change.changers) == 1: 77 | changer = change.changers[0] 78 | else: 79 | changer = change.changers 80 | description = (f"Manual changes performed by {changer}\n\n" 81 | f"{{code:java}}\n{change.diff}\n{{code}}\n") 82 | logger.info(f"Creating ticket: Project: {self.projectkey}, Type: {self.issuetype}") 83 | issue = jira.create_issue(project={"key": self.projectkey}, 84 | summary=summary, 85 | description=description, 86 | issuetype={"name": self.issuetype}, 87 | **self.kwargs) 88 | except Exception as e: 89 | raise JiraNotifierError("Issue creating ticket") from e 90 | # Assign ticket to changer 91 | for changer in change.changers: 92 | try: 93 | logger.info(f"Assigning issue {issue.key} to user {changer}") 94 | issue.update(assignee={'name': changer}) 95 | break 96 | except Exception as e: 97 | logger.warning(f"Unable to assign ticket {issue.key} to changer: {changer}\n{e}") 98 | 99 | 100 | class JiraNotifierError(Exception): 101 | pass 102 | 103 | 104 | def configure_jiranotifier(config: Dict[str, Any], username: str, password: str): 105 | """ 106 | Configures JiraNotifier from config file 107 | :param config: configuration dictionary 108 | :param username: Jira username 109 | :param password: Jira password 110 | """ 111 | if config is None: 112 | return 113 | if not isinstance(config, dict): 114 | raise JiraNotifierError(f"Incorrect Jira Notifier configuration. Should be dict, is {type(config)}") 115 | required_keys = [URL, PROJECTKEY] 116 | if not all(key in config for key in required_keys): 117 | raise JiraNotifierError(f"Missing one of required config keys: {required_keys}") 118 | if config.get(JIRA_IS_TOKEN, "false") != True: 119 | if not username: 120 | raise JiraNotifierError(f"Jira username not set in env var: '{JIRA_USR}' AND not using token auth") 121 | if not password: 122 | raise JiraNotifierError(f"Jira password/token not set in env var: '{JIRA_PSW}'") 123 | extra_fields = config.get(FIELDS, {}) 124 | if not isinstance(extra_fields, dict): 125 | raise JiraNotifierError(f"Incorrect Jira Notifier Fields configuration. " 126 | f"Should be dict, is {type(extra_fields)}") 127 | if FILTERSET in config: 128 | filter_set = config.get(FILTERSET) 129 | if not isinstance(filter_set, dict): 130 | raise JiraNotifierError(f"Incorrect Jira Notifier FilterSet configuration. " 131 | f"Should be dict, is {type(filter_set)}") 132 | if PROJECTIDFILTER not in filter_set: 133 | raise JiraNotifierError(f"Missing required param {PROJECTIDFILTER}") 134 | projectidfilter = filter_set.get(PROJECTIDFILTER) 135 | if not isinstance(projectidfilter, str): 136 | raise JiraNotifierError(f"Incorrect Jira Notifier projectId configuration value. " 137 | f"Should be str, is {type(projectidfilter)}") 138 | else: 139 | filter_set = None 140 | return JiraNotifier(jira_url=config.get(URL), 141 | username=username, 142 | password=password, 143 | issuetype=config.get(ISSUETYPE, DEFAULT_ISSUE_TYPE), 144 | projectkey=config.get(PROJECTKEY), 145 | filter_set=filter_set, 146 | **extra_fields) 147 | -------------------------------------------------------------------------------- /cloudimized/core/result.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import yaml 3 | import csv 4 | from typing import List, Dict 5 | from pathlib import Path 6 | from os.path import isdir 7 | from flatdict import FlatterDict 8 | 9 | from cloudimized.azurecore.azurequery import AzureQuery 10 | from cloudimized.gcpcore.gcpservicequery import GcpServiceQuery 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | PROJECT_ID_KEY = "projectId" #GCP 15 | SUBSCRIPTION_ID_KEY = "subscriptionId" #Azure 16 | AZURE_KEY = "azure" 17 | GCP_KEY = "gcp" 18 | 19 | TARGET_ID_KEY = { 20 | AZURE_KEY: SUBSCRIPTION_ID_KEY, 21 | GCP_KEY: PROJECT_ID_KEY 22 | } 23 | 24 | class QueryResult: 25 | """ 26 | Class representing monitored resource 27 | """ 28 | 29 | def __init__(self): 30 | self.resources = { 31 | AZURE_KEY: {}, 32 | GCP_KEY: {}, 33 | } 34 | 35 | def add_resource(self, resource_name: str, provider: str) -> None: 36 | """ 37 | Adds new resource to results 38 | :param resource_name: name that matches resource_name in query object 39 | :param provider: type of Cloud provider: azure/gcp 40 | :raises QueryResultError 41 | """ 42 | if resource_name in self.resources[provider]: 43 | raise QueryResultError(f"Resource '{resource_name}' is already added in results for provider {provider}") 44 | self.resources[provider][resource_name] = {} 45 | 46 | def add_result(self, resource_name: str, provider: str, target_id: str, result: List[Dict]) -> None: 47 | """ 48 | Adds results for specific resource in given projectId 49 | :param resource_name: resource's result name 50 | :param provider: type of Cloud provider: azure/gcp 51 | :param target_id: projectId(GCP)/subscriptionId(Azure) from which result comes 52 | :param result: result from executing gcp query 53 | """ 54 | if resource_name not in self.resources[provider]: 55 | self.add_resource(resource_name, provider) 56 | self.resources[provider][resource_name][target_id] = result 57 | 58 | def get_result(self, resource_name: str, provider: str, target_id: str) -> List[Dict]: 59 | """ 60 | Get results for specific resoure in given project_id 61 | :param resource_name: resource's result name 62 | :param provider: type of Cloud provider: azure/gcp 63 | :param target_id: projectId(GCP)/subscriptionId(Azure) from which result comes 64 | :return: results from query 65 | """ 66 | return self.resources.get(provider, {}).get(resource_name, {}).get(target_id, None) 67 | 68 | def dump_results(self, directory: str) -> None: 69 | """ 70 | Save results to files 71 | :param directory: root directory to which dump all resources 72 | :raises QueryResultError 73 | """ 74 | if not isdir(directory): 75 | raise QueryResultError(f"Issue dumping results to files. Directory '{directory}' doesn't exist") 76 | for provider, resources in self.resources.items(): 77 | for resource_name, targets_id in resources.items(): 78 | try: 79 | logger.info(f"Creating directory '{directory}/{provider}/{resource_name}'") 80 | Path(f"{directory}/{provider}/{resource_name}").mkdir(parents=True,exist_ok=True) 81 | except Exception as e: 82 | raise QueryResultError(f"Issue creating directory '{directory}/{provider}/{resource_name}'") from e 83 | for target_id, result in targets_id.items(): 84 | # Don't dump files for projects with empty list 85 | if not result: 86 | continue 87 | logger.info(f"Dumping results in {directory}/{provider}/{resource_name}/{target_id}.yaml") 88 | try: 89 | with open(f"{directory}/{provider}/{resource_name}/{target_id}.yaml", "w") as fh: 90 | yaml.dump(result, fh, default_flow_style=False) 91 | except Exception as e: 92 | raise QueryResultError(f"Issue dumping results into file " 93 | f"'{directory}/{provider}/{resource_name}/{target_id}.yaml") 94 | 95 | def dump_results_csv(self, directory: str) -> None: 96 | """ 97 | Save results in CSV files 98 | :param directory: directory to which dump CSV files 99 | :raises QueryResultError 100 | """ 101 | logger.info(f"Dumping results in CSV files") 102 | if not isdir(directory): 103 | raise QueryResultError(f"Issue dumping results to files. Directory '{directory}' doesn't exist") 104 | fieldnames_map = { 105 | AZURE_KEY: {}, 106 | GCP_KEY: {} 107 | } 108 | # Get fieldnames 109 | for provider in [AZURE_KEY, GCP_KEY]: 110 | for resource_name, targets_id in self.resources[provider].items(): 111 | logger.info(f"Discovering fieldnames for provider: {provider}, resource: {resource_name}") 112 | fieldnames_map[provider][resource_name] = set() 113 | for target_id, result in targets_id.items(): 114 | for entry in result: 115 | try: 116 | flatentry = FlatterDict(entry) 117 | fieldnames_map[provider][resource_name].update(flatentry.keys()) 118 | except Exception as e: 119 | logger.warning(f"Unable to get fieldnames for {provider} for resource {resource_name} from entry {entry}") 120 | continue 121 | for resource_name, targets_id in self.resources[provider].items(): 122 | target_id_key = TARGET_ID_KEY[provider] 123 | fieldnames = [target_id_key] + sorted(list(fieldnames_map[provider][resource_name])) 124 | filename = f"{directory}/{provider}/{resource_name}.csv" 125 | logger.info(f"Dumping results in {filename}") 126 | try: 127 | with open(filename, "w", newline="") as csvfile: 128 | writer = csv.DictWriter(csvfile, fieldnames) 129 | writer.writeheader() 130 | for target_id, result in targets_id.items(): 131 | if not result: 132 | continue 133 | for entry in result: 134 | entry[target_id_key] = target_id 135 | flatentry = FlatterDict(entry) 136 | writer.writerow(dict(flatentry)) 137 | except Exception as e: 138 | raise QueryResultError(f"Issue writing results to file {filename}") from e 139 | 140 | 141 | def set_query_results_from_configuration(gcp_services: Dict[str, GcpServiceQuery], 142 | azure_queries: Dict[str, AzureQuery]) -> QueryResult: 143 | """ 144 | Creates and configures QueryResults object based on configuration file 145 | :param gcp_services: service queries configuration stored in GcpOxidized 146 | :return: mapping of resource type and project with result 147 | """ 148 | result = QueryResult() 149 | if gcp_services: 150 | for serviceName, service in gcp_services.items(): 151 | if not service.queries: 152 | raise QueryResultError(f"No queries configured for service '{serviceName}'") 153 | for resource_name in service.queries: 154 | result.add_resource(resource_name=resource_name, provider=GCP_KEY) 155 | if azure_queries: 156 | for resource_name, query in azure_queries.items(): 157 | result.add_resource(resource_name=resource_name, provider=AZURE_KEY) 158 | return result 159 | 160 | 161 | class QueryResultError(Exception): 162 | pass 163 | -------------------------------------------------------------------------------- /cloudimized/core/run.py: -------------------------------------------------------------------------------- 1 | from cloudimized.core.core import execute 2 | 3 | 4 | def run(): 5 | execute() 6 | 7 | 8 | if __name__ == "__main__": 9 | run() 10 | -------------------------------------------------------------------------------- /cloudimized/core/slacknotifier.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from slack_sdk import WebClient 3 | from typing import Dict, Any 4 | 5 | from cloudimized.gitcore.gitchange import GitChange 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | MANUAL_CHANGE_HEADER = ":warning: *MANUAL CHANGE* :warning:" 10 | SLACK_TOKEN = "SLACK_TOKEN" 11 | CHANNEL_ID = "channelID" 12 | REPO_COMMIT_URL = "repoCommitURL" 13 | 14 | 15 | class SlackNotifier: 16 | """ 17 | Sends change notificiations to Slack 18 | """ 19 | 20 | def __init__(self, token: str, channelID: str, repo_commit_url: str): 21 | """ 22 | :param token: bot user OAuth Token 23 | :param channelID: channel ID 24 | :param repo_commit_url: URL base for Git Repository commits 25 | """ 26 | self.channelID = channelID 27 | self.token = token 28 | self.repo_commit_url = repo_commit_url 29 | 30 | def post(self, change: GitChange): 31 | """ 32 | Posts message to Slack 33 | :param change: change to post into Slack 34 | """ 35 | client = WebClient(token=self.token) 36 | comment = "" 37 | if change.manual: 38 | comment = f"{MANUAL_CHANGE_HEADER}\n" 39 | comment += f"{change.message}\n" 40 | 41 | if change.commit: 42 | comment += f"Commit: {self.repo_commit_url}/{change.commit}\n" 43 | else: 44 | comment += f"Unknown commit ID: {self.repo_commit_url}s/master\n" 45 | 46 | try: 47 | response = client.files_upload_v2( 48 | channel = self.channelID, 49 | initial_comment = comment, 50 | file_uploads=[{ 51 | "content": change.diff, 52 | "filename": change.get_filename(), 53 | "title": change.get_filename() 54 | }] 55 | ) 56 | except Exception as e: 57 | raise SlackNotifierError("Issue posting to Slack channel") from e 58 | 59 | 60 | def configure_slack_notifier(config: Dict[str, Any], 61 | token: str) -> SlackNotifier: 62 | """ 63 | Builds Slack Notifier from configuraiton file with sanity check 64 | :param config: Slack Notifier configuration section from config file 65 | :param token: Slack App Bot token passed outside of configuration file 66 | """ 67 | if config is None: 68 | return None 69 | channelID = config.get(CHANNEL_ID, None) 70 | repo_commit_url = config.get(REPO_COMMIT_URL, None) 71 | if not isinstance(channelID, str): 72 | raise SlackNotifierError(f"Incorrect type of config element {CHANNEL_ID}. " 73 | f"Should be str, is {type(CHANNEL_ID)}") 74 | if not isinstance(repo_commit_url, str): 75 | raise SlackNotifierError(f"Incorrect type of config element {REPO_COMMIT_URL}. " 76 | f"Should be str, is {type(REPO_COMMIT_URL)}") 77 | if not isinstance(token, str): 78 | raise SlackNotifierError(f"Missing Slack App Bot token. Set env var {SLACK_TOKEN}" 79 | f" with correct value") 80 | return SlackNotifier(token=token, 81 | channelID=channelID, 82 | repo_commit_url=repo_commit_url) 83 | 84 | 85 | class SlackNotifierError(Exception): 86 | pass 87 | -------------------------------------------------------------------------------- /cloudimized/gcpcore/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/egnyte/cloudimized/b319f4fc8b49008a2b06b30d9122829679218c4c/cloudimized/gcpcore/__init__.py -------------------------------------------------------------------------------- /cloudimized/gcpcore/gcpchangelog.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from typing import List 4 | from datetime import datetime, timedelta 5 | from cloudimized.gcpcore.gcpquery import GcpQuery 6 | from google.cloud import logging as gcp_logging 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | 11 | class GcpChangeLog: 12 | """Individual GCP Log entry representing resource change""" 13 | 14 | def __init__(self, resourceName: str, resourceType: str, changer: str, timestamp: datetime, 15 | methodName: str, requestType: str): 16 | """ 17 | :param resourceName: GCP Resource name 18 | :param resourceType: GCP Resource type 19 | :param changer: Identity of changer 20 | :param timestamp: Timestamp of configuration change 21 | :param methodName: GCP method name for configuration change 22 | :param requestType: GCP request type for configuration change 23 | """ 24 | self.resourceName = resourceName 25 | self.resourceType = resourceType 26 | self.changer = changer 27 | self.timestamp = timestamp 28 | self.methodName = methodName 29 | self.requestType = requestType 30 | 31 | def __eq__(self, other): 32 | # return True 33 | return self.resourceName == other.resourceName and \ 34 | self.resourceType == other.resourceType and \ 35 | self.changer == other.changer and \ 36 | self.timestamp == other.timestamp and \ 37 | self.methodName == other.methodName and \ 38 | self.requestType == other.requestType 39 | 40 | def __repr__(self): 41 | return (f"{self.resourceName} {self.resourceType} {self.changer} " 42 | f"{self.timestamp} {self.methodName} {self.requestType}") 43 | 44 | 45 | def getChangeLogs(project: str, 46 | gcp_query: GcpQuery, 47 | change_time: datetime = None, 48 | time_window: int = 30) -> List[GcpChangeLog]: 49 | """ 50 | :param project: GCP project ID in which query logs 51 | :param gcp_query: GCP Query for which identify change author 52 | :param change_time: point in time from which to look for change 53 | :param time_window: size of time window to look for change (in minutes) 54 | :return: 55 | """ 56 | # Filter string to be used in GCP Logs Explorer 57 | if not change_time: 58 | change_time = datetime.utcnow() 59 | start_time = (change_time - timedelta(minutes=time_window)).strftime("%Y-%m-%dT%H:%M:%S") 60 | filter_str = (f'timestamp>="{start_time}" AND ' 61 | f'logName: "cloudaudit.googleapis.com" AND ' 62 | f'logName: "activity" AND ' 63 | f'resource.type="{gcp_query.gcp_log_resource_type}" AND ' 64 | f'NOT protoPayload.response.@type="type.googleapis.com/error"') 65 | # Running GCP query 66 | changeLogs = [] 67 | gcp_logger = gcp_logging.Client(project=project, _use_grpc=0) # This fixes pagination issue 68 | entries = gcp_logger.list_entries(filter_=filter_str, page_size=6, order_by=gcp_logging.DESCENDING) 69 | for entry in next(entries.pages): 70 | change_log = entry.to_api_repr() 71 | resourceName = change_log.get("protoPayload", {}).get("resourceName", None) 72 | resourceType = change_log.get("resource", {}).get("type", None) 73 | changer = change_log.get("protoPayload", {}).get("authenticationInfo", {}).get("principalEmail", None) 74 | methodName = change_log.get("protoPayload", {}).get("methodName", None) 75 | requestType = change_log.get("protoPayload", {}).get("request", {}).get("@type", None) 76 | try: 77 | timestamp = datetime.strptime(change_log["timestamp"].split(".")[0], "%Y-%m-%dT%H:%M:%S") 78 | except: 79 | logger.warning(f"Issue parsing GCP log timestamp for resource '{resourceName}' for project " 80 | f"{project}") 81 | timestamp = None 82 | changeLogs.append(GcpChangeLog(resourceName, 83 | resourceType, 84 | changer, 85 | timestamp, 86 | methodName, 87 | requestType)) 88 | return changeLogs 89 | -------------------------------------------------------------------------------- /cloudimized/gcpcore/gcpexternaltoken.py: -------------------------------------------------------------------------------- 1 | from google.auth import default 2 | from google.auth.transport.requests import Request 3 | from google.oauth2 import id_token 4 | import logging 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | 9 | def get_idtoken(audience: str): 10 | """ 11 | Use the Google Application Credentials to get ID token 12 | 13 | Args: 14 | audience: The url or target audience to obtain the ID token for 15 | """ 16 | 17 | logger.info(f"Requesting GCP ID token from metatdata server for audience: {audience}") 18 | credentials, _ = default() 19 | logger.info(f"Fetching ID token for account: {credentials.service_account_email}") 20 | id_token_creds = id_token.fetch_id_token(Request(), audience) 21 | return id_token_creds 22 | -------------------------------------------------------------------------------- /cloudimized/gcpcore/gcpquery.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import re 3 | from typing import Callable, Any, Dict, List, Union 4 | from copy import deepcopy 5 | from itertools import filterfalse 6 | from functools import reduce 7 | from operator import methodcaller, itemgetter 8 | 9 | from googleapiclient.discovery import Resource 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | RESOURCE = "resource" 14 | GCP_API_CALL = "gcp_api_call" 15 | GCP_LOG_RESOURCE_TYPE = "gcp_log_resource_type" 16 | RESULT_ITEMS_FIELD = "items" 17 | DEFAULT_RESULT_ITEMS_FILED = "items" 18 | ITEM_EXCLUDE_FILTER = "item_exclude_filter" 19 | NUM_RETRIES = "num_retries" 20 | SORT_FIELDS = "sortFields" 21 | 22 | DEFAULT_NUM_RETRIES = 3 23 | 24 | AGGREGATED_LIST = "aggregatedList" 25 | DEFAULT_SORT_FIELDS = ["name"] 26 | 27 | GCP_PAGE_TOKEN = "pageToken" 28 | GCP_NEXT_PAGE_TOKEN = "nextPageToken" 29 | 30 | class GcpQuery: 31 | """A class for sending list query to GCP""" 32 | 33 | def __init__(self, resource_name: str, 34 | api_call: str, 35 | gcp_log_resource_type: str, 36 | result_items_field: str, 37 | field_exclude_filter: List = None, 38 | field_include_filter: List = None, 39 | item_exclude_filter: List[Dict[str, str]] = None, 40 | num_retries: int = 3, 41 | sort_fields: List = DEFAULT_SORT_FIELDS, 42 | **kwargs): 43 | """ 44 | :param resource_name: user-friendly name to describe queried resource 45 | :param api_call: GCP function call to run on service 46 | :param gcp_log_resource_type: resource type in GCP Log Explorer associated with resource_name 47 | :param result_item_field: key name for items queried in GCP response 48 | :param field_exclude_filter: fields to be excluded from each item 49 | :param field_include_filter: fields to keep for each item 50 | :param item_exclude_filter: regex rules to use for filtering whole items 51 | :param num_retries: number of retry attempts for API calls 52 | :param sort_fields: results sorting fields 53 | :param kwargs: kwargs to pass into gcp function 54 | """ 55 | if field_include_filter and field_exclude_filter: 56 | raise GcpQueryArgumentError(f"Issue for resource_name {resource_name} field_include_filter and " 57 | f"field_exclude_filter are mutually exclusive") 58 | self.resource_name = resource_name 59 | self.api_call = api_call 60 | self.gcp_log_resource_type = gcp_log_resource_type 61 | self.result_items_field = result_items_field 62 | self.result_exclude_filter = field_exclude_filter 63 | self.result_include_filter = field_include_filter 64 | self.result_item_filter = item_exclude_filter 65 | self.num_retries = num_retries 66 | self.sort_fields = sort_fields 67 | self.kwargs = kwargs 68 | 69 | def execute(self, service: Resource, project_id: str) -> List[Dict]: 70 | """ 71 | Sends GCP query that lists resources in project. keyword_arguments should contain project_id entry 72 | where string will be substituted with project_id 73 | :param service: GCP Resource object used to send query 74 | :param project_id: GCP project ID to query 75 | :return: List of resources that were queried 76 | """ 77 | if service is None: 78 | raise GcpQueryError(f"Service not set for '{self.resource_name}'") 79 | logger.info(f"Running query for '{self.resource_name}' in project '{project_id}'") 80 | # Replace in kwargs with actual project_id 81 | query_kwargs = deepcopy(self.kwargs) 82 | # Perform only if project_id is set 83 | ## Don't replace for queries that don't use project_id 84 | if project_id: 85 | for k, v in self.kwargs.items(): 86 | if isinstance(v, str): 87 | query_kwargs[k] = v.replace("", project_id) 88 | logger.debug(f"Replacing string in param_name: {k}, param_value: {v} " 89 | f"with string {project_id}") 90 | try: 91 | # Build function call for resource 92 | # i.e. run service.projects().list() 93 | ## Run query without last call i.e. service.projects() 94 | logger.debug(f"Query GCP Resource object: {service}\n" 95 | f"Query API call: {self.api_call}") 96 | api_last_call_method = self.api_call.split(".")[-1] 97 | query_base = reduce(lambda x, y: methodcaller(y)(x), self.api_call.split(".")[:-1], service) 98 | result = [] 99 | while True: 100 | logger.debug(f"Query base for API call: {query_base}\nAPI call kwargs: {query_kwargs}") 101 | # Run last call on service with arguments if present i.e. (service.projects()).list() 102 | request = methodcaller(api_last_call_method, **query_kwargs)(query_base) 103 | logger.debug(f"API call request object: {request}") 104 | response = request.execute(num_retries=self.num_retries) 105 | logger.debug(f"API call response object: {response}") 106 | current_result = response.get(self.result_items_field, []) 107 | # Separated handling for aggregatedList call (that returned a response) 108 | if api_last_call_method == AGGREGATED_LIST and isinstance(current_result, dict): 109 | current_result = self._parse_aggregated_list(current_result) 110 | result.extend(current_result) 111 | #Continue until there is no further page 112 | next_page_token = response.get(GCP_NEXT_PAGE_TOKEN, None) 113 | if next_page_token is None: 114 | break 115 | else: 116 | query_kwargs[GCP_PAGE_TOKEN] = next_page_token 117 | except Exception as e: 118 | raise GcpQueryError(f"Issue executing call '{self.api_call}' with args '{self.kwargs}'") from e 119 | # Sort result list to get predictable results 120 | # Results from kubernetes cluster list call return non-consistent list order 121 | # Perform sorting based on "name" key if present 122 | self.__sort_result(result, project_id) 123 | if self.result_item_filter: 124 | for filter_condition_set in self.result_item_filter: 125 | result[:] = [i for i in result if self.__filter_item(i, filter_condition_set)] 126 | if self.result_exclude_filter: 127 | return self.__filter_field_exclude(self.result_exclude_filter, result) 128 | elif self.result_include_filter: 129 | return self._filter_field_include(self.result_include_filter, result) 130 | else: 131 | return result 132 | 133 | def __filter_field_exclude(self, fields: List[Union[str, Dict]], result: List[Dict]) -> List[Dict]: 134 | filtered_result = result[:] 135 | for field in fields: 136 | if isinstance(field, str): 137 | for item in filtered_result: 138 | item.pop(field, None) 139 | elif isinstance(field, dict): 140 | for item in filtered_result: 141 | for nested_key, nested_fields in field.items(): 142 | nested_result = item.get(nested_key, {}) 143 | if isinstance(nested_result, dict): 144 | self.__filter_field_exclude(fields=nested_fields, result=[nested_result]) 145 | if isinstance(nested_result, list): 146 | self.__filter_field_exclude(fields=nested_fields, result=nested_result) 147 | return filtered_result 148 | 149 | def _filter_field_include(self, fields: List[Union[str, Dict]], result: List[Dict[str, Any]]) -> List[Dict[str, Any]]: 150 | """ 151 | Filter out all fields except ones specified from each item in results 152 | """ 153 | def filter_dict(entry, filters): 154 | filtered_entry = {} 155 | for key, value in entry.items(): 156 | if isinstance(value, dict): 157 | if key in filters: 158 | if not filters[key]: # Include the entire nested dictionary 159 | filtered_entry[key] = filter_dict(value, {}) 160 | else: 161 | filtered_entry[key] = filter_dict(value, filters[key]) 162 | elif key in filters or not filters: # Include the field if it's in filters or filters are empty 163 | filtered_entry[key] = value 164 | return filtered_entry 165 | 166 | filtered_data = [] 167 | for entry in result: 168 | filtered_entry = filter_dict(entry, fields) 169 | filtered_data.append(filtered_entry) 170 | 171 | return filtered_data 172 | 173 | def __filter_item(self, item: Dict[str, Any], filter_condition_set) -> bool: 174 | for filter_field, filter_condition in filter_condition_set.items(): 175 | if isinstance(filter_condition, str): 176 | field_value = item.get(filter_field, "") 177 | if isinstance(field_value, list): 178 | field_value[:] = [i for i in field_value if not re.match(rf"{filter_condition}", i)] 179 | break 180 | elif isinstance(field_value, str): 181 | if not re.match(rf"{filter_condition}", field_value): 182 | break 183 | elif isinstance(filter_condition, dict): 184 | nested_result = item.get(filter_field, None) 185 | if nested_result is None: 186 | break 187 | if isinstance(nested_result, dict): 188 | return self.__filter_item(nested_result, filter_condition) 189 | elif isinstance(nested_result, list): 190 | nested_result[:] = [i for i in nested_result if self.__filter_item(i, filter_condition)] 191 | break 192 | else: 193 | # If all condition match filter item out 194 | return False 195 | # If there was a break don't filter item out 196 | return True 197 | 198 | def __sort_result(self, result: List[Dict], project_id: str) -> None: 199 | """ 200 | Performs sorting of given result list 201 | :param result: results to be sorted 202 | :param project_id: Project for logging purposes 203 | """ 204 | for sort_field in self.sort_fields: 205 | try: 206 | if isinstance(sort_field, str): 207 | result.sort(key=itemgetter(sort_field)) 208 | elif isinstance(sort_field, dict): 209 | for inner_key, inner_field in sort_field.items(): 210 | for outer_result_item in result: 211 | try: 212 | inner_result = outer_result_item[inner_key] 213 | inner_result.sort(key=itemgetter(inner_field)) 214 | except Exception as e: 215 | logger.warning(f"Unable to sort inner list for {sort_field} fields for project " 216 | f"{project_id}") 217 | except Exception as e: 218 | logger.warning( 219 | f"Issue sorting results for'{self.api_call}' for project '{project_id}' " 220 | f"for sorting field {sort_field}") 221 | logger.debug(f"Reason: {e}") 222 | 223 | def _parse_aggregated_list(self, api_call_result: Dict[str, Dict]) -> List[Dict]: 224 | """ 225 | Parses response from aggregatedList API call into format common to other calls 226 | :param api_call_result: aggregatedList call response 227 | :return: parsed result in common format 228 | """ 229 | result = [] 230 | if not isinstance(api_call_result, dict): 231 | raise GcpQueryError(f"Incorrect result type for aggregatedList parsing. Is {type(api_call_result)}, " 232 | f"should be dict") 233 | try: 234 | logger.debug(f"Retrieving resource name from API call definition: {self.api_call}") 235 | api_result_resource_name = self.api_call.split(".")[-2] 236 | logger.debug(f"Using resource name: {api_result_resource_name} for retrieving " 237 | f"results from aggregatedList call") 238 | except Exception as e: 239 | raise GcpQueryError(f"Issue retrieving resource name from aggregatedList API call {self.api_call}") from e 240 | try: 241 | for region, region_response in api_call_result.items(): 242 | if api_result_resource_name in region_response: 243 | logger.debug(f"Found resource '{api_result_resource_name}' in region '{region}'") 244 | result.extend(region_response[api_result_resource_name]) 245 | return result 246 | except Exception as e: 247 | raise GcpQueryError(f"Issue processing api_call_result") from e 248 | 249 | 250 | def configure_queries(queries: List[Dict[str, Any]]) -> Dict[str, GcpQuery]: 251 | """ 252 | Configures GCP queries objects from configuration 253 | :param queries: per service query configuration 254 | :returns resource name to GCPQuery object mapping 255 | """ 256 | if not isinstance(queries, list): 257 | raise GcpQueryArgumentError(f"Incorrect GCP queries configuration. Should be list, is {type(queries)}") 258 | #TODO better configuraiton file verification 259 | result = {} 260 | for query in queries: 261 | if RESOURCE not in query or GCP_API_CALL not in query: 262 | raise GcpQueryArgumentError(f"Missing required key in query: '{query}'") 263 | if GCP_LOG_RESOURCE_TYPE not in query: 264 | raise GcpQueryArgumentError(f"Missing required key: '{GCP_LOG_RESOURCE_TYPE}' " 265 | f"in query configuration '{query}'") 266 | if ITEM_EXCLUDE_FILTER in query: 267 | item_exclude_filter = query[ITEM_EXCLUDE_FILTER] 268 | if not isinstance(item_exclude_filter, list): 269 | raise GcpQueryArgumentError(f"Incorrect GCP query configuration. Item exclude filter should be list, " 270 | f"is {type(item_exclude_filter)}") 271 | num_retries = query.get(NUM_RETRIES, DEFAULT_NUM_RETRIES) 272 | sort_fields = query.get(SORT_FIELDS, DEFAULT_SORT_FIELDS) 273 | try: 274 | # Create kwargs from only keyword arguments 275 | ## Skip gcp query kwargs and pass everything else to api call 276 | kwargs = dict(filterfalse(lambda x: x[0] not in set([ 277 | "field_exclude_filter", "field_include_filter", "item_exclude_filter", "result_items_field"]), 278 | query.items())) 279 | if "gcp_function_args" in query: 280 | kwargs = {**kwargs, **query["gcp_function_args"]} 281 | results_items_field = query.get(RESULT_ITEMS_FIELD, DEFAULT_RESULT_ITEMS_FILED) 282 | result[query[RESOURCE]] = GcpQuery(resource_name=query[RESOURCE], 283 | api_call=query[GCP_API_CALL], 284 | gcp_log_resource_type=query[GCP_LOG_RESOURCE_TYPE], 285 | result_items_field=results_items_field, 286 | num_retries=num_retries, 287 | sort_fields=sort_fields, 288 | **kwargs) 289 | except Exception as e: 290 | raise GcpQueryArgumentError(f"Issue parsing query config {query}") from e 291 | return result 292 | 293 | 294 | class GcpQueryError(Exception): 295 | pass 296 | 297 | 298 | class GcpQueryArgumentError(GcpQueryError): 299 | pass 300 | -------------------------------------------------------------------------------- /cloudimized/gcpcore/gcpservicequery.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import google.auth 3 | import google.auth.transport.requests 4 | import google_auth_httplib2 5 | import httplib2 6 | from googleapiclient.discovery import Resource 7 | 8 | from os import getenv 9 | from typing import List, Dict 10 | from googleapiclient import discovery 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | SERVICE_SECTION = "gcp_services" 15 | SERVICE_NAME = "serviceName" 16 | VERSION = "version" 17 | QUERIES = "queries" 18 | RETRIES = "retries" 19 | 20 | PROJECTS_DISCOVERY_SERVICE_NAME = "cloudresourcemanager" 21 | PROJECTS_DISCOVERY_SERVICE_CONFIG = [ 22 | { 23 | SERVICE_NAME: PROJECTS_DISCOVERY_SERVICE_NAME, 24 | VERSION: "v1", 25 | QUERIES: [ 26 | 27 | ] 28 | } 29 | ] 30 | 31 | 32 | class GcpServiceQuery: 33 | """Class describing GCP API service and its queries""" 34 | 35 | def __init__(self, serviceName: str, version: str): 36 | """ 37 | :param serviceName: GCP service API name 38 | :param version: GCP API version 39 | :param num_retries: number of retries for each request 40 | """ 41 | self.serviceName = serviceName 42 | self.version = version 43 | self.queries = {} 44 | 45 | def build(self) -> Resource: 46 | """Build resource for interacting with Google Cloud API""" 47 | if getenv("GOOGLE_APPLICATION_CREDENTIALS"): 48 | logger.info("Env var 'GOOGLE_APPLICATION_CREDENTIALS' set. Authenticating using credentials file") 49 | else: 50 | logger.info("Env var 'GOOGLE_APPLICATION_CREDENTIALS' not set. Authenticating using default credentials") 51 | 52 | credentials, _ = google.auth.default(scopes=["https://www.googleapis.com/auth/cloud-platform"]) 53 | authorized_http = google_auth_httplib2.AuthorizedHttp(credentials, http=httplib2.Http(timeout=60)) 54 | service = discovery.build(self.serviceName, 55 | self.version, 56 | http=authorized_http) 57 | return service 58 | 59 | 60 | def configure_services(config: List[Dict]) -> Dict[str, GcpServiceQuery]: 61 | """ 62 | Generate GcpServiceQuery list from config 63 | :param config: list with GcpServieQuery's configuration 64 | :return: mapping of service name to GcpServiceQuery objects 65 | """ 66 | if not isinstance(config, list): 67 | raise GcpServiceQueryConfigError(f"Invalid GcpServiceQuery config {config}") 68 | result = {} 69 | for entry in config: 70 | if not isinstance(entry, dict): 71 | raise GcpServiceQueryConfigError(f"Invalid GcpServiceQuery entry type: '{entry}'. " 72 | f"Should be dict, is {type(entry)}") 73 | serviceName = entry.get(SERVICE_NAME, None) 74 | version = entry.get(VERSION, None) 75 | queries = entry.get(QUERIES, None) 76 | if not serviceName or not version or not queries: 77 | raise GcpServiceQueryConfigError(f"Missing required key for entry {entry}") 78 | gcp_service_query = GcpServiceQuery(serviceName, version) 79 | # Check multiple entries with same name 80 | if serviceName in result: 81 | raise GcpServiceQueryConfigError(f"Multiple GCP service with same name: {serviceName}") 82 | result[serviceName] = gcp_service_query 83 | return result 84 | 85 | 86 | class GcpServiceQueryError(Exception): 87 | pass 88 | 89 | 90 | class GcpServiceQueryConfigError(GcpServiceQueryError): 91 | pass 92 | -------------------------------------------------------------------------------- /cloudimized/gitcore/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/egnyte/cloudimized/b319f4fc8b49008a2b06b30d9122829679218c4c/cloudimized/gitcore/__init__.py -------------------------------------------------------------------------------- /cloudimized/gitcore/gitchange.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from collections import OrderedDict 4 | 5 | logger = logging.getLogger(__name__) 6 | 7 | #TODO: Change this to absract Change class, make GcpChange inherit from it and move it to gcpcore instead 8 | # This will allow to easier expand into other Cloud providers 9 | class GitChange: 10 | """ 11 | Represents configuration change in resources 12 | """ 13 | 14 | def __init__(self, provider: str, resource_type: str, project: str, is_old_structure: str=False): 15 | """ 16 | :param provider: name of provider [azure, gcp] 17 | :param resource_type: GCP/Azure resource type 18 | :param project: GCP/Azure project name 19 | :param is_old_structure: used for transition when Azure support was added 20 | """ 21 | self.provider = provider 22 | self.resource_type = resource_type 23 | self.project = project 24 | self.is_old_structure = is_old_structure 25 | self.message = None 26 | self.diff = None 27 | self.manual = False 28 | self.commit = None 29 | self.changers = None 30 | # List of GCP change logs related to this resource 31 | self.gcp_change_log = [] 32 | # List of Terraform Runs related to this resource 33 | self.terraform_run_log = [] 34 | 35 | def get_filename(self) -> str: 36 | """ 37 | Provides file where configuration is stored 38 | """ 39 | if self.is_old_structure: 40 | return f"{self.resource_type}/{self.project}.yaml" 41 | else: 42 | return f"{self.provider}/{self.resource_type}/{self.project}.yaml" 43 | 44 | def get_commit_message(self) -> str: 45 | """ 46 | Returns Git commit message for this change 47 | """ 48 | basic_msg = f"{self.resource_type.title()} updated in {self.provider.upper()}: {self.project} by" 49 | # Get only unique changer_identities with predicatable order (for passing tests mainly) 50 | unique_changer_identity = OrderedDict.fromkeys([change.changer for change in self.gcp_change_log]) 51 | # No changers identified 52 | if not unique_changer_identity: 53 | message = f"{basic_msg} UNKNOWN" 54 | else: 55 | # TODO: Add addtional info to commit message from gcpchangelog i.e. methodname 56 | message = f"{basic_msg} {list(unique_changer_identity.keys())}" 57 | # Prepend information from Terraform if available 58 | # TODO: Add terraform logic 59 | if self.terraform_run_log: 60 | pass 61 | else: 62 | pass 63 | return message 64 | 65 | def __eq__(self, other): 66 | return self.provider == other.provider and \ 67 | self.resource_type == other.resource_type and \ 68 | self.project == other.project 69 | -------------------------------------------------------------------------------- /cloudimized/gitcore/repo.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import git 4 | from typing import List, Dict 5 | from os.path import exists, isdir, join 6 | from os import listdir 7 | from shutil import rmtree 8 | from cloudimized.gitcore.gitchange import GitChange 9 | from os.path import basename, dirname 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | #TODO Add command line option to set those 14 | # Env name for passing user/pass 15 | GIT_USER = "GIT_USR" 16 | GIT_PASSWORD = "GIT_PSW" 17 | 18 | # Config keys 19 | GIT_SECTION = "git" 20 | GIT_REPO_URL = "remote_url" 21 | GIT_LOCAL_DIR = "local_directory" 22 | 23 | class GitRepo: 24 | """Git repo in which changes are tracked""" 25 | 26 | def __init__(self, user: str, password: str, repo_url: str, directory: str): 27 | """ 28 | :param user: git username 29 | :param password: git password or token 30 | :param repo_url: Git remote repo URL 31 | :param directory: Git local repo directory 32 | """ 33 | self.user = user 34 | self.password = password 35 | self.repo_url = repo_url 36 | self.directory = directory 37 | self.repo = None 38 | 39 | #TODO: Add initial mapping of repo - resource/project - to better compare previous state to current 40 | # This is useful when a resource existed previously and in current run doesn't exist i.e. project was deleted 41 | def setup(self) -> None: 42 | """ 43 | Setup local and remote repo 44 | """ 45 | if exists(self.directory): 46 | logger.info(f"Veryfing Git repo at: '{self.directory}'...") 47 | try: 48 | self.repo = git.Repo(self.directory) 49 | except Exception as e: 50 | raise GitRepoError(f"Directory '{self.directory}' is not a git repo") 51 | try: 52 | logger.info(f"Syncing local repo with remote") 53 | # Remove any local changes, checkout to master and sync with remote 54 | self.repo.git.reset("--hard") 55 | self.repo.git.checkout("master") 56 | self.repo.remotes.origin.fetch() 57 | self.repo.git.reset("--hard", "origin/master") 58 | except Exception as e: 59 | raise GitRepoError(f"Issue syncing remote") from e 60 | else: 61 | if self.repo_url.startswith("https://"): 62 | if self.user is None or self.password is None: 63 | raise GitRepoError("Missing credentials for Git HTTPS method") 64 | # Cloning via HTTPS 65 | method = "HTTPS" 66 | # Add user/pass to URL 67 | repo_url_cred = f"https://{self.user}:{self.password}@{self.repo_url.split('://')[1]}" 68 | elif self.repo_url.startswith("git@"): 69 | # Cloning via SSH 70 | method = "SSH" 71 | repo_url_cred = self.repo_url 72 | else: 73 | raise GitRepoError(f"Incorrect Git URL: '{self.repo_url}'") 74 | try: 75 | logger.info(f"Local Git repo not found. Cloning {self.repo_url} into {self.directory} via {method}") 76 | self.repo = git.Repo.clone_from(repo_url_cred, self.directory) 77 | except Exception as e: 78 | raise GitRepoError(f"Issue cloning Git repo {self.repo_url}: {type(e)}\n{e}") 79 | 80 | def get_changes(self) -> List[GitChange]: 81 | """ 82 | Returns all changes for current repo state 83 | :returns all detected changes 84 | """ 85 | if not self.repo: 86 | raise GitRepoError(f"Git Repo not set up") 87 | file_changes = self.repo.untracked_files + [item.a_path for item in self.repo.index.diff(None)] 88 | result = [] 89 | for filename in file_changes: 90 | #Needed for trasitioning from old directory structure 91 | ## When Azure support was added 92 | dir_path = dirname(filename).split("/") 93 | if len(dir_path) == 2: 94 | provider = dir_path[-2] 95 | is_old_structure = False 96 | else: 97 | provider = "gcp" 98 | is_old_structure = True 99 | resource_type = dir_path[-1] 100 | project = basename(filename).split(".")[0] 101 | result.append(GitChange(provider, resource_type, project, is_old_structure)) 102 | return result 103 | 104 | def commit_change(self, change: GitChange, message: str) -> None: 105 | """ 106 | Commit change with provided message 107 | :param change: single resource/project change 108 | :param message: commit message 109 | :return: 110 | """ 111 | try: 112 | self.repo.git.add(change.get_filename()) 113 | self.repo.git.commit(message) 114 | except Exception as e: 115 | raise GitRepoError(f"Issue commiting change for project '{change.project}' " 116 | f"type '{change.resource_type}'") from e 117 | 118 | def push_changes(self) -> None: 119 | """ 120 | Pushes local changes to remote 121 | """ 122 | try: 123 | self.repo.remotes.origin.push() 124 | except Exception as e: 125 | raise GitRepoError(f"Issue pushing local changes to remote") from e 126 | 127 | def clean_repo(self) -> None: 128 | """ 129 | Removes all files in repository for preparations for new config scanning 130 | :raises GitRepoError 131 | """ 132 | if not self.repo: 133 | raise GitRepoError(f"Repo '{self.repo_url}' needs to be setup first") 134 | try: 135 | # Get all directories in self.directory (to skip files in the list i.e. README.md) 136 | directories = [name for name in listdir(self.directory) if isdir(join(self.directory, name))] 137 | except Exception as e: 138 | raise GitRepoError(f"Issue retrieving directories in directory '{self.directory}'") from e 139 | try: 140 | for directory in directories: 141 | if directory == ".git": #skip git folder 142 | continue 143 | else: 144 | logger.info(f"Removing directory '{self.directory}/{directory}") 145 | rmtree(f"{self.directory}/{directory}") 146 | except Exception as e: 147 | raise GitRepoError(f"Issue removing directories in '{self.directory}'") from e 148 | 149 | 150 | def configure_repo(user: str, password: str, config: Dict[str, str]) -> GitRepo: 151 | """ 152 | Parses configuration file with Git Repo configuration 153 | :param config: repo configuration 154 | :param user: git username for remote repo 155 | :param password: git's password/token for remote repo 156 | :return: Git Repo with parsed configuration 157 | """ 158 | if not config: 159 | raise GitRepoConfigError(f"Missing required git section: '{GIT_SECTION}'") 160 | if not user: 161 | raise GitRepoConfigError(f"Git username not set in env var: '{GIT_USER}'") 162 | if not password: 163 | raise GitRepoConfigError(f"Git password/token not set in env var: '{GIT_PASSWORD}'") 164 | if not isinstance(config, dict): 165 | raise GitRepoConfigError(f"Incorrect type in git configuration section: '{GIT_SECTION}'. " 166 | f"Should be dict, is {type(config)}") 167 | if GIT_REPO_URL not in config: 168 | raise GitRepoConfigError(f"Missing required key in Git configuration: '{GIT_REPO_URL}'") 169 | if GIT_LOCAL_DIR not in config: 170 | raise GitRepoConfigError(f"Missing required key in Git configuration: '{GIT_LOCAL_DIR}'") 171 | return GitRepo(user=user, password=password, repo_url=config[GIT_REPO_URL], directory=config[GIT_LOCAL_DIR]) 172 | 173 | 174 | class GitRepoError(Exception): 175 | pass 176 | 177 | 178 | class GitRepoConfigError(GitRepoError): 179 | pass 180 | -------------------------------------------------------------------------------- /cloudimized/singlerunconfigs/azure/aksClusters.yaml: -------------------------------------------------------------------------------- 1 | resource: aksClusters 2 | field_exclude_filter: 3 | - agent_pool_profiles: 4 | - count 5 | - power_state 6 | - provisioning_state 7 | - power_state 8 | - provisioning_state 9 | #item_exclude_filter: 10 | # - 11 | -------------------------------------------------------------------------------- /cloudimized/singlerunconfigs/azure/networkSecurityGroups.yaml: -------------------------------------------------------------------------------- 1 | resource: networkSecurityGroups 2 | field_exclude_filter: 3 | - default_security_rules: 4 | - etag 5 | - provisioning_state 6 | - etag 7 | - network_interfaces 8 | - provisioning_state 9 | - security_rules: 10 | - etag 11 | - provisioning_state 12 | #item_exclude_filter: 13 | # - 14 | -------------------------------------------------------------------------------- /cloudimized/singlerunconfigs/azure/virtualNetworks.yaml: -------------------------------------------------------------------------------- 1 | resource: virtualNetworks 2 | field_exclude_filter: 3 | - etag 4 | - provisioning_state 5 | - subnets: 6 | - delegations: 7 | - etag 8 | - etag 9 | - ip_configurations 10 | - provisioning_state 11 | - service_association_links: 12 | - etag 13 | - virtual_network_peerings: 14 | - etag 15 | - peering_state 16 | - peering_sync_level 17 | - provisioning_state 18 | #item_exclude_filter: 19 | # - 20 | -------------------------------------------------------------------------------- /cloudimized/singlerunconfigs/azure/vnetGateways.yaml: -------------------------------------------------------------------------------- 1 | resource: vnetGateways 2 | field_exclude_filter: 3 | - etag 4 | - provisioning_state 5 | - ip_configurations: 6 | - etag 7 | - provisioning_state 8 | #item_exclude_filter: 9 | # - 10 | -------------------------------------------------------------------------------- /cloudimized/singlerunconfigs/gcp/addresses.yaml: -------------------------------------------------------------------------------- 1 | serviceName: compute 2 | version: v1 3 | queries: 4 | - resource: addresses 5 | gcp_api_call: addresses.aggregatedList 6 | gcp_function_args: 7 | project: 8 | # filter: addressType=EXTERNAL 9 | gcp_log_resource_type: None #non-applicable 10 | field_include_filter: 11 | - address 12 | - name 13 | - region 14 | - status 15 | # field_exclude_filter: 16 | # - creationTimestamp 17 | # - id 18 | # - kind 19 | # - selfLink 20 | -------------------------------------------------------------------------------- /cloudimized/singlerunconfigs/gcp/forwardingRules.yaml: -------------------------------------------------------------------------------- 1 | serviceName: compute 2 | version: v1 3 | queries: 4 | - resource: forwardingRules 5 | gcp_api_call: forwardingRules.aggregatedList 6 | gcp_function_args: 7 | project: 8 | # filter: 9 | gcp_log_resource_type: None #non-applicable 10 | # field_include_filter: 11 | # - address 12 | # - name 13 | # - region 14 | # - status 15 | field_exclude_filter: 16 | - creationTimestamp 17 | - fingerprint 18 | - id 19 | - kind 20 | - labelFingerprint 21 | - selfLink 22 | -------------------------------------------------------------------------------- /cloudimized/singlerunconfigs/gcp/globalAddresses.yaml: -------------------------------------------------------------------------------- 1 | serviceName: compute 2 | version: v1 3 | queries: 4 | - resource: globalAddresses 5 | gcp_api_call: globalAddresses.list 6 | gcp_function_args: 7 | project: 8 | filter: addressType=EXTERNAL 9 | gcp_log_resource_type: None #non-applicable 10 | field_include_filter: 11 | - address 12 | - name 13 | - status 14 | # field_exclude_filter: 15 | # - id 16 | -------------------------------------------------------------------------------- /cloudimized/singlerunconfigs/gcp/globalPublicDelegatedPrefixes.yaml: -------------------------------------------------------------------------------- 1 | serviceName: compute 2 | version: v1 3 | queries: 4 | - resource: globalPublicDelegatedPrefixes 5 | gcp_api_call: globalPublicDelegatedPrefixes.list 6 | gcp_function_args: 7 | project: 8 | # filter: 9 | gcp_log_resource_type: None #non-applicable 10 | # field_include_filter: 11 | # - address 12 | # - name 13 | # - region 14 | # - status 15 | # field_exclude_filter: 16 | # - creationTimestamp 17 | # - fingerprint 18 | # - id 19 | # - kind 20 | # - labelFingerprint 21 | # - selfLink 22 | -------------------------------------------------------------------------------- /cloudimized/singlerunconfigs/gcp/networks.yaml: -------------------------------------------------------------------------------- 1 | serviceName: compute 2 | version: v1 3 | queries: 4 | - resource: networks 5 | gcp_api_call: networks.list 6 | gcp_function_args: 7 | project: 8 | # filter: 9 | gcp_log_resource_type: None #non-applicable 10 | # field_include_filter: 11 | # - name 12 | field_exclude_filter: 13 | - selfLinkWithId 14 | - networkFirewallPolicyEnforcementOrder 15 | - peerings: 16 | - stateDetails 17 | item_exclude_filter: 18 | - peerings: 19 | name: "servicenetworking-googleapis-com" 20 | # - subnetworks: ".*europe-west[89].*" 21 | -------------------------------------------------------------------------------- /cloudimized/singlerunconfigs/gcp/publicAdvertisedPrefixes.yaml: -------------------------------------------------------------------------------- 1 | serviceName: compute 2 | version: v1 3 | queries: 4 | - resource: publicAdvertisedPrefixes 5 | gcp_api_call: publicAdvertisedPrefixes.list 6 | gcp_function_args: 7 | project: 8 | # filter: 9 | gcp_log_resource_type: None #non-applicable 10 | sortFields: 11 | - name 12 | - publicDelegatedPrefixs: name 13 | # field_include_filter: 14 | # - address 15 | # - name 16 | # - region 17 | # - status 18 | # field_exclude_filter: 19 | # - creationTimestamp 20 | # - fingerprint 21 | # - id 22 | # - kind 23 | # - labelFingerprint 24 | # - selfLink 25 | -------------------------------------------------------------------------------- /cloudimized/singlerunconfigs/gcp/publicDelegatedPrefixes.yaml: -------------------------------------------------------------------------------- 1 | serviceName: compute 2 | version: v1 3 | queries: 4 | - resource: publicDelegatedPrefixes 5 | gcp_api_call: publicDelegatedPrefixes.aggregatedList 6 | gcp_function_args: 7 | project: 8 | # filter: 9 | gcp_log_resource_type: None #non-applicable 10 | # field_include_filter: 11 | # - address 12 | # - name 13 | # - region 14 | # - status 15 | # field_exclude_filter: 16 | # - creationTimestamp 17 | # - fingerprint 18 | # - id 19 | # - kind 20 | # - labelFingerprint 21 | # - selfLink 22 | -------------------------------------------------------------------------------- /cloudimized/singlerunconfigs/gcp/subnetworks.yaml: -------------------------------------------------------------------------------- 1 | serviceName: compute 2 | version: v1 3 | queries: 4 | - resource: subnetworks 5 | gcp_api_call: subnetworks.aggregatedList 6 | gcp_function_args: 7 | project: 8 | # filter: 9 | gcp_log_resource_type: None #non-applicable 10 | # field_include_filter: 11 | # - name 12 | field_exclude_filter: 13 | - stackType 14 | # item_exclude_filter: 15 | # - region: ".*(europe-west[89]|asia-south1).*" 16 | 17 | -------------------------------------------------------------------------------- /cloudimized/singlerunconfigs/gcp/vpnTunnels.yaml: -------------------------------------------------------------------------------- 1 | serviceName: compute 2 | version: v1 3 | queries: 4 | - resource: vpnTunnels 5 | gcp_api_call: vpnTunnels.aggregatedList 6 | gcp_function_args: 7 | project: 8 | # filter: status!=ESTABLISHED 9 | gcp_log_resource_type: None #non-applicable 10 | # field_include_filter: 11 | # - name 12 | # - region 13 | # - status 14 | # - detailedStatus 15 | field_exclude_filter: 16 | - creationTimestamp 17 | - localTrafficSelector 18 | - sharedSecret 19 | - sharedSecretHash 20 | -------------------------------------------------------------------------------- /cloudimized/tfcore/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/egnyte/cloudimized/b319f4fc8b49008a2b06b30d9122829679218c4c/cloudimized/tfcore/__init__.py -------------------------------------------------------------------------------- /cloudimized/tfcore/query.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | 4 | from os import getenv 5 | from typing import Dict, List 6 | from datetime import datetime, timedelta 7 | from terrasnek.api import TFC 8 | from .run import TFRun, parse_tf_runs 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | ORG = 'org' 13 | WORKSPACE = 'workspace' 14 | 15 | TERRAFORM_SECTION = 'terraform' 16 | TERRAFORM_URL = 'url' 17 | TERRAFORM_SERVICE_ACCOUNT_WORKSPACE_MAP = 'service_workspace_map' 18 | TERRAFORM_WORKSPACE_TOKEN_FILE = 'workspace_token_file' 19 | ENV_TERRAFORM_WORKSPACE_TOKEN_FILE = "TERRAFORM_READ_TOKENS" 20 | 21 | class TFQuery: 22 | """ 23 | Query for terraform runs generating changes 24 | """ 25 | 26 | # TODO: add checking of map format 27 | def __init__(self, tf_url: str, sa_org_workspace_map: Dict[str, Dict[str, str]], org_token_map: Dict[str, str]): 28 | """ 29 | :param tf_url: Terraform instance URL 30 | :param sa_org_workspace_map: Map of GCP Service Account to TF organization/workspace 31 | :param org_token_map: Map of TF organization to TF team token 32 | """ 33 | self.tf_url = tf_url 34 | self.sa_org_workspace_map = sa_org_workspace_map 35 | self.org_token_map = org_token_map 36 | 37 | def __get_api(self, gcp_sa: str) -> TFC: 38 | """ 39 | Connect to TF API object for GCP Service account 40 | :param gcp_sa: GCP Service Account which performed change 41 | :raises TFQueryError 42 | :return: Terraform API object 43 | """ 44 | if gcp_sa not in self.sa_org_workspace_map: 45 | raise TFQueryError(f"Unknown GCP ServiceAccount {gcp_sa}") 46 | tf_org = self.sa_org_workspace_map[gcp_sa][ORG] 47 | tf_token = self.org_token_map[tf_org] 48 | tf_api = TFC(tf_token, url=self.tf_url) 49 | tf_api.set_org(tf_org) 50 | return tf_api 51 | 52 | def get_runs(self, gcp_sa: str, 53 | run_limit: int = 10, 54 | change_time: datetime = None, 55 | time_window: int = 30) -> List[TFRun]: 56 | """ 57 | Get TF runs for given GCP Service account at given point in time 58 | :param gcp_sa: GCP Service Account which performed change 59 | :param run_limit: Number of TF run limits to get 60 | :param change_time: point in time from which to look for change 61 | :param time_window: size of time window to look for change (in minutes) 62 | :return: 63 | """ 64 | logger.info(f"Getting TF {run_limit} last runs for workspace connected {gcp_sa}") 65 | tf_api = self.__get_api(gcp_sa) 66 | tf_workspace_names = self.sa_org_workspace_map[gcp_sa][WORKSPACE] 67 | tf_runs = [] 68 | for workspace in tf_workspace_names: 69 | try: 70 | logger.info(f"Getting workspace_id for workspace name {workspace}") 71 | workspace_response = tf_api.workspaces.show(workspace_name=workspace) 72 | tf_workspace_id = workspace_response["data"]["id"] 73 | except Exception as e: 74 | raise TFQueryError(f"Issue getting workspace ID for workspace {workspace}") from e 75 | try: 76 | logger.info(f"Getting {run_limit} TF runs for workspace ID {tf_workspace_id}") 77 | runs_response = tf_api.runs.list(tf_workspace_id, page_size=run_limit, include=["created-by"]) 78 | tf_runs += parse_tf_runs(runs_response, tf_api.get_org(), workspace) 79 | except Exception as e: 80 | raise TFQueryError(f"Issue getting terraform runs") from e 81 | if not change_time: 82 | change_time = datetime.utcnow() 83 | start_time = (change_time - timedelta(minutes=time_window)) 84 | tf_runs = [tf_run for tf_run in tf_runs if tf_run.apply_time >= start_time] 85 | return tf_runs 86 | 87 | 88 | def configure_tfquery(config: Dict) -> TFQuery: 89 | """ 90 | Generates TFQuery based from configuraiton file 91 | :param config: configuration 92 | :return: TFQuery object with parsed config 93 | """ 94 | #TODO Add handling of raised exception 95 | if config is None: 96 | logger.info("No terraform configuration found. Skipping TF querying for additional info") 97 | return None 98 | if not isinstance(config, dict): 99 | raise TFQueryConfigurationError(f"Incorrect configuration type. Should be dict is {type(config)}") 100 | if TERRAFORM_URL not in config: 101 | raise TFQueryConfigurationError(f"Missing required key: {TERRAFORM_URL}") 102 | if TERRAFORM_SERVICE_ACCOUNT_WORKSPACE_MAP not in config: 103 | raise TFQueryConfigurationError(f"Missing required key: {TERRAFORM_SERVICE_ACCOUNT_WORKSPACE_MAP}") 104 | 105 | # Verify url 106 | url = config[TERRAFORM_URL] 107 | if not isinstance(url, str): 108 | raise TFQueryConfigurationError(f"Incorrect value for {TERRAFORM_URL}. Should be str is {type(url)}") 109 | 110 | # Verify service account workspace mapping structure 111 | sa_workspace_map = config[TERRAFORM_SERVICE_ACCOUNT_WORKSPACE_MAP] 112 | if not isinstance(sa_workspace_map, dict): 113 | raise TFQueryConfigurationError(f"Incorrect configuration type in {TERRAFORM_SERVICE_ACCOUNT_WORKSPACE_MAP}. " 114 | f"Should be dict is {type(sa_workspace_map)}") 115 | for key, value in sa_workspace_map.items(): 116 | if not isinstance(key, str): 117 | raise TFQueryConfigurationError(f"Incorrect entry type for {key}. Should be str is {type(key)}") 118 | if not isinstance(value, dict): 119 | raise TFQueryConfigurationError(f"Incorrect entry type for {value}. Should be dict in {type(key)}") 120 | if ORG not in value or WORKSPACE not in value: 121 | raise TFQueryConfigurationError(f"Missing one of required keys: {ORG}, {WORKSPACE} " 122 | f"in {key}") 123 | if not isinstance(value[ORG], str): 124 | raise TFQueryConfigurationError(f"Incorrect value type for {key} {ORG}. " 125 | f"Should be str is {type(value[ORG])}") 126 | if not isinstance(value[WORKSPACE], list): 127 | raise TFQueryConfigurationError(f"Incorrect value type for {key} {WORKSPACE}. " 128 | f"Should be list is {type(value[WORKSPACE])}") 129 | 130 | # Verify token file 131 | token_file = config.get(TERRAFORM_WORKSPACE_TOKEN_FILE, getenv(ENV_TERRAFORM_WORKSPACE_TOKEN_FILE)) 132 | if token_file is None: 133 | raise TFQueryConfigurationError(f"No token file specified in configuration file and no " 134 | f"env var set with file location") 135 | if not isinstance(token_file, str): 136 | raise TFQueryConfigurationError(f"Incorrect value for {TERRAFORM_WORKSPACE_TOKEN_FILE}. " 137 | f"Should be str, is {type(token_file)}") 138 | try: 139 | with open(token_file) as fh: 140 | token_map = json.load(fh) 141 | except Exception as e: 142 | raise TFQueryConfigurationError(f"Issue opening token file {TERRAFORM_WORKSPACE_TOKEN_FILE}") from e 143 | 144 | if not isinstance(token_map, dict): 145 | raise TFQueryConfigurationError(f"Incorrect token file configuration {TERRAFORM_WORKSPACE_TOKEN_FILE} " 146 | f"Should be dict is {type(token_map)}") 147 | for key, value in token_map.items(): 148 | if not isinstance(key, str): 149 | raise TFQueryConfigurationError(f"Incorrect configuration in token file. Workspace names should be string") 150 | if not isinstance(value, str): 151 | raise TFQueryConfigurationError(f"Incorrect configuration in token file. Tokens should be string") 152 | 153 | return TFQuery(tf_url=url, sa_org_workspace_map=sa_workspace_map, org_token_map=token_map) 154 | 155 | 156 | class TFQueryError(Exception): 157 | pass 158 | 159 | 160 | class TFQueryConfigurationError(TFQueryError): 161 | pass 162 | -------------------------------------------------------------------------------- /cloudimized/tfcore/run.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from typing import List, Dict 4 | from itertools import filterfalse 5 | from datetime import datetime, timedelta 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | RUN_CHANGE_STATUS = ['applied', 'errored'] 10 | 11 | 12 | class TFRun: 13 | """ 14 | Represents terraform run related to change 15 | """ 16 | 17 | def __init__(self, message: str, run_id: str, status: str, apply_time: datetime, org: str, workspace: str): 18 | """ 19 | :param message: Terraform Run Message 20 | :param run_id: Terraform Run ID 21 | :param status: Terraform Run Status 22 | :param apply_time: Terraform Run apply time 23 | :param org: Terraform Organization name 24 | :param workspace: Terraform Workspace name 25 | """ 26 | self.message = message 27 | self.run_id = run_id 28 | self.status = status 29 | self.apply_time = apply_time 30 | self.org = org 31 | self.workspace = workspace 32 | 33 | def __repr__(self): 34 | return f"Msg: '{self.message}', RunID: '{self.run_id}', Status:'{self.status}', Applied: '{self.apply_time}'" 35 | 36 | 37 | def parse_tf_runs(runs_response: Dict, org: str, workspace: str) -> List[TFRun]: 38 | """ 39 | Converts TF API runs response into TFRun objects 40 | :param runs_response: TF API response from runs list 41 | :param org: TF org's name related to runs 42 | :param workspace: TF workspace's name related to runs 43 | :returns list of parsed terraform runs 44 | """ 45 | if "data" not in runs_response: 46 | raise TFRunError(f"No 'data' in TF run response") 47 | tf_runs = [] 48 | for run_response in runs_response["data"]: 49 | status = run_response.get("attributes", {}).get("status", None) 50 | # Unknown response structure 51 | if status is None: 52 | logger.warning(f"No status field for TF run\n{run_response}") 53 | continue 54 | # Process only runs that might have changed configuration 55 | if status in ["applied", "errored"]: 56 | status_timestamps = run_response.get("attributes", {}).get("status-timestamps", {}) 57 | apply_time_str = status_timestamps.get("applying-at", None) 58 | if apply_time_str is None and status == "errored": 59 | apply_time_str = status_timestamps.get("errored-at", None) 60 | if apply_time_str is None: 61 | logger.warning(f"No status-timestamps field for TF run\n{run_response}") 62 | continue 63 | try: 64 | apply_time = datetime.strptime(apply_time_str.split("+")[0], "%Y-%m-%dT%H:%M:%S") 65 | except Exception as e: 66 | logger.warning(f"Issue parsing run timestamp\n{type(e)} {e}") 67 | run_id = run_response.get("id", None) 68 | message = run_response.get("attributes", {}).get("message", None) 69 | tf_runs.append(TFRun(message, run_id, status, apply_time, org, workspace)) 70 | return tf_runs 71 | 72 | 73 | def filter_non_change_runs(tf_runs: List[TFRun], change_time: datetime, time_delta: int = 5) -> List[TFRun]: 74 | """ 75 | Filters non-change and non-relevant Terraform Runs 76 | :param tf_runs: list of Terraform Runs 77 | :param change_time: reference timestamp of change made 78 | :param time_delta: time window size in minutes for relevant changes 79 | :return: relevant, change related Terraform Runs 80 | """ 81 | relevant_runs = tf_runs[:] 82 | # Get only change related runs 83 | relevant_runs[:] = filterfalse(__filter_runs_status, relevant_runs) 84 | # Get runs that fall into specified time window 85 | relevant_runs[:] = filterfalse(lambda run: __filter_runs_time(run, change_time, time_delta), relevant_runs) 86 | return relevant_runs 87 | 88 | 89 | def __filter_runs_status(run: TFRun) -> bool: 90 | """ 91 | Filter non-change runs 92 | :param run: Terraform Run object 93 | """ 94 | if run.status is None: 95 | return True 96 | if run.status in RUN_CHANGE_STATUS: 97 | return False 98 | else: 99 | return True 100 | 101 | 102 | def __filter_runs_time(run: TFRun, change_time: datetime, time_delta: int) -> bool: 103 | """ 104 | Filter runs that are outside of specified time window 105 | :param run: TFRun object 106 | :param change_time: Reference time of when change was made 107 | :param time_delta: Size of time window in minutes 108 | """ 109 | if run.apply_time is None: 110 | return True 111 | # Time difference between reference point and apply time 112 | time_diff = abs(change_time - run.apply_time) 113 | # Filter out runs that our outside of window 114 | if time_diff < timedelta(minutes=time_delta): 115 | return False 116 | else: 117 | return True 118 | 119 | 120 | class TFRunError(Exception): 121 | pass 122 | -------------------------------------------------------------------------------- /config-example.yaml: -------------------------------------------------------------------------------- 1 | azure_queries: 2 | - resource: virtualNetworks 3 | # field_exclude_filter: 4 | # - TBD 5 | # item_exclude_filter: 6 | # - TBD 7 | gcp_services: 8 | - serviceName: compute 9 | version: v1 10 | queries: 11 | - resource: networks 12 | gcp_api_call: networks.list 13 | gcp_function_args: 14 | project: 15 | gcp_log_resource_type: gce_network 16 | field_exclude_filter: 17 | - selfLinkWithId 18 | - networkFirewallPolicyEnforcementOrder 19 | - peerings: 20 | - stateDetails 21 | item_exclude_filter: 22 | - peerings: 23 | name: "servicenetworking-googleapis-com" 24 | - resource: subnetworks 25 | gcp_api_call: subnetworks.aggregatedList 26 | gcp_function_args: 27 | project: 28 | gcp_log_resource_type: gce_subnetwork 29 | sortFields: 30 | - name 31 | - secondaryIpRanges: rangeName 32 | - resource: privateServicesAccessRanges 33 | gcp_api_call: globalAddresses.list 34 | gcp_function_args: 35 | project: 36 | filter: purpose=VPC_PEERING 37 | gcp_log_resource_type: gce_reserved_address 38 | - resource: firewalls 39 | gcp_api_call: firewalls.list 40 | gcp_function_args: 41 | project: 42 | gcp_log_resource_type: gce_firewall_rule 43 | item_exclude_filter: 44 | - name: "^k8s.*" 45 | description: '^{"kubernetes.io/' 46 | - resource: routes 47 | gcp_api_call: routes.list 48 | gcp_function_args: 49 | project: 50 | gcp_log_resource_type: gce_route 51 | item_exclude_filter: 52 | - name: ".*(peering|default)-route.*" 53 | - description: "k8s-node-route" 54 | field_exclude_filter: 55 | - warnings 56 | - resource: vpnTunnels 57 | gcp_api_call: vpnTunnels.aggregatedList 58 | gcp_function_args: 59 | project: 60 | gcp_log_resource_type: vpn_tunnel 61 | field_exclude_filter: 62 | - sharedSecretHash 63 | - sharedSecret 64 | - status 65 | - detailedStatus 66 | - serviceName: container 67 | version: v1 68 | queries: 69 | - resource: k8s 70 | gcp_api_call: projects.locations.clusters.list 71 | gcp_function_args: 72 | parent: projects//locations/- 73 | gcp_log_resource_type: gke_cluster 74 | items: clusters 75 | field_exclude_filter: 76 | - currentNodeCount 77 | - status 78 | - nodePools: 79 | - status 80 | git: 81 | remote_url: https://github.com// 82 | local_directory: gcp_config 83 | discover_projects: True 84 | #project_list: 85 | # - my-project-ID 86 | excluded_projects: 87 | - excluded-project-ID 88 | change_processor: 89 | scan_interval: 30 90 | service_account_regex: '^(my-terraform-sa-|\d+@|service-\d+).*' 91 | ticket_regex: "^.*?([a-zA-z]{2,3}[-_][0-9]+).*" 92 | ticket_sys_url: "https://my-tickets.com/list" 93 | slack: 94 | channelID: "C123456789A" 95 | repoCommitURL: "https://github.com///commit" 96 | jira: 97 | url: "https://my.jira.com" 98 | projectKey: "KEY" 99 | terraform: 100 | url: "https://app.terraform.io" 101 | service_workspace_map: 102 | org: my-organization 103 | workspace: ["my-workspace-no1"] 104 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | azure-common==1.1.28 2 | azure-core==1.32.0 3 | azure-identity==1.19.0 4 | azure-mgmt-containerservice==34.2.0 5 | azure-mgmt-core==1.5.0 6 | azure-mgmt-network==28.1.0 7 | azure-mgmt-resource==23.2.0 8 | azure-mgmt-subscription==3.1.1 9 | cachetools==5.5.0 10 | certifi==2024.12.14 11 | cffi==1.17.1 12 | charset-normalizer==3.4.1 13 | cryptography==44.0.1 14 | defusedxml==0.7.1 15 | Deprecated==1.2.15 16 | flatdict==4.0.1 17 | gitdb==4.0.12 18 | GitPython==3.1.44 19 | google-api-core==2.24.0 20 | google-api-python-client==2.156.0 21 | google-auth==2.37.0 22 | google-auth-httplib2==0.2.0 23 | google-cloud-appengine-logging==1.5.0 24 | google-cloud-audit-log==0.3.0 25 | google-cloud-core==2.4.1 26 | google-cloud-logging==3.11.3 27 | googleapis-common-protos==1.66.0 28 | grpc-google-iam-v1==0.13.1 29 | grpcio==1.68.1 30 | grpcio-status==1.68.1 31 | httplib2==0.22.0 32 | idna==3.10 33 | importlib_metadata==8.5.0 34 | isodate==0.7.2 35 | jira==3.8.0 36 | msal==1.31.1 37 | msal-extensions==1.2.0 38 | msrest==0.7.1 39 | oauthlib==3.2.2 40 | opentelemetry-api==1.29.0 41 | packaging==24.2 42 | pillow==11.1.0 43 | pip-install==1.3.5 44 | portalocker==2.10.1 45 | proto-plus==1.25.0 46 | protobuf==5.29.2 47 | pyasn1==0.6.1 48 | pyasn1_modules==0.4.1 49 | pycparser==2.22 50 | PyJWT==2.10.1 51 | pyparsing==3.2.1 52 | PyYAML==6.0.2 53 | requests==2.32.3 54 | requests-oauthlib==2.0.0 55 | requests-toolbelt==1.0.0 56 | rsa==4.9 57 | six==1.17.0 58 | slack_sdk==3.34.0 59 | smmap==5.0.2 60 | terrasnek==0.1.14 61 | typing_extensions==4.12.2 62 | uritemplate==4.1.1 63 | urllib3==2.3.0 64 | wrapt==1.17.0 65 | zipp==3.21.0 66 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import codecs 2 | 3 | from setuptools import setup, find_packages 4 | 5 | with codecs.open("README.md", encoding="utf-8") as f: 6 | readme = f.read() 7 | 8 | # Read dependencies from requirements.txt 9 | with open('requirements.txt') as f: 10 | requirements = f.read().splitlines() 11 | 12 | setup( 13 | name="cloudimized", 14 | version="2.0.1", 15 | description='GCP & Azure configuration scanning tool', 16 | long_description=readme, 17 | long_description_content_type="text/markdown", 18 | url="https://github.com/egnyte/cloudimized", 19 | author="Egnyte and Contributors", 20 | classifiers=[ 21 | "Programming Language :: Python :: 3.9", 22 | "Programming Language :: Python :: 3.10", 23 | "Development Status :: 5 - Production/Stable", 24 | "Intended Audience :: Information Technology", 25 | "Intended Audience :: System Administrators", 26 | "License :: OSI Approved :: MIT License", 27 | "Operating System :: POSIX :: Linux", 28 | "Operating System :: MacOS", 29 | ], 30 | packages=find_packages(), 31 | package_data={ 32 | "": ["LICENSE", "*.md", "config-example.yml"], 33 | "cloudimized": ["singlerunconfigs/*.yaml"] 34 | }, 35 | include_package_data=True, 36 | install_requires=requirements, 37 | extras_require={ 38 | "test": [ 39 | 'mock', 40 | 'time_machine', 41 | ], 42 | }, 43 | entry_points={ 44 | "console_scripts": [ 45 | "cloudimized=cloudimized.core.run:run", 46 | ], 47 | }, 48 | ) 49 | -------------------------------------------------------------------------------- /tests/test_azurevnetgatewaysquery.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import unittest 3 | import mock 4 | from cloudimized.azurecore.azurequery import AzureQueryError 5 | from cloudimized.azurecore.vnetgatewaysquery import VnetGatewaysQuery 6 | 7 | class AzureVnetGatewaysQueryTestCase(unittest.TestCase): 8 | def setUp(self) -> None: 9 | self.patcher = mock.patch("cloudimized.azurecore.vnetgatewaysquery.NetworkManagementClient") 10 | self.mock_client = self.patcher.start() 11 | 12 | logging.disable(logging.WARNING) 13 | self.query = VnetGatewaysQuery( 14 | resource_name="test_name" 15 | ) 16 | mock_result_1 = mock.MagicMock() 17 | mock_result_2 = mock.MagicMock() 18 | mock_result_1.as_dict.return_value = result_resource_1 19 | mock_result_2.as_dict.return_value = result_resource_2 20 | self.mock_client.return_value.virtual_network_gateways.list.side_effect = [ 21 | iter([mock_result_2]), 22 | iter([mock_result_1]) 23 | ] 24 | 25 | def tearDown(self) -> None: 26 | logging.disable(logging.NOTSET) 27 | self.patcher.stop() 28 | 29 | def testArguments(self): 30 | #TODO 31 | pass 32 | 33 | def testExecute(self): 34 | result = self.query.execute(credentials="test_creds", 35 | subscription_id="test_subs_id", 36 | resource_groups=["test_rg_1", "test_rg_2"]) 37 | 38 | self.mock_client.assert_called_with(credential="test_creds", 39 | subscription_id="test_subs_id") 40 | api_calls = [ 41 | mock.call(resource_group_name="test_rg_1"), 42 | mock.call(resource_group_name="test_rg_2") 43 | ] 44 | self.mock_client.return_value.virtual_network_gateways.list.assert_has_calls(api_calls) 45 | 46 | self.assertIsInstance(result, list) 47 | self.assertListEqual(result, query_result_expected) 48 | 49 | def testExecute_query_fail(self): 50 | self.mock_client.return_value.virtual_networks.list_all.side_effect = Exception("test") 51 | with self.assertRaises(AzureQueryError): 52 | self.query.execute(credentials="test_creds", 53 | subscription_id="test_subs_id", 54 | resource_groups=None) 55 | 56 | def testExecute_serializing_fail(self): 57 | mock_raw_result = self.mock_client.return_value.virtual_networks.list_all.return_value 58 | mock_raw_result.__iter__.side_effect = Exception("test") 59 | with self.assertRaises(AzureQueryError): 60 | self.query.execute(credentials="test_creds", 61 | subscription_id="test_subs_id", 62 | resource_groups=None) 63 | self.mock_client.assert_called_with(credential="test_creds", subscription_id="test_subs_id") 64 | 65 | if __name__ == '__main__': 66 | unittest.main() 67 | 68 | result_resource_1 = { 69 | "name": "test1", 70 | "test_str_key": "test_str_value", 71 | "test_list_key": ["test_value1, test_value2"], 72 | "test_dict_key": { 73 | "test_inner_key": "test_inner_value", 74 | "test_inner_list": ["test_inner_value1", "test_inner_value2"] 75 | } 76 | } 77 | 78 | result_resource_2 = { 79 | "name": "test2", 80 | "test_str_key": "test_str_value", 81 | "test_list_key": ["test_value1, test_value2"], 82 | "test_dict_key": { 83 | "test_inner_key": "test_inner_value", 84 | "test_inner_list": ["test_inner_value1", "test_inner_value2"] 85 | } 86 | } 87 | 88 | query_result_expected = [ 89 | result_resource_1, 90 | result_resource_2 91 | ] 92 | -------------------------------------------------------------------------------- /tests/test_azurevnetsquery.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import unittest 3 | import mock 4 | from cloudimized.azurecore.azurequery import AzureQuery, AzureQueryError, AzureQueryArgumentError 5 | from cloudimized.azurecore.virtualnetworksquery import VirtualNetworksQuery 6 | 7 | class AzureVnetsQueryTestCase(unittest.TestCase): 8 | def setUp(self) -> None: 9 | self.patcher = mock.patch("cloudimized.azurecore.virtualnetworksquery.NetworkManagementClient") 10 | self.mock_client = self.patcher.start() 11 | 12 | logging.disable(logging.WARNING) 13 | self.query = VirtualNetworksQuery( 14 | resource_name="test_name" 15 | ) 16 | mock_result_1 = mock.MagicMock() 17 | mock_result_2 = mock.MagicMock() 18 | mock_result_1.as_dict.return_value = result_resource_1 19 | mock_result_2.as_dict.return_value = result_resource_2 20 | mock_raw_result = mock.MagicMock() 21 | mock_raw_result.__iter__.return_value = iter([mock_result_2, mock_result_1]) #Reverse order to check sorting 22 | self.mock_client.return_value.virtual_networks.list_all.return_value = mock_raw_result 23 | 24 | def tearDown(self) -> None: 25 | logging.disable(logging.NOTSET) 26 | self.patcher.stop() 27 | 28 | def testArguments(self): 29 | #TODO 30 | pass 31 | 32 | def testExecute(self): 33 | result = self.query.execute(credentials="test_creds", 34 | subscription_id="test_subs_id", 35 | resource_groups=None) 36 | 37 | self.mock_client.assert_called_with(credential="test_creds", 38 | subscription_id="test_subs_id") 39 | self.mock_client.return_value.virtual_networks.list_all.assert_called_with() 40 | 41 | self.assertIsInstance(result, list) 42 | self.assertListEqual(result, query_result_expected) 43 | 44 | def testExecute_query_fail(self): 45 | self.mock_client.return_value.virtual_networks.list_all.side_effect = Exception("test") 46 | with self.assertRaises(AzureQueryError): 47 | self.query.execute(credentials="test_creds", 48 | subscription_id="test_subs_id", 49 | resource_groups=None) 50 | 51 | def testExecute_serializing_fail(self): 52 | mock_raw_result = self.mock_client.return_value.virtual_networks.list_all.return_value 53 | mock_raw_result.__iter__.side_effect = Exception("test") 54 | with self.assertRaises(AzureQueryError): 55 | self.query.execute(credentials="test_creds", 56 | subscription_id="test_subs_id", 57 | resource_groups=None) 58 | self.mock_client.assert_called_with(credential="test_creds", subscription_id="test_subs_id") 59 | 60 | if __name__ == '__main__': 61 | unittest.main() 62 | 63 | result_resource_1 = { 64 | "name": "test1", 65 | "test_str_key": "test_str_value", 66 | "test_list_key": ["test_value1, test_value2"], 67 | "test_dict_key": { 68 | "test_inner_key": "test_inner_value", 69 | "test_inner_list": ["test_inner_value1", "test_inner_value2"] 70 | } 71 | } 72 | 73 | result_resource_2 = { 74 | "name": "test2", 75 | "test_str_key": "test_str_value", 76 | "test_list_key": ["test_value1, test_value2"], 77 | "test_dict_key": { 78 | "test_inner_key": "test_inner_value", 79 | "test_inner_list": ["test_inner_value1", "test_inner_value2"] 80 | } 81 | } 82 | 83 | query_result_expected = [ 84 | result_resource_1, 85 | result_resource_2 86 | ] 87 | -------------------------------------------------------------------------------- /tests/test_gcpchangelog.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import unittest 3 | import mock 4 | import datetime as dt 5 | 6 | import time_machine 7 | 8 | from cloudimized.gcpcore.gcpchangelog import GcpChangeLog, getChangeLogs 9 | 10 | 11 | class GcpChangeLogTestCase(unittest.TestCase): 12 | @time_machine.travel(dt.datetime(1985, 10, 26, 1, 24)) #This is needed to mock datetime 13 | def setUp(self): 14 | logging.disable(logging.WARNING) 15 | 16 | def tearDown(self) -> None: 17 | logging.disable(logging.NOTSET) 18 | 19 | # @time_machine.travel(dt.datetime(1985, 10, 26, 1, 24)) #This is needed to mock datetime 20 | @mock.patch("cloudimized.gcpcore.gcpchangelog.gcp_logging") 21 | def testAllFields(self, mock_gcp_logging): 22 | # Mock GcpQuery object 23 | gcp_query = mock.MagicMock() 24 | type(gcp_query).gcp_log_resource_type = mock.PropertyMock(return_value="gce_route") 25 | type(gcp_query).resource_name = mock.PropertyMock(return_value="route") 26 | 27 | # Mock entry.to_api_repr() 28 | mock_gcp_log_entry = mock.MagicMock() 29 | mock_gcp_log_entry.to_api_repr.return_value = test_entry_to_api 30 | # Mock gcp_logging.Client, gcp_logger.list_entries 31 | # Complex return value needed for 'next(entries.pages)' 32 | type(mock_gcp_logging.Client.return_value.list_entries.return_value).pages =\ 33 | mock.PropertyMock(return_value=iter([[mock_gcp_log_entry]])) 34 | result = getChangeLogs(project='test-project', 35 | gcp_query=gcp_query, 36 | change_time=dt.datetime.utcnow(), 37 | time_window=30) 38 | expected_gcpchangelog = [GcpChangeLog( 39 | resourceName="projects/test-project-123/global/routes/test-route", 40 | resourceType="gce_route", 41 | changer="user@example.com", 42 | timestamp=dt.datetime.strptime("1985-10-26T01:15:58", "%Y-%m-%dT%H:%M:%S"), 43 | methodName="v1.compute.routes.delete", 44 | requestType="type.googleapis.com/compute.routes.delete")] 45 | self.assertEqual(expected_gcpchangelog, result) 46 | 47 | 48 | # @time_machine.travel(dt.datetime(1985,10,26,1,24)) 49 | @mock.patch("cloudimized.gcpcore.gcpchangelog.gcp_logging") 50 | def testEmptyFields(self, mock_gcp_logging): 51 | # Mock GcpQuery object 52 | gcp_query = mock.MagicMock() 53 | type(gcp_query).gcp_log_resource_type = mock.PropertyMock(return_value="gce_route") 54 | type(gcp_query).resource_name = mock.PropertyMock(return_value="route") 55 | 56 | # Mock entry.to_api_repr() 57 | mock_gcp_log_entry = mock.MagicMock() 58 | mock_gcp_log_entry.to_api_repr.return_value = test_entry_empty_fields 59 | # Mock gcp_logging.Client, gcp_logger.list_entries 60 | # Complex return value needed for 'next(entries.pages)' 61 | type(mock_gcp_logging.Client.return_value.list_entries.return_value).pages =\ 62 | mock.PropertyMock(return_value=iter([[mock_gcp_log_entry]])) 63 | result = getChangeLogs(project='test-project', 64 | gcp_query=gcp_query, 65 | change_time=dt.datetime.utcnow(), 66 | time_window=30) 67 | expected_gcpchangelog = [GcpChangeLog ( 68 | resourceName=None, 69 | resourceType=None, 70 | changer=None, 71 | timestamp=None, 72 | methodName=None, 73 | requestType=None)] 74 | self.assertEqual(expected_gcpchangelog, result) 75 | 76 | #TODO Add a test that verifies there was a call with proper filter string 77 | 78 | 79 | 80 | 81 | test_entry_to_api = { 82 | 'logName': 'projects/test-project/logs/cloudaudit.googleapis.com%2Factivity', 83 | 'resource': { 84 | 'type': 'gce_route', 85 | 'labels': { 86 | 'project_id': 'test-project-123', 87 | 'route_id': '0000000000000000000' 88 | } 89 | }, 90 | 'insertId': '000000aaaaaa', 91 | 'severity': 'NOTICE', 92 | 'timestamp': '1985-10-26T01:15:58.465104Z', 93 | 'operation': { 94 | 'id': 'operation-11111111111111-1111111111111-1111111-11111', 95 | 'producer': 'compute.googleapis.com', 96 | 'last': True 97 | }, 98 | 'protoPayload': { 99 | '@type': 'type.googleapis.com/google.cloud.audit.AuditLog', 100 | 'authenticationInfo': { 101 | 'principalEmail': 'user@example.com' 102 | }, 103 | 'requestMetadata': { 104 | 'callerIp': '8.8.8.8', 105 | 'callerSuppliedUserAgent': 'TestAgentChrome' 106 | }, 107 | 'serviceName': 'compute.googleapis.com', 108 | 'methodName': 'v1.compute.routes.delete', 109 | 'resourceName': 'projects/test-project-123/global/routes/test-route', 110 | 'request': { 111 | '@type': 'type.googleapis.com/compute.routes.delete' 112 | } 113 | } 114 | } 115 | 116 | test_entry_empty_fields = {} 117 | -------------------------------------------------------------------------------- /tests/test_gcpservicequery.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import mock 3 | 4 | from cloudimized.gcpcore.gcpservicequery import GcpServiceQuery, configure_services 5 | from cloudimized.gcpcore.gcpservicequery import GcpServiceQueryError, GcpServiceQueryConfigError 6 | from google_auth_httplib2 import AuthorizedHttp 7 | from googleapiclient.discovery import Resource 8 | 9 | 10 | class GcpServiceQueryTestCase(unittest.TestCase): 11 | def setUp(self): 12 | self.gcpservice = GcpServiceQuery(serviceName="compute", version="v1") 13 | test_queries = { 14 | "test_resource": mock.MagicMock() 15 | } 16 | self.gcpservice.queries = test_queries 17 | 18 | @mock.patch("cloudimized.gcpcore.gcpservicequery.google_auth_httplib2.AuthorizedHttp", spec=AuthorizedHttp) 19 | @mock.patch("cloudimized.gcpcore.gcpservicequery.google.auth") 20 | @mock.patch("cloudimized.gcpcore.gcpservicequery.discovery") 21 | @mock.patch("cloudimized.gcpcore.gcpservicequery.getenv") 22 | def testBuild(self, mock_getenv, mock_discovery, mock_auth, mock_authhttp): 23 | self.gcpservice.queries 24 | mock_getenv.return_value = "/creds/secret.file" 25 | mock_discovery.build.return_value = mock.MagicMock(spec=Resource) 26 | mock_auth.default.return_value = (mock.MagicMock(), mock.MagicMock()) 27 | mock_authhttp_object = mock.MagicMock(spec=AuthorizedHttp) 28 | mock_authhttp.return_value = mock_authhttp_object 29 | service = self.gcpservice.build() 30 | mock_getenv.assert_called_with("GOOGLE_APPLICATION_CREDENTIALS") 31 | mock_discovery.build.assert_called_with(self.gcpservice.serviceName, 32 | self.gcpservice.version, 33 | http=mock_authhttp_object) 34 | self.assertIsInstance(service, Resource) 35 | 36 | mock_getenv.reset_mock(return_value=True, side_effect=True) 37 | mock_discovery.reset_mock(return_value=True, side_effect=True) 38 | mock_auth.reset_mock(return_value=True, side_effect=True) 39 | mock_getenv.return_value = None 40 | mock_discovery.build.return_value = mock.MagicMock(spec=Resource) 41 | mock_auth.default.return_value = (mock.MagicMock(), mock.MagicMock()) 42 | service = self.gcpservice.build() 43 | mock_auth.default.assert_called_with(scopes=["https://www.googleapis.com/auth/cloud-platform"]) 44 | mock_discovery.build.assert_called_with(self.gcpservice.serviceName, 45 | self.gcpservice.version, 46 | http=mock_authhttp_object) 47 | self.assertIsInstance(service, Resource) 48 | 49 | def test_configure_services(self): 50 | # Config needs to be list 51 | with self.assertRaises(GcpServiceQueryConfigError): 52 | configure_services("invalid argument") 53 | # Each entry in list need to be dict 54 | with self.assertRaises(GcpServiceQueryConfigError): 55 | configure_services(test_queries_config_incorrect_entry_type) 56 | # Each entry needs to have required keys 57 | with self.assertRaises(GcpServiceQueryConfigError): 58 | configure_services(test_queries_config_incorrect_entry_details) 59 | 60 | correct_result = configure_services(test_queries_config_correct) 61 | self.assertIsInstance(correct_result, dict) 62 | self.assertEqual(len(correct_result), 2) 63 | for key, value in correct_result.items(): 64 | self.assertIsInstance(key, str) 65 | self.assertIsInstance(value, GcpServiceQuery) 66 | 67 | 68 | if __name__ == '__main__': 69 | unittest.main() 70 | 71 | test_queries_compute = [ 72 | { 73 | "resource": "network", 74 | "gcp_call": "network", # this for now point to serviceMap 75 | "field_exclude_filter": ["creationTimestamp"], 76 | "gcp_function_args": { 77 | "project": "" 78 | } 79 | }, 80 | { 81 | "resource": "staticRoute", 82 | "gcp_call": "staticRoute", # this for now point to serviceMap 83 | "field_exclude_filter": ["creationTimestamp"], 84 | "gcp_function_args": { 85 | "project": "" 86 | } 87 | }, 88 | { 89 | "resource": "project", 90 | "gcp_call": "project", 91 | } 92 | ] 93 | 94 | test_queries_project = [ 95 | { 96 | "resource": "project", 97 | "gcp_call": "project", 98 | "gcp_function_args": { 99 | "project": "" 100 | } 101 | } 102 | ] 103 | 104 | test_queries_config_correct = [ 105 | { 106 | "serviceName": "compute", 107 | "version": "v1", 108 | "queries": test_queries_compute 109 | }, 110 | { 111 | "serviceName": "project", 112 | "version": "v1", 113 | "queries": test_queries_project 114 | } 115 | ] 116 | 117 | test_queries_config_incorrect_entry_type = [ 118 | { 119 | "valid": "valid" 120 | }, 121 | "invalid" 122 | ] 123 | 124 | test_queries_config_incorrect_entry_details = [ 125 | { 126 | "serviceName": "test", 127 | "version": "test", 128 | "queries": [] 129 | }, 130 | { 131 | "test": [ 132 | { 133 | "requiredKey": "missing" 134 | } 135 | ] 136 | } 137 | ] 138 | -------------------------------------------------------------------------------- /tests/test_gitchange.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import mock 4 | 5 | from cloudimized.gitcore.gitchange import GitChange 6 | 7 | class MyTestCase(unittest.TestCase): 8 | def testGetFilename(self): 9 | expected_result = "azure/firewall/project1.yaml" 10 | gitchange = GitChange("azure", "firewall", "project1") 11 | result = gitchange.get_filename() 12 | self.assertEqual(result, expected_result) 13 | 14 | def testGetCommitMsg(self): 15 | #No changers 16 | gitchange = GitChange("gcp", "testresource", "testproject") 17 | expected_result = "Testresource updated in GCP: testproject by UNKNOWN" 18 | self.assertEqual(gitchange.get_commit_message(), expected_result) 19 | 20 | # Multiple unique changers 21 | gitchange = GitChange("gcp", "testresource", "testproject") 22 | mock_gitchangelog1 = mock.MagicMock() 23 | type(mock_gitchangelog1).changer = mock.PropertyMock(return_value="testuser") 24 | mock_gitchangelog2 = mock.MagicMock() 25 | type(mock_gitchangelog2).changer = mock.PropertyMock(return_value="anothertestuser") 26 | gitchange.gcp_change_log += [mock_gitchangelog1, mock_gitchangelog2] 27 | expected_result = "Testresource updated in GCP: testproject by ['testuser', 'anothertestuser']" 28 | self.assertEqual(gitchange.get_commit_message(), expected_result) 29 | 30 | # Multiple non-unique changer 31 | gitchange = GitChange("gcp", "testresource", "testproject") 32 | mock_gitchangelog1 = mock.MagicMock() 33 | type(mock_gitchangelog1).changer = mock.PropertyMock(return_value="testuser") 34 | mock_gitchangelog2 = mock.MagicMock() 35 | type(mock_gitchangelog2).changer = mock.PropertyMock(return_value="anothertestuser") 36 | mock_gitchangelog3 = mock.MagicMock() 37 | type(mock_gitchangelog3).changer = mock.PropertyMock(return_value="testuser") 38 | gitchange.gcp_change_log += [mock_gitchangelog1, mock_gitchangelog2, mock_gitchangelog3] 39 | expected_result = "Testresource updated in GCP: testproject by ['testuser', 'anothertestuser']" 40 | self.assertEqual(gitchange.get_commit_message(), expected_result) 41 | 42 | 43 | 44 | if __name__ == '__main__': 45 | unittest.main() 46 | 47 | modified_files = [ 48 | "firewall/project1.yaml", 49 | "vpn/project2.yaml" 50 | ] 51 | 52 | modified_files_empty = [] 53 | -------------------------------------------------------------------------------- /tests/test_gitrepo.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import mock 3 | 4 | import cloudimized.gitcore.repo as repo 5 | from cloudimized.gitcore.repo import GitRepo, GitRepoError, configure_repo 6 | from cloudimized.gitcore.gitchange import GitChange 7 | 8 | class GitRepoCase(unittest.TestCase): 9 | def setUp(self) -> None: 10 | self.gitrepo = GitRepo("user", "password", "https://example.com/test/repo.git", "localRepo") 11 | 12 | @mock.patch("cloudimized.gitcore.repo.git") 13 | @mock.patch("cloudimized.gitcore.repo.exists") 14 | def test_setup_local_exists(self, mock_exists, mock_git): 15 | mock_exists.return_value = True 16 | # Fetching remote 17 | self.gitrepo.setup() 18 | mock_git.Repo().remotes.origin.fetch.assert_called_once() 19 | mock_git.reset_mock() 20 | # Issue fetching remote 21 | with self.assertRaises(repo.GitRepoError): 22 | mock_git.Repo().remotes.origin.fetch.side_effect = Exception() 23 | self.gitrepo.setup() 24 | 25 | @mock.patch("cloudimized.gitcore.repo.git") 26 | @mock.patch("cloudimized.gitcore.repo.exists") 27 | def test_setup_ssh_cloning(self, mock_exists, mock_git): 28 | mock_exists.return_value = False 29 | self.gitrepo.repo_url = "git@git.example.com:test/repo.git" 30 | self.gitrepo.setup() 31 | mock_git.Repo.clone_from.assert_called_with("git@git.example.com:test/repo.git", "localRepo") 32 | 33 | @mock.patch("cloudimized.gitcore.repo.git") 34 | @mock.patch("cloudimized.gitcore.repo.exists") 35 | def test_setup_https_cloning(self, mock_exists, mock_git): 36 | mock_exists.return_value = False 37 | # Test correct url with creds 38 | self.gitrepo.setup() 39 | mock_git.Repo.clone_from.assert_called_with("https://user:password@example.com/test/repo.git", 40 | "localRepo") 41 | mock_git.reset_mock() 42 | 43 | # Test missing password 44 | with self.assertRaises(repo.GitRepoError): 45 | self.gitrepo.user = None 46 | self.gitrepo.setup() 47 | mock_git.reset_mock() 48 | 49 | # Test incorrect URL 50 | with self.assertRaises(repo.GitRepoError): 51 | self.gitrepo.user = "user" 52 | self.gitrepo.repo_url = "incorrect_url" 53 | self.gitrepo.setup() 54 | 55 | def test_get_changes_repo_not_setup(self): 56 | with self.assertRaises(repo.GitRepoError) as cm: 57 | self.gitrepo.get_changes() 58 | self.assertEqual("Git Repo not set up", str(cm.exception)) 59 | 60 | def test_get_changes_success(self): 61 | git_mock = mock.MagicMock() 62 | type(git_mock).untracked_files = mock.PropertyMock(return_value=["test_resource1/test_untracked_file1.yaml", 63 | "test_resource2/test_untracked_file2.yaml"]) 64 | index_mock = mock.MagicMock() 65 | item_0 = mock.MagicMock() 66 | type(item_0).a_path = mock.PropertyMock(return_value="test_resource1/test_change_file3.yaml") 67 | item_1 = mock.MagicMock() 68 | type(item_1).a_path = mock.PropertyMock(return_value="test_resource3/test_change_file4.yaml") 69 | index_mock.diff.return_value = [item_0, item_1] 70 | type(git_mock).index = mock.PropertyMock(return_value=index_mock) 71 | self.gitrepo.repo = git_mock 72 | result = self.gitrepo.get_changes() 73 | self.assertIsInstance(result, list) 74 | self.assertEqual(len(result), 4) 75 | for change in result: 76 | self.assertIsInstance(change, GitChange) 77 | self.assertEqual(result[0].resource_type, "test_resource1") 78 | self.assertEqual(result[0].project, "test_untracked_file1") 79 | self.assertEqual(result[1].resource_type, "test_resource2") 80 | self.assertEqual(result[1].project, "test_untracked_file2") 81 | self.assertEqual(result[2].resource_type, "test_resource1") 82 | self.assertEqual(result[2].project, "test_change_file3") 83 | self.assertEqual(result[3].resource_type, "test_resource3") 84 | self.assertEqual(result[3].project, "test_change_file4") 85 | 86 | def test_commit_change_exception(self): 87 | repo_mock = mock.MagicMock() 88 | repo_mock.git.add.side_effect = Exception() 89 | self.gitrepo.repo = repo_mock 90 | with self.assertRaises(GitRepoError) as cm: 91 | self.gitrepo.commit_change(GitChange(provider="azure", resource_type="test_resource", project="test_project"), "test_message") 92 | self.assertEqual("Issue commiting change for project 'test_project' type 'test_resource'", str(cm.exception)) 93 | 94 | def test_commit_change_success(self): 95 | repo_mock = mock.MagicMock() 96 | self.gitrepo.repo = repo_mock 97 | self.gitrepo.commit_change(GitChange(provider="gcp", resource_type="test_resource", project="test_project"), "test_message") 98 | repo_mock.git.add.assert_called_with("gcp/test_resource/test_project.yaml") 99 | repo_mock.git.commit.assert_called_with("test_message") 100 | 101 | def test_push_changes_issue(self): 102 | repo_mock = mock.MagicMock() 103 | repo_mock.remotes.origin.push.side_effect = Exception() 104 | self.gitrepo.repo = repo_mock 105 | with self.assertRaises(GitRepoError) as cm: 106 | self.gitrepo.push_changes() 107 | self.assertEqual("Issue pushing local changes to remote", str(cm.exception)) 108 | 109 | def test_push_changes_success(self): 110 | repo_mock = mock.MagicMock() 111 | self.gitrepo.repo = repo_mock 112 | self.gitrepo.push_changes() 113 | repo_mock.remotes.origin.push.assert_called_once() 114 | 115 | @mock.patch("cloudimized.gitcore.repo.GitRepo", spec=GitRepo) 116 | def test_configure_repo(self, mock_gitrepo): 117 | # Missing username 118 | with self.assertRaises(repo.GitRepoConfigError): 119 | configure_repo(user=None, password=None, config={}) 120 | # Missing password 121 | with self.assertRaises(repo.GitRepoConfigError): 122 | configure_repo(user="test_user", password=None, config={}) 123 | # Incorrect config format 124 | with self.assertRaises(repo.GitRepoConfigError): 125 | configure_repo(user="test_user", password="secret", config=[]) 126 | # Missing required key in dictionary 127 | with self.assertRaises(repo.GitRepoConfigError): 128 | configure_repo(user="test_user", password="secret", config={}) 129 | # Missing required key in dictionary 130 | with self.assertRaises(repo.GitRepoConfigError): 131 | configure_repo(user="test_user", password="secret", config=no_local_dir_config) 132 | 133 | result = configure_repo(user="test_user", password="secret", config=local_dir_config) 134 | self.assertIsInstance(result, GitRepo) 135 | mock_gitrepo.assert_called_with("test_user", "secret", "test_url", "/local/dir") 136 | 137 | @mock.patch("cloudimized.gitcore.repo.rmtree") 138 | @mock.patch("cloudimized.gitcore.repo.listdir") 139 | def test_clean_repo_not_setup(self, mock_listdir, mock_rmtree): 140 | with self.assertRaises(GitRepoError) as cm: 141 | self.gitrepo.clean_repo() 142 | self.assertEqual("Repo 'https://example.com/test/repo.git' needs to be setup first", 143 | str(cm.exception)) 144 | 145 | @mock.patch("cloudimized.gitcore.repo.rmtree") 146 | @mock.patch("cloudimized.gitcore.repo.listdir") 147 | def test_clean_repo_listing_issue(self, mock_listdir, mock_rmtree): 148 | self.gitrepo.repo = mock.MagicMock() 149 | mock_listdir.side_effect = Exception("issue") 150 | with self.assertRaises(GitRepoError) as cm: 151 | self.gitrepo.clean_repo() 152 | self.assertEqual("Issue retrieving directories in directory 'localRepo'", 153 | str(cm.exception)) 154 | 155 | @mock.patch("cloudimized.gitcore.repo.isdir") 156 | @mock.patch("cloudimized.gitcore.repo.rmtree") 157 | @mock.patch("cloudimized.gitcore.repo.listdir") 158 | def test_clean_repo_remove_issue(self, mock_listdir, mock_rmtree, mock_isdir): 159 | self.gitrepo.repo = mock.MagicMock() 160 | mock_listdir.return_value = ["test_directory1", "test_directory2"] 161 | mock_isdir.side_effect = [True, True] 162 | mock_rmtree.side_effect = Exception("issue") 163 | with self.assertRaises(GitRepoError) as cm: 164 | self.gitrepo.clean_repo() 165 | self.assertEqual("Issue removing directories in 'localRepo'", 166 | str(cm.exception)) 167 | 168 | @mock.patch("cloudimized.gitcore.repo.isdir") 169 | @mock.patch("cloudimized.gitcore.repo.rmtree") 170 | @mock.patch("cloudimized.gitcore.repo.listdir") 171 | def test_clean_repo_only_git_folder(self, mock_listdir, mock_rmtree, mock_isdir): 172 | self.gitrepo.repo = mock.MagicMock() 173 | mock_listdir.return_value = [".git"] 174 | mock_isdir.side_effect = [True] 175 | self.gitrepo.clean_repo() 176 | mock_rmtree.assert_not_called() 177 | 178 | @mock.patch("cloudimized.gitcore.repo.isdir") 179 | @mock.patch("cloudimized.gitcore.repo.rmtree") 180 | @mock.patch("cloudimized.gitcore.repo.listdir") 181 | def test_clean_repo_success(self, mock_listdir, mock_rmtree, mock_isdir): 182 | self.gitrepo.repo = mock.MagicMock() 183 | mock_listdir.return_value = [".git", "test_directory1", "test_directory2", "README.md"] 184 | mock_isdir.side_effect = [True, True, True, False] 185 | self.gitrepo.clean_repo() 186 | calls = [ 187 | mock.call("localRepo/test_directory1"), 188 | mock.call("localRepo/test_directory2") 189 | ] 190 | mock_rmtree.assert_has_calls(calls) 191 | 192 | if __name__ == '__main__': 193 | unittest.main() 194 | 195 | 196 | missing_url_config = { 197 | } 198 | 199 | no_local_dir_config = { 200 | repo.GIT_REPO_URL: "test_url" 201 | } 202 | 203 | local_dir_config = { 204 | repo.GIT_REPO_URL: "test_url", 205 | repo.GIT_LOCAL_DIR: "/local/dir" 206 | } 207 | -------------------------------------------------------------------------------- /tests/test_jiranotifier.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import mock 4 | from jira import JIRA 5 | 6 | from cloudimized.core.jiranotifier import configure_jiranotifier, JiraNotifier, JiraNotifierError, logger 7 | from cloudimized.gitcore.gitchange import GitChange 8 | 9 | 10 | class JiraNotifierTestCase(unittest.TestCase): 11 | def setUp(self) -> None: 12 | kwargs = {"test_field": [{"name": "test_value"}]} 13 | self.jiranotifier = JiraNotifier(jira_url="test_url", 14 | projectkey="test_key", 15 | username="test_user", 16 | password="test_pass", 17 | issuetype="test_type", 18 | filter_set=None, 19 | **kwargs) 20 | self.gitchange = GitChange(provider="azure", 21 | resource_type="test_resource", 22 | project="test_project") 23 | self.gitchange.diff = "TEST_CHANGE_DIFF" 24 | self.gitchange.changers = ["test_changer"] 25 | 26 | def test_configure_incorrect_config_type(self): 27 | with self.assertRaises(JiraNotifierError) as cm: 28 | configure_jiranotifier(config="incorrect_config_type", 29 | username="test_user", 30 | password="test_password") 31 | self.assertEqual(f"Incorrect Jira Notifier configuration. Should be dict, is ", str(cm.exception)) 32 | 33 | def test_configure_missing_required_key(self): 34 | with self.assertRaises(JiraNotifierError) as cm: 35 | configure_jiranotifier(config={"missing_required_key": ""}, 36 | username="test_user", 37 | password="test_password") 38 | self.assertEqual(f"Missing one of required config keys: ['url', 'projectKey']", str(cm.exception)) 39 | 40 | def test_configure_missing_credentials(self): 41 | with self.assertRaises(JiraNotifierError) as cm: 42 | configure_jiranotifier(config={"url": "", "projectKey": ""}, 43 | username="test_user", 44 | password="") 45 | self.assertEqual(f"Jira password/token not set in env var: 'JIRA_PSW'", 46 | str(cm.exception)) 47 | 48 | def test_configure_missing_token(self): 49 | with self.assertRaises(JiraNotifierError) as cm: 50 | configure_jiranotifier(config={"url": "", "projectKey": "", "isToken": True}, 51 | username="", 52 | password="") 53 | self.assertEqual(f"Jira password/token not set in env var: 'JIRA_PSW'", 54 | str(cm.exception)) 55 | 56 | def test_configure_incorrect_fields_type(self): 57 | with self.assertRaises(JiraNotifierError) as cm: 58 | configure_jiranotifier(config={"url": "", "projectKey": "", 59 | "fields": "incorrect_type"}, 60 | username="test_user", 61 | password="test_password") 62 | self.assertEqual((f"Incorrect Jira Notifier Fields configuration. " 63 | f"Should be dict, is "), str(cm.exception)) 64 | 65 | def test_configure_incorrect_filterset_type(self): 66 | with self.assertRaises(JiraNotifierError) as cm: 67 | configure_jiranotifier(config={"url": "", "projectKey": "", 68 | "filterSet": "incorrect_type"}, 69 | username="test_user", 70 | password="test_password") 71 | self.assertEqual((f"Incorrect Jira Notifier FilterSet configuration. " 72 | f"Should be dict, is "), str(cm.exception)) 73 | 74 | def test_configure_incorrect_projectidfilter_type(self): 75 | with self.assertRaises(JiraNotifierError) as cm: 76 | configure_jiranotifier(config={"url": "", "projectKey": "", 77 | "filterSet": {"missing_key": None}}, 78 | username="test_user", 79 | password="test_password") 80 | self.assertEqual(f"Missing required param projectId", str(cm.exception)) 81 | 82 | def test_configure_incorrect_projectidfilter_value(self): 83 | with self.assertRaises(JiraNotifierError) as cm: 84 | configure_jiranotifier(config={"url": "", "projectKey": "", 85 | "filterSet": {"projectId": []}}, 86 | username="test_user", 87 | password="test_password") 88 | self.assertEqual((f"Incorrect Jira Notifier projectId configuration value. " 89 | f"Should be str, is "), str(cm.exception)) 90 | 91 | @mock.patch("cloudimized.core.jiranotifier.JiraNotifier", spec=JiraNotifier) 92 | def test_configure_correct_result(self, mock_jiranotifier): 93 | result = configure_jiranotifier(config={"url": "test_url", 94 | "projectKey": "TEST", 95 | "fields": { 96 | "extra": "testField" 97 | }}, 98 | username="test_user", 99 | password="test_password") 100 | self.assertIsInstance(result, JiraNotifier) 101 | mock_jiranotifier.assert_called_with(jira_url="test_url", 102 | username="test_user", 103 | password="test_password", 104 | issuetype="Task", 105 | projectkey="TEST", 106 | filter_set=None, 107 | extra="testField") 108 | 109 | @mock.patch("cloudimized.core.jiranotifier.JiraNotifier", spec=JiraNotifier) 110 | def test_configure_correct_result_with_token_auth(self, mock_jiranotifier): 111 | result = configure_jiranotifier(config={"url": "test_url", 112 | "projectKey": "TEST", 113 | "isToken": True, 114 | "fields": { 115 | "extra": "testField" 116 | }}, 117 | username="", 118 | password="test_password") 119 | self.assertIsInstance(result, JiraNotifier) 120 | mock_jiranotifier.assert_called_with(jira_url="test_url", 121 | username="", 122 | password="test_password", 123 | issuetype="Task", 124 | projectkey="TEST", 125 | filter_set=None, 126 | extra="testField") 127 | 128 | @mock.patch("cloudimized.core.jiranotifier.JIRA", spec=JIRA) 129 | def test_post_non_manual_change(self, mock_jira): 130 | self.gitchange.manual = False 131 | result = self.jiranotifier.post(self.gitchange) 132 | self.assertIsNone(result) 133 | mock_jira.assert_not_called() 134 | 135 | @mock.patch("cloudimized.core.jiranotifier.JIRA", spec=JIRA) 136 | def test_post_non_matching_filter(self, mock_jira): 137 | self.gitchange.manual = True 138 | filter_set = {"projectId": "NO_MATCH"} 139 | self.jiranotifier.filter_set = filter_set 140 | result = self.jiranotifier.post(self.gitchange) 141 | self.assertIsNone(result) 142 | mock_jira.assert_not_called() 143 | 144 | @mock.patch("cloudimized.core.jiranotifier.JIRA", spec=JIRA) 145 | def test_post_authentication_issue(self, mock_jira): 146 | self.gitchange.manual = True 147 | mock_jira.side_effect = Exception("Auth Issue") 148 | with self.assertRaises(JiraNotifierError) as cm: 149 | self.jiranotifier.post(self.gitchange) 150 | self.assertEqual(f"Issue creating ticket\nAuth Issue", f"{str(cm.exception)}\n{str(cm.exception.__cause__)}") 151 | 152 | @mock.patch("cloudimized.core.jiranotifier.JIRA", spec=JIRA) 153 | def test_post_creating_issue_issue(self, mock_jira): 154 | self.gitchange.manual = True 155 | mock_jira_object = mock.MagicMock() 156 | mock_jira_object.create_issue.side_effect = Exception("Ticket create issue") 157 | mock_jira.return_value = mock_jira_object 158 | with self.assertRaises(JiraNotifierError) as cm: 159 | self.jiranotifier.post(self.gitchange) 160 | self.assertEqual(f"Issue creating ticket\nTicket create issue", 161 | f"{str(cm.exception)}\n{str(cm.exception.__cause__)}") 162 | mock_jira_object.create_issue.assert_called_with(project={"key": "test_key"}, 163 | summary=(f"GCP manual change detected - project: " 164 | f"test_project, resource: test_resource"), 165 | description=(f"Manual changes performed by test_changer\n\n" 166 | f"{{code:java}}\nTEST_CHANGE_DIFF\n{{code}}\n"), 167 | issuetype={"name": "test_type"}, 168 | test_field=[{"name": "test_value"}]) 169 | 170 | @mock.patch("cloudimized.core.jiranotifier.JIRA", spec=JIRA) 171 | def test_post_update_assignee_issue(self, mock_jira): 172 | self.gitchange.manual = True 173 | mock_issue_object = mock.MagicMock() 174 | mock_issue_object.update.side_effect = Exception("Update Issue") 175 | type(mock_issue_object).key = mock.PropertyMock(return_value="TEST_KEY") 176 | mock_jira_object = mock.MagicMock() 177 | mock_jira_object.create_issue.return_value = mock_issue_object 178 | mock_jira.return_value = mock_jira_object 179 | with self.assertLogs(logger, level="WARNING") as cm: 180 | self.jiranotifier.post(self.gitchange) 181 | self.assertEqual(f"WARNING:cloudimized.core.jiranotifier:Unable to assign ticket TEST_KEY to " 182 | f"changer: test_changer\nUpdate Issue", 183 | cm.output[0]) 184 | mock_jira_object.create_issue.assert_called_with(project={"key": "test_key"}, 185 | summary=(f"GCP manual change detected - project: " 186 | f"test_project, resource: test_resource"), 187 | description=(f"Manual changes performed by test_changer\n\n" 188 | f"{{code:java}}\nTEST_CHANGE_DIFF\n{{code}}\n"), 189 | issuetype={"name": "test_type"}, 190 | test_field=[{"name": "test_value"}]) 191 | mock_issue_object.update.assert_called_with(assignee={"name": "test_changer"}) 192 | 193 | @mock.patch("cloudimized.core.jiranotifier.JIRA", spec=JIRA) 194 | def test_post_update_success(self, mock_jira): 195 | self.gitchange.manual = True 196 | filter_set = {"projectId": ".est_pro.*"} 197 | self.jiranotifier.filter_set = filter_set 198 | mock_issue_object = mock.MagicMock() 199 | type(mock_issue_object).key = mock.PropertyMock(return_value="TEST_KEY") 200 | mock_issue_object.__str__.return_value = "test_issue_str" 201 | mock_jira_object = mock.MagicMock() 202 | mock_jira_object.create_issue.return_value = mock_issue_object 203 | mock_jira.return_value = mock_jira_object 204 | with self.assertLogs(logger, level="INFO") as cm: 205 | self.jiranotifier.post(self.gitchange) 206 | self.assertEqual(f"INFO:cloudimized.core.jiranotifier:Assigning issue TEST_KEY to user test_changer", 207 | cm.output[-1]) 208 | mock_jira_object.create_issue.assert_called_with(project={"key": "test_key"}, 209 | summary=(f"GCP manual change detected - project: " 210 | f"test_project, resource: test_resource"), 211 | description=(f"Manual changes performed by test_changer\n\n" 212 | f"{{code:java}}\nTEST_CHANGE_DIFF\n{{code}}\n"), 213 | issuetype={"name": "test_type"}, 214 | test_field=[{"name": "test_value"}]) 215 | mock_issue_object.update.assert_called_with(assignee={"name": "test_changer"}) 216 | 217 | 218 | if __name__ == '__main__': 219 | unittest.main() 220 | -------------------------------------------------------------------------------- /tests/test_result.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import mock 3 | from cloudimized.core.result import QueryResult, QueryResultError, set_query_results_from_configuration, AZURE_KEY, GCP_KEY 4 | from cloudimized.azurecore.virtualnetworksquery import VirtualNetworksQuery 5 | from cloudimized.gcpcore.gcpservicequery import GcpServiceQuery 6 | 7 | 8 | class QueryResultTestCase(unittest.TestCase): 9 | def setUp(self) -> None: 10 | self.queryresult = QueryResult() 11 | 12 | def test_add_resource_success(self): 13 | self.queryresult.add_resource("test-resource", provider="azure") 14 | self.assertIn("azure", self.queryresult.resources) 15 | self.assertIn("gcp", self.queryresult.resources) 16 | self.assertIn("test-resource", self.queryresult.resources["azure"]) 17 | self.assertIsInstance(self.queryresult.resources["azure"], dict) 18 | self.assertIsInstance(self.queryresult.resources["gcp"], dict) 19 | self.assertIsInstance(self.queryresult.resources["azure"]["test-resource"], dict) 20 | 21 | def test_add_resource_resource_already_there(self): 22 | self.queryresult.add_resource("test-resource", provider="gcp") 23 | self.queryresult.add_resource("test-resource", provider="azure") 24 | with self.assertRaises(QueryResultError) as cm: 25 | self.queryresult.add_resource("test-resource", provider="gcp") 26 | self.assertEqual(f"Resource 'test-resource' is already added in results for provider gcp", str(cm.exception)) 27 | 28 | def test_add_result(self): 29 | self.queryresult.add_result(resource_name="test_resource1", 30 | provider="azure", 31 | target_id="test_subscriptionID", 32 | result=test_result) 33 | self.queryresult.add_result(resource_name="test_resource2", 34 | provider="gcp", 35 | target_id="test_projectID", 36 | result=test_result) 37 | resources = self.queryresult.resources 38 | self.assertIn("azure", resources) 39 | self.assertIn("gcp", resources) 40 | self.assertIn("test_resource1", resources["azure"]) 41 | self.assertIn("test_resource2", resources["gcp"]) 42 | subscriptions = resources["azure"]["test_resource1"] 43 | projects = resources["gcp"]["test_resource2"] 44 | self.assertIn("test_projectID", projects) 45 | self.assertIs(test_result, projects["test_projectID"]) 46 | self.assertIn("test_subscriptionID", subscriptions) 47 | self.assertIs(test_result, subscriptions["test_subscriptionID"]) 48 | 49 | def test_get_results_existing(self): 50 | self.queryresult.add_result(resource_name="test_resource", 51 | provider="gcp", 52 | target_id="test_projectID", 53 | result=test_result) 54 | existing_result = self.queryresult.get_result(resource_name="test_resource", 55 | provider="gcp", 56 | target_id="test_projectID") 57 | self.assertIs(test_result, existing_result) 58 | 59 | def test_get_results_no_resource(self): 60 | result = self.queryresult.get_result(resource_name="non-existing", 61 | provider="azure", 62 | target_id="test_project") 63 | self.assertIsNone(result) 64 | 65 | def test_get_results_no_project(self): 66 | self.queryresult.add_result(resource_name="test_resource", 67 | provider="azure", 68 | target_id="test_projectID", 69 | result=test_result) 70 | result = self.queryresult.get_result(resource_name="test_resource", 71 | provider="azure", 72 | target_id="non_existing") 73 | self.assertIsNone(result) 74 | 75 | def test_set_query_results_from_configuration_no_queris(self): 76 | mock_gcpservicequery = mock.MagicMock(spec=GcpServiceQuery) 77 | mock_gcpservicequery.queries = {} 78 | test_gcp_services = { 79 | "test_serviceName": mock_gcpservicequery 80 | } 81 | with self.assertRaises(QueryResultError) as cm: 82 | set_query_results_from_configuration( 83 | gcp_services=test_gcp_services, 84 | azure_queries=None) 85 | self.assertEqual("No queries configured for service 'test_serviceName'", str(cm.exception)) 86 | 87 | def test_set_query_results_from_configuration_success(self): 88 | mock_gcpservicequery = mock.MagicMock(spec=GcpServiceQuery) 89 | mock_gcpservicequery.queries = {"test_resource": "query_configuration"} 90 | test_gcp_services = { 91 | "test_serviceName": mock_gcpservicequery 92 | } 93 | test_azure_queries = { 94 | "test_query": None 95 | } 96 | result = set_query_results_from_configuration( 97 | gcp_services=test_gcp_services, 98 | azure_queries=test_azure_queries) 99 | self.assertIsInstance(result, QueryResult) 100 | resources = result.resources 101 | self.assertIn("test_resource", result.resources[GCP_KEY]) 102 | self.assertIn("test_query", result.resources[AZURE_KEY]) 103 | projects = resources[GCP_KEY]["test_resource"] 104 | self.assertEqual(len(projects), 0) 105 | 106 | @mock.patch("cloudimized.core.result.isdir") 107 | def test_dump_results_not_directory(self, mock_isdir): 108 | mock_isdir.return_value = False 109 | with self.assertRaises(QueryResultError) as cm: 110 | self.queryresult.dump_results("test_directory") 111 | self.assertEqual("Issue dumping results to files. Directory 'test_directory' doesn't exist", 112 | str(cm.exception)) 113 | 114 | @mock.patch("cloudimized.core.result.Path") 115 | @mock.patch("cloudimized.core.result.isdir") 116 | def test_dump_results_issue_creating_subdirectory(self, mock_isdir, mock_path): 117 | mock_isdir.return_value = True 118 | mock_mkdir = mock.MagicMock() 119 | mock_mkdir.mkdir.side_effect = Exception("Issue creating test directory") 120 | mock_path.return_value = mock_mkdir 121 | self.queryresult.add_result(resource_name="test_resource", 122 | provider="gcp", 123 | target_id="test_project", 124 | result=test_result) 125 | with self.assertRaises(QueryResultError) as cm: 126 | self.queryresult.dump_results("test_directory") 127 | self.assertEqual("Issue creating directory 'test_directory/gcp/test_resource'", 128 | str(cm.exception)) 129 | 130 | @mock.patch("builtins.open") 131 | @mock.patch("cloudimized.core.result.Path") 132 | @mock.patch("cloudimized.core.result.isdir") 133 | def test_dump_results_issue_creating_files(self, mock_isdir, mock_path, mock_open): 134 | mock_isdir.return_value = True 135 | mock_open.side_effect = Exception("issue opening file") 136 | self.queryresult.add_result(resource_name="test_resource", project_id="test_project", result=test_result) 137 | with self.assertRaises(QueryResultError) as cm: 138 | self.queryresult.dump_results("test_directory") 139 | self.assertEqual("Issue dumping results into file 'test_directory/test_resource/test_project.yaml", 140 | str(cm.exception)) 141 | 142 | @mock.patch("cloudimized.core.result.yaml.dump") 143 | @mock.patch("builtins.open") 144 | @mock.patch("cloudimized.core.result.Path") 145 | @mock.patch("cloudimized.core.result.isdir") 146 | def test_dump_results_issue_creating_files(self, mock_isdir, mock_path, mock_b_open, mock_dump): 147 | mock_isdir.return_value = True 148 | mock_temp = mock.MagicMock() 149 | mock_fh = mock.MagicMock() 150 | mock_fh.__enter__.return_value = mock_temp 151 | mock_b_open.return_value = mock_fh 152 | self.queryresult.add_result(resource_name="test_resource", 153 | provider="azure", 154 | target_id="test_project", 155 | result=test_result) 156 | self.queryresult.dump_results("test_directory") 157 | mock_b_open.assert_called_with("test_directory/azure/test_resource/test_project.yaml", "w") 158 | mock_dump.assert_called_with(test_result, mock_temp, default_flow_style=False) 159 | 160 | @mock.patch("cloudimized.core.result.yaml.dump") 161 | @mock.patch("builtins.open") 162 | @mock.patch("cloudimized.core.result.Path") 163 | @mock.patch("cloudimized.core.result.isdir") 164 | def test_dump_results_empty_list_result(self, mock_isdir, mock_path, mock_b_open, mock_dump): 165 | mock_isdir.return_value = True 166 | mock_temp = mock.MagicMock() 167 | mock_fh = mock.MagicMock() 168 | mock_fh.__enter__.return_value = mock_temp 169 | mock_b_open.return_value = mock_fh 170 | self.queryresult.add_result(resource_name="test_resource", 171 | provider="azure", 172 | target_id="test_project", result=[]) 173 | self.queryresult.dump_results("test_directory") 174 | mock_b_open.assert_not_called() 175 | mock_dump.assert_not_called() 176 | 177 | @mock.patch("cloudimized.core.result.isdir") 178 | def test_dump_results_csv_not_directory(self, mock_isdir): 179 | mock_isdir.return_value = False 180 | with self.assertRaises(QueryResultError) as cm: 181 | self.queryresult.dump_results_csv("test_directory") 182 | self.assertEqual("Issue dumping results to files. Directory 'test_directory' doesn't exist", 183 | str(cm.exception)) 184 | 185 | #TODO Test exceptions in dump_results_csv 186 | 187 | @mock.patch("cloudimized.core.result.csv.DictWriter") 188 | @mock.patch("builtins.open") 189 | @mock.patch("cloudimized.core.result.isdir") 190 | def test_dump_results_success(self, mock_isdir, mock_b_open, mock_dictwriter): 191 | mock_isdir.return_value = True 192 | mock_fh = mock.MagicMock() 193 | mock_fh.__enter__.return_value = mock.MagicMock() 194 | mock_b_open.return_value = mock_fh 195 | mock_writer = mock.MagicMock() 196 | mock_dictwriter.return_value = mock_writer 197 | self.queryresult.resources = test_dump_result 198 | self.queryresult.dump_results_csv("test_directory") 199 | mock_isdir.assert_called_once() 200 | mock_b_open.assert_has_calls( 201 | [ 202 | mock.call("test_directory/azure/test_resource.csv", 203 | "w", 204 | newline=""), 205 | mock.call("test_directory/gcp/test_resource.csv", 206 | "w", 207 | newline=""), 208 | ], 209 | any_order=True 210 | ) 211 | mock_dictwriter.assert_has_calls( 212 | [ 213 | mock.call(mock_fh.__enter__.return_value, 214 | ["projectId", 215 | "id", 216 | "name", 217 | "test_field1", 218 | "test_field2", ]), 219 | mock.call(mock_fh.__enter__.return_value, 220 | ["subscriptionId", 221 | "id", 222 | "name", 223 | "test_field1", 224 | "test_field2", ]), 225 | ], 226 | any_order=True 227 | ) 228 | mock_writer.writeheader.call_count == 2 229 | #TODO finish test 230 | 231 | 232 | if __name__ == '__main__': 233 | unittest.main() 234 | 235 | test_result = [ 236 | {"entry_name": "test_1"}, 237 | {"entry_name": "test_2"} 238 | ] 239 | 240 | test_gcp_services = { 241 | "test_serviceName": mock.MagicMock(spec=GcpServiceQuery) 242 | } 243 | 244 | test_dump_result = { 245 | "azure": { 246 | "test_resource": { 247 | "test_project_1": [ 248 | {"name": "test_name1", "id": "test_id1", "test_field1": "test_value1"}, 249 | {"name": "test_name2", "id": "test_id2", "test_field2": "test_value2"} 250 | ], 251 | } 252 | }, 253 | "gcp": { 254 | "test_resource": { 255 | "test_subscription_1": [ 256 | {"name": "test_name1", "id": "test_id1", "test_field1": "test_value1"}, 257 | {"name": "test_name2", "id": "test_id2", "test_field2": "test_value2"} 258 | ], 259 | } 260 | }, 261 | } 262 | -------------------------------------------------------------------------------- /tests/test_slacknotifier.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import unittest 3 | 4 | class SlackNotifierTestCase(unittest.TestCase): 5 | pass 6 | -------------------------------------------------------------------------------- /tests/test_tfquery.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import mock 3 | import datetime as dt 4 | 5 | import time_machine 6 | 7 | from cloudimized.tfcore.query import TFQuery, TFQueryError, TFQueryConfigurationError, configure_tfquery 8 | import cloudimized.tfcore.query as q 9 | from cloudimized.tfcore.run import TFRun 10 | 11 | class TFQueryTestCase(unittest.TestCase): 12 | def setUp(self) -> None: 13 | self.tf_query = TFQuery("https://terraform.test", test_sa_org_workspace_map, test_org_token_map) 14 | 15 | @time_machine.travel(dt.datetime(1985, 10, 26, 1, 40)) 16 | def test_unknown_service_account(self): 17 | with self.assertRaises(TFQueryError): 18 | self.tf_query.get_runs("unknown_sa") 19 | 20 | #Query resolving workspace name to id fails 21 | @time_machine.travel(dt.datetime(1985, 10, 26, 1, 40)) 22 | @mock.patch("cloudimized.tfcore.query.TFC") 23 | def test_workspace_id_fail(self, mock_tfc): 24 | mock_tf_api = mock.MagicMock() 25 | mock_tf_api.workspaces.show.side_effect = Exception() 26 | mock_tfc.return_value = mock_tf_api 27 | with self.assertRaises(TFQueryError): 28 | self.tf_query.get_runs("sa-test-project1") 29 | mock_tf_api.workspaces.show.assert_called_with(workspace_name="test_workspace1") 30 | 31 | @time_machine.travel(dt.datetime(1985, 10, 26, 1, 40)) 32 | @mock.patch("cloudimized.tfcore.query.TFC") 33 | def test_runs_list_fail(self, mock_tfc): 34 | mock_workspace_response = mock.MagicMock() 35 | mock_workspace_response.__getitem__.side_effect = test_workspace_response.__getitem__ 36 | mock_tf_api = mock.MagicMock() 37 | mock_tf_api.workspaces.show.return_value = mock_workspace_response 38 | mock_tf_api.runs.list.side_effect = Exception() 39 | mock_tfc.return_value = mock_tf_api 40 | with self.assertRaises(TFQueryError): 41 | self.tf_query.get_runs("sa-test-project1") 42 | mock_tf_api.workspaces.show.assert_called_with(workspace_name="test_workspace1") 43 | mock_tf_api.runs.list.assert_called_with("id_test_workspace1", page_size=10, include=["created-by"]) 44 | 45 | @time_machine.travel(dt.datetime(1985, 10, 26, 1, 40)) 46 | @mock.patch("cloudimized.tfcore.query.parse_tf_runs") 47 | @mock.patch("cloudimized.tfcore.query.TFC") 48 | def test_runs_parse_fails(self, mock_tfc, mock_parse): 49 | mock_workspace_response = mock.MagicMock() 50 | mock_workspace_response.__getitem__.side_effect = test_workspace_response.__getitem__ 51 | mock_tf_api = mock.MagicMock() 52 | mock_tf_api.workspaces.show.return_value = mock_workspace_response 53 | mock_tf_api.runs.list.return_value = test_runs_response 54 | mock_tf_api.get_org.return_value = "test_org1" 55 | mock_tfc.return_value = mock_tf_api 56 | mock_parse.side_effect = Exception() 57 | with self.assertRaises(TFQueryError): 58 | self.tf_query.get_runs("sa-test-project1") 59 | mock_tf_api.workspaces.show.assert_called_with(workspace_name="test_workspace1") 60 | mock_tf_api.runs.list.assert_called_with("id_test_workspace1", page_size=10, include=["created-by"]) 61 | mock_parse.assert_called_with(test_runs_response, "test_org1", "test_workspace1") 62 | 63 | @time_machine.travel(dt.datetime(1985, 10, 26, 1, 40)) 64 | @mock.patch("cloudimized.tfcore.query.parse_tf_runs") 65 | @mock.patch("cloudimized.tfcore.query.TFC") 66 | def test_runs_success(self, mock_tfc, mock_parse): 67 | mock_workspace_response = mock.MagicMock() 68 | mock_workspace_response.__getitem__.side_effect = test_workspace_response.__getitem__ 69 | mock_tf_api = mock.MagicMock() 70 | mock_tf_api.workspaces.show.return_value = mock_workspace_response 71 | mock_tf_api.runs.list.return_value = test_runs_response 72 | mock_tf_api.get_org.return_value = "test_org2" 73 | mock_tfc.return_value = mock_tf_api 74 | mock_parse.return_value = test_get_runs_result 75 | result = self.tf_query.get_runs("sa-test-project2") 76 | 77 | calls_workspace_show = [ 78 | mock.call(workspace_name="test_workspace2"), 79 | mock.call(workspace_name="test_workspace3") 80 | ] 81 | calls_runs_list = [ 82 | mock.call("id_test_workspace1", page_size=10, include=["created-by"]), 83 | mock.call("id_test_workspace1", page_size=10, include=["created-by"]) 84 | ] 85 | calls_parse = [ 86 | mock.call(test_runs_response, "test_org2", "test_workspace2"), 87 | mock.call(test_runs_response, "test_org2", "test_workspace3") 88 | ] 89 | 90 | mock_tf_api.workspaces.show.assert_has_calls(calls_workspace_show, any_order=True) 91 | mock_tf_api.runs.list.assert_has_calls(calls_runs_list) 92 | mock_parse.assert_has_calls(calls_parse) 93 | 94 | self.assertIsInstance(result, list) 95 | #Only 2 elements in list 96 | self.assertEqual(len(result), 4) 97 | self.assertEqual(result[0].message, "test-message1") 98 | self.assertEqual(result[1].message, "test-message3") 99 | self.assertEqual(result[2].message, "test-message1") 100 | self.assertEqual(result[3].message, "test-message3") 101 | 102 | 103 | @mock.patch("cloudimized.tfcore.query.getenv") 104 | @mock.patch("cloudimized.tfcore.query.TFQuery", spec=TFQuery) 105 | @mock.patch("cloudimized.tfcore.query.json.load") 106 | @mock.patch("builtins.open") 107 | def test_configure_tfquery(self, mock_open, mock_json_load, mock_tfquery, mock_getenv): 108 | mock_json_load.return_value = token_file_correct_data 109 | # No terraform configuration 110 | self.assertIsNone(configure_tfquery(None)) 111 | 112 | # Incorrect type 113 | with self.assertRaises(TFQueryConfigurationError): 114 | configure_tfquery(config="incorrect type") 115 | 116 | # Missing keys 117 | ## url 118 | with self.assertRaises(TFQueryConfigurationError): 119 | configure_tfquery(config={}) 120 | ## service account workspace map 121 | with self.assertRaises(TFQueryConfigurationError): 122 | configure_tfquery(config={ 123 | q.TERRAFORM_URL: "test_url" 124 | }) 125 | ## token file 126 | with self.assertRaises(TFQueryConfigurationError): 127 | configure_tfquery(config={ 128 | q.TERRAFORM_URL: "test_url", 129 | q.TERRAFORM_SERVICE_ACCOUNT_WORKSPACE_MAP: {} 130 | }) 131 | 132 | # Incorrect value 133 | ## url 134 | with self.assertRaises(TFQueryConfigurationError): 135 | configure_tfquery(config={ 136 | q.TERRAFORM_URL: 0, #incorrect type 137 | q.TERRAFORM_SERVICE_ACCOUNT_WORKSPACE_MAP: {}, 138 | q.TERRAFORM_WORKSPACE_TOKEN_FILE: "test_file" 139 | }) 140 | ## service workspace map 141 | ### type 142 | with self.assertRaises(TFQueryConfigurationError): 143 | configure_tfquery(config={ 144 | q.TERRAFORM_URL: "test_url", 145 | q.TERRAFORM_SERVICE_ACCOUNT_WORKSPACE_MAP: "incorrect type", 146 | q.TERRAFORM_WORKSPACE_TOKEN_FILE: "test_file" 147 | }) 148 | ### key in map 149 | with self.assertRaises(TFQueryConfigurationError): 150 | configure_tfquery(config={ 151 | q.TERRAFORM_URL: "test_url", 152 | q.TERRAFORM_SERVICE_ACCOUNT_WORKSPACE_MAP: { 153 | 1: {} 154 | }, 155 | q.TERRAFORM_WORKSPACE_TOKEN_FILE: "test_file" 156 | }) 157 | ### value in map 158 | with self.assertRaises(TFQueryConfigurationError): 159 | configure_tfquery(config={ 160 | q.TERRAFORM_URL: "test_url", 161 | q.TERRAFORM_SERVICE_ACCOUNT_WORKSPACE_MAP: { 162 | "test": "incorrect_value" 163 | }, 164 | q.TERRAFORM_WORKSPACE_TOKEN_FILE: "test_file" 165 | }) 166 | ### missing required key arg in map 167 | with self.assertRaises(TFQueryConfigurationError): 168 | configure_tfquery(config={ 169 | q.TERRAFORM_URL: "test_url", 170 | q.TERRAFORM_SERVICE_ACCOUNT_WORKSPACE_MAP: { 171 | "test_sa_1": { 172 | "org": "test_org" # missing workspace 173 | } 174 | }, 175 | q.TERRAFORM_WORKSPACE_TOKEN_FILE: "test_file" 176 | }) 177 | ### incorrect type of required args workspace 178 | with self.assertRaises(TFQueryConfigurationError): 179 | configure_tfquery(config={ 180 | q.TERRAFORM_URL: "test_url", 181 | q.TERRAFORM_SERVICE_ACCOUNT_WORKSPACE_MAP: { 182 | "test_sa_1": { 183 | "org": "test_org", 184 | "workspace": 1 185 | } 186 | }, 187 | q.TERRAFORM_WORKSPACE_TOKEN_FILE: "test_file" 188 | }) 189 | ### incorrect type of required args org 190 | with self.assertRaises(TFQueryConfigurationError): 191 | configure_tfquery(config={ 192 | q.TERRAFORM_URL: "test_url", 193 | q.TERRAFORM_SERVICE_ACCOUNT_WORKSPACE_MAP: { 194 | "test_sa_1": { 195 | "org": 1, 196 | "workspace": "test_workspace" 197 | } 198 | }, 199 | q.TERRAFORM_WORKSPACE_TOKEN_FILE: "test_file" 200 | }) 201 | ## token file 202 | ### type 203 | with self.assertRaises(TFQueryConfigurationError): 204 | configure_tfquery(config={ 205 | q.TERRAFORM_URL: "test_url", 206 | q.TERRAFORM_SERVICE_ACCOUNT_WORKSPACE_MAP: {}, 207 | q.TERRAFORM_WORKSPACE_TOKEN_FILE: 0 #incorrect type 208 | }) 209 | ### issue opening file 210 | mock_open.side_effect = Exception("open issue") 211 | with self.assertRaises(TFQueryConfigurationError): 212 | configure_tfquery(config={ 213 | q.TERRAFORM_URL: "test_url", 214 | q.TERRAFORM_SERVICE_ACCOUNT_WORKSPACE_MAP: {}, 215 | q.TERRAFORM_WORKSPACE_TOKEN_FILE: "test/file" 216 | }) 217 | ### issue loading json file 218 | mock_open.side_effect = None 219 | mock_json_load.side_effect = Exception("load issue") 220 | with self.assertRaises(TFQueryConfigurationError): 221 | configure_tfquery(config={ 222 | q.TERRAFORM_URL: "test_url", 223 | q.TERRAFORM_SERVICE_ACCOUNT_WORKSPACE_MAP: {}, 224 | q.TERRAFORM_WORKSPACE_TOKEN_FILE: 'test/file' 225 | }) 226 | ## issue with token file structure 227 | ### incorrect key 228 | mock_open.side_effect = None 229 | mock_json_load.side_effect = None 230 | mock_json_load.return_value = token_file_incorrect_key 231 | with self.assertRaises(TFQueryConfigurationError): 232 | configure_tfquery(config={ 233 | q.TERRAFORM_URL: "test_url", 234 | q.TERRAFORM_SERVICE_ACCOUNT_WORKSPACE_MAP: {}, 235 | q.TERRAFORM_WORKSPACE_TOKEN_FILE: 'test/file' 236 | }) 237 | mock_json_load.return_value = token_file_incorrect_value 238 | with self.assertRaises(TFQueryConfigurationError): 239 | configure_tfquery(config={ 240 | q.TERRAFORM_URL: "test_url", 241 | q.TERRAFORM_SERVICE_ACCOUNT_WORKSPACE_MAP: {}, 242 | q.TERRAFORM_WORKSPACE_TOKEN_FILE: 'test/file' 243 | }) 244 | 245 | ### no token file in config and env var 246 | mock_open.side_effect = None 247 | mock_json_load.side_effect = None 248 | mock_json_load.return_value = token_file_incorrect_key 249 | with self.assertRaises(TFQueryConfigurationError): 250 | configure_tfquery(config={ 251 | q.TERRAFORM_URL: "test_url", 252 | q.TERRAFORM_SERVICE_ACCOUNT_WORKSPACE_MAP: {} 253 | }) 254 | 255 | # Correct data 256 | mock_json_load.return_value = token_file_correct_data 257 | result = configure_tfquery(config={ 258 | q.TERRAFORM_URL: "test_url", 259 | q.TERRAFORM_SERVICE_ACCOUNT_WORKSPACE_MAP: test_sa_org_workspace_map, 260 | q.TERRAFORM_WORKSPACE_TOKEN_FILE: 'test/file' 261 | }) 262 | self.assertIsInstance(result, TFQuery) 263 | mock_tfquery.assert_called_with("test_url", test_sa_org_workspace_map, token_file_correct_data) 264 | 265 | # Correct data with env var 266 | mock_json_load.return_value = token_file_correct_data 267 | mock_getenv.return_value = 'test/file' 268 | result = configure_tfquery(config={ 269 | q.TERRAFORM_URL: "test_url", 270 | q.TERRAFORM_SERVICE_ACCOUNT_WORKSPACE_MAP: test_sa_org_workspace_map 271 | }) 272 | self.assertIsInstance(result, TFQuery) 273 | mock_tfquery.assert_called_with("test_url", test_sa_org_workspace_map, token_file_correct_data) 274 | 275 | 276 | 277 | 278 | if __name__ == '__main__': 279 | unittest.main() 280 | 281 | test_sa_org_workspace_map = { 282 | "sa-test-project1": { 283 | "org": "test_org1", 284 | "workspace": ["test_workspace1"] 285 | }, 286 | "sa-test-project2": { 287 | "org": "test_org2", 288 | "workspace": ["test_workspace2", "test_workspace3"] 289 | } 290 | } 291 | 292 | test_org_token_map = { 293 | "test_org1": "secret_token1", 294 | "test_org2": "secret_token2" 295 | } 296 | 297 | test_workspace_response = { 298 | "data": { 299 | "id": "id_test_workspace1" 300 | } 301 | } 302 | 303 | test_runs_response = { 304 | "data": "test" 305 | } 306 | 307 | token_file_incorrect_key = { 308 | 1: "incorrect key" 309 | } 310 | 311 | token_file_incorrect_value = { 312 | "key": 1 313 | } 314 | 315 | token_file_correct_data = { 316 | "test_org_1": "test_token_1", 317 | "test_org_2": "test_token_2" 318 | } 319 | 320 | test_get_runs_result = [ 321 | TFRun("test-message1", 322 | "test-id1", 323 | "applied", 324 | dt.datetime.strptime("1985-10-26T01:24:00", "%Y-%m-%dT%H:%M:%S"), 325 | "test_org", 326 | "test_workspace"), 327 | TFRun("test-message3", 328 | "test-id3", 329 | "errored", 330 | dt.datetime.strptime("1985-10-26T01:29:00", "%Y-%m-%dT%H:%M:%S"), 331 | "test_org", 332 | "test_workspace"), 333 | TFRun("test-message4", 334 | "test-id4", 335 | "applied", 336 | dt.datetime.strptime("1985-10-25T01:22:00", "%Y-%m-%dT%H:%M:%S"), 337 | "test_org", 338 | "test_workspace") 339 | ] 340 | -------------------------------------------------------------------------------- /tests/test_tfrun.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import mock 3 | import logging 4 | 5 | from datetime import datetime 6 | from cloudimized.tfcore.run import TFRun, TFRunError, parse_tf_runs, filter_non_change_runs 7 | 8 | class TFRunTestCase(unittest.TestCase): 9 | def setUp(self) -> None: 10 | logging.disable(logging.WARNING) 11 | 12 | def tearDown(self) -> None: 13 | logging.disable(logging.NOTSET) 14 | 15 | def test_parse_no_data(self): 16 | with self.assertRaises(TFRunError): 17 | parse_tf_runs({}, "test_org", "test_workspace") 18 | 19 | def test_parse_missing_status(self): 20 | # Assume 21 | result = parse_tf_runs(tf_run_missing_status, "test_org", "test_workspace") 22 | self.assertEqual(result, []) 23 | 24 | def test_parse_non_relevat_status(self): 25 | result = parse_tf_runs(tf_run_status_planning, "test_org", "test_workspace") 26 | self.assertEqual(result, []) 27 | 28 | @mock.patch("cloudimized.tfcore.run.TFRun") 29 | def test_parse_relevant_runs(self, mock_tfrun): 30 | result = parse_tf_runs(tf_run_status_relevant, "test_org", "test_workspace") 31 | # Assert that those two TFRun objects were created and returned in list 32 | calls = [ 33 | mock.call("test_msg_1", 34 | "test_id_1", 35 | "applied", 36 | datetime.strptime("2001-01-01T00:00:01", "%Y-%m-%dT%H:%M:%S"), 37 | "test_org", 38 | "test_workspace"), 39 | mock.call("test_msg_3", 40 | "test_id_3", 41 | "errored", 42 | datetime.strptime("2001-01-01T00:00:03", "%Y-%m-%dT%H:%M:%S"), 43 | "test_org", 44 | "test_workspace") 45 | ] 46 | mock_tfrun.assert_has_calls(calls) 47 | self.assertEqual(len(result), len(calls)) 48 | 49 | def test_filter_non_change_runs(self): 50 | change_time = datetime.strptime("2001-01-01T00:02:00", "%Y-%m-%dT%H:%M:%S") 51 | result = filter_non_change_runs(tf_runs=tf_runs_test, change_time=change_time) 52 | self.assertEqual(len(result), 2) 53 | self.assertIs(result[0], tf_runs_test[0]) 54 | self.assertIs(result[1], tf_runs_test[2]) 55 | 56 | 57 | if __name__ == '__main__': 58 | unittest.main() 59 | 60 | tf_run_missing_status = { 61 | "data": [{}] 62 | } 63 | 64 | tf_run_status_planning = { 65 | "data": [{ 66 | "attributes": { 67 | "status": "planing" 68 | } 69 | }] 70 | } 71 | 72 | tf_run_status_relevant = { 73 | "data": [ 74 | { 75 | "attributes": { 76 | "status": "applied", 77 | "message": "test_msg_1", 78 | "status-timestamps": { 79 | "applying-at": "2001-01-01T00:00:01+00:00" 80 | } 81 | }, 82 | "id": "test_id_1" 83 | }, 84 | { 85 | "attributes": { 86 | "status": "pending", 87 | "message": "test_msg_2", 88 | "status-timestamps": {} 89 | }, 90 | "id": "test_id_2" 91 | }, 92 | { 93 | "attributes": { 94 | "status": "errored", 95 | "message": "test_msg_3", 96 | "status-timestamps": { 97 | "errored-at": "2001-01-01T00:00:03+00:00" 98 | } 99 | }, 100 | "id": "test_id_3" 101 | }, 102 | ] 103 | } 104 | 105 | tf_runs_test = [ 106 | TFRun("test-message1", 107 | "test-id1", 108 | "applied", 109 | datetime.strptime("2001-01-01T00:00:01", "%Y-%m-%dT%H:%M:%S"), 110 | "test_org", 111 | "test_workspace"), 112 | TFRun("test-message2", 113 | "test-id2", 114 | "pending", 115 | datetime.strptime("2001-01-01T00:00:02", "%Y-%m-%dT%H:%M:%S"), 116 | "test_org", 117 | "test_workspace"), 118 | TFRun("test-message3", 119 | "test-id3", 120 | "errored", 121 | datetime.strptime("2001-01-01T00:00:03", "%Y-%m-%dT%H:%M:%S"), 122 | "test_org", 123 | "test_workspace"), 124 | TFRun("test-message4", 125 | "test-id4", 126 | "applied", 127 | datetime.strptime("2000-12-30T00:00:01", "%Y-%m-%dT%H:%M:%S"), 128 | "test_org", 129 | "test_workspace"), 130 | ] 131 | --------------------------------------------------------------------------------