├── src └── cloudminer │ ├── __init__.py │ ├── exceptions.py │ ├── logger.py │ ├── cloud_miner.py │ ├── utils.py │ ├── scripts_executor.py │ └── azure_automation_session.py ├── resources ├── pip │ ├── src │ │ └── pip │ │ │ └── __init__.py │ └── setup.py ├── random_whl-0.0.1-py3-none-any.whl └── README.md ├── images ├── cloud-miner-usage-python.png └── cloud-miner-usage-powershell.png ├── requirements.txt ├── .gitignore ├── setup.py ├── LICENSE └── README.md /src/cloudminer/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resources/pip/src/pip/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "23.2.1" -------------------------------------------------------------------------------- /src/cloudminer/exceptions.py: -------------------------------------------------------------------------------- 1 | class CloudMinerException(Exception): 2 | pass -------------------------------------------------------------------------------- /images/cloud-miner-usage-python.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SafeBreach-Labs/CloudMiner/HEAD/images/cloud-miner-usage-python.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | certifi==2023.7.22 2 | charset-normalizer==3.2.0 3 | idna==3.4 4 | requests==2.31.0 5 | urllib3==2.0.4 6 | wheel==0.41.2 -------------------------------------------------------------------------------- /images/cloud-miner-usage-powershell.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SafeBreach-Labs/CloudMiner/HEAD/images/cloud-miner-usage-powershell.png -------------------------------------------------------------------------------- /resources/random_whl-0.0.1-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SafeBreach-Labs/CloudMiner/HEAD/resources/random_whl-0.0.1-py3-none-any.whl -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | .vscode 3 | **venv** 4 | **dist** 5 | **build** 6 | **egg-info** 7 | resources/pip/src/* 8 | !resources/README.md 9 | !resources/pip/src/pip/__init__.py -------------------------------------------------------------------------------- /resources/README.md: -------------------------------------------------------------------------------- 1 | # Folder content 2 | ### Pip folder - contains the 'pip' package to replace the original 'pip' package in the Automation Account 3 | 4 | ### random_whl-0.0.1-py3-none-any.whl - is a dummy whl file we use to upload and trigger the code execution -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | with open("requirements.txt") as f: 4 | required = f.read().splitlines() 5 | 6 | setup( 7 | name="CloudMiner", 8 | version="1.0.0", 9 | package_dir={"": "src"}, 10 | packages=find_packages(where="src"), 11 | install_requires=required) -------------------------------------------------------------------------------- /resources/pip/setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | Setup file for installing our custom pip package 3 | """ 4 | from setuptools import find_packages, setup 5 | 6 | setup( 7 | name="pip", 8 | version="1.0.0", 9 | description="CloudMiner custom pip", 10 | entry_points={ 11 | "console_scripts": [ 12 | "pip=pip.main:_main", 13 | ], 14 | }, 15 | package_dir={"": "src"}, 16 | packages=find_packages("src"), 17 | ) 18 | -------------------------------------------------------------------------------- /src/cloudminer/logger.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import logging 3 | 4 | INDENT_CHAR = "\t" 5 | 6 | class CloudMinerlogger(logging.Logger): 7 | """ 8 | The main program logging 9 | """ 10 | def __init__(self, name: str, level: int = 0) -> None: 11 | super().__init__(name, level) 12 | self.indent = 0 13 | 14 | def _log(self, level, msg, *args, **kwargs) -> None: 15 | if level == logging.INFO: 16 | bullet = '[+]' 17 | elif level == logging.DEBUG: 18 | bullet = '[*]' 19 | elif level == logging.ERROR or level == logging.WARNING: 20 | bullet = '[!]' 21 | elif level == 100: #header 22 | bullet = '[#]' 23 | else: 24 | bullet = '[~]' 25 | 26 | msg = f"{INDENT_CHAR*self.indent}{bullet} {msg}" 27 | return super()._log(level, msg, *args, **kwargs) 28 | 29 | def add_indent(self): 30 | self.indent += 1 31 | 32 | def remove_indent(self): 33 | self.indent -= 1 34 | 35 | 36 | def init_logging() -> CloudMinerlogger: 37 | logger = CloudMinerlogger("root") 38 | handler = logging.StreamHandler(sys.stdout) 39 | logger.addHandler(handler) 40 | return logger 41 | 42 | logger = init_logging() -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2023, SafeBreach Labs 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | 1. Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 11 | 2. Redistributions in binary form must reproduce the above copyright notice, 12 | this list of conditions and the following disclaimer in the documentation 13 | and/or other materials provided with the distribution. 14 | 15 | 3. Neither the name of the copyright holder nor the names of its 16 | contributors may be used to endorse or promote products derived from 17 | this software without specific prior written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 23 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 25 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 26 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CloudMiner 2 | Execute code within Azure Automation service without getting charged 3 | 4 | ## Description 5 | CloudMiner is a tool designed to get free computing power within Azure Automation service. The tool utilizes the upload module/package flow to execute code which is totally free to use. This tool is intended for educational and research purposes only and should be used responsibly and with proper authorization. 6 | 7 | * This flow was reported to Microsoft on 3/23 which decided to not change the service behavior as it's considered as "by design". As for 3/9/23, this tool can still be used without getting charged. 8 | 9 | * Each execution is limited to 3 hours 10 | 11 | ## Requirements 12 | 1. Python 3.8+ with the libraries mentioned in the file `requirements.txt` 13 | 2. Configured Azure CLI - https://learn.microsoft.com/en-us/cli/azure/install-azure-cli 14 | - Account must be logged in before using this tool 15 | 16 | ## Installation 17 | ```pip install .``` 18 | 19 | ## Usage 20 | ``` 21 | usage: cloud_miner.py [-h] --path PATH --id ID -c COUNT [-t TOKEN] [-r REQUIREMENTS] [-v] 22 | 23 | CloudMiner - Free computing power in Azure Automation Service 24 | 25 | optional arguments: 26 | -h, --help show this help message and exit 27 | --path PATH the script path (Powershell or Python) 28 | --id ID id of the Automation Account - /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Automation/a 29 | utomationAccounts/{automationAccountName} 30 | -c COUNT, --count COUNT 31 | number of executions 32 | -t TOKEN, --token TOKEN 33 | Azure access token (optional). If not provided, token will be retrieved using the Azure CLI 34 | -r REQUIREMENTS, --requirements REQUIREMENTS 35 | Path to requirements file to be installed and use by the script (relevant to Python scripts only) 36 | -v, --verbose Enable verbose mode 37 | ``` 38 | 39 | ## Example usage 40 | ### Python 41 | ![Alt text](images/cloud-miner-usage-python.png?raw=true "Usage Example") 42 | ### Powershell 43 | ![Alt text](images/cloud-miner-usage-powershell.png?raw=true "Usage Example") 44 | 45 | ## License 46 | CloudMiner is released under the BSD 3-Clause License. 47 | Feel free to modify and distribute this tool responsibly, while adhering to the license terms. 48 | 49 | ## Author - Ariel Gamrian 50 | * LinkedIn - [Ariel Gamrian](https://www.linkedin.com/in/ariel-gamrian/) -------------------------------------------------------------------------------- /src/cloudminer/cloud_miner.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import logging 4 | import argparse 5 | 6 | import cloudminer.utils as utils 7 | from cloudminer.logger import logger 8 | from cloudminer.exceptions import CloudMinerException 9 | from azure_automation_session import AzureAutomationSession 10 | from scripts_executor import PowershellScriptExecutor, PythonScriptExecutor 11 | 12 | 13 | def get_access_token_from_cli() -> str: 14 | """ 15 | Retrieve Azure access token using Azure CLI 16 | 17 | :raises CloudMinerException: If Azure CLI is not installed or not in PATH environment variable 18 | If account is not logged in via Azure CLI 19 | If failed to retrieve account access token 20 | """ 21 | logger.info("Retrieving access token using Azure CLI...") 22 | try: 23 | # Check if user is logged in 24 | process = utils.run_command(["az", "account", "show"]) 25 | if process.returncode != 0: 26 | raise CloudMinerException(f"Account must be logged in via Azure CLI") 27 | 28 | process = utils.run_command(["az", "account", "get-access-token"]) 29 | if process.returncode != 0: 30 | raise CloudMinerException(f"Failed to retrieve access token using Azure CLI. Error: {process.stderr}") 31 | 32 | except FileNotFoundError: 33 | raise CloudMinerException("Azure CLI is not installed on the system or not in PATH environment variable") 34 | 35 | return json.loads(process.stdout)["accessToken"] 36 | 37 | 38 | def parse_args() -> argparse.Namespace: 39 | """ 40 | Parse command line arguments 41 | """ 42 | parser = argparse.ArgumentParser(description="CloudMiner - Free computing power in Azure Automation Service") 43 | parser.add_argument("--path", type=str, help="the script path (Powershell or Python)", required=True) 44 | parser.add_argument("--id", type=str, help="id of the Automation Account - /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Automation/automationAccounts/{automationAccountName}", required=True) 45 | parser.add_argument("-c","--count", type=int, help="number of executions", required=True) 46 | parser.add_argument("-t","--token", type=str, help="Azure access token (optional). If not provided, token will be retrieved using the Azure CLI") 47 | parser.add_argument("-r","--requirements", type=str, help="Path to requirements file to be installed and use by the script (relevant to Python scripts only)") 48 | parser.add_argument('-v', '--verbose', action='store_true', help='Enable verbose mode') 49 | return parser.parse_args() 50 | 51 | 52 | def main(): 53 | args = parse_args() 54 | level = logging.DEBUG if args.verbose else logging.INFO 55 | logger.setLevel(level) 56 | logger.info(utils.PROJECT_BANNER) 57 | 58 | if not os.path.exists(args.path): 59 | raise CloudMinerException(f"Script path '{args.path}' does not exist!") 60 | 61 | if args.requirements and not os.path.exists(args.requirements): 62 | raise CloudMinerException(f"Requirements path '{args.requirements}' does not exist!") 63 | 64 | access_token = args.token or get_access_token_from_cli() 65 | automation_session = AzureAutomationSession(args.id, access_token) 66 | 67 | file_extension = utils.get_file_extension(args.path).lower() 68 | if file_extension == PowershellScriptExecutor.EXTENSION: 69 | logger.info(f"File type detected - Powershell") 70 | executor = PowershellScriptExecutor(automation_session, args.path) 71 | 72 | elif file_extension == PythonScriptExecutor.EXTENSION: 73 | logger.info(f"File type detected - Python") 74 | executor = PythonScriptExecutor(automation_session, args.path, args.requirements) 75 | 76 | else: 77 | raise CloudMinerException(f"File extension {file_extension} is not supported") 78 | 79 | 80 | executor.execute_script(args.count) 81 | 82 | logger.info("CloudMiner finished successfully :)") 83 | 84 | if __name__ == "__main__": 85 | main() 86 | -------------------------------------------------------------------------------- /src/cloudminer/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import zipfile 4 | import tempfile 5 | import subprocess 6 | from typing import List 7 | 8 | from cloudminer.logger import logger 9 | from cloudminer.exceptions import CloudMinerException 10 | 11 | RESOURCES_DIRECTORY = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))), "resources") 12 | PROJECT_BANNER = """ 13 | /$$$$$$ /$$ /$$ /$$ /$$ /$$ /$$ 14 | /$$__ $$| $$ | $$| $$$ /$$$|__/ /$$$$$$ 15 | | $$ \__/| $$ /$$$$$$ /$$ /$$ /$$$$$$$| $$$$ /$$$$ /$$ /$$$$$$$ /$$$$$$ /$$$$$$ /$$__ $$ 16 | | $$ | $$ /$$__ $$| $$ | $$ /$$__ $$| $$ $$/$$ $$| $$| $$__ $$ /$$__ $$ /$$__ $$ | $$ \__/ 17 | | $$ | $$| $$ \ $$| $$ | $$| $$ | $$| $$ $$$| $$| $$| $$ \ $$| $$$$$$$$| $$ \__/ | $$$$$$ 18 | | $$ $$| $$| $$ | $$| $$ | $$| $$ | $$| $$\ $ | $$| $$| $$ | $$| $$_____/| $$ \____ $$ 19 | | $$$$$$/| $$| $$$$$$/| $$$$$$/| $$$$$$$| $$ \/ | $$| $$| $$ | $$| $$$$$$$| $$ /$$ \ $$ 20 | \______/ |__/ \______/ \______/ \_______/|__/ |__/|__/|__/ |__/ \_______/|__/ | $$$$$$/ 21 | \_ $$_/ 22 | \__/ 23 | 24 | \n-- CloudMiner: v1.0.0 (SafeBreach Labs) --\n""" 25 | 26 | def get_temp_file_path(file_name: str) -> str: 27 | """ 28 | Returns the user temp directory 29 | """ 30 | temp_dir = tempfile.gettempdir() 31 | return os.path.join(temp_dir, file_name) 32 | 33 | 34 | def get_file_extension(file_path: str): 35 | """ 36 | Returns the file extension from file path 37 | """ 38 | return os.path.basename(os.path.splitext(file_path)[1]) 39 | 40 | 41 | def get_file_name(file_path: str): 42 | """ 43 | Returns the file name from full file path 44 | """ 45 | return os.path.basename(os.path.splitext(file_path)[0]) 46 | 47 | 48 | def zip_file(source_file: str, file_name_within_archive: str = None, zip_path: str = None) -> str: 49 | """ 50 | Zip a file 51 | 52 | :param source_file: File path to zip 53 | :param file_name_within_archive: Name of file within the zip archive 54 | :param zip_path: Path to the zipped file. If None, will use the temp directory 55 | 56 | :return: zip file path 57 | """ 58 | src_file_name = get_file_name(source_file) 59 | if zip_path is None: 60 | zip_path = get_temp_file_path(f"{src_file_name}.zip") 61 | 62 | if file_name_within_archive is None: 63 | file_name_within_archive = src_file_name 64 | 65 | with zipfile.ZipFile(zip_path, "w") as zipf: 66 | zipf.write(source_file, arcname=file_name_within_archive) 67 | 68 | logger.debug(f"Zip file created at '{zip_path}'") 69 | return zip_path 70 | 71 | 72 | def run_command(cmd: List[str], shell: bool = True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs) -> subprocess.CompletedProcess: 73 | """ 74 | Helper function to run commands 75 | """ 76 | logger.debug(f"Running command: '{subprocess.list2cmdline(cmd)}'") 77 | return subprocess.run(cmd, 78 | shell=shell, 79 | text=text, 80 | stdout=stdout, 81 | stderr=stderr, 82 | **kwargs) 83 | 84 | 85 | def package_to_whl(package_path: str) -> str: 86 | """ 87 | Create a whl file from a given Python package 88 | whl file wil lbe saved in the dist directory 89 | 90 | :raises CloudMinerException: If failed to create whl file 91 | 92 | :return: whl file path 93 | """ 94 | package_name = os.path.basename(package_path) 95 | setup_file = os.path.join(package_path, "setup.py") 96 | dist_dir_path = os.path.join(package_path, "dist") 97 | 98 | logger.debug(f"Creating a .whl file for package - '{package_name}'") 99 | process = run_command(["python", setup_file, "bdist_wheel"], cwd=package_path) 100 | if process.returncode != 0: 101 | raise CloudMinerException(f"Failed to create .whl file. Error: {process.stderr}") 102 | 103 | try: 104 | whl_file_name = os.listdir(dist_dir_path)[0] 105 | except IndexError: 106 | raise CloudMinerException(f"Failed to find the creatd .whl file in folder '{dist_dir_path}'.") 107 | 108 | logger.info(f"whl file successfully created - '{whl_file_name}'") 109 | return os.path.join(dist_dir_path, whl_file_name) 110 | -------------------------------------------------------------------------------- /src/cloudminer/scripts_executor.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import uuid 4 | import shutil 5 | from typing import List 6 | from abc import ABC, abstractmethod 7 | 8 | import cloudminer.utils as utils 9 | from cloudminer.logger import logger 10 | from cloudminer.exceptions import CloudMinerException 11 | from azure_automation_session import UPLOAD_STATE, UPLOAD_TIMEOUT, AzureAutomationSession 12 | 13 | 14 | class ScriptExecutor(ABC): 15 | 16 | EXTENSION: str 17 | 18 | def __init__(self, automation_session: AzureAutomationSession, script_path: str) -> None: 19 | """ 20 | :param automation_session: Automation account session to use 21 | :param script_path: Script to execute within Automation Account 22 | """ 23 | super().__init__() 24 | self.automation_session = automation_session 25 | self.script_path = script_path 26 | 27 | @abstractmethod 28 | def execute_script(self, count: int): 29 | """ 30 | Executes a script within Azure Automation 31 | 32 | :param count: Number of executions 33 | """ 34 | pass 35 | 36 | 37 | class PowershellScriptExecutor(ScriptExecutor): 38 | 39 | EXTENSION = ".ps1" 40 | 41 | def execute_script(self, count: int): 42 | """ 43 | Executes Powershell module within Azure Automation 44 | 45 | :param count: Number of executions 46 | """ 47 | for index in range(count): 48 | logger.info(f"Triggering Powershell execution - {index+1}/{count}:") 49 | logger.add_indent() 50 | module_name = str(uuid.uuid4()) 51 | zipped_ps_module = utils.zip_file(self.script_path, f"{module_name}.psm1") 52 | self.automation_session.upload_powershell_module(module_name, zipped_ps_module) 53 | logger.info(f"Triggered module import flow in Automation Account. Code execution will be triggered in a few minutes...") 54 | logger.remove_indent() 55 | 56 | 57 | class PythonScriptExecutor(ScriptExecutor): 58 | """ 59 | ScriptExecutor class to execute Python scripts 60 | """ 61 | EXTENSION = ".py" 62 | PIP_PACKAGE_NAME = "pip" 63 | UPLOAD_STATE_CHECK_INTERVAL_SECONDS = 20 64 | CUSTOM_PIP_PATH = os.path.join(utils.RESOURCES_DIRECTORY, PIP_PACKAGE_NAME) 65 | DUMMY_WHL_PATH = os.path.join(utils.RESOURCES_DIRECTORY, "random_whl-0.0.1-py3-none-any.whl") 66 | 67 | def __init__(self, automation_session: AzureAutomationSession, script_path: str, requirements_file: str = None) -> None: 68 | """ 69 | :param automation_session: Automation account session to use 70 | :param script_path: Script to execute within Automation Account 71 | :param requirements_path: Path to requirements file to be installed and use by the script 72 | """ 73 | super().__init__(automation_session, script_path) 74 | self.requirements_file = requirements_file 75 | 76 | def _delete_pip_if_exists(self): 77 | """ 78 | Validate 'pip' package does not exist 79 | 80 | :param delete_if_exists: If True and package exists, deletes the package 81 | 82 | :raises CloudMinerException: If package exists and 'delete_if_exists' is False 83 | """ 84 | pip_package = self.automation_session.get_python_package(PythonScriptExecutor.PIP_PACKAGE_NAME) 85 | if pip_package: 86 | logger.warning(f"Package '{PythonScriptExecutor.PIP_PACKAGE_NAME}' already exists in Automation Account. Deleting package") 87 | self.automation_session.delete_python_package(PythonScriptExecutor.PIP_PACKAGE_NAME) 88 | 89 | def _wait_for_package_upload(self, package_name: str, timeout_seconds: int = UPLOAD_TIMEOUT): 90 | """ 91 | Wait until the package upload flow is finished or until timeout (Blocking) 92 | 93 | :param package_name: Python package name to wait for 94 | :param timeout_seconds: Maximum time to wait for the upload 95 | 96 | :raises CloudMinerException: If the upload flow has not started for the given package 97 | If upload flow has finished with an error 98 | If timeout is reached 99 | """ 100 | logger.info(f"Waiting for package to finish upload. This might take a few minutes...") 101 | logger.add_indent() 102 | start_time = time.time() 103 | end_time = start_time + timeout_seconds 104 | while time.time() < end_time: 105 | package_data = self.automation_session.get_python_package(package_name) 106 | if not package_data: 107 | raise CloudMinerException(f"Upload flow for package '{package_name}' has failed to be started") 108 | 109 | upload_state = package_data["properties"]["provisioningState"] 110 | if upload_state == UPLOAD_STATE.SUCCEEDED: 111 | logger.remove_indent() 112 | break 113 | elif upload_state == UPLOAD_STATE.FAILED: 114 | error = package_data["properties"]["error"]["message"] 115 | raise CloudMinerException("Python package upload failed. Error: ", error) 116 | else: 117 | logger.debug(f"Upload state - '{upload_state}'") 118 | time.sleep(PythonScriptExecutor.UPLOAD_STATE_CHECK_INTERVAL_SECONDS) 119 | else: 120 | raise CloudMinerException("Python package upload failed due to timeout") 121 | 122 | def _wrap_script(self) -> List[str]: 123 | """ 124 | Construct lines of code for installing Python packages 125 | """ 126 | INSTALL_REQUIREMENTS_CODE = [] 127 | if self.requirements_file: 128 | with open(self.requirements_file, 'r') as f: 129 | requirements = [line.replace('\n', '') for line in f.readlines()] 130 | INSTALL_REQUIREMENTS_CODE = ["import requests, subprocess, sys, os, tempfile", 131 | "tmp_folder = tempfile.gettempdir()", 132 | "sys.path.append(tmp_folder)", 133 | "tmp_pip = requests.get('https://bootstrap.pypa.io/get-pip.py').content", 134 | "open(os.path.join(tmp_folder, 'tmp_pip.py'), 'wb+').write(tmp_pip)", 135 | f"subprocess.run(f'{{sys.executable}} {{os.path.join(tmp_folder, \"tmp_pip.py\")}} {' '.join(requirements)} --target {{tmp_folder}}', shell=True)"] 136 | 137 | return '\n'.join(["# Auto added by CloudMiner", 138 | "######################################################################################"] + 139 | INSTALL_REQUIREMENTS_CODE + 140 | ["def _main():\n\tpass", 141 | "######################################################################################\n"]) 142 | 143 | 144 | def _create_whl_for_upload(self) -> str: 145 | """ 146 | Creates a Python package whl using the given Python script 147 | 148 | :raises CloudMinerException: If failed to create .whl file 149 | """ 150 | main_file_path = os.path.join(PythonScriptExecutor.CUSTOM_PIP_PATH, "src", PythonScriptExecutor.PIP_PACKAGE_NAME, "main.py") 151 | shutil.copyfile(self.script_path, main_file_path) 152 | 153 | #Add a main function to the file to be used as the entry point 154 | with open(main_file_path, 'r') as f: 155 | raw_main_file = f.read() 156 | 157 | wrapped_main_file = self._wrap_script() + raw_main_file 158 | 159 | with open(main_file_path, 'w') as f: 160 | f.write(wrapped_main_file) 161 | 162 | return utils.package_to_whl(PythonScriptExecutor.CUSTOM_PIP_PATH) 163 | 164 | def execute_script(self, count: int): 165 | """ 166 | Executes Python script within Azure Automation 167 | 168 | :param script_path: .whl file path. Get using 'prepare_file_for_upload' 169 | :param count: Number of executions 170 | """ 171 | self._delete_pip_if_exists() 172 | whl_path = self._create_whl_for_upload() 173 | logger.info(f"Replacing the default 'pip' package present in the Automation account:") 174 | logger.add_indent() 175 | self.automation_session.upload_python_package(PythonScriptExecutor.PIP_PACKAGE_NAME, whl_path) 176 | self._wait_for_package_upload(PythonScriptExecutor.PIP_PACKAGE_NAME) 177 | logger.remove_indent() 178 | 179 | logger.info("Successfully replaced the pip package!") 180 | for index in range(count): 181 | logger.info(f"Triggering Python execution - {index+1}/{count}:") 182 | logger.add_indent() 183 | package_name = str(uuid.uuid4()) 184 | self.automation_session.upload_python_package(package_name, PythonScriptExecutor.DUMMY_WHL_PATH) 185 | logger.info(f"Code execution will be triggered in a few minutes...") 186 | logger.remove_indent() 187 | -------------------------------------------------------------------------------- /src/cloudminer/azure_automation_session.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import requests 4 | import posixpath 5 | import urllib.parse 6 | from enum import Enum 7 | from http import HTTPStatus 8 | from datetime import datetime 9 | from datetime import timedelta 10 | from requests.exceptions import ReadTimeout, ChunkedEncodingError 11 | 12 | from cloudminer.logger import logger 13 | from cloudminer.exceptions import CloudMinerException 14 | 15 | URL_GET_STORAGE_BLOB = "https://s2.automation.ext.azure.com/api/Orchestrator/GenerateSasLinkUri?accountId={account_id}&assetType=Module" 16 | AZURE_MANAGEMENT_URL = "https://management.azure.com" 17 | DEFAULT_API_VERSION = "2018-06-30" 18 | UPLOAD_TIMEOUT = 300 19 | SLEEP_BETWEEN_ERROR_SECONDS = 10 20 | TIME_BETWEEN_REQUESTS_SECONDS = 0.5 21 | TEMP_STORAGE_VALID_SAFETY_SECONDS = 60 22 | HTTP_REQUEST_TIMEOUT = 5 23 | 24 | class UPLOAD_STATE(str, Enum): 25 | """ 26 | Package/Module upload state 27 | """ 28 | FAILED = "Failed" 29 | CREATING = "Creating" 30 | SUCCEEDED = "Succeeded" 31 | CONTENT_VALIDATED = "ContentValidated" 32 | CONTENT_DOWNLOADED = "ContentDownloaded" 33 | CONNECTION_TYPE_IMPORTED = "ConnectionTypeImported" 34 | RUNNING_IMPORT_MODULE_RUNBOOK = "RunningImportModuleRunbook" 35 | 36 | class AzureAutomationSession: 37 | """ 38 | Represents a session of Azure Automation 39 | """ 40 | def __init__(self, account_id: str, access_token: str) -> None: 41 | """ 42 | Initiate an Azure Automation session 43 | Validates that the Automation Account exists and the access token is valid 44 | 45 | :param account_id: Automation account ID - /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Automation/automationAccounts/{automationAccountName} 46 | :param access_token: Azure access token 47 | 48 | :raises CloudMinerException: If access token proivided is not valid 49 | If Automation Account ID is not valid 50 | If Automation Account provided does not exist 51 | """ 52 | self.__account_id = account_id 53 | self.__access_token = access_token 54 | self.__next_request_time = 0 55 | try: 56 | self.__http_request("GET", self.__get_url()) 57 | logger.info("Access token is valid") 58 | except requests.HTTPError as e: 59 | if e.response.status_code == HTTPStatus.UNAUTHORIZED: 60 | raise CloudMinerException("Access token provided is not valid") from e 61 | if e.response.status_code == HTTPStatus.BAD_REQUEST: 62 | raise CloudMinerException(f"Automation Account ID provided is not valid - '{account_id}'") from e 63 | if e.response.status_code == HTTPStatus.NOT_FOUND: 64 | raise CloudMinerException(f"Automation Account does not exists - '{account_id}'") from e 65 | raise 66 | 67 | def __get_url(self, path: str = "") -> str: 68 | """ 69 | Construct a url for the given path in the Automation Account 70 | """ 71 | return posixpath.join(AZURE_MANAGEMENT_URL, 72 | self.__account_id[1:], 73 | path) + f"?api-version={DEFAULT_API_VERSION}" 74 | 75 | def __wait_for_next_request(self): 76 | """ 77 | Helper function to make sure we wait before each request 78 | """ 79 | current_time = time.time() 80 | time_gap = self.__next_request_time - current_time 81 | if time_gap > 0: 82 | time.sleep(time_gap) 83 | 84 | self.__next_request_time = time.time() + TIME_BETWEEN_REQUESTS_SECONDS 85 | 86 | def __http_request(self, 87 | http_method: str, 88 | url: str, 89 | headers: dict = None, 90 | add_auth_info: bool = True, 91 | retries: int = 5, 92 | timeout: int = HTTP_REQUEST_TIMEOUT, 93 | **kwargs) -> requests.Response: 94 | """ 95 | Safe HTTP request to Azure services 96 | 97 | :param http_method: HTTP method of the request 98 | :param url: URL of the request 99 | :param headers: Headers of the request 100 | :param authorization: If True, set the 'Authorization' header 101 | :param retries: Retries count on a bad server response 102 | :return: Response object 103 | 104 | :raises HTTPError: If bad response is received 105 | """ 106 | self.__wait_for_next_request() 107 | 108 | if headers is None: 109 | headers = {} 110 | 111 | if add_auth_info: 112 | headers["Authorization"] = f"Bearer {self.__access_token}" 113 | 114 | for _ in range(retries): 115 | resp = None 116 | try: 117 | resp = requests.request(http_method, url, headers=headers, timeout=timeout, **kwargs) 118 | except (ReadTimeout, ChunkedEncodingError, ConnectionError): # Bad response from server 119 | pass 120 | 121 | if resp is None or resp.status_code in [HTTPStatus.TOO_MANY_REQUESTS, 122 | HTTPStatus.GATEWAY_TIMEOUT, 123 | HTTPStatus.SERVICE_UNAVAILABLE]: 124 | 125 | logger.warning(f"Too many requests. Retrying in {SLEEP_BETWEEN_ERROR_SECONDS} seconds...") 126 | time.sleep(SLEEP_BETWEEN_ERROR_SECONDS) 127 | else: 128 | resp.raise_for_status() 129 | return resp 130 | else: 131 | raise CloudMinerException(f"Failed to send HTTP request - Reached maximum retries. "\ 132 | f"Method - '{http_method}', url - '{url}'") 133 | 134 | def __upload_file_to_temp_storage(self, file_path: str) -> str: 135 | """ 136 | Create a temp storage and upload a file 137 | 138 | :param file_path: File path to upload 139 | :return: Temp storage url 140 | """ 141 | url = URL_GET_STORAGE_BLOB.format(account_id=self.__account_id) 142 | self.__current_temp_storage_url = self.__http_request("GET", url).json() 143 | logger.debug("Temporary blob storage created successfully") 144 | 145 | with open(file_path, "rb") as f: 146 | file_data = f.read() 147 | 148 | self.__http_request("PUT", 149 | self.__current_temp_storage_url, 150 | headers={"x-ms-blob-type": "BlockBlob"}, 151 | add_auth_info=False, 152 | data=file_data) 153 | 154 | file_name = os.path.basename(file_path) 155 | logger.debug(f"File '{file_name}' uploaded to temporary storage") 156 | return self.__current_temp_storage_url 157 | 158 | def upload_powershell_module(self, module_name: str, zipped_ps_module: str): 159 | """ 160 | Upload a Powershell module to the Automation Account 161 | """ 162 | logger.info(f"Uploading Powershell module '{module_name}'") 163 | 164 | temp_storage_url = self.__upload_file_to_temp_storage(zipped_ps_module) 165 | url = self.__get_url(f"modules/{module_name}") 166 | request_data = { 167 | "properties": { 168 | "contentLink": { 169 | "uri": temp_storage_url 170 | } 171 | } 172 | } 173 | self.__http_request("PUT", url, json=request_data) 174 | 175 | def upload_python_package(self, package_name: str, whl_path: str): 176 | """ 177 | Upload a Python package from a given blob storage 178 | """ 179 | logger.info(f"Uploading Python package - '{package_name}':") 180 | 181 | temp_storage_url = self.__upload_file_to_temp_storage(whl_path) 182 | url = self.__get_url(f"python3Packages/{package_name}") 183 | request_data = { 184 | "properties": { 185 | "contentLink": { 186 | "uri": temp_storage_url 187 | } 188 | } 189 | } 190 | self.__http_request("PUT", url, json=request_data) 191 | logger.info(f"Triggered package import flow in Automation Account.") 192 | 193 | def get_python_package(self, package_name: str) -> dict: 194 | """ 195 | Retrieve a Python package. Return None if does not exist 196 | """ 197 | url = self.__get_url(f"python3Packages/{package_name}") 198 | try: 199 | package_data = self.__http_request("GET", url).json() 200 | except requests.HTTPError as e: 201 | if e.response.status_code == HTTPStatus.NOT_FOUND: 202 | return None 203 | else: 204 | raise 205 | 206 | return package_data 207 | 208 | def delete_python_package(self, package_name: str): 209 | """ 210 | Delete a Python package 211 | 212 | :raises CloudMinerException: If the given package does not exists 213 | """ 214 | url = self.__get_url(f"python3Packages/{package_name}") 215 | try: 216 | self.__http_request("DELETE", url) 217 | except requests.HTTPError as e: 218 | if e.response.status_code == HTTPStatus.NOT_FOUND: 219 | raise CloudMinerException(f"Failed to delete package {package_name}. Package does not exist") 220 | else: 221 | raise --------------------------------------------------------------------------------