├── .gitignore ├── README.md ├── bob ├── bob-config.json.sample ├── bob.py ├── clients │ ├── __init__.py │ ├── azure │ │ ├── __init__.py │ │ ├── azure_blueprint_factory.py │ │ ├── azure_client.py │ │ └── blueprints │ │ │ ├── __init__.py │ │ │ ├── azure_blueprint.py │ │ │ ├── azure_build.py │ │ │ ├── azure_download.py │ │ │ └── build_instance.py │ └── client_factory.py ├── config.py └── menu.py └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | db.sqlite3 61 | db.sqlite3-journal 62 | 63 | # Flask stuff: 64 | instance/ 65 | .webassets-cache 66 | 67 | # Scrapy stuff: 68 | .scrapy 69 | 70 | # Sphinx documentation 71 | docs/_build/ 72 | 73 | # PyBuilder 74 | target/ 75 | 76 | # Jupyter Notebook 77 | .ipynb_checkpoints 78 | 79 | # IPython 80 | profile_default/ 81 | ipython_config.py 82 | 83 | # pyenv 84 | .python-version 85 | 86 | # pipenv 87 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 88 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 89 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 90 | # install all needed dependencies. 91 | #Pipfile.lock 92 | 93 | # celery beat schedule file 94 | celerybeat-schedule 95 | 96 | # SageMath parsed files 97 | *.sage.py 98 | 99 | # Environments 100 | .env 101 | .venv 102 | env/ 103 | venv/ 104 | ENV/ 105 | env.bak/ 106 | venv.bak/ 107 | 108 | # Spyder project settings 109 | .spyderproject 110 | .spyproject 111 | 112 | # Rope project settings 113 | .ropeproject 114 | 115 | # mkdocs documentation 116 | /site 117 | 118 | # mypy 119 | .mypy_cache/ 120 | .dmypy.json 121 | dmypy.json 122 | 123 | # Pyre type checker 124 | .pyre/ 125 | 126 | # Config file 127 | bob-config.json 128 | build_configs/ 129 | .DS_Store 130 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Bob the Builder 2 | 3 | Can we build it? Yes we can! Bob is a framework for automating builds for payloads in a "scriptably" configurable way. Read more at [https://coffeegist.com/security/automation-through-azure-with-bob/](https://coffeegist.com/security/automation-through-azure-with-bob/). 4 | 5 | ## Dependencies 6 | 7 | To ensure use of python virtual environments on Ubuntu, run the following command to install the necessary packages. 8 | ```bash 9 | sudo apt-get install python3-venv 10 | ``` 11 | 12 | ## Installation 13 | 14 | ```bash 15 | git clone git@github.com:coffeegist/bob-the-builder.git 16 | cd bob-the-builder 17 | python3 -m venv venv 18 | . ./venv/bin/activate 19 | pip install -r requirements.txt 20 | ``` 21 | 22 | ## Setup 23 | 24 | Copy and edit the `bob-config.json` file to include a personal access token and organization url to the Azure DevOps account of your choosing. 25 | 26 | ```bash 27 | cd bob 28 | cp bob-config.json.sample bob-config.json 29 | ``` 30 | 31 | ## Running 32 | 33 | For this project to function, you need to have access to an Azure DevOps account that has pre-existing pipelines configured to build payloads. 34 | 35 | ### Creating a token 36 | 37 | Login to https://dev.azure.com/YOUR_USERNAME/_usersSettings/tokens 38 | - Click **+ New Token** 39 | - Name: “Bob” 40 | - Organization: "\" 41 | - Scope: Full Access 42 | - Click **Create** 43 | 44 | ### Configuring a blueprint 45 | 46 | Once you have access, run the following command to configure a blueprint: 47 | 48 | ```bash 49 | python bob.py configure -f my-blueprint.json 50 | ``` 51 | 52 | ### Queueing a blueprint 53 | 54 | Feel free to modify your blueprint manually, especially if you are building a pipeline that has queue-time variables. Then, using your newly configured blueprint, `my-blueprint.json`: 55 | 56 | ```bash 57 | python bob.py run -f my-blueprint.json -o ./build-artifacts 58 | ``` 59 | 60 | ## When you're finished... 61 | 62 | This project obviously uses virtual environments (venv). When you're done with Bob, just deactivate your virtual environment by issuing the following command: 63 | 64 | ```bash 65 | deactivate 66 | ``` 67 | 68 | ## That's it!! 69 | 70 | Please post any issues you come across! 71 | -------------------------------------------------------------------------------- /bob/bob-config.json.sample: -------------------------------------------------------------------------------- 1 | { 2 | "azure_personal_access_token": "SOME_ACCESS_TOKEN", 3 | "azure_organization_url": "https://dev.azure.com/YOUR_ORGANIZATION_HERE" 4 | } 5 | -------------------------------------------------------------------------------- /bob/bob.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | from menu import Menu 4 | from clients.client_factory import ClientFactory 5 | 6 | 7 | def get_client(client_name, config_file): 8 | return ClientFactory.get_client(client_name, config_file) 9 | 10 | 11 | def configure_cmd(args): 12 | blueprints = [] 13 | 14 | while True: 15 | blueprints.extend(args._client.create_blueprints()) 16 | args._client.save_blueprints(blueprints, args.filename) 17 | if not Menu.yes_or_no("\nAdd another job?"): 18 | break 19 | 20 | 21 | def run_cmd(args): 22 | blueprints = args._client.load_blueprints(args.filename) 23 | 24 | for blueprint in blueprints: 25 | args._client.execute_blueprint(blueprint, args.output_directory) 26 | 27 | 28 | def main(): 29 | parser = argparse.ArgumentParser(description='Bob the Builder') 30 | parser.add_argument('-c', '--config', metavar='FILE', 31 | required=False, default='bob-config.json', help='Bob config file location') 32 | 33 | subparsers = parser.add_subparsers() 34 | 35 | # "configure" 36 | configure_parser = subparsers.add_parser('configure', help="Generate a blueprint for future jobs") 37 | configure_parser.set_defaults(dispatch=configure_cmd) 38 | configure_parser.add_argument('-f', '--filename', metavar='FILE', 39 | required=True, help='File to write blueprint to') 40 | 41 | # "run" 42 | run_parser = subparsers.add_parser('run', help="Queue a pre-generated blueprint") 43 | run_parser.set_defaults(dispatch=run_cmd) 44 | run_parser.add_argument('-f', '--filename', metavar='FILE', 45 | required=True, help='File to read blueprint from') 46 | run_parser.add_argument('-o', '--output-directory', metavar='DIR', 47 | required=False, default='.', help='Directory to output artifacts to') 48 | 49 | args = parser.parse_args() 50 | args._client = get_client('Azure', args.config) 51 | 52 | if 'dispatch' in args: 53 | cmd = args.dispatch 54 | del args.dispatch 55 | cmd(args) 56 | else: 57 | parser.print_usage() 58 | 59 | 60 | if __name__ == "__main__": 61 | main() 62 | -------------------------------------------------------------------------------- /bob/clients/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coffeegist/bob-the-builder/7095983e0729535bedb5ad08b61da3d38ef7fcb6/bob/clients/__init__.py -------------------------------------------------------------------------------- /bob/clients/azure/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coffeegist/bob-the-builder/7095983e0729535bedb5ad08b61da3d38ef7fcb6/bob/clients/azure/__init__.py -------------------------------------------------------------------------------- /bob/clients/azure/azure_blueprint_factory.py: -------------------------------------------------------------------------------- 1 | from azure.devops.credentials import BasicAuthentication 2 | from azure.devops.connection import Connection 3 | 4 | from menu import Menu 5 | from .blueprints import * 6 | 7 | class AzureBlueprintFactory(): 8 | 9 | 10 | def __init__(self, connection): 11 | self._connection = connection 12 | self._BLUEPRINT_ACTION_MAP = { 13 | 'Build' : self.create_build_blueprints, 14 | 'Download' : self.create_download_blueprints 15 | } 16 | 17 | 18 | def create_blueprints(self): 19 | project = self._select_project() 20 | action = self._select_action() 21 | 22 | return self._BLUEPRINT_ACTION_MAP[action](project) 23 | 24 | 25 | def create_build_blueprints(self, project): 26 | blueprints = [] 27 | definitions = self._select_definition(project, multiple=True) 28 | for definition in definitions: 29 | print("\n--- Configuring {} ---".format(definition.name)) 30 | azure_blueprint = AzureBuild() 31 | azure_blueprint.set_project(project.name) 32 | azure_blueprint.set_definition(definition.name) 33 | azure_blueprint.add_build_instance( 34 | self._select_definition_queue_time_variables(definition)) 35 | 36 | if Menu.yes_or_no('Use default agent queue?'): 37 | agent_queue = definition.queue 38 | else: 39 | agent_queue = self._select_agent_queue(project) 40 | 41 | azure_blueprint.set_agent_queue(agent_queue.name) 42 | 43 | if Menu.yes_or_no('Use default agent specification?'): 44 | try: 45 | agent_specification = (definition.additional_properties 46 | ['process']['target']['agentSpecification']['identifier']) 47 | except: 48 | agent_specification = None 49 | else: 50 | agent_specification = self._select_agent_specification(agent_queue) 51 | 52 | azure_blueprint.set_agent_specification(agent_specification) 53 | 54 | if Menu.yes_or_no('Download build artifacts?'): 55 | azure_blueprint.set_download_artifacts(True) 56 | else: 57 | azure_blueprint.set_download_artifacts(False) 58 | 59 | blueprints.append(azure_blueprint) 60 | 61 | return blueprints 62 | 63 | 64 | def create_download_blueprints(self, project): 65 | blueprints = [] 66 | definitions = self._select_definition(project, multiple=True) 67 | for definition in definitions: 68 | azure_blueprint = AzureDownload() 69 | azure_blueprint.set_project(project.name) 70 | azure_blueprint.set_definition(definition.name) 71 | blueprints.append(azure_blueprint) 72 | 73 | return blueprints 74 | 75 | 76 | def load_blueprints(self, filename): 77 | return AzureBlueprint.load_from_file(filename) 78 | 79 | 80 | def _select_project(self): 81 | core_client = self._connection.clients.get_core_client() 82 | projects = core_client.get_projects() 83 | print("\n-- Projects --") 84 | return Menu.choose_from_list(projects, "project") 85 | 86 | 87 | def _select_definition(self, project, multiple=False): 88 | build_client = self._connection.clients_v5_1.get_build_client() 89 | definitions = build_client.get_definitions(project.name, include_all_properties=True) 90 | print("\n-- Definitions --") 91 | if multiple: 92 | return Menu.choose_multiple_from_list(definitions, "definition") 93 | else: 94 | return Menu.choose_from_list(definitions, "definition") 95 | 96 | 97 | def _select_agent_pool(self, project): 98 | task_agent_client = self._connection.clients.get_task_agent_client() 99 | agent_pools = task_agent_client.get_agent_pools() 100 | print("\n-- Agent Pools --") 101 | return Menu.choose_from_list(agent_pools, "Agent Pool") 102 | 103 | 104 | def _select_action(self): 105 | print("\n-- Actions --") 106 | return Menu.choose_from_list( 107 | list(self._BLUEPRINT_ACTION_MAP.keys()), "action", None 108 | ) 109 | 110 | 111 | def _select_agent_queue(self, project): 112 | task_agent_client = self._connection.clients_v5_1.get_task_agent_client() 113 | agent_queues = task_agent_client.get_agent_queues(project.id) 114 | print("\n-- Agent Queues --") 115 | return Menu.choose_from_list(agent_queues, "Agent Queue") 116 | 117 | 118 | def _select_agent_specification(self, queue): 119 | # Agent strings should not change according to guy @ microsoft 120 | # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops 121 | 122 | agent_specification = Menu.choose_from_list([ 123 | "windows-latest", 124 | "windows-2019", 125 | "vs2017-win2016", 126 | "vs2015-win2012r2", 127 | "win1803", 128 | "ubuntu-latest", 129 | "ubuntu-16.04", 130 | "macOS-latest", 131 | "macOS-10.14", 132 | "macOS-10.13" 133 | ], "Agent Specifications", field=None) 134 | 135 | return agent_specification 136 | 137 | 138 | def _select_definition_queue_time_variables(self, definition): 139 | azure_build_instance = AzureBuildInstance(name="Default") 140 | 141 | if "variables" in definition.additional_properties: 142 | for key, value in definition.additional_properties["variables"].items(): 143 | if "allowOverride" in value: 144 | if value["allowOverride"]: 145 | azure_build_instance.add_queue_time_variable(key, value["value"]) 146 | 147 | return azure_build_instance 148 | -------------------------------------------------------------------------------- /bob/clients/azure/azure_client.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import time 4 | 5 | from azure.devops.credentials import BasicAuthentication 6 | from azure.devops.connection import Connection 7 | from azure.devops.v5_1.build.models import AgentSpecification, Build 8 | 9 | from .azure_blueprint_factory import AzureBlueprintFactory 10 | from .blueprints import * 11 | 12 | class AzureClient: 13 | 14 | def __init__(self, config): 15 | self._credentials = BasicAuthentication('', config['azure_personal_access_token']) 16 | self._connection = Connection(base_url=config['azure_organization_url'], creds=self._credentials) 17 | 18 | 19 | def create_blueprints(self): 20 | return AzureBlueprintFactory(self._connection).create_blueprints() 21 | 22 | 23 | def load_blueprints(self, filename): 24 | return AzureBlueprint.load_from_file(filename) 25 | 26 | 27 | def save_blueprints(self, blueprints, filename): 28 | AzureBlueprint.save_blueprints_to_file(blueprints, filename) 29 | 30 | 31 | def execute_blueprint(self, blueprint, output_directory): 32 | if isinstance(blueprint, AzureBuild): 33 | self._execute_azure_build_blueprint(blueprint, output_directory) 34 | elif isinstance(blueprint, AzureDownload): 35 | self._execute_azure_download_blueprint(blueprint, output_directory) 36 | else: 37 | print("Unknown blueprint type: {}".format(type(blueprint.__name__))) 38 | 39 | print() 40 | 41 | 42 | def download_build_artifacts(self, azure_build, download_dir='.', name=None): 43 | build_client = self._connection.clients.get_build_client() 44 | artifacts = build_client.get_artifacts(azure_build.project.name, azure_build.id) 45 | for artifact in artifacts: 46 | if artifact.resource.download_url is None: 47 | continue 48 | 49 | extension = self._get_extension_from_artifact_download_url(artifact.resource.download_url) 50 | 51 | filename = "{}_{}.{}".format(artifact.name, azure_build.id, extension) 52 | if name is not None: 53 | filename = "{}_{}".format(name, filename) 54 | 55 | if not os.path.exists(download_dir): 56 | os.makedirs(download_dir) 57 | 58 | filepath = os.path.join(download_dir, filename) 59 | print ("Downloading {} ...".format(filepath), end='', flush=True) 60 | 61 | with open(filepath, 'wb') as f: 62 | for chunk in build_client.get_artifact_content_zip(azure_build.project.name, azure_build.id, artifact.name): 63 | f.write(chunk) 64 | 65 | print(" Finished!") 66 | 67 | 68 | def _build_definition(self, project, definition, queue, agent_specification, source_branch, parameters, tags): 69 | build_client = self._connection.clients.get_build_client() 70 | new_build = Build(definition=definition, queue=queue, agent_specification=agent_specification, source_branch=source_branch, parameters=parameters) 71 | build = build_client.queue_build(new_build, project.id) 72 | 73 | previous_status = None 74 | first_newline = '' 75 | while build.status != "completed": 76 | if previous_status != build.status: 77 | print("{}Status - {}".format(first_newline, build.status), end='') 78 | previous_status = build.status 79 | first_newline = '\n' 80 | else: 81 | print(".", end='', flush=True) 82 | build = build_client.get_build(project.id, build.id) 83 | time.sleep(2) 84 | 85 | print() 86 | build_client.add_build_tags(tags, project.id, build.id) 87 | 88 | return build 89 | 90 | 91 | def _execute_azure_build_blueprint(self, blueprint, output_directory): 92 | project = self.get_project_by_name(blueprint.get_project()) 93 | definition = self.get_definition_by_name( 94 | project.name, blueprint.get_definition() 95 | ) 96 | 97 | print('\nStarting {} for {}->{} ...'.format( 98 | blueprint.__class__.__name__, 99 | blueprint.get_project(), 100 | blueprint.get_definition())) 101 | 102 | queue = self.get_agent_queue_by_name( 103 | project.name, blueprint.get_agent_queue() 104 | ) 105 | if blueprint.get_agent_specification() != None: 106 | agent_specification = AgentSpecification( 107 | identifier=blueprint.get_agent_specification() 108 | ) 109 | else: 110 | agent_specification = None 111 | 112 | for instance in blueprint.get_build_instances(): 113 | parameters = json.dumps(instance.get_queue_time_variables()) 114 | tags = instance.get_tags() 115 | instance_name = instance.get_name() 116 | if instance_name is None: 117 | instance_name = "Default" 118 | print("\nBuilding instance {}...".format(instance_name)) 119 | 120 | build = self._build_definition( 121 | project, definition, queue, agent_specification, 122 | blueprint.get_source_branch(), parameters, tags 123 | ) 124 | 125 | if build.result == AzureBuild.RESULT_SUCCESS: 126 | print("Build succeeded!") 127 | 128 | if blueprint.get_download_artifacts(): 129 | download_name = self._normalize_filename( 130 | '_'.join(filter(None, [ 131 | blueprint.get_definition(), 132 | instance_name, 133 | '_'.join(tags) 134 | ]))) 135 | 136 | self.download_build_artifacts( 137 | build, output_directory, download_name 138 | ) 139 | else: 140 | print("Skipping download...") 141 | else: 142 | print("Build failed with: {}!".format(build.result)) 143 | 144 | 145 | def _normalize_filename(self, name): 146 | return name.strip().lower().replace(' ', '-') 147 | 148 | 149 | def _execute_azure_download_blueprint(self, blueprint, output_directory): 150 | project = self.get_project_by_name(blueprint.get_project()) 151 | definition = self.get_definition_by_name(project.name, blueprint.get_definition()) 152 | print('\nStarting {} for {}->{}...\n'.format( 153 | blueprint.__class__.__name__, 154 | blueprint.get_project(), 155 | blueprint.get_definition())) 156 | 157 | build_client = self._connection.clients.get_build_client() 158 | 159 | kwargs = { 160 | 'project': project.name, 161 | 'definitions': [definition.id], 162 | 'branch_name': 'refs/heads/{}'.format(blueprint.get_source_branch()), 163 | 'status_filter': 'completed', 164 | 'result_filter': 'succeeded', 165 | 'top': 1 166 | } 167 | 168 | if len(blueprint.get_tags()) > 0: 169 | kwargs['tag_filters'] = blueprint.get_tags(); 170 | 171 | builds = build_client.get_builds(**kwargs) 172 | if len(builds) == 0: 173 | print("No builds found! Skipping...") 174 | else: 175 | for build in builds: 176 | download_name = self._normalize_filename( 177 | '_'.join(filter(None, [ 178 | blueprint.get_definition(), 179 | '_'.join(blueprint.get_tags()) 180 | ]))) 181 | self.download_build_artifacts(build, output_directory, download_name) 182 | 183 | 184 | def _get_extension_from_artifact_download_url(self, url): 185 | extension = None 186 | format_split = url.split("format=") 187 | if len(format_split) > 0: 188 | extension = format_split[1].split("&")[0] 189 | else: 190 | extension = "zip" 191 | return extension 192 | 193 | 194 | def _get_build_artifact_download_links(self, build): 195 | links = [] 196 | build_client = self._connection.clients.get_build_client() 197 | artifacts = build_client.get_artifacts(build.project.name, build.id) 198 | for artifact in artifacts: 199 | links.append(artifact.resource.download_url) 200 | return links 201 | 202 | 203 | def get_project_by_name(self, project_name): 204 | core_client = self._connection.clients.get_core_client() 205 | return core_client.get_project(project_name) 206 | 207 | 208 | def get_definition_by_name(self, project_name, definition_name): 209 | build_client = self._connection.clients_v5_1.get_build_client() 210 | definitions = build_client.get_definitions(project_name, name=definition_name, include_all_properties=True) 211 | if len(definitions) > 1: 212 | print("Duplicate definitions exist for {}".format(definition_name)) 213 | sys.exit(0) 214 | elif len(definitions) == 0: 215 | print("No definitions exist for {}".format(definition_name)) 216 | sys.exit(0) 217 | return definitions[0] 218 | 219 | 220 | def get_agent_queue_by_name(self, project_name, queue_name): 221 | task_agent_client = self._connection.clients_v5_1.get_task_agent_client() 222 | queues = task_agent_client.get_agent_queues(project_name, queue_name=queue_name) 223 | if len(queues) > 1: 224 | print("Duplicate queues exist for {}".format(queue_name)) 225 | sys.exit(0) 226 | elif len(queues) == 0: 227 | print("No queues exist for {}".format(queue_name)) 228 | sys.exit(0) 229 | return queues[0] 230 | -------------------------------------------------------------------------------- /bob/clients/azure/blueprints/__init__.py: -------------------------------------------------------------------------------- 1 | from .azure_blueprint import AzureBlueprint 2 | from .azure_build import AzureBuild 3 | from .azure_download import AzureDownload 4 | from .build_instance import AzureBuildInstance 5 | -------------------------------------------------------------------------------- /bob/clients/azure/blueprints/azure_blueprint.py: -------------------------------------------------------------------------------- 1 | import json 2 | import pprint 3 | 4 | class AzureBlueprint(): 5 | TYPE_KEY = "__type__" 6 | PROJECT_KEY = "project" 7 | DEFINITION_KEY = "definition" 8 | SOURCE_BRANCH_KEY = "source_branch" 9 | 10 | def __init__(self, project=None, definition=None): 11 | self._type = 'NONE' 12 | self._project = project 13 | self._definition = definition 14 | self._source_branch = 'master' 15 | 16 | def get_actions(self): 17 | return self._actions 18 | 19 | 20 | def set_actions(self, actions): 21 | self._actions = actions 22 | 23 | 24 | def get_project(self): 25 | return self._project 26 | 27 | 28 | def set_project(self, project): 29 | self._project = project 30 | 31 | 32 | def get_definition(self): 33 | return self._definition 34 | 35 | 36 | def set_definition(self, definition): 37 | self._definition = definition 38 | 39 | 40 | def get_source_branch(self): 41 | return self._source_branch 42 | 43 | 44 | def set_source_branch(self, branch): 45 | self._source_branch = branch 46 | 47 | 48 | def save_to_file(self, filename): 49 | file_data = self.to_dict() 50 | with open(filename, 'w') as config_fp: 51 | json.dump(file_data, config_fp, sort_keys=True, indent=4) 52 | 53 | 54 | def to_dict(self): 55 | dict = { 56 | AzureBlueprint.TYPE_KEY: type(self).__name__, 57 | AzureBlueprint.PROJECT_KEY: self._project, 58 | AzureBlueprint.DEFINITION_KEY: self._definition, 59 | AzureBlueprint.SOURCE_BRANCH_KEY: self._source_branch 60 | } 61 | 62 | return dict 63 | 64 | 65 | def populate_from_dict(self, config): 66 | if AzureBlueprint.PROJECT_KEY in config.keys(): 67 | self._project = config[AzureBlueprint.PROJECT_KEY] 68 | if AzureBlueprint.DEFINITION_KEY in config.keys(): 69 | self._definition = config[AzureBlueprint.DEFINITION_KEY] 70 | if AzureBlueprint.SOURCE_BRANCH_KEY in config.keys(): 71 | self._source_branch = config[AzureBlueprint.SOURCE_BRANCH_KEY] 72 | 73 | 74 | @staticmethod 75 | def save_blueprints_to_file(blueprints, filename): 76 | file_data = [] 77 | for blueprint in blueprints: 78 | file_data.append(blueprint.to_dict()) 79 | 80 | with open(filename, 'w') as config_fp: 81 | json.dump(file_data, config_fp, sort_keys=True, indent=4) 82 | 83 | 84 | @staticmethod 85 | def load_from_file(filename): 86 | data = None 87 | blueprints = [] 88 | blueprint_type_map = {} 89 | 90 | for blueprint_type in AzureBlueprint.__subclasses__(): 91 | blueprint_type_map[blueprint_type.__name__] = blueprint_type 92 | 93 | with open(filename) as data_fp: 94 | data = json.load(data_fp) 95 | 96 | if type(data) != list: 97 | data = [data] 98 | 99 | for config in data: 100 | if AzureBlueprint.TYPE_KEY in config.keys(): 101 | blueprint_type = config[AzureBlueprint.TYPE_KEY] 102 | if blueprint_type in blueprint_type_map.keys(): 103 | blueprint = blueprint_type_map[blueprint_type]() 104 | blueprint.populate_from_dict(config) 105 | blueprints.append(blueprint) 106 | else: 107 | print("Unknown blueprint type: {}".format(blueprint_type)) 108 | 109 | return blueprints 110 | -------------------------------------------------------------------------------- /bob/clients/azure/blueprints/azure_build.py: -------------------------------------------------------------------------------- 1 | import json 2 | import pprint 3 | 4 | from .azure_blueprint import AzureBlueprint 5 | from .build_instance import AzureBuildInstance 6 | 7 | class AzureBuild(AzureBlueprint): 8 | RESULT_SUCCESS = "succeeded" 9 | 10 | BUILD_INSTANCES_KEY = "build_instances" 11 | AGENT_QUEUE_KEY = "agent_queue" 12 | AGENT_SPECIFICATION_KEY = "agent_specification" 13 | DOWNLOAD_ARTIFACTS_KEY = "download_artifacts" 14 | 15 | 16 | def __init__(self): 17 | super().__init__() 18 | self._build_instances = [] 19 | self._agent_queue = None 20 | self._agent_specification = None 21 | self._download_artifacts = True 22 | 23 | 24 | def get_agent_queue(self): 25 | return self._agent_queue 26 | 27 | 28 | def set_agent_queue(self, agent_queue): 29 | self._agent_queue = agent_queue 30 | 31 | 32 | def get_agent_specification(self): 33 | return self._agent_specification 34 | 35 | 36 | def set_agent_specification(self, agent_specification): 37 | self._agent_specification = agent_specification 38 | 39 | 40 | def get_build_instances(self): 41 | return self._build_instances 42 | 43 | 44 | def get_download_artifacts(self): 45 | return self._download_artifacts 46 | 47 | 48 | def set_download_artifacts(self, download_artifacts): 49 | self._download_artifacts = download_artifacts 50 | 51 | 52 | def add_build_instance(self, build_instance): 53 | self._build_instances.append(build_instance) 54 | 55 | 56 | def to_dict(self): 57 | dict = super().to_dict() 58 | 59 | updates = { 60 | AzureBuild.AGENT_QUEUE_KEY: self._agent_queue, 61 | AzureBuild.AGENT_SPECIFICATION_KEY: self._agent_specification, 62 | AzureBuild.DOWNLOAD_ARTIFACTS_KEY: self._download_artifacts, 63 | AzureBuild.BUILD_INSTANCES_KEY: [] 64 | } 65 | 66 | for build_instance in self._build_instances: 67 | updates[AzureBuild.BUILD_INSTANCES_KEY].append(build_instance.to_dict()) 68 | 69 | dict.update(updates) 70 | 71 | return dict 72 | 73 | 74 | def populate_from_dict(self, config): 75 | super().populate_from_dict(config) 76 | 77 | if AzureBuild.AGENT_QUEUE_KEY in config.keys(): 78 | self._agent_queue = config[AzureBuild.AGENT_QUEUE_KEY] 79 | if AzureBuild.AGENT_SPECIFICATION_KEY in config.keys(): 80 | self._agent_specification = config[AzureBuild.AGENT_SPECIFICATION_KEY] 81 | if AzureBuild.DOWNLOAD_ARTIFACTS_KEY in config.keys(): 82 | self._download_artifacts = config[AzureBuild.DOWNLOAD_ARTIFACTS_KEY] 83 | 84 | if AzureBuild.BUILD_INSTANCES_KEY in config.keys(): 85 | for instance in config[AzureBuild.BUILD_INSTANCES_KEY]: 86 | build_instance = AzureBuildInstance.from_dict(instance) 87 | self.add_build_instance(build_instance) 88 | build_instance = None 89 | -------------------------------------------------------------------------------- /bob/clients/azure/blueprints/azure_download.py: -------------------------------------------------------------------------------- 1 | import json 2 | import pprint 3 | 4 | from .azure_blueprint import AzureBlueprint 5 | 6 | class AzureDownload(AzureBlueprint): 7 | TAGS_KEY = "tags" 8 | 9 | def __init__(self): 10 | super().__init__() 11 | self._tags = [] 12 | 13 | 14 | def get_tags(self): 15 | return self._tags 16 | 17 | 18 | def set_tags(self, tags): 19 | self._tags = tags 20 | 21 | 22 | def add_tag(self, tag): 23 | self._tags.append(tag) 24 | 25 | 26 | def to_dict(self): 27 | dict = super().to_dict() 28 | 29 | updates = { 30 | AzureDownload.TAGS_KEY: self._tags 31 | } 32 | 33 | dict.update(updates) 34 | 35 | return dict 36 | 37 | 38 | def populate_from_dict(self, config): 39 | super().populate_from_dict(config) 40 | 41 | if AzureDownload.TAGS_KEY in config.keys(): 42 | self._tags = config[AzureDownload.TAGS_KEY] 43 | -------------------------------------------------------------------------------- /bob/clients/azure/blueprints/build_instance.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | class AzureBuildInstance(): 4 | NAME_KEY = "name" 5 | QUEUE_TIME_VARIABLES_KEY = "queue_time_variables" 6 | TAGS_KEY = "tags" 7 | 8 | 9 | def __init__(self, name="Default"): 10 | self._name = name 11 | self._queue_time_variables = {} 12 | self._tags = [] 13 | 14 | 15 | def get_name(self): 16 | return self._name 17 | 18 | 19 | def get_queue_time_variables(self): 20 | return self._queue_time_variables 21 | 22 | 23 | def get_tags(self): 24 | return self._tags 25 | 26 | 27 | def add_queue_time_variable(self, key, value): 28 | self._queue_time_variables[key] = value 29 | 30 | 31 | def add_tag(self, tag): 32 | self._tags.append(tag) 33 | 34 | 35 | def add_tags(self, tag_list): 36 | self._tags += tag_list 37 | 38 | 39 | def to_dict(self): 40 | dict = { 41 | AzureBuildInstance.NAME_KEY: self._name, 42 | AzureBuildInstance.QUEUE_TIME_VARIABLES_KEY: {}, 43 | AzureBuildInstance.TAGS_KEY: self._tags 44 | } 45 | for key, value in self._queue_time_variables.items(): 46 | try: 47 | value = json.loads(value) 48 | except ValueError: 49 | pass 50 | dict[AzureBuildInstance.QUEUE_TIME_VARIABLES_KEY][key] = value 51 | return dict 52 | 53 | 54 | @staticmethod 55 | def from_dict(dict): 56 | build_instance = AzureBuildInstance(dict[AzureBuildInstance.NAME_KEY]) 57 | build_instance.add_tags(dict[AzureBuildInstance.TAGS_KEY]) 58 | for key, value in dict[AzureBuildInstance.QUEUE_TIME_VARIABLES_KEY].items(): 59 | try: 60 | if (not isinstance(value, str)): 61 | value = json.dumps(value) 62 | except ValueError: 63 | pass 64 | build_instance.add_queue_time_variable(key, value) 65 | return build_instance 66 | -------------------------------------------------------------------------------- /bob/clients/client_factory.py: -------------------------------------------------------------------------------- 1 | from config import Config 2 | from clients.azure.azure_client import AzureClient 3 | 4 | class ClientFactory(): 5 | _client_map = { 6 | 'Azure': AzureClient 7 | } 8 | 9 | def get_client_list(): 10 | return list(ClientFactory._client_map.keys()) 11 | 12 | def get_client(client_name, config_file): 13 | client_type = ClientFactory._client_map[client_name] 14 | return client_type(ClientFactory._get_config(config_file)) 15 | 16 | def _get_config(config_file): 17 | return Config(config_file) 18 | -------------------------------------------------------------------------------- /bob/config.py: -------------------------------------------------------------------------------- 1 | """ 2 | https://github.com/microsoft/azure-devops-python-samples/blob/master/src/config.py 3 | """ 4 | 5 | import json 6 | import os 7 | import pathlib 8 | import sys 9 | import pprint 10 | 11 | DEFAULT_CONFIG_FILE_NAME = "bob-config.json" 12 | CONFIG_KEYS = [ 13 | 'azure_organization_url', 14 | 'azure_personal_access_token', 15 | ] 16 | 17 | 18 | def emit(msg, *args): 19 | print(msg % args) 20 | 21 | 22 | class Config(): 23 | def __init__(self, filename=None): 24 | if not filename: 25 | runner_path = (pathlib.Path(os.getcwd()) / pathlib.Path(sys.argv[0])).resolve() 26 | filename = runner_path.parents[0] / pathlib.Path(DEFAULT_CONFIG_FILE_NAME) 27 | 28 | self._filename = filename 29 | 30 | try: 31 | with open(filename) as config_fp: 32 | self._config = json.load(config_fp) 33 | except FileNotFoundError: 34 | emit("warning: no config file found.") 35 | self._config = {} 36 | except json.JSONDecodeError: 37 | emit("possible bug: config file exists but isn't parseable") 38 | self._config = {} 39 | 40 | def __getitem__(self, name): 41 | self._check_if_name_valid(name) 42 | return self._config.get(name, None) 43 | 44 | def __setitem__(self, name, value): 45 | self._check_if_name_valid(name) 46 | self._config[name] = value 47 | 48 | def __delitem__(self, name): 49 | self._check_if_name_valid(name) 50 | self._config.pop(name, None) 51 | 52 | def __len__(self): 53 | return len(CONFIG_KEYS) 54 | 55 | def __iter__(self): 56 | for key in CONFIG_KEYS: 57 | yield key 58 | 59 | def save(self): 60 | with open(self._filename, 'w') as config_fp: 61 | json.dump(self._config, config_fp, sort_keys=True, indent=4) 62 | 63 | def _check_if_name_valid(self, name): 64 | if name not in CONFIG_KEYS: 65 | raise KeyError("{0} is not a valid config key".format(name)) 66 | -------------------------------------------------------------------------------- /bob/menu.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | class Menu: 4 | 5 | @staticmethod 6 | def choose_from_list(items, item_type="item", field="name"): 7 | while True: 8 | try: 9 | print() 10 | for index, item in enumerate(items, start=1): 11 | if field is None: 12 | item_title = item 13 | else: 14 | item_title = getattr(item, field) 15 | print("{0} - {1}".format(index, item_title)) 16 | 17 | if len(items) == 1: 18 | selection = 1 19 | else: 20 | selection = int(input("\nChoose {} number (0 to quit): ".format(item_type))) 21 | if selection <= len(items) and selection > 0: 22 | return items[selection - 1] 23 | elif selection == 0: 24 | sys.exit(0) 25 | except KeyboardInterrupt: 26 | sys.exit(0) 27 | except ValueError: 28 | print("Invalid selection.") 29 | 30 | 31 | @staticmethod 32 | def choose_multiple_from_list(items, item_type="item", field="name"): 33 | selected_items = [] 34 | 35 | while True: 36 | try: 37 | print() 38 | for index, item in enumerate(items, start=1): 39 | if field is None: 40 | item_title = item 41 | else: 42 | item_title = getattr(item, field) 43 | print("{0} - {1}".format(index, item_title)) 44 | 45 | if len(items) == 1: 46 | selections = ['1'] 47 | else: 48 | selections = input("\nChoose multiple comma-separated {} numbers (0 to quit): ".format(item_type)) 49 | 50 | for selection in selections.split(','): 51 | selection = int(selection) 52 | if selection <= len(items) and selection > 0: 53 | selected_items.append(items[selection - 1]) 54 | elif selection == 0: 55 | sys.exit(0) 56 | 57 | return selected_items 58 | except KeyboardInterrupt: 59 | sys.exit(0) 60 | except ValueError: 61 | print("Invalid selection.") 62 | 63 | 64 | @staticmethod 65 | def yes_or_no(question): 66 | result = True 67 | 68 | reply = str(input(question+' (Y/n): ')).lower().strip() 69 | 70 | if len(reply) > 0 and reply[0] == 'n': 71 | result = False 72 | 73 | return result 74 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | azure-devops==5.1.0b1 2 | certifi==2019.6.16 3 | chardet==3.0.4 4 | idna==2.8 5 | isodate==0.6.0 6 | msrest==0.6.9 7 | oauthlib==3.1.0 8 | requests==2.22.0 9 | requests-oauthlib==1.2.0 10 | six==1.12.0 11 | urllib3==1.26.5 12 | --------------------------------------------------------------------------------