├── .gitignore ├── LICENSE ├── README.md ├── bot.py ├── exceptions.py ├── requirements.txt ├── settings_example.py ├── tests.py └── wait_strategy.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | 103 | # pytest 104 | .pytest_cache 105 | 106 | # customized 107 | settings.py 108 | cloned_repos 109 | source_repos 110 | .pytest_data -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Victor Hu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Python Support](https://img.shields.io/badge/python-3.6-blue.svg)]() 2 | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/79fdd815b4f740a39fc6ec4093493410)](https://www.codacy.com/app/seLain/ghsp2ofer?utm_source=github.com&utm_medium=referral&utm_content=seLain/ghsp2ofer&utm_campaign=Badge_Grade) 3 | 4 | ## What's This 5 | 6 | On the way to build a fully-automated coding bot. 7 | 8 | ## For Now 9 | 10 | It's a functional prototype bot which can replicate code files randomly from designated source. 11 | The replicated code is then committed to local git repo and pushed to Github repo. 12 | 13 | ## How2Start 14 | 15 | First, be sure you have Python 3.6+ installed. (Python 3.6.5rc1+ recommended) 16 | 17 | Optionally create a virtual environment for this bot, and dive into this virtual environment. 18 | 19 | `python36 -m venv py36_venv\ghsp2ofer` 20 | `py36_venv\ghsp2ofer\Scripts\activate` (on Windows) 21 | 22 | Download **ghsp2ofer** and extract to a specific folder, such as `ghsp2ofer_bot`. change your working directory to `ghsp2ofer_bot`. 23 | 24 | Install required packages. 25 | 26 | `(ghsp2ofer) pip install -r requirements` 27 | 28 | Create two necessary directories: `ghsp2ofer_bot\cloned_repos`, `ghsp2ofer_bot\source_repos`. 29 | 30 | Create an empty repository on Github (wihtout .gitignore). For example, new repository `Spoofer`. 31 | 32 | Rename `ghsp2ofer_bot\settings_example.py` as `ghsp2ofer_bot\settings.py`, put necessary information in to `ghsp2ofer_bot\settings.py`. 33 | * Set settings.DEFAULT_REPO as `Spoofer` ( or whatever repository name you created earlier) 34 | * Set settings.WAIT_STRATEGY, currently two strategies are available: 35 | 1. RandomWaitStrategy: delays next commit by random(settings.RANDOM_MIN_MINUTES, settings.RANDOM_MAX_MINUTES) minutes. 36 | 2. RandomWorkHourStrategy: delays next commit by random(settings.RANDOM_MIN_MINUTES, settings.RANDOM_MAX_MINUTES) minutes if and only if the next commit time lays in settings.WORK_HOURS 37 | * Set settings.RANDOM_MIN_MINUTES, settings.RANDOM_MAX_MINUTES. The bot will be triggered and sleep for random(settings.RANDOM_MIN_MINUTES, settings.RANDOM_MAX_MINUTES) minutes. 38 | * Set settings.WORK_HOURS which specifies working hours of this bot. 39 | 40 | Create directory `ghsp2ofer_bot\source_repos\Spoofer`, and put in code files to be replicated. 41 | 42 | Final step, 43 | 44 | `(ghsp2ofer) python bot.py` 45 | -------------------------------------------------------------------------------- /bot.py: -------------------------------------------------------------------------------- 1 | import os, logging, shutil, glob, random, time 2 | from datetime import datetime 3 | from github import Github, InputGitTreeElement 4 | from git import Repo 5 | from git.exc import GitCommandError 6 | from exceptions import ClonedRepoExistedError, BranchUpToDateException,\ 7 | DefaultCommitToolException 8 | import importlib 9 | import settings 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | class Bot(object): 14 | 15 | def __init__(self): 16 | self.github = None 17 | self.url = None 18 | 19 | def login(self, token=None): 20 | if token: 21 | self.github = Github(token) 22 | else: 23 | self.github = Github(settings.ACCESS_TOKEN) 24 | # update self.url 25 | self.url = self.github.get_user().url 26 | 27 | def get_repo_names(self): 28 | return [r.name for r in self.github.get_user().get_repos()] 29 | 30 | def create_issue(self, repo_name, issue): 31 | repo = self.github.get_user().get_repo(name=repo_name) 32 | repo.create_issue(title=issue['title'], body=issue['body']) 33 | 34 | # param state:: 'open', 'closed', 'all' 35 | def get_issues(self, repo_name, state='open'): 36 | repo = self.github.get_user().get_repo(name=repo_name) 37 | return repo.get_issues(state=state) 38 | 39 | def report_status(self, repo_name): 40 | repo = self.github.get_user().get_repo(name=repo_name) 41 | return '\n'.join([ 42 | 'Bot connected to repository: %s' % repo.full_name, 43 | ' clone_url: %s' % repo.clone_url 44 | ]) 45 | 46 | def repo_clone(self, repo_name, root_dir): 47 | repo = self.github.get_user().get_repo(name=repo_name) 48 | repo_dir = os.sep.join([root_dir, repo_name]) 49 | try: 50 | Repo.clone_from(repo.clone_url, repo_dir) 51 | except GitCommandError: 52 | raise ClonedRepoExistedError 53 | 54 | def remote_addfiles_commit(self, repo_name, file_list, message=''): 55 | # load repo and get meta 56 | repo = self.github.get_user().get_repo(name=repo_name) 57 | master_ref = repo.get_git_ref('heads/master') 58 | master_sha = master_ref.object.sha 59 | base_tree = repo.get_git_tree(master_sha) 60 | # enroll the files to be committed 61 | element_list = list() 62 | for entry in file_list: 63 | with open(os.sep.join([settings.DEFAULT_SOURCE_ROOT_DIR, repo_name, entry]), 'r') as input_file: 64 | data = input_file.read() 65 | element = InputGitTreeElement(entry.replace('\\', '/'), '100644', 'blob', data) 66 | element_list.append(element) 67 | # prepare commit trees and make commit request 68 | tree = repo.create_git_tree(element_list, base_tree) 69 | parent = repo.get_git_commit(master_sha) 70 | commit = repo.create_git_commit(message, tree, [parent]) 71 | master_ref.edit(commit.sha) 72 | 73 | def addfiles_commit_push_remote(self, repo_name, root_dir, file_list, message='', remote_name='origin'): 74 | # load repo 75 | repo_dir = os.sep.join([root_dir, repo_name]) 76 | repo = Repo(repo_dir) 77 | # always pull first 78 | remote = repo.remote(remote_name) 79 | remote.set_url('https://%s:%s@github.com/%s/%s.git' %\ 80 | (settings.USERNAME, settings.PASSWORD, settings.USERNAME, repo_name)) 81 | remote.pull() 82 | # copy and add files specified to local repository 83 | for file in file_list: 84 | src = os.sep.join([settings.DEFAULT_SOURCE_ROOT_DIR, repo_name, file]) 85 | dest = os.sep.join([repo_dir, file]) 86 | try: 87 | shutil.copy2(src, dest) 88 | except IOError as e: # parent directory not exists or something wrong 89 | # creating parent directories 90 | os.makedirs(os.path.dirname(dest)) 91 | shutil.copy2(src, dest) 92 | try: 93 | repo.git.add(file) 94 | except GitCommandError: 95 | print('File %s can not be added. Possibly being ruled out by .gitignore. This file is passed.' % file) 96 | # make commit 97 | try: 98 | repo.git.commit('-m %s' % message) 99 | remote.push() 100 | except GitCommandError as e: 101 | print(e) 102 | raise BranchUpToDateException 103 | 104 | def random_auto_commit(self): 105 | # prepare all files abailable 106 | search_for = os.sep.join([settings.DEFAULT_SOURCE_ROOT_DIR, 107 | settings.DEFAULT_REPO, 108 | '**', '*.*']) 109 | prefix = settings.DEFAULT_SOURCE_ROOT_DIR + os.sep + settings.DEFAULT_REPO + os.sep 110 | files = [f.replace(prefix, '') for f in glob.glob(search_for, recursive=True)] 111 | # randomly choose one and make commit 112 | chosen_files = list(random.sample(set(files), random.randint(1, 4))) 113 | message = ' '.join(['add files:']+[os.path.basename(f) for f in chosen_files]) 114 | # perform commit and push to remote 115 | if settings.DEFAULT_COMMIT_TOOL == 'GitPython': 116 | try: 117 | self.addfiles_commit_push_remote(repo_name=settings.DEFAULT_REPO, 118 | root_dir=settings.DEFAULT_CLONE_ROOT_DIR, 119 | file_list=chosen_files, 120 | message=message) 121 | except BranchUpToDateException: 122 | print('up to date. nothing to do. pass.') 123 | elif settings.DEFAULT_COMMIT_TOOL == 'PyGithub': 124 | self.remote_addfiles_commit(repo_name=settings.DEFAULT_REPO, 125 | file_list=chosen_files, 126 | message=message) 127 | else: 128 | raise DefaultCommitToolException 129 | 130 | def auto_pull_request(self, title, message, base_branch, head_branch, can_modify=True): 131 | """ 132 | Create a PR from head branch to base branch. ex, from 'head:bug-fix' to 'base:master'. 133 | Please note that `auto_pull_request` can not create cross-repo pull request, 134 | such as making a PR to upstream repository. 135 | :param title: string - title of PR 136 | :param message: string - the message body of PR 137 | :param base_branch: string - the base branch of this PR, ex: 'master' 138 | :param head_branch: string - the target branch of this PR, ex: 'bug-fix' 139 | :param can_modify: bool - mark if repo maitainer can modify this PR 140 | """ 141 | repo = self.github.get_user().get_repo(name=settings.DEFAULT_REPO) 142 | repo.create_pull(title, message, base_branch, head_branch, can_modify) 143 | 144 | def run(self): 145 | try: 146 | bot.repo_clone(repo_name=settings.DEFAULT_REPO, 147 | root_dir=settings.DEFAULT_CLONE_ROOT_DIR) 148 | except ClonedRepoExistedError: 149 | print('Repository existed. Does not clone.') 150 | while True: 151 | print('auto commit.') 152 | self.random_auto_commit() 153 | print('done. going sleep...') 154 | # dynamic strategy loading according to settings.py 155 | strategy_module = importlib.import_module('wait_strategy') 156 | strategy_class = getattr(strategy_module, settings.WAIT_STRATEGY) 157 | sleep_minutes, awake_time = strategy_class().get_awake_time(datetime.now()) 158 | print('scheduled to sleep %s minutes. next awake: %s' % (sleep_minutes, awake_time)) 159 | time.sleep(60*sleep_minutes) 160 | print('awake.') 161 | 162 | if __name__ == "__main__": 163 | bot = Bot() 164 | bot.login() 165 | #print(bot.report_status(repo_name=settings.DEFAULT_REPO)) 166 | bot.run() 167 | ''' # make a local clone repo, make changes, and commit -> push 168 | try: 169 | bot.repo_clone(repo_name=settings.DEFAULT_REPO, 170 | root_dir=settings.DEFAULT_CLONE_ROOT_DIR) 171 | except ClonedRepoExistedError: 172 | print('Repository existed. Does not clone.') 173 | try: 174 | bot.addfiles_commit_push_remote(repo_name=settings.DEFAULT_REPO, 175 | root_dir=settings.DEFAULT_CLONE_ROOT_DIR, 176 | file_list=['exceptions.py'], 177 | message='commit_push_remote') 178 | except BranchUpToDateException: 179 | print('Branch up to date. Does not commit.') 180 | ''' 181 | ''' # make remote commit through GitHub API 182 | file_list = ['settings.py',] 183 | bot.remote_addfiles_commit(repo_name=settings.DEFAULT_REPO, 184 | file_list = file_list, 185 | message='add files %s' % file_list) 186 | ''' 187 | -------------------------------------------------------------------------------- /exceptions.py: -------------------------------------------------------------------------------- 1 | 2 | class ClonedRepoExistedError(Exception): 3 | """Base class for other exceptions""" 4 | pass 5 | 6 | class BranchUpToDateException(Exception): 7 | pass 8 | 9 | class DefaultCommitToolException(Exception): 10 | pass 11 | 12 | class PotentialInfiniteLoopException(Exception): 13 | pass -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | PyGithub==1.37 2 | PyJWT==1.6.0 3 | GitPython==2.1.8 -------------------------------------------------------------------------------- /settings_example.py: -------------------------------------------------------------------------------- 1 | # GitHub repository access token 2 | USERNAME = '' 3 | PASSWORD = '' 4 | EMAIL = '' 5 | ACCESS_TOKEN = '' 6 | 7 | # Name of default repository 8 | DEFAULT_REPO = '' 9 | 10 | # Default root dir to put cloned repositories 11 | DEFAULT_CLONE_ROOT_DIR = 'cloned_repos' 12 | 13 | # Default root dir to put source repositories 14 | DEFAULT_SOURCE_ROOT_DIR = 'source_repos' 15 | 16 | # Choose commit tool 17 | # The value can only be one of ['GitPython', 'PyGithub'] 18 | DEFAULT_COMMIT_TOOL = 'GitPython' 19 | 20 | # WaitStrategy selection. The value must be one of the concrete wait strategies in wait_strategy.py 21 | WAIT_STRATEGY = 'RandomWorkHourStrategy' 22 | 23 | # Random commit interval 24 | RANDOM_MIN_MINUTES = 60 25 | RANDOM_MAX_MINUTES = 120 26 | 27 | # Bot working hours 28 | WORK_HOURS = [9, 18] # from 9:00AM~6:00PM -------------------------------------------------------------------------------- /tests.py: -------------------------------------------------------------------------------- 1 | import time, pathlib, os 2 | from unittest import TestCase 3 | from bot import Bot 4 | import settings 5 | from exceptions import ClonedRepoExistedError 6 | 7 | class BotTestCase(TestCase): 8 | 9 | def setUp(self): 10 | # set up needed testing data folder 11 | pathlib.Path('/test_data').mkdir(parents=True, exist_ok=True) 12 | # set up bot instance 13 | self.bot = Bot() 14 | self.bot.login() 15 | 16 | def test_get_repo_names(self): 17 | self.assertEqual(settings.DEFAULT_REPO in self.bot.get_repo_names(), True) 18 | self.assertEqual('MadeUpRepo' in self.bot.get_repo_names(), False) 19 | 20 | def test_create_issue_get_issues(self): 21 | issue = {'title': 'test issue', 22 | 'body': 'this is a test issue.'} 23 | self.bot.create_issue(settings.DEFAULT_REPO, issue) 24 | existing_issues = self.bot.get_issues(settings.DEFAULT_REPO, state='open') 25 | self.assertEqual(issue['title'] in [i.title for i in existing_issues], True) 26 | self.assertEqual(issue['body'] in [i.body for i in existing_issues], True) 27 | 28 | def test_repo_clone(self): 29 | expected_dir = os.sep.join(['.pytest_data', settings.DEFAULT_REPO]) 30 | # make sure the clone dir does not exist yet 31 | self.assertEqual(os.path.isdir(expected_dir), False) 32 | self.bot.repo_clone(repo_name=settings.DEFAULT_REPO, root_dir='.pytest_data') 33 | self.assertEqual(os.path.isdir(expected_dir), True) 34 | # test if expected_dir exist already. if yes, ClonedRepoExistedError should be thrown 35 | try: 36 | self.bot.repo_clone(repo_name=settings.DEFAULT_REPO, root_dir='.pytest_data') 37 | except ClonedRepoExistedError: 38 | self.assertEqual(True, True) -------------------------------------------------------------------------------- /wait_strategy.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta, abstractmethod 2 | from datetime import timedelta 3 | from exceptions import PotentialInfiniteLoopException 4 | import random 5 | import settings 6 | 7 | class WaitStrategy(metaclass=ABCMeta): 8 | ''' 9 | Abstract method to get next akake time 10 | :param Datetime current_time: a given current time 11 | :return: sleep_minutes, awake_time 12 | ''' 13 | @abstractmethod 14 | def get_awake_time(self, current_time): 15 | pass 16 | 17 | class RandomWaitStrategy(WaitStrategy): 18 | def get_awake_time(self, current_time): 19 | sleep_minutes = random.randint(settings.RANDOM_MIN_MINUTES, settings.RANDOM_MAX_MINUTES) 20 | awake_time = current_time + timedelta(minutes=sleep_minutes) 21 | return sleep_minutes, awake_time 22 | 23 | class RandomWorkHourStrategy(WaitStrategy): 24 | def get_awake_time(self, current_time): 25 | work_hours = settings.WORK_HOURS 26 | infinite_loop_sentinel = 0 27 | while True: 28 | if infinite_loop_sentinel > 100: 29 | raise PotentialInfiniteLoopException 30 | sleep_minutes = random.randint(settings.RANDOM_MIN_MINUTES, settings.RANDOM_MAX_MINUTES) 31 | awake_time = current_time + timedelta(minutes=sleep_minutes) 32 | if awake_time.hour in range(work_hours[0], work_hours[1]): 33 | return sleep_minutes, awake_time 34 | else: 35 | infinite_loop_sentinel += 1 --------------------------------------------------------------------------------