├── .gitignore ├── LICENSE ├── README.md ├── crawler.py ├── git.py ├── main.py └── option.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Private Data 2 | info.json 3 | option.json 4 | 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | MANIFEST 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *.cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | 63 | # Flask stuff: 64 | instance/ 65 | .webassets-cache 66 | 67 | # Scrapy stuff: 68 | .scrapy 69 | 70 | # Sphinx documentation 71 | docs/_build/ 72 | 73 | # PyBuilder 74 | target/ 75 | 76 | # Jupyter Notebook 77 | .ipynb_checkpoints 78 | 79 | # pyenv 80 | .python-version 81 | 82 | # celery beat schedule file 83 | celerybeat-schedule 84 | 85 | # SageMath parsed files 86 | *.sage.py 87 | 88 | # Environments 89 | .env 90 | .venv 91 | env/ 92 | venv/ 93 | ENV/ 94 | env.bak/ 95 | venv.bak/ 96 | 97 | # Spyder project settings 98 | .spyderproject 99 | .spyproject 100 | 101 | # Rope project settings 102 | .ropeproject 103 | 104 | # mkdocs documentation 105 | /site 106 | 107 | # mypy 108 | .mypy_cache/ 109 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018-2020 Minho Kim 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Codeforces-AutoCommit 2 | ========== 3 | [![](http://st.codeforces.com/s/74869/images/codeforces-logo-with-telegram.png)](http://codeforces.com/) 4 | ---------- 5 | `Codeforces-AutoCommit` is a tool that solves various algorithmic problems in [Codeforces](http://codeforces.com/) and automatically pushes source code to remote repositories such as [Github](https://github.com) if you get the correct answer. This tool uses your handle to search and analyze the source code in Codeforces. If the source code does not exist in your local repository, download the source code and save it to your local repository and use `Git` to automatically add, commit, and push to the remote repository. 6 | 7 | Installation 8 | ---------- 9 | ``` 10 | $ git clone https://github.com/ISKU/Codeforces-AutoCommit 11 | ``` 12 | 13 | **Dependency** 14 | ``` 15 | $ pip3 install requests 16 | $ pip3 install bs4 17 | ``` 18 | 19 | - [Python](https://www.python.org/) 20 | - [Git](https://git-scm.com/) 21 | 22 | How to use 23 | ---------- 24 | - If you do not want to enter user information every time you run the tool, create `info.json` as in the following example: 25 | ``` json 26 | { 27 | "handle": "my_codeforces_handle", 28 | "git_id": "my_github_id", 29 | "git_pw": "my_github_password", 30 | "remote_url": "https://github.com/ISKU/Algorithm" 31 | } 32 | ``` 33 | 34 | - Make sure to enter the user information correctly when running the tool or in info.json. Then run the tool as follows. 35 | ``` 36 | $ python3 main.py 37 | ``` 38 | 39 | - This tool has a very long wait time. It is recommended to run in `Background` as follows. 40 | ``` 41 | $ nohup python3 main.py & 42 | ``` 43 | 44 | Default 45 | ---------- 46 | - **Commit message** is "Add solution for [contest_id][problem_index]". 47 | - **Create a directory** named [contest_id] and **save the source code file** named [problem_index] in that directory. 48 | - Search the source code every **10 minutes**. 49 | - Search **all submitted** source code and analyze the correct source code. 50 | - If there are multiple correct source codes, select the **last submitted** source code. 51 | 52 | Extension 53 | ---------- 54 | - You can freely manage your own remote repositories by extending the tool. 55 | - In the `option.json` file, enter the options you want, as in the following example: 56 | 57 | ``` json 58 | { 59 | "commit_message": "Add for [CONTEST][INDEX] [TITLE]", 60 | "source_tree": "Algorithm/Codeforces/src", 61 | "mkdir": true, 62 | "dir_name": "[CONTEST]", 63 | "source_name": "[INDEX]", 64 | "poll": 600, 65 | "lang": "GNU C++17" 66 | } 67 | ``` 68 | > :bulb: Unused options must be cleared. 69 |
70 | 71 | **Key Options:** 72 | 73 | | **Key** | **Description** 74 | |:-------------------|:------------------------------------------------------------------------------------------- 75 | | **commit_message** | Set the commit message. 76 | | **source_tree** | Save the source code in that path. (The starting directory must match the repository name.) 77 | | **mkdir** | Decide if you want to create a directory when you save the source code.
(false: dir_name option is ignored.) 78 | | **dir_name** | Set the name of the directory where the source code is saved. 79 | | **source_name** | Set the name of the source code file. 80 | | **poll** | Set the source code search cycle in seconds in Codeforces. 81 | | **lang** | Only the languages you submit in that language will be pushed. 82 | 83 | > :bulb: [CONTEST]: If the content contains [CONTEST], it is replaced by contest_id.
84 | > :bulb: [INDEX]: If the content contains [INDEX], it is replaced by problem_index.
85 | > :bulb: [TITLE]: If the content contains [TITLE], it is replaced by problem_title. 86 | 87 | Example 88 | ---------- 89 | - https://github.com/ISKU/Algorithm 90 | - The above repository uses [Codeforces-AutoCommit](https://github.com/ISKU/Codeforces-AutoCommit) to manage the source code. Option used is as follows. 91 | 92 | ``` json 93 | { 94 | "commit_message": "Codeforces #[CONTEST][INDEX]: [TITLE]", 95 | "source_tree": "Algorithm/Codeforces", 96 | "mkdir": true, 97 | "dir_name": "[CONTEST]", 98 | "source_name": "[INDEX]", 99 | "poll": 600, 100 | "lang": "Java 8" 101 | } 102 | ``` 103 | 104 | License 105 | ---------- 106 | - [MIT](LICENSE) 107 | 108 | Author 109 | ---------- 110 | - Minho Kim ([ISKU](https://github.com/ISKU)) 111 | - http://codeforces.com/profile/isku 112 | - **E-mail:** minho.kim093@gmail.com 113 | -------------------------------------------------------------------------------- /crawler.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import requests 3 | from bs4 import BeautifulSoup 4 | 5 | USER_STATUS_URL = 'http://codeforces.com/api/user.status?' 6 | SUBMISSION_URL_FORMAT = 'http://codeforces.com/contest/%d/submission/%d' 7 | 8 | class Crawler: 9 | 10 | def get_user_status(self, handle, start=0, offset=0): 11 | params = {'handle' : handle} 12 | if (not start == 0) or (not offset == 0): 13 | params['from'] = start 14 | params['count'] = offset 15 | 16 | res = requests.get(USER_STATUS_URL, params) 17 | 18 | if res.status_code == 400: 19 | return True, res.json()['comment'] 20 | if res.status_code == 200: 21 | return False, res.json()['result'] 22 | 23 | return True, str(res.status_code) 24 | 25 | def get_source(self, submission_id, contest_id): 26 | res = requests.get(SUBMISSION_URL_FORMAT % (contest_id, submission_id)) 27 | 28 | if not res.status_code == 200: 29 | return True, str(res.status_code) 30 | 31 | soup = BeautifulSoup(res.text, 'html.parser') 32 | source = soup.find(id='program-source-text') 33 | 34 | if source is None: 35 | return True, 'faild to parse source' 36 | return False, source.get_text() 37 | -------------------------------------------------------------------------------- /git.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import subprocess 4 | 5 | ERROR_FORMAT = '* Error: %s' 6 | 7 | class Git: 8 | 9 | def __init__(self): 10 | self.pipe = subprocess.PIPE 11 | 12 | def clone(self, repo_name, repo_url): 13 | if os.path.isdir(repo_name): 14 | print("* Repository '%s' already exists." % (repo_name)) 15 | return 16 | 17 | command = ['git', 'clone', repo_url] 18 | proc = subprocess.Popen(command, stdout=self.pipe, stderr=self.pipe) 19 | stdout, stderr = proc.communicate() 20 | 21 | if not stdout is None: 22 | print(stdout.decode()) 23 | if not stderr is None: 24 | print(stderr.decode()) 25 | if not proc.returncode == 0: 26 | sys.exit(ERROR_FORMAT % ('git clone, ' + proc.returncode)) 27 | 28 | def pull(self, cwd, repo_url): 29 | command = ['git', 'pull', repo_url] 30 | proc = subprocess.Popen(command, cwd=cwd, stdout=self.pipe, stderr=self.pipe) 31 | stdout, stderr = proc.communicate() 32 | 33 | if not stdout is None: 34 | print(stdout.decode()) 35 | if not stderr is None: 36 | print(stderr.decode()) 37 | if not proc.returncode == 0: 38 | sys.exit(ERROR_FORMAT % ('git pull, ' + proc.returncode)) 39 | 40 | def add(self, cwd, file_path): 41 | command = ['git', 'add', file_path] 42 | proc = subprocess.Popen(command, cwd=cwd, stdout=self.pipe, stderr=self.pipe) 43 | stdout, stderr = proc.communicate() 44 | 45 | if not stdout is None: 46 | print(stdout.decode()) 47 | if not stderr is None: 48 | print(stderr.decode()) 49 | if not proc.returncode == 0: 50 | sys.exit(ERROR_FORMAT % ('git add, ' + proc.returncode)) 51 | 52 | def commit(self, cwd, commit_message): 53 | command = ['git', 'commit', '-m', commit_message] 54 | proc = subprocess.Popen(command, cwd=cwd, stdout=self.pipe, stderr=self.pipe) 55 | stdout, stderr = proc.communicate() 56 | 57 | if not stdout is None: 58 | print(stdout.decode()) 59 | if not stderr is None: 60 | print(stderr.decode()) 61 | if not proc.returncode == 0: 62 | sys.exit(ERROR_FORMAT % ('git commit, ' + proc.returncode)) 63 | 64 | def push(self, cwd, remote_url, branch): 65 | command = ['git', 'push', remote_url, branch] 66 | proc = subprocess.Popen(command, cwd=cwd, stdout=self.pipe, stderr=self.pipe) 67 | stdout, stderr = proc.communicate() 68 | 69 | if not proc.returncode == 0: 70 | sys.exit(ERROR_FORMAT % ('git push, ' + proc.returncode)) 71 | 72 | def all(self, repo_name, remote_url, branch, file_path, commit_message): 73 | self.add(repo_name, file_path) 74 | self.commit(repo_name, commit_message) 75 | self.push(repo_name, remote_url, branch) 76 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import json 4 | import getpass 5 | import time 6 | from crawler import Crawler 7 | from git import Git 8 | from option import Option 9 | 10 | ERROR_FORMAT = '* Error: %s' 11 | 12 | class Main: 13 | 14 | def __init__(self, crawler, git, option, user_info): 15 | self.crawler = crawler 16 | self.git = git 17 | self.option = option 18 | self.user_info = user_info 19 | 20 | self.user_name = user_info['user_name'] 21 | self.repo_url = user_info['repo_url'] 22 | self.repo_name = user_info['repo_name'] 23 | 24 | self.solved_list = [] 25 | 26 | def run(self): 27 | self.init_clone() 28 | 29 | while True: 30 | self.find_solved_problem() 31 | self.push_source() 32 | self.idle() 33 | 34 | def init_clone(self): 35 | git_id = self.user_info['git_id'] 36 | git_pw = self.user_info['git_pw'] 37 | remote_url = 'https://%s:%s@github.com/%s/%s' % (git_id, git_pw, git_id, self.repo_name) 38 | 39 | git.clone(self.repo_name, self.repo_url) 40 | git.pull(self.repo_name, remote_url) 41 | 42 | def find_solved_problem(self): 43 | self.solved_list = [] 44 | solved_set = set() 45 | 46 | user_status_error, user_status = self.crawler.get_user_status(self.user_name) 47 | if user_status_error: 48 | print(ERROR_FORMAT % ('get_user_status, ' + user_status)) 49 | return 50 | 51 | for submission in user_status: 52 | if not submission['verdict'] == 'OK': 53 | continue 54 | 55 | key = str(submission['contestId']) + submission['problem']['index'] 56 | if key in solved_set: 57 | continue 58 | solved_set.add(key) 59 | 60 | solved = {} 61 | solved['submission_id'] = submission['id'] 62 | solved['contest_id'] = submission['contestId'] 63 | solved['problem_id'] = submission['problem']['index'] 64 | solved['problem_title'] = submission['problem']['name'] 65 | solved['language'] = submission['programmingLanguage'] 66 | solved['cp_id'] = key 67 | self.solved_list.append(solved) 68 | 69 | def push_source(self): 70 | for solved in self.solved_list: 71 | submission_id = solved['submission_id'] 72 | contest_id = solved['contest_id'] 73 | problem_id = solved['problem_id'] 74 | language = solved['language'] 75 | 76 | source_tree = self.option.source_tree(solved) 77 | source_name = self.option.source_name(solved) 78 | file_path = '%s%s%s' % (source_tree, source_name, self.option.get_ext(language)) 79 | 80 | support, same = self.option.lang(solved) 81 | if support: 82 | if not same: 83 | print("* '%s' language is not supported (submission: %d, contest: %d, index: %s)" % (language, submission_id, contest_id, problem_id)) 84 | continue 85 | 86 | if os.path.exists(file_path): 87 | print('* source already exists (submission: %d, contest: %d, index: %s)' % (submission_id, contest_id, problem_id)) 88 | continue 89 | 90 | print('* Downloading source (submission: %d, contest: %d, index: %s)' % (submission_id, contest_id, problem_id)) 91 | source_error, source = self.crawler.get_source(submission_id, contest_id) 92 | if source_error: 93 | print(ERROR_FORMAT % ('get_source, ' + source)) 94 | print('* Failed to download the source (submission: %d, contest: %d, index: %s)\n' % (submission_id, contest_id, problem_id)) 95 | continue 96 | print(source) 97 | 98 | self.save_source(source_tree, file_path, source) 99 | print("* Successfully saved the '%s'" % (file_path)) 100 | 101 | self.git_all(file_path, solved) 102 | print('* Successfully pushed the source (submission: %d, contest: %d, index: %s)\n' % (submission_id, contest_id, problem_id)) 103 | 104 | def save_source(self, source_tree, file_path, source): 105 | if not os.path.isdir(source_tree): 106 | os.makedirs(source_tree) 107 | 108 | f = open(file_path, 'w') 109 | f.write(source) 110 | f.close() 111 | 112 | def git_all(self, file_path, solved): 113 | git_id = self.user_info['git_id'] 114 | git_pw = self.user_info['git_pw'] 115 | remote_url = 'https://%s:%s@github.com/%s/%s' % (git_id, git_pw, git_id, self.repo_name) 116 | file_path = file_path.replace(self.repo_name, '.', 1) 117 | commit_message = self.option.commit_message(solved) 118 | 119 | self.git.all(self.repo_name, remote_url, 'master', file_path, commit_message) 120 | 121 | def idle(self): 122 | poll = self.option.poll() 123 | 124 | print('* Wait %d seconds... \n' % (poll)) 125 | time.sleep(poll) 126 | print('* Restart work') 127 | 128 | if __name__ == '__main__': 129 | user_info = {} 130 | if os.path.isfile('option.json'): 131 | user_info = json.loads(open('option.json', 'r').read()) 132 | 133 | if os.path.isfile('info.json'): 134 | private_info = json.loads(open('info.json', 'r').read()) 135 | user_info['user_name'] = private_info['handle'] 136 | user_info['repo_url'] = private_info['remote_url'] 137 | user_info['repo_name'] = private_info['remote_url'].split('/')[-1].split('.')[0] 138 | user_info['git_id'] = private_info['git_id'] 139 | user_info['git_pw'] = private_info['git_pw'] 140 | else: 141 | user_info['user_name'] = input('* Handle for Codeforces: ') 142 | user_info['repo_url'] = input('* URL for a remote repository: ') 143 | user_info['repo_name'] = user_info['repo_url'].split('/')[-1].split('.')[0] 144 | user_info['git_id'] = input("* Username for 'https://github.com': ") 145 | user_info['git_pw'] = getpass.getpass("* Password for 'https://%s@github.com': " % (user_info['git_id'])) 146 | 147 | crawler = Crawler() 148 | git = Git() 149 | option = Option(user_info) 150 | try: 151 | Main(crawler, git, option, user_info).run() 152 | except KeyboardInterrupt: 153 | print('\n* bye\n') 154 | -------------------------------------------------------------------------------- /option.py: -------------------------------------------------------------------------------- 1 | class Option: 2 | 3 | def __init__(self, user_info): 4 | self.user_info = user_info 5 | 6 | def commit_message(self, solved): 7 | if not 'commit_message' in self.user_info: 8 | return 'Add solution for %d%s' % (solved['contest_id'], solved['problem_id']) 9 | return self.replace_info(self.user_info['commit_message'], solved) 10 | 11 | def source_tree(self, solved): 12 | if not 'source_tree' in self.user_info: 13 | if self.mkdir(): 14 | return '%s/%s/' % (self.user_info['repo_name'], self.dir_name(solved)) 15 | return '%s/' % (self.user_info['repo_name']) 16 | 17 | if self.user_info['source_tree'][-1] == '/': 18 | if self.mkdir(): 19 | return '%s%s/' % (self.replace_info(self.user_info['source_tree'], solved), self.dir_name(solved)) 20 | return self.replace_info(self.user_info['source_tree']) 21 | 22 | if self.mkdir(): 23 | return '%s/%s/' % (self.replace_info(self.user_info['source_tree'], solved), self.dir_name(solved)) 24 | return '%s/' % (self.replace_info(self.user_info['source_tree'])) 25 | 26 | def mkdir(self): 27 | if not 'mkdir' in self.user_info: 28 | return True 29 | return self.user_info['mkdir'] 30 | 31 | def dir_name(self, solved): 32 | if not 'dir_name' in self.user_info: 33 | return solved['contest_id'] 34 | return self.replace_info(self.user_info['dir_name'], solved) 35 | 36 | def source_name(self, solved): 37 | if not 'source_name' in self.user_info: 38 | return solved['problem_id'] 39 | return self.replace_info(self.user_info['source_name'], solved) 40 | 41 | def poll(self): 42 | if not 'poll' in self.user_info: 43 | return 600 44 | return self.user_info['poll'] 45 | 46 | def lang(self, solved): 47 | if not 'lang' in self.user_info: 48 | return False, None 49 | if not self.user_info['lang'] == solved['language']: 50 | return True, False 51 | return True, True 52 | 53 | def replace_info(self, value, solved): 54 | value = value.replace('[CONTEST]', str(solved['contest_id'])) 55 | value = value.replace('[INDEX]', solved['problem_id']) 56 | value = value.replace('[TITLE]', solved['problem_title']) 57 | return value 58 | 59 | def get_ext(self, language): 60 | return { 61 | 'GNU C': '.c', 62 | 'GNU C11': '.c', 63 | 'Clang++17 Diagnostics': '.cpp', 64 | 'GNU C++': '.cpp', 65 | 'GNU C++11': '.cpp', 66 | 'GNU C++14': '.cpp', 67 | 'GNU C++17': '.cpp', 68 | 'GNU C++17 Diagnostics': '.cpp', 69 | 'MS C++': '.cpp', 70 | 'Mono C#': '.cs', 71 | 'D': '.d', 72 | 'Go': '.go', 73 | 'Haskell': '.hs', 74 | 'Java 8': '.java', 75 | 'Kotlin': '.kt', 76 | 'Ocaml': '.ml', 77 | 'Delphi': '.dpr', 78 | 'FPC': '.pas', 79 | 'PascalABC.NET': '.pas', 80 | 'Perl': '.pl', 81 | 'PHP': '.php', 82 | 'Python 2': '.py', 83 | 'Python 3': '.py', 84 | 'PyPy 2': '.py', 85 | 'PyPy 3': '.py', 86 | 'Ruby': '.rb', 87 | 'Rust': '.rs', 88 | 'Scala': '.scala', 89 | 'JavaScript': '.js', 90 | 'Node.js': '.js', 91 | }[language] 92 | --------------------------------------------------------------------------------