├── requirements.txt
├── .idea
├── vcs.xml
├── misc.xml
├── .gitignore
├── inspectionProfiles
│ ├── profiles_settings.xml
│ └── Project_Default.xml
├── modules.xml
└── Upcode.iml
├── UploadToGithub.py
├── README.md
├── AtcoderScraper.py
├── CodeChefScraper.py
├── main.py
└── CodeForcesScraper.py
/requirements.txt:
--------------------------------------------------------------------------------
1 | PyGithub~=1.55
2 | grequests~=0.6.0
3 | requests~=2.27.1
4 | beautifulsoup4~=4.11.1
5 | lxml~=4.9.2
6 | selenium~=4.2.0
7 | webdriver_manager~=3.7.0
--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
--------------------------------------------------------------------------------
/.idea/.gitignore:
--------------------------------------------------------------------------------
1 | # Default ignored files
2 | /shelf/
3 | /workspace.xml
4 | # Editor-based HTTP Client requests
5 | /httpRequests/
6 | # Datasource local storage ignored files
7 | /dataSources/
8 | /dataSources.local.xml
9 |
--------------------------------------------------------------------------------
/.idea/inspectionProfiles/profiles_settings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/.idea/Upcode.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/UploadToGithub.py:
--------------------------------------------------------------------------------
1 | from github import GithubException
2 |
3 |
4 | def upload_to_github(repo, git_path, content, problem_info=''):
5 | folder_path = '/'.join(git_path.split('/')[:-1])
6 |
7 | try:
8 | all_paths = [file.path for file in repo.get_contents(folder_path)]
9 |
10 | except GithubException:
11 | if problem_info:
12 | repo.create_file(f'{folder_path}/__info.txt', "Created info.txt", problem_info, branch="main")
13 |
14 | all_paths = []
15 |
16 | if git_path not in all_paths:
17 | repo.create_file(git_path, 'Committing files', content, branch="main")
18 | print(git_path, 'CREATED')
19 | else:
20 | print(git_path, 'ALREADY EXISTS')
21 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # UpCode
2 | A software for uploading all your accepted solutions from CodeChef, CodeForces, and Atcoder to Github with no hassles, and fully automated using Python.
3 |
4 | ## How to use:
5 | * Generate an API key from https://github.com/settings/tokens. Make sure the repo section is checked.
6 | * Download the project and extract the zip file. Navigate to the extracted folder and run the following command in terminal:
7 |
8 | ```
9 | pip install -r requirements.txt
10 | ```
11 | * To start using the project, run the following command in terminal:
12 |
13 | ```
14 | python3 main.py
15 | ```
16 |
17 | ### Modules used:
18 | * `requests` and `grequests` to get the html
19 | * BeautifulSoup4 (`bs4`) to parse the html
20 | * `selenium` to make CodeForces scraper more reliable
21 | * `webdriver_manager` to automatically create the chromium executable
22 | * `PyGithub` to access the GitHub API
23 | * `json` to parse CodeForces API
24 | * `multiprocessing` to parallelize CodeForces and CodeChef uploads
25 | * Misc: `time`, `logging`, `dotenv`, `inspect`
26 |
--------------------------------------------------------------------------------
/.idea/inspectionProfiles/Project_Default.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
--------------------------------------------------------------------------------
/AtcoderScraper.py:
--------------------------------------------------------------------------------
1 | import grequests
2 | import requests
3 | import json
4 | from time import sleep
5 | from bs4 import BeautifulSoup
6 |
7 |
8 | headers = {
9 | 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.63 Safari/537.36',
10 | }
11 |
12 |
13 | def get_submission_info(username):
14 | cur = 0
15 |
16 | while True:
17 | submissions = json.loads(requests.get(f'https://kenkoooo.com/atcoder/atcoder-api/v3/user/submissions?user={username}&from_second={cur}').text)
18 | if not submissions:
19 | break
20 |
21 | for submission in submissions:
22 | if submission['result'] == 'AC':
23 | try:
24 | yield {
25 | 'language': submission['language'],
26 | 'problem_code': submission['problem_id'],
27 | 'solution_id': submission['id'],
28 | 'problem_link': f'https://atcoder.jp/contests/{submission["contest_id"]}/tasks/{submission["problem_id"]}',
29 | 'link': f'https://atcoder.jp/contests/{submission["contest_id"]}/submissions/{submission["id"]}',
30 | }
31 |
32 | except KeyError:
33 | pass
34 |
35 | cur = submission['epoch_second'] + 1
36 |
37 | sleep(1)
38 |
39 |
40 | def get_code(html):
41 | soup = BeautifulSoup(html, 'lxml')
42 | return soup.select_one('#submission-code').text
43 |
44 |
45 | def get_solutions(username, all_info=None):
46 | if all_info is None:
47 | all_info = list(get_submission_info(username))[::-1]
48 |
49 | responses = grequests.imap(grequests.get(info['link'], headers=headers) for info in all_info)
50 | for response, info in zip(responses, all_info):
51 | code = get_code(response.text)
52 | yield {
53 | 'language': info['language'],
54 | 'problem_code': info['problem_code'],
55 | 'solution_id': info['solution_id'],
56 | 'problem_link': info['problem_link'],
57 | 'link': info['link'],
58 | 'solution': code,
59 | }
60 |
--------------------------------------------------------------------------------
/CodeChefScraper.py:
--------------------------------------------------------------------------------
1 | import grequests
2 | import requests
3 | import json
4 | from bs4 import BeautifulSoup, SoupStrainer
5 |
6 | headers = {
7 | 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36',
8 | }
9 |
10 |
11 | def get_links(username):
12 | html = requests.get(f'https://www.codechef.com/users/{username}').text
13 | soup = BeautifulSoup(html, 'lxml')
14 |
15 | for link in soup.select('body > main > div > div > div > div > div > section.rating-data-section.problems-solved > div > article:nth-child(2) > p > span > a'):
16 | yield 'https://www.codechef.com' + link['href'] + '?status=FullAC'
17 |
18 |
19 | def get_submission_links(html):
20 | soup = BeautifulSoup(html, 'lxml', parse_only=SoupStrainer('td'))
21 | return [f'https://www.codechef.com/viewsolution/{obj.text}' for obj in
22 | soup.select('#content > div > div > div.tablebox-section.l-float > table > tbody > tr > td:nth-child(1)')]
23 |
24 |
25 | def get_info(solution_id):
26 | link1 = f'https://www.codechef.com/api/submission-code/{solution_id}'
27 | link2 = f'https://www.codechef.com/api/submission-details/{solution_id}'
28 | data1 = requests.get(link1, headers=headers).text
29 | data2 = requests.get(link2, headers=headers).text
30 | json_data1 = json.loads(data1)
31 | json_data2 = json.loads(data2)
32 |
33 | contest_code = json_data2['data']['other_details']['contestCode']
34 | problem_code = json_data2['data']['other_details']['problemCode']
35 |
36 | return {
37 | 'language': json_data1['data']['language']['short_name'],
38 | 'problem_code': problem_code,
39 | 'solution_id': solution_id,
40 | 'problem_link': f'https://www.codechef.com/{contest_code}/problems/{problem_code}',
41 | 'link': f'https://www.codechef.com/viewsolution/{solution_id}',
42 | 'solution': json_data1['data']['code'],
43 | }
44 |
45 |
46 | def get_solutions(username):
47 | links = list(get_links(username))[::-1]
48 | responses = grequests.imap(grequests.get(u) for u in links)
49 |
50 | submission_links = (link for response in responses for link in get_submission_links(response.content))
51 | submission_responses = grequests.imap(grequests.get(u, headers=headers) for u in submission_links)
52 |
53 | for response in submission_responses:
54 | solution_id = (response.url.split('/')[-1])
55 | yield get_info(solution_id)
56 |
--------------------------------------------------------------------------------
/main.py:
--------------------------------------------------------------------------------
1 | import CodeChefScraper
2 | import CodeForcesScraper
3 | import AtcoderScraper
4 | from time import sleep
5 | import logging
6 | from github import Github
7 | from github.GithubException import UnknownObjectException
8 | from UploadToGithub import upload_to_github
9 | import inspect
10 | # from multiprocessing import Process
11 |
12 | EXTENSIONS = {
13 | 'c++': 'cpp',
14 | 'clang': 'cpp',
15 | 'gcc': 'c',
16 | 'py': 'py',
17 | 'javascript': 'js',
18 | 'java': 'java',
19 | 'c#': 'cs',
20 | 'go': 'go',
21 | 'haskell': 'hs',
22 | 'kotlin': 'kt',
23 | 'delphi': 'dpr',
24 | 'pascal': 'pas',
25 | 'perl': 'pl',
26 | 'php': 'php',
27 | 'rust': 'rs',
28 | 'scala': 'sc',
29 | 'node': 'js',
30 | }
31 |
32 |
33 | def upload_solution(website, solution, repo):
34 | try:
35 | s = solution["language"].lower()
36 |
37 | extension = 'txt'
38 | for key, value in EXTENSIONS.items():
39 | if key in s:
40 | extension = value
41 | break
42 |
43 | if solution.get('problem_name', ''):
44 | path = f'{website}/{solution["language"]}/{solution["problem_code"]} | {solution["problem_name"]}/{solution["solution_id"]}.{extension}'
45 | else:
46 | path = f'{website}/{solution["language"]}/{solution["problem_code"]}/{solution["solution_id"]}.{extension}'
47 |
48 | problem_info = ''
49 | if solution.get('problem_link', ''):
50 | if not solution.get('problem_name', ''):
51 | problem_info = f"""
52 | # This text file was auto-generated by UpCode
53 |
54 | Link: {solution['problem_link']}
55 | """
56 |
57 | else:
58 | problem_info = f"""
59 | # This text file was auto-generated by UpCode
60 |
61 | Problem: {solution['problem_name']}
62 | Link: {solution['problem_link']}
63 | """
64 |
65 | upload_to_github(repo, path, solution['solution'], inspect.cleandoc(problem_info))
66 | return True
67 |
68 | except Exception as e:
69 | logging.error(f'{e} FOR {solution}')
70 | return False
71 |
72 |
73 | def codeforces_uploader(codeforces_username, repo):
74 | failed_codeforces = []
75 | for solution in CodeForcesScraper.get_solutions(codeforces_username):
76 | if not upload_solution('CodeForces', solution, repo):
77 | failed_codeforces.append(solution)
78 |
79 | for _ in range(3):
80 | if failed_codeforces:
81 | sleep(180)
82 |
83 | new_failed_codeforces = []
84 | for solution in CodeForcesScraper.get_solutions(codeforces_username, failed_codeforces):
85 | if not upload_solution('CodeForces', solution, repo):
86 | new_failed_codeforces.append(solution)
87 |
88 | failed_codeforces = new_failed_codeforces
89 |
90 |
91 | def codechef_uploader(codechef_username, repo):
92 | for solution in CodeChefScraper.get_solutions(codechef_username):
93 | upload_solution('CodeChef', solution, repo)
94 |
95 |
96 | def atcoder_uploader(atcoder_username, repo):
97 | for solution in AtcoderScraper.get_solutions(atcoder_username):
98 | upload_solution('Atcoder', solution, repo)
99 |
100 |
101 | def main():
102 | codeforces_username = input('Enter codeforces username (Press enter if N/A): ')
103 | codechef_username = input('Enter codechef username (Press enter if N/A): ')
104 | atcoder_username = input('Enter atcoder username (Press enter if N/A): ')
105 | access_token = input('Enter github access token: ')
106 |
107 | repo_name = input('Enter repository name (Press enter to use "CP-Solutions"): ')
108 | if repo_name.isspace() or not repo_name:
109 | repo_name = 'CP-Solutions'
110 |
111 | g = Github(access_token)
112 |
113 | try:
114 | repo = g.get_user().get_repo(repo_name)
115 |
116 | except UnknownObjectException:
117 | repo = g.get_user().create_repo(repo_name, private=True)
118 |
119 | if atcoder_username:
120 | atcoder_uploader(atcoder_username, repo)
121 | # atcoder_process = Process(target=atcoder_uploader, args=(atcoder_username, repo))
122 | # atcoder_process.start()
123 |
124 | if codechef_username:
125 | codechef_uploader(codechef_username, repo)
126 | # codechef_process = Process(target=codechef_uploader, args=(codechef_username, repo))
127 | # codechef_process.start()
128 |
129 | if codeforces_username:
130 | codeforces_uploader(codeforces_username, repo)
131 | # codeforces_process = Process(target=codeforces_uploader, args=(codeforces_username, repo))
132 | # codeforces_process.start()
133 |
134 |
135 | if __name__ == '__main__':
136 | main()
137 |
--------------------------------------------------------------------------------
/CodeForcesScraper.py:
--------------------------------------------------------------------------------
1 | import requests
2 | import json
3 | from json.decoder import JSONDecodeError
4 | import logging
5 | from time import sleep
6 |
7 | from selenium import webdriver
8 | from selenium.webdriver.chrome.options import Options
9 | from webdriver_manager.chrome import ChromeDriverManager
10 | from selenium.webdriver.common.by import By
11 | from selenium.webdriver.support.ui import Select
12 | from selenium.common.exceptions import *
13 |
14 | logging.getLogger('WDM').setLevel(logging.NOTSET)
15 |
16 | headers = {
17 | 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.63 Safari/537.36',
18 | }
19 |
20 |
21 | def get_submission_info(username):
22 | submissions = json.loads(requests.get(f'https://codeforces.com/api/user.status?handle={username}').text)['result']
23 |
24 | for submission in submissions:
25 | if submission['verdict'] == 'OK':
26 | try:
27 | if len(str(submission["problem"]["contestId"])) <= 4 and len(submission["author"]["members"]) == 1:
28 | yield {
29 | 'language': submission['programmingLanguage'],
30 | 'problem_code': f'{submission["problem"]["contestId"]}{submission["problem"]["index"]}',
31 | 'solution_id': submission['id'],
32 | 'problem_name': submission['problem']['name'] if 'name' in submission['problem'] else '',
33 | 'problem_link': f'https://codeforces.com/contest/{submission["problem"]["contestId"]}/problem/{submission["problem"]["index"]}',
34 | 'link': f'https://codeforces.com/contest/{submission["contestId"]}/submission/{submission["id"]}?f0a28=2',
35 | }
36 |
37 | except KeyError:
38 | pass
39 |
40 |
41 | def get_code(driver):
42 | lines = driver.find_elements(By.CSS_SELECTOR, '#program-source-text > ol > li')
43 | return '\n'.join(line.text for line in lines)
44 |
45 |
46 | def get_solutions(username, all_info=None):
47 | try:
48 | if all_info is None:
49 | all_info = list(get_submission_info(username))
50 |
51 | except JSONDecodeError:
52 | logging.error("CodeForces API is currently unavailable. Please try again later.")
53 | return
54 |
55 | sub_id_info = {info['solution_id']: info for info in all_info}
56 | for info in all_info:
57 | sub_id_info[info['solution_id']] = info
58 |
59 | options = Options()
60 | options.add_argument('headless')
61 | options.add_argument('window-size=1920x1080')
62 | options.add_argument('disable-gpu')
63 |
64 | driver = webdriver.Chrome(ChromeDriverManager().install(), options=options)
65 | driver.get(f'https://codeforces.com/submissions/{username}')
66 |
67 | sleep(1)
68 |
69 | select = Select(driver.find_element(By.ID, 'verdictName'))
70 | select.select_by_value('OK')
71 |
72 | driver.find_element(By.CSS_SELECTOR, 'input[value=Apply]').click()
73 |
74 | sub_ids = [info['solution_id'] for info in all_info]
75 |
76 | try:
77 | pages = int(driver.find_elements(By.CSS_SELECTOR, '#pageContent > div > ul > li > span')[-1].text)
78 |
79 | except IndexError:
80 | pages = 1
81 |
82 | index = 1
83 |
84 | driver.get(f'https://codeforces.com/submissions/{username}/page/{index}')
85 |
86 | prev = {}
87 |
88 | fail_counter = 0
89 |
90 | failed = []
91 | for sub_id in sub_ids:
92 | if index > pages:
93 | break
94 |
95 | if fail_counter:
96 | try:
97 | driver.get(sub_id_info[sub_id]['link'])
98 |
99 | code = get_code(driver)
100 | sub_id_info[sub_id]['solution'] = code
101 | prev[code] = sub_id_info[sub_id]
102 | yield sub_id_info[sub_id]
103 |
104 | fail_counter -= 1
105 | if fail_counter == 0:
106 | driver.get(f'https://codeforces.com/submissions/{username}/page/{index}')
107 |
108 | sleep(0.3)
109 | continue
110 |
111 | except NoSuchElementException:
112 | failed.append((sub_id, index))
113 | sleep(120)
114 |
115 | driver.quit()
116 |
117 | driver = webdriver.Chrome(ChromeDriverManager().install(), options=options)
118 | driver.get(f'https://codeforces.com/submissions/{username}/page/{index}')
119 | fail_counter = 0
120 |
121 | for _ in range(3):
122 | try:
123 | element = driver.find_element(By.PARTIAL_LINK_TEXT, str(sub_id))
124 | driver.execute_script("arguments[0].click();", element)
125 | sleep(0.3)
126 |
127 | counter = 0
128 |
129 | cur = []
130 | while not cur or ((cur in prev) and ((
131 | sub_id_info[sub_id]['problem_code'] != prev[cur]['problem_code']) or
132 | (sub_id_info[sub_id]['problem_code'][-1].isdigit() and prev[cur]['problem_code'][-1].isdigit()))
133 | ):
134 | sleep(0.3)
135 | cur = '\n'.join(ele.text for ele in driver.find_elements(By.CSS_SELECTOR, '#facebox > div > div > div > pre > code > ol > li'))
136 |
137 | counter += 1
138 | if counter % 3 == 0:
139 | driver.refresh()
140 | driver.execute_script("arguments[0].click();", element)
141 |
142 | sleep(0.3)
143 | cur = '\n'.join(ele.text for ele in driver.find_elements(By.CSS_SELECTOR, '#facebox > div > div > div > pre > code > ol > li'))
144 |
145 | code = cur
146 |
147 | sub_id_info[sub_id]['solution'] = code.replace('\u00a0', '\n')
148 | prev[code] = sub_id_info[sub_id]
149 | yield sub_id_info[sub_id]
150 |
151 | break
152 |
153 | except NoSuchElementException:
154 | if driver.current_url == 'https://codeforces.com/':
155 | sleep(60)
156 |
157 | else:
158 | index += 1
159 | driver.get(f'https://codeforces.com/submissions/{username}/page/{index}')
160 | break
161 |
162 | except StaleElementReferenceException:
163 | driver.execute_script("location.reload(true);")
164 | sleep(2)
165 |
166 | else:
167 | fail_counter = 5
168 |
--------------------------------------------------------------------------------