├── .gitignore ├── requirements.txt ├── result.png ├── README.md └── versionshaker.py /.gitignore: -------------------------------------------------------------------------------- 1 | .tmp/ 2 | .idea/ -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | urllib3 2 | requests 3 | rich 4 | GitPython -------------------------------------------------------------------------------- /result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Orange-Cyberdefense/versionshaker/HEAD/result.png -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # versionshaker 2 | 3 | Versionshaker is a tool to find a remote website version based on a git repository 4 | This tool will help you to find the website version of the github project in use based on static files like js,css,xml... 5 | 6 | To use this tool you will need : 7 | - Identify the git repository to use based on usual fingerprint (wordpress/joomla/magento/prestashop/drupal/...) 8 | - Identify the interesting statics files (js, css, xml, ...) 9 | - run with : `python3 versionshaker.py -c -u -f ` 10 | 11 | [![asciicast](result.png)](https://asciinema.org/a/WYD8WtfnULpbfkz5uKMmqLExF) 12 | 13 | ## Install 14 | 15 | ``` 16 | python3 -m pip install -r requirements.txt 17 | ``` 18 | 19 | ## Run 20 | 21 | You can run the project with the following options : 22 | - -u : the remote website url 23 | - -c or -l : use a remote git project to clone locally (inside the .tmp/ folder) or use an existing folder (be careful, this options will force checkout the different tags and so this can break unsaved work) 24 | - -f : the list of files to check (comma separated) 25 | - -v : add verbosity (optional) 26 | - -p : path corresponding to the web folder inside the git repository (useful in case of public/ folder by example) 27 | - -P : add proxy options (optional) 28 | - -t : the list of tags to check coma separated instead of the (optional) 29 | 30 | Example : 31 | ``` 32 | python3 versionshaker.py -c https://github.com/WordPress/WordPress -u https://wordpress.website.example/ -f wp-includes/css/dist/block-library/style.min.css,wp-includes/js/jquery/jquery.min.js 33 | ``` 34 | -------------------------------------------------------------------------------- /versionshaker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # coding: utf-8 3 | 4 | import argparse 5 | import requests 6 | import os 7 | import sys 8 | from rich import print 9 | from rich.progress import Progress 10 | from rich.table import Table 11 | from rich.console import Console 12 | from os.path import isfile 13 | from git import Repo 14 | from difflib import SequenceMatcher 15 | from urllib3.exceptions import InsecureRequestWarning 16 | 17 | # Suppress only the single warning from urllib3 needed. 18 | requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) 19 | 20 | 21 | def parser(): 22 | parser = argparse.ArgumentParser( 23 | description='Version checker 0.1', 24 | epilog='Example : python3 version_checker.py -c https://github.com/repo -u http://url/ -f js/admin.js,js/tools.js') 25 | parser.add_argument('-u', '--url', help="target url ended with a /", required=True) 26 | 27 | group = parser.add_mutually_exclusive_group(required=True) 28 | group.add_argument('-c', '--clone', help="git url to clone locally") 29 | group.add_argument('-l', '--local', help="local git repository to compare") 30 | 31 | group2 = parser.add_mutually_exclusive_group(required=True) 32 | # group2.add_argument('-x', '--extensions', help="Extensions to check (default : js,css)", required=False) 33 | group2.add_argument('-f', '--files', 34 | help="files relative path separated by ,", 35 | required=False) 36 | 37 | parser.add_argument('-t', '--tags', help="force tags to use coma separated", required=False) 38 | parser.add_argument('-p', '--path', help="git repository web folder location (useful in case of public folder)", 39 | required=False) 40 | parser.add_argument('-v', '--verbose', help="Give more verbosity", action="store_true") 41 | parser.add_argument('-P', '--proxy', help="use proxy to launch request to url", required=False) 42 | return parser.parse_args() 43 | 44 | 45 | def color_ratio(ratio): 46 | """ 47 | define the color for ratio 48 | :param ratio: 49 | :return: string (color) 50 | """ 51 | if ratio == 100: 52 | color = 'green' 53 | elif ratio >= 95: 54 | color = 'blue' 55 | elif ratio >= 90: 56 | color = 'yellow' 57 | else: 58 | color = 'red' 59 | return color 60 | 61 | 62 | class VersionChecker: 63 | 64 | def __init__(self, url, git=None, local=None, verbose=False, files=None, extensions=None, proxy=None, tags=None, 65 | web_folder=None): 66 | """ 67 | Init the class 68 | :param url: the remote url to check 69 | :param git: the git repository url to clone 70 | :param local: the local git repository (precedence on git) 71 | :param verbose: more output 72 | :param files: file list to check 73 | :param extensions: extension list to automatic get files (not implemented yet) 74 | :param proxy: proxy parameter value 75 | :param tags: list of tags to check (by default all tags are verified) 76 | """ 77 | self.url = url 78 | if self.url[-1:] != '/': 79 | self.url += '/' 80 | self.git = git 81 | self.local = local 82 | self.verbose = verbose 83 | self.repo_local_path = '.tmp/' 84 | if web_folder is None: 85 | self.web_folder = '' 86 | else: 87 | self.web_folder = web_folder 88 | if self.web_folder != '': 89 | if self.web_folder[-1:] != '/': 90 | self.web_folder += '/' 91 | if tags is not None: 92 | self.tags = [x.strip() for x in tags.split(',')] 93 | else: 94 | self.tags = None 95 | self.repo = None 96 | self.files_to_check = None 97 | if files is not None: 98 | self.files = [x.strip() for x in files.split(',')] 99 | self.extensions = None 100 | else: 101 | self.files = None 102 | if extensions is None: 103 | self.extensions = ['js', 'css'] 104 | else: 105 | self.extensions = [x.strip() for x in extensions.split(',')] 106 | if proxy is not None: 107 | self.proxy = { 108 | 'http': proxy, 109 | 'https': proxy 110 | } 111 | else: 112 | self.proxy = None 113 | 114 | def clone(self): 115 | """ 116 | Clone git repository locally inside the .tmp folder 117 | :return: 118 | """ 119 | tmp_folder = self.repo_local_path 120 | print("[blue] Clone the git %s repository [/blue]" % self.git) 121 | try: 122 | os.makedirs(tmp_folder) 123 | except: 124 | print( 125 | "[yellow] [-] We use %s to clone repository, this directory already exist[/yellow] " % tmp_folder) 126 | inp = input(" Delete ? [Y/n] ") 127 | if inp.lower() == "" or inp.lower() == "y": 128 | os.system('rm -rf ' + tmp_folder) 129 | os.makedirs(tmp_folder) 130 | else: 131 | self.repo = Repo(tmp_folder) 132 | return 133 | print("[blue] [+] Cloning repo please wait... [/blue] ") 134 | Repo.clone_from(self.git, tmp_folder) 135 | if self.verbose: 136 | print("[blue] [+] Cloned ! [/blue]") 137 | self.repo = Repo(tmp_folder) 138 | 139 | def check_url(self): 140 | """ 141 | Check the remote URL 142 | exit if the response code is not 200 143 | :return: response text 144 | """ 145 | r = requests.get(self.url, proxies=self.proxy, verify=False) 146 | if r.status_code != 200: 147 | print('[red] url status code %i is not 200 exit [/red]' % r.status_code) 148 | sys.exit(1) 149 | return r.text 150 | 151 | def auto_discover_files(self, url_text_content): 152 | """ 153 | TODO : Automatic static files discovery based on the url and the extensions 154 | :return: list of valid files 155 | """ 156 | print('[red] recon not implemented yet [/red]') 157 | return [] 158 | 159 | def check_files_exists(self): 160 | """ 161 | Check if the files are available on the remote target 162 | :return: list of files with response code 200 163 | """ 164 | file_list = [] 165 | for file in self.files: 166 | r = requests.get(self.url + file, proxies=self.proxy, verify=False) 167 | if r.status_code == 200: 168 | print('[green] [+] file %s found on server [/green]' % file) 169 | file_list.append((file, r.text)) 170 | else: 171 | print('[yellow] [-] file %s not found on server skip [/yellow]' % file) 172 | return file_list 173 | 174 | def process_tag(self, tag_name, files): 175 | results = {} 176 | self.repo.git.checkout(tag_name, force=True) 177 | for (file, text) in files: 178 | if isfile(self.repo_local_path + self.web_folder + file): 179 | try: 180 | with open(self.repo_local_path + self.web_folder + file, 'r') as f: 181 | git_file = f.read().encode('utf-8') 182 | web_file = text.encode('utf-8') 183 | ratio = SequenceMatcher(None, git_file, web_file).quick_ratio() 184 | ratio *= 100.00 185 | color = color_ratio(ratio) 186 | results[file] = ratio 187 | ratio = '[' + color + '] ' + str(ratio) + '[/' + color + ']' 188 | if self.verbose: 189 | print('[green] %s ratio for file %s is %s[/green]' % (tag_name, file, ratio)) 190 | except UnicodeDecodeError: 191 | results[file] = -1 192 | else: 193 | if self.verbose: 194 | print('[red] file %s not found [/red]' % file) 195 | results[file] = -1 196 | return results 197 | 198 | def check_diff(self, tags, files): 199 | """ 200 | Check the difference between the file on remote and on git tag 201 | :param tags: list of tags to verify 202 | :param files: list of files to check 203 | :return: result dict format : result[tag][file]=ratio 204 | """ 205 | results = {} 206 | if not self.verbose: 207 | with Progress() as progress: 208 | task1 = progress.add_task("[cyan]checking tags...", total=len(tags)) 209 | for tag_name in tags: 210 | results[tag_name] = self.process_tag(tag_name, files) 211 | progress.update(task1, advance=1, description='[cyan]Checking tags :[/cyan] %15s' % tag_name) 212 | else: 213 | for tag_name in tags: 214 | if self.verbose: 215 | print('[blue] --- tag : %s --- [/blue]' % tag_name) 216 | results[tag_name] = self.process_tag(tag_name, files) 217 | return results 218 | 219 | def compile_tags_ratio(self, results): 220 | """ 221 | Compile the results {tag => {file => ratio} } to get the bests match 222 | :param results: 223 | :return: dict { tag => ratio checked files} 224 | """ 225 | bests = {} 226 | for tag, result in results.items(): 227 | tag_ratio = 0 228 | nb_files = 0 229 | for file, ratio in result.items(): 230 | tag_ratio += ratio 231 | nb_files += 1 232 | bests[tag] = tag_ratio / nb_files 233 | return bests 234 | 235 | def compile_tags_ratio_total(self, results, total_files): 236 | """ 237 | Compile the results {tag => {file => ratio} } to get the bests match 238 | :param total_files: total number of file to check 239 | :param results: 240 | :return: dict { tag => (ratio checked files, ratio total files)} 241 | """ 242 | bests = {} 243 | for tag, result in results.items(): 244 | tag_ratio = 0 245 | nb_files = 0 246 | for file, ratio in result.items(): 247 | tag_ratio += ratio 248 | nb_files += 1 249 | bests[tag] = tag_ratio / total_files 250 | return bests 251 | 252 | def compile_files_best_tags(self, results): 253 | """ 254 | Compile the results {tag => {file => ratio} } to get the bests match 255 | :param results: 256 | :return: dict { file => ratio, tags} 257 | """ 258 | bests = {} 259 | for tag, result in results.items(): 260 | for file, ratio in result.items(): 261 | if file in bests: 262 | (best_ratio, best_tag) = bests[file] 263 | if ratio > best_ratio: 264 | bests[file] = (ratio, tag) 265 | elif ratio == best_ratio: 266 | bests[file] = (ratio, tag + ', ' + best_tag) 267 | else: 268 | bests[file] = (ratio, tag) 269 | return bests 270 | 271 | def compile_tag_nb_best_matching_files(self, bests_files): 272 | """ 273 | Return a tag dict with the number of best match in files checked 274 | :param bests_files: 275 | :return: dict : {tag => nb_files_best_match} 276 | """ 277 | # print tags results 278 | tag_nb_best_match = {} 279 | best = 0 280 | for file, values in bests_files.items(): 281 | (ratio, tags) = values 282 | for tag in [x.strip() for x in tags.split(',')]: 283 | if tag not in tag_nb_best_match: 284 | tag_nb_best_match[tag] = 1 285 | else: 286 | tag_nb_best_match[tag] += 1 287 | return tag_nb_best_match 288 | 289 | def compile_find_best_tag(self, tags_ratio_checked, tags_ratio_total, tags_nb_best_match): 290 | """ 291 | Return a list of best tags based on ratio total, ratio on file checked and nb best file match 292 | :param tags_ratio_checked: 293 | :param tags_ratio_total: 294 | :param tags_nb_best_match: 295 | :return: list of best tags 296 | """ 297 | best_tags = [] 298 | best = 0 299 | max_ratio_checked_tag = max(tags_ratio_checked, key=tags_ratio_checked.get) 300 | max_ratio_total_tag = max(tags_ratio_total, key=tags_ratio_total.get) 301 | 302 | max_files_match = max(tags_nb_best_match.values()) 303 | 304 | for tag, nb in tags_nb_best_match.items(): 305 | if nb == max_files_match: 306 | best_tags.append(tag) 307 | 308 | best_tags.append(max_ratio_checked_tag) 309 | best_tags.append(max_ratio_total_tag) 310 | return best_tags 311 | 312 | def print_results(self, results, nb_files): 313 | """ 314 | print the check results 315 | :param nb_files: total files number to check 316 | :param results: {tag => {file => ratio} } 317 | """ 318 | bests_files = self.compile_files_best_tags(results) 319 | tags_ratio_checked = self.compile_tags_ratio(results) 320 | tags_ratio_total = self.compile_tags_ratio_total(results, nb_files) 321 | tags_nb_best_match = self.compile_tag_nb_best_matching_files(bests_files) 322 | best_tags = self.compile_find_best_tag(tags_ratio_checked, tags_ratio_total, tags_nb_best_match) 323 | 324 | print('\n[blue] --- RESULTS by files --- [/blue]') 325 | for file, value in bests_files.items(): 326 | (ratio, tag) = value 327 | color = color_ratio(ratio) 328 | r = '[' + color + '] ' + str(ratio) + '[/' + color + ']' 329 | print('file %s : %s (%s)' % (file, tag, r)) 330 | 331 | console = Console() 332 | print('\n[blue] --- RESULTS by tags --- [/blue]') 333 | table = Table(show_header=True, header_style="bold magenta") 334 | table.add_column("Tag") 335 | table.add_column("Ratio on checked files") 336 | table.add_column("Ratio on all files") 337 | table.add_column("Number of best choice on %i files" % nb_files) 338 | for tag, nb_file_best in sorted(tags_nb_best_match.items()): 339 | tag_ratio_checked = tags_ratio_checked[tag] 340 | tag_ratio_total = tags_ratio_total[tag] 341 | ratio_checked = '[' + color_ratio(tag_ratio_checked) + '] ' + str(tag_ratio_checked) + '[/' + color_ratio( 342 | tag_ratio_checked) + ']' 343 | ratio_total = '[' + color_ratio(tag_ratio_total) + '] ' + str(tag_ratio_total) + '[/' + color_ratio( 344 | tag_ratio_total) + ']' 345 | if tag in best_tags: 346 | table.add_row('[green]' + tag + '[/green]', ratio_checked, ratio_total, 347 | '[green]' + str(nb_file_best) + '[/green]') 348 | else: 349 | table.add_row(tag, ratio_checked, ratio_total, str(nb_file_best)) 350 | console.print(table) 351 | 352 | def init_git_repository(self): 353 | """ 354 | Init the git repository with local folder or clone remote repository 355 | :return: repository 356 | """ 357 | if self.local is None: 358 | self.clone() 359 | else: 360 | try: 361 | self.repo = Repo(self.local) 362 | if self.local[-1:] != '/': 363 | self.repo_local_path = self.local + '/' 364 | else: 365 | self.repo_local_path = self.local 366 | except: 367 | print('[red]Local git repository (%s) not found[/red]' % self.local) 368 | sys.exit(1) 369 | return self.repo 370 | 371 | def init_files_to_check(self, response): 372 | """ 373 | Init the file list to check 374 | :param response: 375 | :return: list of files path 376 | """ 377 | if self.files is None: 378 | self.files_to_check = self.auto_discover_files(response) 379 | else: 380 | self.files_to_check = self.check_files_exists() 381 | return self.files_to_check 382 | 383 | def init_tag_list(self): 384 | """ 385 | Init the list of tags to check 386 | :return: list of tags 387 | """ 388 | if self.tags is not None: 389 | tags = self.tags 390 | else: 391 | tags = [tag.name for tag in self.repo.tags] 392 | 393 | if self.verbose: 394 | print('[blue] --- tag list --- [/blue]') 395 | print("[green]" + ", ".join(tags) + "[/green]") 396 | return tags 397 | 398 | def execute(self): 399 | """ 400 | run function 401 | """ 402 | response = self.check_url() 403 | files = self.init_files_to_check(response) 404 | self.init_git_repository() 405 | tags = self.init_tag_list() 406 | results = self.check_diff(tags, files) 407 | self.print_results(results, len(files)) 408 | 409 | 410 | if __name__ == "__main__": 411 | arg = parser() 412 | checker = VersionChecker(arg.url, arg.clone, arg.local, verbose=arg.verbose, files=arg.files, proxy=arg.proxy, 413 | tags=arg.tags, web_folder=arg.path) 414 | checker.execute() 415 | --------------------------------------------------------------------------------