├── .gitignore ├── Client └── disfuzz.py ├── LICENSE ├── README.md └── Server ├── .htaccess ├── _config.php ├── _func.php ├── baseline ├── example │ ├── bin │ │ ├── COPYING │ │ ├── README │ │ ├── afl-fuzz │ │ ├── instrumented_cmp │ │ └── instrumented_cmp.c │ ├── init.sh │ ├── input │ │ └── test │ ├── run.sh │ └── upgrade.sh └── index.html ├── data └── index.html ├── download.php ├── files.php ├── index.php ├── session.php └── submit.php /.gitignore: -------------------------------------------------------------------------------- 1 | */.DS_Store 2 | .idea/* -------------------------------------------------------------------------------- /Client/disfuzz.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import multiprocessing 4 | import socket 5 | import urllib 6 | import json 7 | import sys 8 | import random 9 | import os 10 | import shutil 11 | import time 12 | import hashlib 13 | import subprocess 14 | import psutil 15 | import traceback 16 | import requests 17 | import tmuxp 18 | 19 | from glob import glob 20 | 21 | DISFUZZ_HOST = "https://www.example.com/api/v1" 22 | DISFUZZ_VERSION = "0.1b" 23 | 24 | host_config = { 25 | "hostname": socket.gethostname(), 26 | "processors": multiprocessing.cpu_count(), 27 | "amount_auto_projects": multiprocessing.cpu_count(), 28 | "projects_folder": "./projects", 29 | "sync_sleep": 120 30 | } 31 | 32 | # --------------------------------------------------------- 33 | 34 | def list_projects(): 35 | print "Available projects:" 36 | 37 | for project_name in available_projects: 38 | if is_current_project(project_name): 39 | if is_project_available(project_name): 40 | 41 | 42 | try: 43 | if is_project_update_available(project_name): 44 | print "-> " + project_name + " (" + available_projects[project_name][ 45 | "name"] + ") New version available!" 46 | else: 47 | print "-> " + project_name + " (" + available_projects[project_name]["name"] + ") " + \ 48 | "Version " + get_current_project_version(project_name) + " " + \ 49 | "Last update " + time.ctime( 50 | os.path.getmtime(get_project_meta_path(project_name) + "/.version")) 51 | except ApiException: 52 | print "-> " + project_name + " (" + available_projects[project_name]["name"] + ")" 53 | else: 54 | print "-> " + project_name + " (" + available_projects[project_name]["name"] + ") Not longer available!" 55 | else: 56 | print " " + project_name + " (" + available_projects[project_name]["name"] + ")" 57 | 58 | 59 | def init_project(project_name): 60 | print "Init project " + project_name + "..." 61 | 62 | projects = get_available_projects() 63 | 64 | if not project_name in projects: 65 | print "Error: Unknown project " + project_name + "!" 66 | return 67 | 68 | try: 69 | project_info = do_api_request(projects[project_name]["setup_url"]) 70 | except ApiException as e: 71 | print "Init of project failed!" 72 | print e 73 | return 74 | 75 | if is_current_project(project_name): 76 | print "WARNING: continuing will remove your progress for this project!" 77 | time.sleep(3) 78 | 79 | shutil.rmtree(get_project_path(project_name)) 80 | 81 | os.mkdir(get_project_path(project_name)) 82 | os.mkdir(get_project_meta_path(project_name)) 83 | os.mkdir(get_project_meta_path(project_name) + "/submission") 84 | os.mkdir(get_project_path(project_name) + "/input") 85 | os.mkdir(get_project_path(project_name) + "/output") 86 | os.mkdir(get_project_path(project_name) + "/output/import") 87 | os.mkdir(get_project_path(project_name) + "/output/import/queue") 88 | 89 | store_current_project_info(project_name, project_info) 90 | update_project_files(project_name) 91 | 92 | if os.path.exists(get_project_path(project_name) + "/init.sh"): 93 | os.chmod(get_project_path(project_name) + "/init.sh", 0744) 94 | 95 | psutil.Popen("./init.sh", shell=True, cwd=get_project_path(project_name)).wait() 96 | 97 | print "Project " + project_name + " is now ready to run!" 98 | 99 | 100 | def update_project(project_name): 101 | print "Update project " + project_name + "..." 102 | 103 | if not project_name in get_available_projects(): 104 | print "Error: Project " + project_name + " is not longer available!" 105 | return 106 | 107 | try: 108 | project_info = get_latest_project_info(project_name) 109 | except ApiException as e: 110 | print "Update of project failed!" 111 | print e 112 | return 113 | 114 | current_version = get_current_project_version(project_name) 115 | 116 | store_current_project_info(project_name, project_info) 117 | update_project_files(project_name) 118 | 119 | if os.path.exists(get_project_path(project_name) + "/upgrade.sh"): 120 | os.chmod(get_project_path(project_name) + "/upgrade.sh", 0744) 121 | 122 | psutil.Popen("./upgrade.sh", shell=True, cwd=get_project_path(project_name)).wait() 123 | 124 | if current_version != get_current_project_version(project_name): 125 | print "Updated " + project_name + " to version " + get_current_project_version(project_name) + "!" 126 | else: 127 | print "Nothing to update" 128 | 129 | 130 | def update_project_files(project_name): 131 | try: 132 | project_files_info = get_latest_project_file_info(project_name) 133 | except ApiException as e: 134 | print "Update of project files failed!" 135 | print e 136 | return 137 | 138 | for file_info in project_files_info["files"]: 139 | if not os.path.exists(get_project_path(project_name) + file_info["path"]): 140 | try: 141 | os.stat(get_project_path(project_name) + os.path.dirname(file_info["path"])) 142 | except: 143 | os.makedirs(get_project_path(project_name) + os.path.dirname(file_info["path"])) 144 | 145 | print "Downloading " + file_info["path"] + "..." 146 | 147 | urllib.urlretrieve(file_info["url"], 148 | get_project_path(project_name) + file_info["path"]) 149 | elif md5sum(get_project_path(project_name) + file_info["path"]) != file_info["md5sum"]: 150 | print "Updating " + file_info["path"] + "..." 151 | 152 | os.unlink(get_project_path(project_name) + file_info["path"]) 153 | 154 | urllib.urlretrieve(file_info["url"], 155 | get_project_path(project_name) + file_info["path"]) 156 | else: 157 | print "Skipped " + file_info["path"] 158 | 159 | store_current_project_version(project_name, project_files_info["version"]) 160 | 161 | 162 | def update_project_testcases(project_name): 163 | try: 164 | project_queue_submissions = get_current_project_queue_submission_state(project_name) 165 | except: 166 | project_queue_submissions = [] 167 | 168 | try: 169 | project_testcases_info = get_latest_project_testcases_info(project_name) 170 | except ApiException as e: 171 | print "Update of project testcases failed!" 172 | print e 173 | return 174 | 175 | for file_info in project_testcases_info["files"]: 176 | while "::" in file_info["path"]: 177 | file_info["path"] = file_info["path"].split("::")[1] 178 | 179 | if file_info["md5sum"] not in project_queue_submissions: # We don't want our own queue submissions back 180 | if not os.path.exists(get_project_path(project_name) + "/output/import/queue/" + file_info["path"]): 181 | try: 182 | os.stat( 183 | get_project_path(project_name) + "/output/import/queue/" + os.path.dirname(file_info["path"])) 184 | except: 185 | os.makedirs( 186 | get_project_path(project_name) + "/output/import/queue/" + os.path.dirname(file_info["path"])) 187 | 188 | print "Downloading " + file_info["path"] + "..." 189 | 190 | urllib.urlretrieve(file_info["url"], 191 | get_project_path(project_name) + "/output/import/queue/" + file_info["path"]) 192 | 193 | 194 | def sync_project(project_name, exclude_testcases=False): 195 | if not exclude_testcases: 196 | update_project_testcases(project_name) 197 | 198 | submit_project_queue(project_name) 199 | submit_project_hangs(project_name) 200 | submit_project_crashes(project_name) 201 | submit_project_instance_stats(project_name) 202 | 203 | 204 | def submit_project_hangs(project_name): 205 | project_info = get_current_project_info(project_name) 206 | 207 | try: 208 | project_hang_submissions = get_current_project_hang_submission_state(project_name) 209 | except: 210 | project_hang_submissions = [] 211 | 212 | for instance_path in glob(get_project_path(project_name) + "/output/*"): 213 | if os.path.isdir(instance_path) and os.path.basename(instance_path) != "import": 214 | for hang_file_path in glob(instance_path + "/hangs/*"): 215 | hang_file_path_md5sum = md5sum(hang_file_path) 216 | 217 | if hang_file_path_md5sum not in project_hang_submissions: 218 | print "Submitting " + hang_file_path + "..." 219 | 220 | requests.post(project_info["hang_submit_url"], 221 | files={"file": open(hang_file_path, 'rb')}) 222 | 223 | project_hang_submissions.append(hang_file_path_md5sum) 224 | else: # we have seen this or submitted this before, delete it 225 | os.unlink(hang_file_path) 226 | 227 | store_current_project_hang_submission_state(project_name, project_hang_submissions) 228 | 229 | 230 | def submit_project_crashes(project_name): 231 | project_info = get_current_project_info(project_name) 232 | 233 | try: 234 | project_crash_submissions = get_current_project_crash_submission_state(project_name) 235 | except: 236 | project_crash_submissions = [] 237 | 238 | for instance_path in glob(get_project_path(project_name) + "/output/*"): 239 | if os.path.isdir(instance_path) and os.path.basename(instance_path) != "import": 240 | for crash_file_path in glob(instance_path + "/crashes/*"): 241 | crash_file_path_md5sum = md5sum(crash_file_path) 242 | 243 | if crash_file_path_md5sum not in project_crash_submissions: 244 | print "Submitting " + crash_file_path + "..." 245 | 246 | requests.post(project_info["crash_submit_url"], 247 | files={"file": open(crash_file_path, 'rb')}) 248 | 249 | project_crash_submissions.append(crash_file_path_md5sum) 250 | else: # we have seen this or submitted this before, delete it 251 | os.unlink(crash_file_path) 252 | 253 | store_current_project_crash_submission_state(project_name, project_crash_submissions) 254 | 255 | 256 | def submit_project_queue(project_name): 257 | project_info = get_current_project_info(project_name) 258 | 259 | try: 260 | project_queue_submissions = get_current_project_queue_submission_state(project_name) 261 | except: 262 | project_queue_submissions = [] 263 | 264 | for instance_path in glob(get_project_path(project_name) + "/output/*"): 265 | if os.path.isdir(instance_path) and os.path.basename(instance_path) != "import": 266 | for queue_file_path in glob(instance_path + "/queue/*"): 267 | queue_file_path_md5sum = md5sum(queue_file_path) 268 | 269 | if queue_file_path_md5sum not in project_queue_submissions: 270 | print "Submitting " + queue_file_path + "..." 271 | 272 | requests.post(project_info["queue_submit_url"], 273 | files={"file": open(queue_file_path, 'rb')}) 274 | 275 | project_queue_submissions.append(queue_file_path_md5sum) 276 | 277 | store_current_project_queue_submission_state(project_name, project_queue_submissions) 278 | 279 | 280 | def submit_project_instance_stats(project_name): 281 | project_info = get_current_project_info(project_name) 282 | 283 | for fp in glob(get_project_path(project_name) + "/output/*/fuzzer_stats"): 284 | requests.post(project_info["session_submit_url"], 285 | files={"file": (os.path.basename(os.path.dirname(fp)), open(fp, 'rb'))}) 286 | 287 | 288 | def is_current_project(project_name): 289 | try: 290 | os.stat(get_project_path(project_name)) 291 | return True 292 | except: 293 | return False 294 | 295 | 296 | def is_project_update_available(project_name): 297 | return get_latest_project_version(project_name) != get_current_project_version(project_name) 298 | 299 | 300 | def is_project_running(project_name): 301 | try: 302 | project_state = get_current_project_state(project_name) 303 | 304 | is_running = False 305 | 306 | for pid in project_state["pid"]: 307 | if psutil.pid_exists(pid): 308 | is_running = True 309 | 310 | return is_running 311 | except: 312 | return False 313 | 314 | 315 | def is_project_finished(project_name): 316 | try: 317 | os.stat(get_project_path(project_name) + "/.meta/.finished") 318 | return True 319 | except: 320 | return False 321 | 322 | 323 | def start_project_instance(project_name, start_as_master=False, use_tmux=False): 324 | if not os.path.exists(get_project_path(project_name) + "/run.sh"): 325 | raise Exception("Missing run.sh!") 326 | 327 | try: 328 | if os.path.exists(get_project_meta_path(project_name) + "/.last_pid"): 329 | os.unlink(get_project_meta_path(project_name) + "/.last_pid") 330 | 331 | os.chmod(get_project_path(project_name) + "/run.sh", 0744) 332 | 333 | if use_tmux: 334 | start_cmd = "exec ./run.sh" 335 | window_name = "slave" 336 | 337 | if start_as_master: 338 | start_cmd += " -master" 339 | window_name = "master" 340 | 341 | tmux_server = tmuxp.Server(socket_path=get_project_meta_path(project_name) + "/.tmux_socket") 342 | 343 | if tmux_server.has_session(project_name): 344 | tmux_session = tmux_server.findWhere({"session_name": project_name}) 345 | tmux_window = tmux_session.new_window(attach=True) 346 | tmux_pane = tmux_window.attached_pane() 347 | else: 348 | tmux_session = tmux_server.new_session(session_name=project_name) 349 | tmux_window = tmux_session.attached_window() 350 | tmux_pane = tmux_window.attached_pane() 351 | 352 | tmux_window.rename_window(window_name) 353 | tmux_pane.send_keys("cd " + get_project_path(project_name)) 354 | tmux_pane.send_keys("echo $$ > ./.meta/.last_pid") 355 | tmux_pane.send_keys(start_cmd) 356 | 357 | while not os.path.exists(get_project_meta_path(project_name) + "/.last_pid"): 358 | pass 359 | 360 | with open(get_project_meta_path(project_name) + "/.last_pid", "r") as f: 361 | last_pid = int(f.read().strip()) 362 | 363 | tmux_window.rename_window(window_name + " (" + str(last_pid) + ")") 364 | 365 | handle = psutil.Process(last_pid) 366 | else: 367 | start_cmd = ["exec ./run.sh"] 368 | 369 | if start_as_master: 370 | start_cmd.append("-master") 371 | 372 | handle = psutil.Popen(start_cmd, 373 | shell=True, 374 | cwd=get_project_path(project_name), 375 | stdout=subprocess.PIPE) # stderr=subprocess.PIPE 376 | 377 | # Add the new pid to the state file so we can kill it in the future 378 | state = {"pid": []} 379 | 380 | try: 381 | state = get_current_project_state(project_name) 382 | except: 383 | pass 384 | 385 | state["pid"].append(handle.pid) 386 | 387 | store_current_project_state(project_name, state) 388 | except Exception as e: 389 | print e 390 | return False 391 | 392 | return handle 393 | 394 | 395 | def stop_project_all_instances(project_name): 396 | try: 397 | project_state = get_current_project_state(project_name) 398 | 399 | for pid in project_state["pid"]: 400 | if psutil.pid_exists(pid): 401 | print "Killing " + str(pid) + "..." 402 | 403 | kill_process_with_children(psutil.Process(pid)) 404 | 405 | # kill the tmux server if it's running 406 | tmux_server = tmuxp.Server(socket_path=get_project_meta_path(project_name) + "/.tmux_socket") 407 | 408 | if tmux_server.has_session(project_name): 409 | tmux_server.kill_session(project_name) 410 | 411 | project_state["pid"] = [] 412 | 413 | store_current_project_state(project_name, project_state) 414 | except Exception as e: 415 | print e 416 | pass 417 | 418 | 419 | def stop_project_instance(project_name, handle): 420 | if handle.is_running(): 421 | print "Killing " + str(handle.pid) + "..." 422 | 423 | kill_process_with_children(handle) 424 | 425 | cleanup_project_instance(project_name, handle) 426 | 427 | 428 | def cleanup_project_instance(project_name, handle): 429 | project_state = get_current_project_state(project_name) 430 | 431 | if handle.pid in project_state["pid"]: 432 | project_state["pid"].remove(handle.pid) 433 | 434 | store_current_project_state(project_name, project_state) 435 | 436 | 437 | def get_project_path(project_name): 438 | return host_config["projects_folder"] + "/" + project_name 439 | 440 | 441 | def get_project_meta_path(project_name): 442 | return get_project_path(project_name) + "/.meta" 443 | 444 | 445 | def get_current_project_info(project_name): 446 | with open(get_project_meta_path(project_name) + "/.session", "r") as f: 447 | return json.load(f) 448 | 449 | 450 | def store_current_project_info(project_name, project_info): 451 | with open(get_project_meta_path(project_name) + "/.session", "w") as f: 452 | json.dump(project_info, f) 453 | 454 | 455 | def get_current_project_version(project_name): 456 | with open(get_project_meta_path(project_name) + "/.version", "r") as f: 457 | return json.load(f) 458 | 459 | 460 | def store_current_project_version(project_name, version_info): 461 | with open(get_project_meta_path(project_name) + "/.version", "w") as f: 462 | json.dump(version_info, f) 463 | 464 | 465 | def get_current_project_state(project_name): 466 | with open(get_project_meta_path(project_name) + "/.state", "r") as f: 467 | return json.load(f) 468 | 469 | 470 | def store_current_project_state(project_name, state): 471 | with open(get_project_meta_path(project_name) + "/.state", "w") as f: 472 | json.dump(state, f) 473 | 474 | 475 | def get_current_project_hang_submission_state(project_name): 476 | with open(get_project_meta_path(project_name) + "/submission/.hangs", "r") as f: 477 | return json.load(f) 478 | 479 | 480 | def store_current_project_hang_submission_state(project_name, state): 481 | with open(get_project_meta_path(project_name) + "/submission/.hangs", "w") as f: 482 | json.dump(state, f) 483 | 484 | 485 | def get_current_project_crash_submission_state(project_name): 486 | with open(get_project_meta_path(project_name) + "/submission/.crashes", "r") as f: 487 | return json.load(f) 488 | 489 | 490 | def store_current_project_crash_submission_state(project_name, state): 491 | with open(get_project_meta_path(project_name) + "/submission/.crashes", "w") as f: 492 | json.dump(state, f) 493 | 494 | 495 | def get_current_project_queue_submission_state(project_name): 496 | with open(get_project_meta_path(project_name) + "/submission/.queue", "r") as f: 497 | return json.load(f) 498 | 499 | 500 | def store_current_project_queue_submission_state(project_name, state): 501 | with open(get_project_meta_path(project_name) + "/submission/.queue", "w") as f: 502 | json.dump(state, f) 503 | 504 | 505 | available_projects = {} 506 | 507 | 508 | def get_available_projects(): 509 | return available_projects 510 | 511 | 512 | def update_available_projects(): 513 | global available_projects 514 | 515 | available_projects = do_api_request(DISFUZZ_HOST + "?c=" + host_config["hostname"]) 516 | 517 | 518 | def amount_available_projects(): 519 | return len(available_projects) 520 | 521 | 522 | def is_project_available(project_name): 523 | return project_name in available_projects 524 | 525 | 526 | def get_current_projects(): 527 | project_folders = glob(host_config["projects_folder"] + "/*/.meta/.session") 528 | projects = [] 529 | 530 | for f in project_folders: 531 | projects.append( 532 | os.path.dirname(os.path.dirname(f)).replace(host_config["projects_folder"], "").replace("/", "")) 533 | 534 | return projects 535 | 536 | 537 | def get_latest_project_info(project_name): 538 | project_info = get_current_project_info(project_name) 539 | 540 | return do_api_request(project_info["session_update_url"]) 541 | 542 | 543 | def get_latest_project_file_info(project_name): 544 | project_info = get_current_project_info(project_name) 545 | 546 | return do_api_request(project_info["files_url"]) 547 | 548 | 549 | def get_latest_project_testcases_info(project_name): 550 | project_info = get_current_project_info(project_name) 551 | 552 | return do_api_request(project_info["queue_download_url"]) 553 | 554 | 555 | def get_latest_project_version(project_name): 556 | project_file_info = get_latest_project_file_info(project_name) 557 | 558 | return project_file_info["version"] 559 | 560 | 561 | class ApiException(Exception): 562 | pass 563 | 564 | 565 | def do_api_request(url): 566 | api_result = json.loads(urllib.urlopen(url).read()) 567 | 568 | if "error" in api_result: 569 | raise ApiException("Error: Can't load data from API!\nAPI error: " + api_result["error"]) 570 | 571 | return api_result 572 | 573 | 574 | def md5sum(p): 575 | m = hashlib.md5() 576 | 577 | with open(p, 'rb') as f: 578 | while True: 579 | d = f.read(8192) 580 | if not d: 581 | break 582 | m.update(d) 583 | 584 | return m.hexdigest() 585 | 586 | 587 | def sha1sum(p): 588 | m = hashlib.sha1() 589 | 590 | with open(p, 'rb') as f: 591 | while True: 592 | d = f.read(8192) 593 | if not d: 594 | break 595 | m.update(d) 596 | 597 | return m.hexdigest() 598 | 599 | 600 | def kill_process_with_children(process_handle): 601 | for child_handle in process_handle.children(recursive=True): 602 | try: 603 | child_handle.kill() 604 | except psutil.NoSuchProcess: 605 | pass 606 | 607 | try: 608 | process_handle.kill() 609 | except psutil.NoSuchProcess: 610 | pass 611 | 612 | 613 | # --------------------------------------------------------- 614 | 615 | print "DisFuzz " + DISFUZZ_VERSION 616 | 617 | if not os.path.exists(host_config["projects_folder"]): 618 | os.makedirs(host_config["projects_folder"]) 619 | 620 | try: 621 | update_available_projects() 622 | except Exception as e: 623 | print "It was not possible to load the list with available projects!" 624 | print e 625 | sys.exit() 626 | 627 | if len(sys.argv) == 2 and sys.argv[1] == "list": 628 | list_projects() 629 | elif len(sys.argv) == 3 and sys.argv[1] == "init": 630 | init_project(sys.argv[2]) 631 | elif len(sys.argv) >= 2 and sys.argv[1] == "sync": 632 | if len(sys.argv) == 2 or (len(sys.argv) == 3 and sys.argv[2] == "all"): 633 | for p in get_current_projects(): 634 | print "Sync " + p + "..." 635 | 636 | sync_project(p) 637 | 638 | print "Sync completed!" 639 | elif is_current_project(sys.argv[2]): 640 | sync_project(sys.argv[2]) 641 | 642 | print "Sync completed!" 643 | else: 644 | print "Unknown project " + sys.argv[2] + "!" 645 | elif len(sys.argv) >= 2 and sys.argv[1] == "monitor": 646 | if len(sys.argv) == 2 or (len(sys.argv) == 3 and sys.argv[2] == "all"): 647 | print "Monitoring:" 648 | for p in get_current_projects(): 649 | if is_project_running(p): 650 | print p 651 | 652 | while True: 653 | try: 654 | time.sleep(host_config["sync_sleep"]) 655 | 656 | for p in get_current_projects(): 657 | if is_project_running(p): 658 | sync_project(p) 659 | 660 | if is_project_update_available(p): 661 | print "Update available for " + p + "!" 662 | except KeyboardInterrupt: 663 | sys.exit() 664 | except Exception as e: 665 | print "Unknown error! Ignore for now..." 666 | print e 667 | traceback.print_exc() 668 | elif is_current_project(sys.argv[2]): 669 | print "Monitoring of " + sys.argv[2] + " started!" 670 | 671 | while True: 672 | try: 673 | time.sleep(host_config["sync_sleep"]) 674 | 675 | if is_project_running(sys.argv[2]): 676 | sync_project(sys.argv[2]) 677 | 678 | if is_project_update_available(sys.argv[2]): 679 | print "Update available for " + sys.argv[2] + "!" 680 | except KeyboardInterrupt: 681 | sys.exit() 682 | except Exception as e: 683 | print "Unknown error! Ignore for now..." 684 | print e 685 | traceback.print_exc() 686 | else: 687 | print "Unknown project " + sys.argv[2] + "!" 688 | elif len(sys.argv) >= 2 and sys.argv[1] == "update": 689 | if len(sys.argv) == 2 or (len(sys.argv) == 3 and sys.argv[2] == "all"): 690 | for p in get_current_projects(): 691 | if not is_project_running(p): 692 | update_project(p) 693 | else: 694 | print "Project " + p + " is running, update skipped." 695 | elif is_current_project(sys.argv[2]): 696 | if not is_project_running(sys.argv[2]): 697 | update_project(sys.argv[2]) 698 | else: 699 | print "Project " + sys.argv[2] + " is running, update skipped." 700 | else: 701 | print "Unknown project " + sys.argv[2] + "!" 702 | elif len(sys.argv) >= 2 and sys.argv[1] == "sessions": 703 | if len(sys.argv) == 2 or (len(sys.argv) == 3 and sys.argv[2] == "all"): 704 | for p in get_current_projects(): 705 | tmux_server = tmuxp.Server(socket_path=get_project_meta_path(p) + "/.tmux_socket") 706 | 707 | if is_project_running(p) and tmux_server.has_session(p): 708 | print p + " instances:" 709 | 710 | tmux_session = tmux_server.findWhere({"session_name": p}) 711 | 712 | for w in tmux_session.list_windows(): 713 | print "\t" + w.get("window_name") 714 | elif is_current_project(sys.argv[2]): 715 | tmux_server = tmuxp.Server(socket_path=get_project_meta_path(sys.argv[2]) + "/.tmux_socket") 716 | 717 | if is_project_running(sys.argv[2]) and tmux_server.has_session(sys.argv[2]): 718 | print sys.argv[2] + " instances:" 719 | 720 | tmux_session = tmux_server.findWhere({"session_name": sys.argv[2]}) 721 | 722 | for w in tmux_session.list_windows(): 723 | print "\t" + w.get("window_name") 724 | else: 725 | print "Unknown project " + sys.argv[2] + "!" 726 | elif len(sys.argv) >= 3 and sys.argv[1] == "start": 727 | if not tmuxp.util.has_required_tmux_version(): 728 | print "tmux version too old!" 729 | sys.exit() 730 | 731 | if is_current_project(sys.argv[2]): 732 | # if not is_project_running(sys.argv[2]): 733 | if len(sys.argv) == 4 and sys.argv[3] == "-m": 734 | if start_project_instance(sys.argv[2], use_tmux=True, start_as_master=True): 735 | print "Started!\nDon't forget to start a monitor session." 736 | else: 737 | print "Failed to start " + sys.argv[2] + "!" 738 | else: 739 | if start_project_instance(sys.argv[2], use_tmux=True): 740 | print "Started!\nDon't forget to start a monitor session." 741 | else: 742 | print "Failed to start " + sys.argv[2] + "!" 743 | # else: 744 | # print "Project " + sys.argv[2] + " is currently already running!" 745 | else: 746 | print "Unknown project " + sys.argv[2] + "!" 747 | elif len(sys.argv) >= 2 and sys.argv[1] == "stop": 748 | if len(sys.argv) == 2 or (len(sys.argv) == 3 and sys.argv[2] == "all"): 749 | for p in get_current_projects(): 750 | # if is_project_running(p): 751 | print "Stopping " + p + "..." 752 | stop_project_all_instances(p) 753 | elif is_current_project(sys.argv[2]): 754 | # if is_project_running(sys.argv[2]): 755 | print "Stopping " + sys.argv[2] + "..." 756 | 757 | stop_project_all_instances(sys.argv[2]) 758 | # else: 759 | # print "Project " + sys.argv[2] + " is currently not running!" 760 | else: 761 | print "Unknown project " + sys.argv[2] + "!" 762 | elif len(sys.argv) == 3 and sys.argv[1] == "console": 763 | if is_current_project(sys.argv[2]): 764 | tmux_server = tmuxp.Server(socket_path=get_project_meta_path(sys.argv[2]) + "/.tmux_socket") 765 | 766 | if is_project_running(sys.argv[2]) and tmux_server.has_session(sys.argv[2]): 767 | tmux_session = tmux_server.findWhere({"session_name": sys.argv[2]}) 768 | 769 | tmux_session.attach_session() 770 | else: 771 | print "Project " + sys.argv[2] + " is not running!" 772 | else: 773 | print "Unknown project " + sys.argv[2] + "!" 774 | elif len(sys.argv) == 2 and sys.argv[1] == "auto": 775 | print "Detecting host information..." 776 | print "Amount processors: " + str(host_config["processors"]) 777 | print "Hostname: " + host_config["hostname"] 778 | 779 | # update or init a few projects 780 | current_projects = get_current_projects() 781 | 782 | if len(current_projects): 783 | print "Current projects:" 784 | 785 | for p in current_projects: 786 | print "-> " + p 787 | 788 | print "Updating..." 789 | 790 | for p in current_projects: 791 | update_project(p) 792 | else: 793 | amount_available_projects = amount_available_projects() 794 | max_projects = host_config["amount_auto_projects"] 795 | 796 | list_projects() 797 | 798 | print "Looks this is the first time you use this. Lets bootstrap (maximum) " + str( 799 | host_config["amount_auto_projects"]) + " random from a total of " + str( 800 | amount_available_projects) + " projects..." 801 | 802 | projects = [] 803 | for project_name in get_available_projects(): 804 | available_projects[project_name]["__internal_name"] = project_name 805 | projects.append(available_projects[project_name]) 806 | 807 | random.shuffle(projects) 808 | 809 | i = 0; 810 | while i < max_projects and len(projects) > 0: 811 | init_project(projects[0]["__internal_name"]) 812 | 813 | del projects[0] 814 | 815 | i += 1 816 | 817 | # now its time to start them 818 | print "Downloading queue..." 819 | 820 | for p in current_projects: 821 | update_project_testcases(p) 822 | 823 | print "Start projects..." 824 | 825 | running_project_handles = [] 826 | for p in get_current_projects(): 827 | if not is_project_running(p): 828 | print "Starting " + p + "..." 829 | 830 | project_handle = start_project_instance(p) 831 | 832 | if project_handle: 833 | running_project_handles.append({ 834 | "project_name": p, 835 | "handle": project_handle 836 | }) 837 | 838 | time.sleep(10) 839 | else: 840 | print "Failed to start " + p + "!" 841 | else: 842 | print "Skipped " + p + "! Is it still running?\n" 843 | 844 | print str(len(running_project_handles)) + " instance(s) started!" 845 | 846 | print "WARNING: Leave this process running!" 847 | 848 | while True: 849 | try: 850 | time.sleep(host_config["sync_sleep"]) 851 | 852 | # Check for crashed projects 853 | for rpi in running_project_handles: 854 | if not rpi["handle"].is_running(): 855 | cleanup_project_instance(rpi["project_name"], rpi["handle"]) 856 | 857 | if not is_project_finished(rpi["project_name"]): 858 | print "Project " + rpi["project_name"] + " has stopped prematurely!" 859 | print "Return code: " + str(rpi["handle"].returncode) 860 | 861 | running_project_handles.remove(rpi) 862 | 863 | print "Restarting " + p + "..." 864 | 865 | project_handle = start_project_instance(p) 866 | 867 | if project_handle: 868 | running_project_handles.append({ 869 | "project_name": p, 870 | "handle": project_handle 871 | }) 872 | else: 873 | print "Failed to start " + p + "!" 874 | 875 | # Some small maintenance 876 | for p in get_current_projects(): 877 | if is_project_running(p): 878 | sync_project(p) 879 | 880 | if is_project_update_available(p): 881 | print "Update available for " + p + "!" 882 | print "Restarting " + p + "..." 883 | 884 | stop_project_all_instances(p) 885 | 886 | for rpi in running_project_handles: 887 | if rpi["project_name"] == p: 888 | running_project_handles.remove(rpi) 889 | 890 | update_project(p) 891 | 892 | project_handle = start_project_instance(p) 893 | 894 | if project_handle: 895 | running_project_handles.append({ 896 | "project_name": p, 897 | "handle": project_handle 898 | }) 899 | else: 900 | print "Failed to start " + p + "!" 901 | except KeyboardInterrupt: 902 | print "Uploading the queue, hangs & crashes..." 903 | 904 | for p in get_current_projects(): 905 | if is_project_running(p): 906 | sync_project(p, exclude_testcases=True) 907 | 908 | print "Killing the running processes..." 909 | 910 | for rpi in running_project_handles: 911 | print "Stopping " + rpi["project_name"] + "..." 912 | 913 | if rpi["handle"].is_running(): 914 | stop_project_instance(rpi["project_name"], rpi["handle"]) 915 | 916 | sys.exit() 917 | except Exception as e: 918 | print "Unknown error! Ignore for now..." 919 | print e 920 | traceback.print_exc() 921 | else: 922 | print 923 | print "Usage: " + sys.argv[0] + " [command]\n" 924 | print "Commands:\n" 925 | print "auto\t\t\t\tInit or update a couple of projects and start fuzzing" 926 | print 927 | print "list\t\t\t\tShow the available projects." 928 | print "sessions []\tList the instances of a project" 929 | print "console \t\tOpen a console to the project instances" 930 | print 931 | print "init \t\tInit a project" 932 | print "update [all|]\tUpdate the project to the latest version" 933 | print 934 | print "start [-m]\tStart a new instance (use -m to start a master instance)" 935 | print "stop [all|]\tStop all instances" 936 | print 937 | print "sync [all|]\tSync all or a single instance" 938 | print "monitor [all|]\tContinuously sync all or a single running instance" 939 | print -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Martijn Bogaard 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Distributed Fuzzing for afl 2 | 3 | This project is a quick and very dirty attempt (it's written in a few evenings) to combine the computing power of many machines that are outside my control for fuzzing with the great tool "afl". It works by a server that provides the projects to fuzz and a client running on the machines that downloads 1 or more projects (depending on the amount of CPU cores), downloads the queue and starts fuzzing. The new generated testcases are uploaded to the queue to sync to other clients and the crashed and hangs (after deduplication) are uploaded so they can be manually analyzed. The client will also check on a regular base if the project is modified. It will upgrade and restart the project when a update is detected. The client itself is not (yet) self updating. 4 | 5 | The server is able to support without any problem tens of clients, however with very big queues the sync algorithm will become a bottleneck. The calculation of the hashes of the queue and other folders is (currently) not cached. 6 | 7 | **Warning: Use this only with (semi) trusted clients and never connect to a untrusted server. The client will download and run arbitrary executable code from the server and nothing will prevent a client to submit a malicious sample to the queue that actually exploits a vulnerability.** 8 | 9 | ## Todo 10 | 11 | - Make the client self-updating 12 | - Make it possible to have a project run multiple times on the same client (a quick workaround is to copy the project multiple times and use $aggregateMap to combine the queues) 13 | - Upload more metadata so it can plot fancy graphs based on all clients 14 | - Rewrite the server to something decent or at least do a serious clean up. (It's also not very secure atm, but you should only use this in a trusted environment as malicious clients are a much bigger risk) 15 | - Add some kind of admin panel 16 | 17 | ## Install @ client 18 | 19 | sudo apt-get install python python-dev python-pip tmux screen 20 | sudo pip install requests psutil tmuxp 21 | mkdir ./fuzzing 22 | sudo mount -t tmpfs -o size=2G tmpfs ./fuzzing 23 | cd ./fuzzing 24 | wget https://www.example.com/disfuzz.py 25 | chmod +x disfuzz.py 26 | 27 | ## Install @ server 28 | 29 | - Upload the provided PHP scripts to a webserver 30 | - Give write access to the data folder 31 | - Add your project to the baseline folder (an example is provided) 32 | - Modify the config to include your project 33 | 34 | ## Usage: Automated fuzzing 35 | 36 | screen ./disfuzz.py auto 37 | 38 | Close now your terminal window or use ^a + d to detach! 39 | 40 | ## Usage: Manual fuzzing 41 | 42 | ./disfuzz.py list 43 | ./disfuzz.py init 44 | ./disfuzz.py start [-m] 45 | screen ./disfuzz.py monitor 46 | 47 | Close now your terminal window or use ^a + d to detach! 48 | 49 | Note: The first but ONLY the first client must start itself as master instance 50 | Note: Make sure that the monitor system is always running! Otherwise your results are not uploaded and the results of other instances not imported. 51 | 52 | ## Security 53 | 54 | - Provide the server component over HTTPS when its used over an untrusted network as otherwise an attacker can VERY easily get RCE on the nodes by a MITM attack as the client is self updating and will execute update.sh as part of the update process. 55 | - Client will download and execute code coming from the server, so only connect to known and trusted servers 56 | - The server will accept any submitted testcase which will be spread to all connected clients. Malicious clients could use this to exploit a vulnerability and get RCE on other clients or the machine of the administrator testing the submitted crash case. 57 | -------------------------------------------------------------------------------- /Server/.htaccess: -------------------------------------------------------------------------------- 1 | Options -Indexes -------------------------------------------------------------------------------- /Server/_config.php: -------------------------------------------------------------------------------- 1 | array('another-project'), 12 | ); 13 | 14 | // Do we allow new clients to join? (And get a session token) 15 | $acceptNewSessionIds = true; 16 | 17 | // Where to find the baseline data (the project that is downloaded to the client) 18 | $baselineFolder = __DIR__ . '/baseline'; 19 | 20 | // Where should the data be stored. 21 | $dataFolder = __DIR__ . '/data'; 22 | 23 | // Path to this script 24 | $baseUrl = 'https://www.example.com/api/v1'; 25 | 26 | // Most likely you dont want to touch this (or the client is also modified for it) 27 | $projectSubmitTargets = array( 28 | 'session' => array( 29 | 'folder' => 'sessions', 30 | 'can_download' => false, 31 | 'aggregate' => false, 32 | 'allow_overwrite' => true, 33 | 'allow_duplicates' => true, 34 | ), 35 | 36 | 'queue' => array( 37 | 'folder' => 'queue', 38 | 'can_download' => true, 39 | 'aggregate' => true, 40 | 'allow_overwrite' => false, 41 | 'allow_duplicates' => false, 42 | ), 43 | 'hang' => array( 44 | 'folder' => 'hangs', 45 | 'can_download' => false, 46 | 'aggregate' => false, 47 | 'allow_overwrite' => false, 48 | 'allow_duplicates' => false, 49 | ), 50 | 'crash' => array( 51 | 'folder' => 'crashes', 52 | 'can_download' => false, 53 | 'aggregate' => false, 54 | 'allow_overwrite' => false, 55 | 'allow_duplicates' => false, 56 | ) 57 | ); 58 | -------------------------------------------------------------------------------- /Server/_func.php: -------------------------------------------------------------------------------- 1 | 6 | 7 | Copyright 2013, 2014, 2015 Google Inc. All rights reserved. 8 | Released under terms and conditions of Apache License, Version 2.0. 9 | 10 | For new versions and additional information, check out: 11 | http://lcamtuf.coredump.cx/afl/ 12 | 13 | To compare notes with other users or get notified about major new features, 14 | send a mail to . -------------------------------------------------------------------------------- /Server/baseline/example/bin/afl-fuzz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MartijnB/disfuzz-afl/61f04aecf38e443fa9274df3604691ef98a637bf/Server/baseline/example/bin/afl-fuzz -------------------------------------------------------------------------------- /Server/baseline/example/bin/instrumented_cmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MartijnB/disfuzz-afl/61f04aecf38e443fa9274df3604691ef98a637bf/Server/baseline/example/bin/instrumented_cmp -------------------------------------------------------------------------------- /Server/baseline/example/bin/instrumented_cmp.c: -------------------------------------------------------------------------------- 1 | /* 2 | Example testcase from american fuzzy lop 3 | -------------------------------------------------------- 4 | 5 | Written and maintained by Michal Zalewski 6 | 7 | Copyright 2014 Google Inc. All rights reserved. 8 | 9 | Licensed under the Apache License, Version 2.0 (the "License"); 10 | you may not use this file except in compliance with the License. 11 | You may obtain a copy of the License at: 12 | 13 | http://www.apache.org/licenses/LICENSE-2.0 14 | 15 | */ 16 | 17 | /* 18 | 19 | A simple proof-of-concept for instrumented strcpy() or memcpy(). 20 | 21 | Normally, afl-fuzz will have difficulty ever reaching the code behind 22 | something like: 23 | 24 | if (!strcmp(password, "s3cr3t!")) ... 25 | 26 | This is because the strcmp() operation is completely opaque to the tool. 27 | A simple and non-invasive workaround that doesn't require complex code 28 | analysis is to replace strcpy(), memcpy(), and equivalents with 29 | inlined, non-optimized coe. 30 | 31 | I am still evaluating the value of doing this, but for time being, here's 32 | a quick demo of how it may work. To test: 33 | 34 | $ ./afl-gcc instrumented_cmp.c 35 | $ mkdir test_in 36 | $ printf xxxxxxxxxxxxxxxx >test_in/input 37 | $ ./afl-fuzz -i test_in -o test_out ./a.out 38 | 39 | */ 40 | 41 | #include 42 | #include 43 | #include 44 | 45 | /* Naive instrumented memcmp(). */ 46 | 47 | int my_memcmp(char* ptr1, char* ptr2, int len) 48 | __attribute__((always_inline)); 49 | 50 | int my_memcmp(char* ptr1, char* ptr2, int len) { 51 | 52 | while (len--) if (*(ptr1++) ^ *(ptr2++)) return 1; 53 | return 0; 54 | 55 | } 56 | 57 | #define memcmp my_memcmp 58 | 59 | /* Normal program. */ 60 | 61 | char tmp[16]; 62 | 63 | int main(int argc, char** argv) { 64 | 65 | int len = read(0, tmp, sizeof(tmp)); 66 | 67 | if (len != sizeof(tmp)) { 68 | 69 | printf("Truncated file!\n"); 70 | exit(1); 71 | 72 | } 73 | 74 | if (!memcmp(tmp + 5, "sesame", 6)) { 75 | 76 | /* Simulated "faulty" code path. */ 77 | 78 | int* x = (int*)0x12345678; 79 | *x = 1; 80 | 81 | } else printf("Bad password.\n"); 82 | 83 | return 0; 84 | 85 | } 86 | -------------------------------------------------------------------------------- /Server/baseline/example/init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | chmod +x ./bin/afl-fuzz 4 | chmod +x ./bin/instrumented_cmp 5 | -------------------------------------------------------------------------------- /Server/baseline/example/input/test: -------------------------------------------------------------------------------- 1 | test -------------------------------------------------------------------------------- /Server/baseline/example/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export LD_LIBRARY_PATH=./libs:${LD_LIBRARY_PATH} 4 | 5 | FUZZID=`date "+%Y%m%d%H%M%S"` 6 | 7 | echo "Fuzz ID: $FUZZID" >&2 8 | 9 | if [ "$#" -eq 1 ]; then 10 | if [ "$1" == "-master" ]; then 11 | ./bin/afl-fuzz -i input -o output -M "$FUZZID" ./bin/instrumented_cmp 12 | else 13 | ./bin/afl-fuzz -i input -o output -S "$FUZZID" ./bin/instrumented_cmp 14 | fi 15 | else 16 | ./bin/afl-fuzz -i input -o output -S "$FUZZID" ./bin/instrumented_cmp 17 | fi -------------------------------------------------------------------------------- /Server/baseline/example/upgrade.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | chmod +x ./bin/afl-fuzz 4 | chmod +x ./bin/instrumented_cmp 5 | -------------------------------------------------------------------------------- /Server/baseline/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MartijnB/disfuzz-afl/61f04aecf38e443fa9274df3604691ef98a637bf/Server/baseline/index.html -------------------------------------------------------------------------------- /Server/data/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MartijnB/disfuzz-afl/61f04aecf38e443fa9274df3604691ef98a637bf/Server/data/index.html -------------------------------------------------------------------------------- /Server/download.php: -------------------------------------------------------------------------------- 1 | 'Invalid project!')); 12 | exit(); 13 | } 14 | 15 | $fileType = request_filetype(); 16 | 17 | if ($fileType == 'baseline') { 18 | $rootFolder = $baselineFolder . DIRECTORY_SEPARATOR . $projectName; 19 | } 20 | else { 21 | if (!$projectSubmitTargets[$fileType]['can_download']) { 22 | header('Content-Type: application/json'); 23 | 24 | echo json_encode(array('error' => 'File downloading not allowed!')); 25 | exit(); 26 | } 27 | 28 | if (!isset($_GET['c']) || empty($_GET['c']) || preg_match('/^([a-zA-Z0-9-]+\.)*[a-zA-Z0-9]+$/', $_GET['c']) < 1) { 29 | header('Content-Type: application/json'); 30 | 31 | echo json_encode(array('error' => 'Invalid client name!')); 32 | exit(); 33 | } 34 | 35 | $clientName = $_GET['c']; 36 | 37 | if (!file_exists($dataFolder . DIRECTORY_SEPARATOR . $projectName . DIRECTORY_SEPARATOR . $clientName)) { 38 | header('Content-Type: application/json'); 39 | 40 | echo json_encode(array('error' => 'Invalid client name!')); 41 | exit(); 42 | } 43 | 44 | if (!isset($_GET['s']) || empty($_GET['s']) || preg_match('/^[a-zA-Z0-9]+$/', $_GET['s']) < 1) { 45 | header('Content-Type: application/json'); 46 | 47 | echo json_encode(array('error' => 'Invalid session token!')); 48 | exit(); 49 | } 50 | 51 | $sessionId = $_GET['s']; 52 | 53 | if (!file_exists($dataFolder . DIRECTORY_SEPARATOR . $projectName . DIRECTORY_SEPARATOR . $clientName . DIRECTORY_SEPARATOR . $sessionId)) { 54 | header('Content-Type: application/json'); 55 | 56 | echo json_encode(array('error' => 'Invalid session token!')); 57 | exit(); 58 | } 59 | 60 | if ($projectSubmitTargets[$fileType]['aggregate']) { 61 | $rootFolder = $dataFolder . DIRECTORY_SEPARATOR . $projectName; 62 | 63 | if (isset($aggregateMap[$projectName])) { 64 | $aggregatedFolders = $aggregateMap[$projectName]; 65 | } 66 | else { 67 | $aggregatedFolders = array(); 68 | } 69 | 70 | $aggregatedFolders[] = $projectName; 71 | } 72 | else { 73 | $rootFolder = $dataFolder . DIRECTORY_SEPARATOR . $projectName . DIRECTORY_SEPARATOR . $clientName . DIRECTORY_SEPARATOR . $sessionId . DIRECTORY_SEPARATOR . $projectSubmitTargets[$fileType]['folder']; 74 | } 75 | } 76 | 77 | if (!file_exists($rootFolder)) { 78 | http_response_code(500); 79 | 80 | header('Content-Type: application/json'); 81 | 82 | echo json_encode(array('error' => 'Invalid project!')); 83 | exit(); 84 | } 85 | 86 | if ($projectSubmitTargets[$fileType]['aggregate']) { 87 | if (substr_count($_GET['f'], "::") == 2) { 88 | list($projectBucket, $bucket, $relFilePath) = explode('::', $_GET['f']); 89 | } 90 | else{ 91 | list($bucket, $relFilePath) = explode('::', $_GET['f']); 92 | } 93 | 94 | $bucketFound = false; 95 | foreach ($aggregatedFolders as $folderName) { 96 | $clientIterator = new DirectoryIterator($dataFolder . DIRECTORY_SEPARATOR . $folderName); 97 | 98 | if (sha1($folderName) == $projectBucket) { 99 | foreach($clientIterator as $path => $o) { 100 | if (!$o->isDot() && $o->isDir()) { 101 | $sessionIterator = new DirectoryIterator($dataFolder . DIRECTORY_SEPARATOR . $folderName . DIRECTORY_SEPARATOR . $o->getFileName()); 102 | foreach($sessionIterator as $path => $os) { 103 | if (!$os->isDot() && $os->isDir()) { 104 | $folder = $dataFolder . DIRECTORY_SEPARATOR . $folderName . DIRECTORY_SEPARATOR . $o->getFileName() . DIRECTORY_SEPARATOR . $os->getFileName() . DIRECTORY_SEPARATOR . $projectSubmitTargets[$fileType]['folder']; 105 | 106 | if (sha1(str_replace($dataFolder . DIRECTORY_SEPARATOR . $folderName, '', $folder)) == $bucket) { 107 | $bucketFound = true; 108 | $rootFolder = $dataFolder . DIRECTORY_SEPARATOR . $folderName; 109 | break 3; 110 | } 111 | } 112 | } 113 | } 114 | } 115 | } 116 | } 117 | 118 | if (!$bucketFound) { 119 | header('Content-Type: application/json'); 120 | 121 | echo json_encode(array('error' => 'Invalid bucket!')); 122 | exit(); 123 | } 124 | 125 | $filePath = realpath($folder . DIRECTORY_SEPARATOR . $relFilePath); 126 | } 127 | else { 128 | $filePath = realpath($rootFolder . DIRECTORY_SEPARATOR . $_GET['f']); 129 | } 130 | 131 | if ($rootFolder != substr($filePath, 0, strlen($rootFolder))) { 132 | header('Content-Type: application/json'); 133 | 134 | echo json_encode(array('error' => 'Invalid file path!')); 135 | exit(); 136 | } 137 | 138 | if (!file_exists($filePath)) { 139 | header('Content-Type: application/json'); 140 | 141 | echo json_encode(array('error' => 'File missing!')); 142 | exit(); 143 | } 144 | 145 | header('Content-Description: File Transfer'); 146 | header('Content-Type: application/octet-stream'); 147 | header('Content-Disposition: attachment; filename='.basename($filePath)); 148 | header('Expires: 0'); 149 | header('Cache-Control: must-revalidate'); 150 | header('Pragma: public'); 151 | header('Content-Length: ' . filesize($filePath)); 152 | readfile($filePath); 153 | exit; -------------------------------------------------------------------------------- /Server/files.php: -------------------------------------------------------------------------------- 1 | 'Invalid project!')); 12 | exit(); 13 | } 14 | 15 | $fileType = request_filetype(); 16 | 17 | if ($fileType == 'baseline') { 18 | $downloadUrlAccessToken = ''; 19 | $rootFolder = $baselineFolder . DIRECTORY_SEPARATOR . $projectName; 20 | } 21 | else { 22 | if (!isset($_GET['c']) || empty($_GET['c']) || preg_match('/^([a-zA-Z0-9-]+\.)*[a-zA-Z0-9]+$/', $_GET['c']) < 1) { 23 | header('Content-Type: application/json'); 24 | 25 | echo json_encode(array('error' => 'Invalid client name!')); 26 | exit(); 27 | } 28 | 29 | $clientName = $_GET['c']; 30 | 31 | if (!file_exists($dataFolder . DIRECTORY_SEPARATOR . $projectName . DIRECTORY_SEPARATOR . $clientName)) { 32 | header('Content-Type: application/json'); 33 | 34 | echo json_encode(array('error' => 'Invalid client name!')); 35 | exit(); 36 | } 37 | 38 | if (!isset($_GET['s']) || empty($_GET['s']) || preg_match('/^[a-zA-Z0-9]+$/', $_GET['s']) < 1) { 39 | header('Content-Type: application/json'); 40 | 41 | echo json_encode(array('error' => 'Invalid session token!')); 42 | exit(); 43 | } 44 | 45 | $sessionId = $_GET['s']; 46 | 47 | if (!file_exists($dataFolder . DIRECTORY_SEPARATOR . $projectName . DIRECTORY_SEPARATOR . $clientName . DIRECTORY_SEPARATOR . $sessionId)) { 48 | header('Content-Type: application/json'); 49 | 50 | echo json_encode(array('error' => 'Invalid session token!')); 51 | exit(); 52 | } 53 | 54 | $downloadUrlAccessToken = '&c='.$clientName.'&s='.$sessionId; 55 | 56 | if ($projectSubmitTargets[$fileType]['aggregate']) { 57 | $rootFolder = $dataFolder . DIRECTORY_SEPARATOR . $projectName; 58 | 59 | if (isset($aggregateMap[$projectName])) { 60 | $aggregatedFolders = $aggregateMap[$projectName]; 61 | } 62 | else { 63 | $aggregatedFolders = array(); 64 | } 65 | 66 | $aggregatedFolders[] = $projectName; 67 | } 68 | else { 69 | $rootFolder = $dataFolder . DIRECTORY_SEPARATOR . $projectName . DIRECTORY_SEPARATOR . $clientName . DIRECTORY_SEPARATOR . $sessionId . DIRECTORY_SEPARATOR . $projectSubmitTargets[$fileType]['folder']; 70 | } 71 | } 72 | 73 | if (!file_exists($dataFolder . DIRECTORY_SEPARATOR . $projectName)) { 74 | http_response_code(500); 75 | 76 | header('Content-Type: application/json'); 77 | 78 | echo json_encode(array('error' => 'Invalid project!')); 79 | exit(); 80 | } 81 | 82 | $files = array(); 83 | $versionHash = sha1(''); 84 | 85 | if ($fileType != 'baseline' && $projectSubmitTargets[$fileType]['aggregate']) { 86 | foreach ($aggregatedFolders as $folderName) { 87 | $clientIterator = new DirectoryIterator($dataFolder . DIRECTORY_SEPARATOR . $folderName); 88 | 89 | foreach($clientIterator as $path => $o) { 90 | if (!$o->isDot() && $o->isDir()) { 91 | $sessionIterator = new DirectoryIterator($dataFolder . DIRECTORY_SEPARATOR . $folderName . DIRECTORY_SEPARATOR . $o->getFileName()); 92 | foreach($sessionIterator as $path => $os) { 93 | if (!$os->isDot() && $os->isDir()) { 94 | $folder = $dataFolder . DIRECTORY_SEPARATOR . $folderName . DIRECTORY_SEPARATOR . $o->getFileName() . DIRECTORY_SEPARATOR . $os->getFileName() . DIRECTORY_SEPARATOR . $projectSubmitTargets[$fileType]['folder']; 95 | 96 | if (file_exists($folder) && is_dir($folder)) { 97 | $fileIterator = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($folder, FilesystemIterator::SKIP_DOTS)); 98 | 99 | foreach($fileIterator as $path => $o) { 100 | $md5Hash = md5_file($path); 101 | $sha1Hash = sha1_file($path); 102 | 103 | $files[] = array( 104 | 'path' => str_replace($folder, '', $path), 105 | 'size' => filesize($path), 106 | 'url' => $baseUrl . DIRECTORY_SEPARATOR . 'download.php?p='.$projectName . $downloadUrlAccessToken .'&t='.$fileType.'&f=' . urlencode(sha1($folderName) . '::'. sha1(str_replace($dataFolder . DIRECTORY_SEPARATOR . $folderName, '', $folder)) . '::'.str_replace($folder, '', $path)), 107 | 'md5sum' => $md5Hash, 108 | 'sha1sum' => $sha1Hash 109 | ); 110 | 111 | $versionHash = sha1($versionHash . str_replace($dataFolder . DIRECTORY_SEPARATOR . $folderName, '', $path) . ':' . filesize($path) . ':' . $sha1Hash); 112 | } 113 | } 114 | } 115 | } 116 | } 117 | } 118 | } 119 | } 120 | else { 121 | if (!file_exists($rootFolder)) { 122 | http_response_code(500); 123 | 124 | header('Content-Type: application/json'); 125 | 126 | echo json_encode(array('error' => 'Invalid project!')); 127 | exit(); 128 | } 129 | 130 | $fileIterator = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($rootFolder, FilesystemIterator::SKIP_DOTS)); 131 | 132 | foreach($fileIterator as $path => $o) { 133 | $md5Hash = md5_file($path); 134 | $sha1Hash = sha1_file($path); 135 | 136 | $files[] = array( 137 | 'path' => str_replace($rootFolder, '', $path), 138 | 'size' => filesize($path), 139 | 'url' => $baseUrl . DIRECTORY_SEPARATOR . 'download.php?p='.$projectName . $downloadUrlAccessToken .'&t='.$fileType.'&f=' . urlencode(str_replace($rootFolder, '', $path)), 140 | 'md5sum' => md5_file($path), 141 | 'sha1sum' => sha1_file($path) 142 | ); 143 | 144 | $versionHash = sha1($versionHash . str_replace($rootFolder, '', $path) . ':' . filesize($path) . ':' . $sha1Hash); 145 | } 146 | } 147 | 148 | header('Content-Type: application/json'); 149 | 150 | echo json_encode(array('files' => $files, 'version' => $versionHash)); -------------------------------------------------------------------------------- /Server/index.php: -------------------------------------------------------------------------------- 1 | 'Invalid client name!')); 10 | exit(); 11 | } 12 | 13 | $clientName = $_GET['c']; 14 | 15 | $projectsInfo = array(); 16 | 17 | foreach ($currentProjects as $projectName) { 18 | $projectsInfo[$projectName] = array( 19 | 'name' => $projectName, 20 | 'setup_url' => $baseUrl . DIRECTORY_SEPARATOR . 'session.php?c='.$clientName.'&p='.$projectName 21 | ); 22 | } 23 | 24 | header('Content-Type: application/json'); 25 | 26 | echo json_encode($projectsInfo); -------------------------------------------------------------------------------- /Server/session.php: -------------------------------------------------------------------------------- 1 | 'New sessionIds are not available!')); 12 | exit(); 13 | } 14 | 15 | if (($projectName = request_project_name()) === false) { 16 | http_response_code(500); 17 | 18 | header('Content-Type: application/json'); 19 | 20 | echo json_encode(array('error' => 'Invalid project!')); 21 | exit(); 22 | } 23 | 24 | if (!isset($_GET['c']) || empty($_GET['c']) || preg_match('/^([a-zA-Z0-9-]+\.)*[a-zA-Z0-9]+$/', $_GET['c']) < 1) { 25 | header('Content-Type: application/json'); 26 | 27 | echo json_encode(array('error' => 'Invalid client name!')); 28 | exit(); 29 | } 30 | 31 | $clientName = $_GET['c']; 32 | 33 | if (!file_exits($dataFolder . DIRECTORY_SEPARATOR . $projectName)) { 34 | if (!mkdir($dataFolder . DIRECTORY_SEPARATOR . $projectName)) { 35 | header('Content-Type: application/json'); 36 | 37 | echo json_encode(array('error' => 'Can\'t add client!')); 38 | exit(); 39 | } 40 | } 41 | 42 | if (!file_exists($dataFolder . DIRECTORY_SEPARATOR . $projectName . DIRECTORY_SEPARATOR . $clientName)) { 43 | if (!mkdir($dataFolder . DIRECTORY_SEPARATOR . $projectName . DIRECTORY_SEPARATOR . $clientName)) { 44 | header('Content-Type: application/json'); 45 | 46 | echo json_encode(array('error' => 'Can\'t add client!')); 47 | exit(); 48 | } 49 | } 50 | 51 | // If a session token is give, we update the session 52 | if (isset($_GET['s'])) { 53 | if (!isset($_GET['s']) || empty($_GET['s']) || preg_match('/^[a-zA-Z0-9]+$/', $_GET['s']) < 1) { 54 | header('Content-Type: application/json'); 55 | 56 | echo json_encode(array('error' => 'Invalid session token!')); 57 | exit(); 58 | } 59 | 60 | $sessionId = $_GET['s']; 61 | 62 | if (!file_exists($dataFolder . DIRECTORY_SEPARATOR . $projectName . DIRECTORY_SEPARATOR . $clientName . DIRECTORY_SEPARATOR . $sessionId)) { 63 | header('Content-Type: application/json'); 64 | 65 | echo json_encode(array('error' => 'Invalid session token!')); 66 | exit(); 67 | } 68 | } 69 | else { 70 | $sessionId = sha1(uniqid($_SERVER['REMOTE_ADDR'], true)); 71 | 72 | if (!file_exists($dataFolder . DIRECTORY_SEPARATOR . $projectName . DIRECTORY_SEPARATOR . $clientName . DIRECTORY_SEPARATOR . $sessionId)) { 73 | if (!mkdir($dataFolder . DIRECTORY_SEPARATOR . $projectName . DIRECTORY_SEPARATOR . $clientName . DIRECTORY_SEPARATOR . $sessionId)) { 74 | header('Content-Type: application/json'); 75 | 76 | echo json_encode(array('error' => 'Can\'t add client!')); 77 | exit(); 78 | } 79 | } 80 | 81 | foreach ($projectSubmitTargets as $targetName => $targetInfo) { 82 | if (!mkdir($dataFolder . DIRECTORY_SEPARATOR . $projectName . DIRECTORY_SEPARATOR . $clientName . DIRECTORY_SEPARATOR . $sessionId . DIRECTORY_SEPARATOR . $targetInfo['folder'])) { 83 | header('Content-Type: application/json'); 84 | 85 | echo json_encode(array('error' => 'Can\'t add client!')); 86 | exit(); 87 | } 88 | } 89 | } 90 | 91 | $sessionInfo = array( 92 | 'project_name' => $projectName, 93 | 'client_name' => $clientName, 94 | 'session_id' => $sessionId, 95 | 'session_update_url' => $baseUrl . DIRECTORY_SEPARATOR . 'session.php?c='.$clientName.'&p='.$projectName . '&s=' . $sessionId, 96 | 'files_url' => $baseUrl . DIRECTORY_SEPARATOR . 'files.php?c='.$clientName.'&p='.$projectName . '&s=' . $sessionId 97 | ); 98 | 99 | foreach ($projectSubmitTargets as $targetName => $targetInfo) { 100 | if ($targetInfo['can_download']) { 101 | $sessionInfo[$targetName . '_download_url'] = $baseUrl . DIRECTORY_SEPARATOR . 'files.php?c='.$clientName.'&p='.$projectName.'&t=' . $targetName . '&s=' . $sessionId; 102 | } 103 | 104 | $sessionInfo[$targetName . '_submit_url'] = $baseUrl . DIRECTORY_SEPARATOR . 'submit.php?c='.$clientName.'&p='.$projectName.'&t=' . $targetName . '&s=' . $sessionId; 105 | } 106 | 107 | header('Content-Type: application/json'); 108 | 109 | echo json_encode($sessionInfo); -------------------------------------------------------------------------------- /Server/submit.php: -------------------------------------------------------------------------------- 1 | 'Invalid project!')); 12 | exit(); 13 | } 14 | 15 | $fileType = request_filetype(); 16 | 17 | if (!isset($_GET['c']) || empty($_GET['c']) || preg_match('/^([a-zA-Z0-9-]+\.)*[a-zA-Z0-9]+$/', $_GET['c']) < 1) { 18 | header('Content-Type: application/json'); 19 | 20 | echo json_encode(array('error' => 'Invalid client name!')); 21 | exit(); 22 | } 23 | 24 | $clientName = $_GET['c']; 25 | 26 | if (!file_exists($dataFolder . DIRECTORY_SEPARATOR . $projectName . DIRECTORY_SEPARATOR . $clientName)) { 27 | header('Content-Type: application/json'); 28 | 29 | echo json_encode(array('error' => 'Invalid client name!')); 30 | exit(); 31 | } 32 | 33 | if (!isset($_GET['s']) || empty($_GET['s']) || preg_match('/^[a-zA-Z0-9]+$/', $_GET['s']) < 1) { 34 | header('Content-Type: application/json'); 35 | 36 | echo json_encode(array('error' => 'Invalid session token!')); 37 | exit(); 38 | } 39 | 40 | $sessionId = $_GET['s']; 41 | 42 | if (!file_exists($dataFolder . DIRECTORY_SEPARATOR . $projectName . DIRECTORY_SEPARATOR . $clientName . DIRECTORY_SEPARATOR . $sessionId)) { 43 | header('Content-Type: application/json'); 44 | 45 | echo json_encode(array('error' => 'Invalid session token!')); 46 | exit(); 47 | } 48 | 49 | $rootFolder = $dataFolder . DIRECTORY_SEPARATOR . $projectName . DIRECTORY_SEPARATOR . $clientName . DIRECTORY_SEPARATOR . $sessionId . DIRECTORY_SEPARATOR . $projectSubmitTargets[$fileType]['folder']; 50 | 51 | if (!file_exists($rootFolder)) { 52 | http_response_code(500); 53 | 54 | header('Content-Type: application/json'); 55 | 56 | echo json_encode(array('error' => 'Invalid project!')); 57 | exit(); 58 | } 59 | 60 | if (count($_FILES) === 0) { 61 | http_response_code(500); 62 | 63 | header('Content-Type: application/json'); 64 | 65 | echo json_encode(array('error' => 'File(s) missing!')); 66 | exit(); 67 | } 68 | 69 | if (!file_exists($dataFolder . DIRECTORY_SEPARATOR . $projectName . DIRECTORY_SEPARATOR . '.meta')) { 70 | mkdir($dataFolder . DIRECTORY_SEPARATOR . $projectName . DIRECTORY_SEPARATOR . '.meta'); 71 | } 72 | 73 | if (!$projectSubmitTargets[$fileType]['allow_duplicates']) { 74 | if (!file_exists($dataFolder . DIRECTORY_SEPARATOR . $projectName . DIRECTORY_SEPARATOR . '.meta' . DIRECTORY_SEPARATOR . $projectSubmitTargets[$fileType]['folder'].'.dedup')) { 75 | file_put_contents($dataFolder . DIRECTORY_SEPARATOR . $projectName . DIRECTORY_SEPARATOR . '.meta' . DIRECTORY_SEPARATOR . $projectSubmitTargets[$fileType]['folder'].'.dedup', serialize(array())); 76 | } 77 | 78 | $dedupData = unserialize(file_get_contents($dataFolder . DIRECTORY_SEPARATOR . $projectName . DIRECTORY_SEPARATOR . '.meta' . DIRECTORY_SEPARATOR . $projectSubmitTargets[$fileType]['folder'].'.dedup')); 79 | } 80 | 81 | $amountFiles = 0; 82 | foreach ($_FILES as $formFileName => $formFileInfo) { 83 | if ($formFileInfo['error'] == UPLOAD_ERR_OK) { 84 | if (strpos($formFileInfo['name'], '..') !== false || strpos($formFileInfo['name'], '/') !== false) { 85 | header('Content-Type: application/json'); 86 | 87 | echo json_encode(array('error' => 'Invalid filename!')); 88 | exit(); 89 | } 90 | 91 | $md5Hash = md5_file($formFileInfo['tmp_name']); 92 | 93 | if (!$projectSubmitTargets[$fileType]['allow_duplicates']) { 94 | if (array_key_exists($md5Hash, $dedupData)) { 95 | // A file with this hash already exists 96 | continue; 97 | } 98 | } 99 | 100 | $filePath = $rootFolder . DIRECTORY_SEPARATOR . $formFileInfo['name']; 101 | 102 | if ($rootFolder != substr($filePath, 0, strlen($rootFolder))) { 103 | header('Content-Type: application/json'); 104 | 105 | echo json_encode(array('error' => 'Invalid path!')); 106 | exit(); 107 | } 108 | 109 | if (file_exists($filePath) && !$projectSubmitTargets[$fileType]['allow_overwrite']) { 110 | if (md5_file($filePath) != $md5Hash) { 111 | $i = 1; 112 | do { 113 | $tmpFilePath = dirname($filePath) . DIRECTORY_SEPARATOR . basename($filePath) . ':'.$i; 114 | $i++; 115 | } 116 | while (file_exists($tmpFilePath)); 117 | 118 | $filePath = $tmpFilePath; 119 | } 120 | } 121 | 122 | move_uploaded_file($formFileInfo['tmp_name'], $filePath); 123 | 124 | if (!$projectSubmitTargets[$fileType]['allow_duplicates']) { 125 | $dedupData[$md5Hash] = array( 126 | 'file' => str_replace($rootFolder, '', $filePath) 127 | ); 128 | } 129 | 130 | $amountFiles++; 131 | } 132 | } 133 | 134 | if (!$projectSubmitTargets[$fileType]['allow_duplicates']) { 135 | file_put_contents($dataFolder . DIRECTORY_SEPARATOR . $projectName . DIRECTORY_SEPARATOR . '.meta' . DIRECTORY_SEPARATOR . $projectSubmitTargets[$fileType]['folder'].'.dedup', serialize($dedupData)); 136 | } 137 | 138 | header('Content-Type: application/json'); 139 | 140 | echo json_encode(array('error' => $amountFiles . ' file(s) uploaded.')); 141 | exit(); --------------------------------------------------------------------------------