├── README.md ├── proxy.txt └── svn_git_scan_mul_proxy.py /README.md: -------------------------------------------------------------------------------- 1 | 用于全球svn，git信息泄露扫描，基于多线程，带宽控制，代理线程池支持 2 | -------------------------------------------------------------------------------- /proxy.txt: -------------------------------------------------------------------------------- 1 | 61.234.249.128:8118 2 | 116.246.6.51:80 3 | 116.246.6.52:80 4 | 124.254.57.150:8118 5 | 218.78.210.190:8080 6 | 114.112.91.97:90 7 | 124.202.171.2:8118 8 | 223.203.0.4:80 9 | 124.200.100.74:8118 10 | 101.4.136.66:80 11 | 183.131.144.204:80 12 | 111.13.136.58:80 13 | 218.89.170.114:8888 14 | 117.79.131.109:8080 15 | 120.203.158.149:8118 16 | 222.45.85.53:8118 17 | 101.226.249.237:80 18 | 222.45.85.210:8118 19 | 122.96.59.106:81 20 | 120.203.158.148:8118 21 | 211.141.130.106:8118 22 | 122.96.59.106:843 23 | -------------------------------------------------------------------------------- /svn_git_scan_mul_proxy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | 4 | # 5 | # Note that you have to run the script in the same version of python which 6 | # was used to generate the exe. Otherwise unmarshalling will fail. 7 | # 8 | 9 | #import marshal, imp 10 | # 11 | #f = open( 'PYTHONSCRIPT', 'rb' ) 12 | ## 13 | ## struct Header 14 | ## { 15 | ## unsigned int tag; 16 | ## unsigned int optimize; 17 | ## unsigned int unbuffered; 18 | ## unsigned int data_bytes; 19 | ## unsigned char zippath[VARIABLE_SIZE] 20 | ## }; 21 | ## 22 | ## Skip the header, you have to know the header size beforehand. 23 | ## 24 | #f.seek( 0x11 ) 25 | #ob = marshal.load( f ) 26 | # 27 | #for i in xrange( 0, len( ob ) ) : 28 | # open( str( i ) + '.pyc', 'wb' ).write( imp.get_magic() + '\0' * 4 + marshal.dumps( ob[i] ) ) 29 | # 30 | #f.close() 31 | # Embedded file name: rom0scan.py 32 | import sys, string, getopt, os, struct, socket, random, inspect, re 33 | import pycurl, Queue, itertools, collections 34 | from threading import Thread, RLock 35 | from cStringIO import StringIO 36 | 37 | class MyException(Exception): 38 | 39 | def __init__(self, value = None): 40 | self.value = value 41 | 42 | def __str__(self): 43 | return str(self.value) 44 | 45 | def __repr__(self): 46 | return repr(self.value) 47 | 48 | 49 | def itos32(num): 50 | return struct.pack('=I', int(num & 4294967295L)) 51 | 52 | 53 | def uint16(num): 54 | return int(struct.unpack('=H', struct.pack('=H', int(num & 65535)))[0]) 55 | 56 | 57 | def uint32(num): 58 | return int(struct.unpack('=I', struct.pack('=I', int(num & 4294967295L)))[0]) 59 | 60 | 61 | def readb16(buf, index): 62 | if index < 0: 63 | index += len(buf) 64 | return int(struct.unpack('>H', buf[index:index + 2])[0]) 65 | 66 | 67 | def readstr(buf, index): 68 | if index < 0: 69 | index += len(buf) 70 | return buf[index:].split('\x00')[0] 71 | 72 | 73 | def resolvehost(host): 74 | try: 75 | ret = uint32(int(struct.unpack('>I', socket.inet_aton(socket.gethostbyname(host)))[0])) 76 | except: 77 | ret = 0 78 | 79 | return ret 80 | 81 | 82 | def dosomething(str): 83 | str += '\x00' 84 | xxx = '' 85 | i = 0 86 | while i < len(str) - 1: 87 | if '\\' != str[i]: 88 | xxx += str[i] 89 | elif i + 1 >= len(str): 90 | xxx += str[i] 91 | elif '\\' == str[i + 1]: 92 | xxx += '\\' 93 | i += 1 94 | elif 'r' == str[i + 1]: 95 | xxx += '\r' 96 | i += 1 97 | elif 'n' == str[i + 1]: 98 | xxx += '\n' 99 | i += 1 100 | elif 't' == str[i + 1]: 101 | xxx += '\t' 102 | i += 1 103 | elif '0' == str[i + 1]: 104 | xxx += '\x00' 105 | i += 1 106 | elif 'x' == str[i + 1]: 107 | if str[i + 2] >= '0' and str[i + 2] <= '9' or str[i + 2] >= 'a' and str[i + 2] <= 'f' or str[i + 2] >= 'A' and str[i + 2] <= 'F': 108 | i += 2 109 | tmp = str[i] 110 | if str[i + 1] >= '0' and str[i + 1] <= '9' or str[i + 1] >= 'a' and str[i + 1] <= 'f' or str[i + 1] >= 'A' and str[i + 1] <= 'F': 111 | i += 1 112 | tmp += str[i] 113 | else: 114 | tmp = '0' + tmp 115 | xxx += tmp.decode('hex_codec') 116 | else: 117 | xxx += str[i] 118 | else: 119 | xxx += str[i] 120 | i += 1 121 | 122 | return xxx 123 | 124 | 125 | def AnyToSth(src, dst = 'utf_8'): 126 | codelist = ['utf_8', 127 | 'gbk', 128 | 'gb18030', 129 | 'big5', 130 | 'latin_1'] 131 | ret = '(error)' 132 | for c in codelist: 133 | try: 134 | ret = src.decode(c).encode(dst) 135 | break 136 | except UnicodeDecodeError: 137 | pass 138 | except UnicodeEncodeError: 139 | pass 140 | 141 | return ret 142 | 143 | 144 | class BitReader(): 145 | 146 | def __init__(self, bytes): 147 | self.__bits__ = collections.deque() 148 | for byte in bytes: 149 | byte = ord(byte) 150 | for i in xrange(8): 151 | self.__bits__.append(byte >> 7 - i & 1) 152 | 153 | def getBit(self): 154 | return self.__bits__.popleft() 155 | 156 | def getBits(self, num): 157 | ret = 0 158 | for i in xrange(num): 159 | ret += self.getBit() << num - 1 - i 160 | 161 | return ret 162 | 163 | def getLength(self): 164 | length = 2 165 | while True: 166 | i = self.getBits(2) 167 | length += i 168 | if not (3 == i and length < 8): 169 | break 170 | 171 | if 8 == length: 172 | while True: 173 | i = self.getBits(4) 174 | length += i 175 | if 15 != i: 176 | break 177 | 178 | return length 179 | 180 | 181 | def LZSDecompress(data): 182 | reader = BitReader(data) 183 | result = '' 184 | while True: 185 | bit = reader.getBit() 186 | if not bit: 187 | byte = reader.getBits(8) 188 | result += chr(byte) 189 | continue 190 | bit = reader.getBit() 191 | if 1 == bit: 192 | offset = reader.getBits(7) 193 | else: 194 | offset = reader.getBits(11) 195 | if 1 == bit and 0 == offset: 196 | break 197 | if offset > len(result): 198 | break 199 | length = reader.getLength() 200 | for i in xrange(length): 201 | result += result[-offset] 202 | 203 | return result 204 | 205 | 206 | QueueInWorkerExit = False 207 | 208 | class QueueInWorker(Thread): 209 | 210 | def __init__(self, queue_in, queue_out, httpheader = [], proxypool = [], noproxy = 'localhost,127.0.0.1',\ 211 | referer = '', followlocation = 1, maxredirs = 5, redir_protocols = 3, protocols = 3, connecttimeout = 30,\ 212 | timeout = 30, connect_max = 16, monitor = False, port = None): 213 | super(QueueInWorker, self).__init__() 214 | self.queue_in = queue_in 215 | self.queue_out = queue_out 216 | self.httpheader = httpheader 217 | self.proxypool = proxypool 218 | self.noproxy = noproxy 219 | self.referer = referer 220 | self.followlocation = followlocation 221 | self.maxredirs = maxredirs 222 | self.redir_protocols = redir_protocols 223 | self.protocols = protocols 224 | self.connecttimeout = connecttimeout 225 | self.timeout = timeout 226 | self.connect_max = connect_max 227 | self.monitor = monitor 228 | self.port = port 229 | self.m = pycurl.CurlMulti() 230 | self.m.cpool = [] 231 | self.setDaemon(True) 232 | 233 | def __del__(self): 234 | if self.m is not None: 235 | for c in self.m.cpool: 236 | if c.host is not None: 237 | c.host = None 238 | if c.head is not None: 239 | c.head.close() 240 | c.head = None 241 | if c.body is not None: 242 | c.body.close() 243 | c.body = None 244 | self.m.remove_handle(c) 245 | self.m.cpool.remove(c) 246 | c.close() 247 | 248 | self.m.cpool = [] 249 | self.m.close() 250 | self.m = None 251 | return 252 | 253 | def c_init(self, c): 254 | c.setopt(pycurl.HTTPHEADER, self.httpheader) 255 | if len(self.proxypool) > 0: 256 | c.setopt(pycurl.PROXY, random.choice(self.proxypool)) 257 | #c.setopt(pycurl.NOPROXY, self.noproxy) 258 | if self.referer is None: 259 | c.setopt(pycurl.AUTOREFERER, 0) 260 | elif '' == self.referer: 261 | c.setopt(pycurl.AUTOREFERER, 1) 262 | else: 263 | c.setopt(pycurl.REFERER, self.referer) 264 | c.setopt(pycurl.FOLLOWLOCATION, self.followlocation) 265 | # c.setopt(pycurl.MAXREDIRS, self.maxredirs) 266 | c.setopt(pycurl.MAXREDIRS, 0)#不进行跳转 267 | #c.setopt(pycurl.REDIR_PROTOCOLS, self.redir_protocols) 268 | #c.setopt(pycurl.PROTOCOLS, self.protocols) 269 | c.setopt(pycurl.NOSIGNAL, 1) 270 | c.setopt(pycurl.CONNECTTIMEOUT, self.connecttimeout) 271 | c.setopt(pycurl.TIMEOUT, self.timeout) 272 | c.setopt(pycurl.ENCODING, '') 273 | # if self.port is not None: 274 | # c.setopt(pycurl.PORT, self.port) 275 | return 276 | 277 | def run(self): 278 | global QueueInWorkerExit 279 | random.seed() 280 | payloads = ['/.git/config','/.svn/entries'] 281 | while True: 282 | num = len(self.m.cpool) 283 | # print 'while in master,num is ', num 284 | # if num >= self.connect_max: 285 | # for c in self.m.cpool: 286 | # print c.getinfo(pycurl.EFFECTIVE_URL) 287 | if not QueueInWorkerExit and num < self.connect_max: 288 | num = self.connect_max - num 289 | for i in xrange(num): 290 | try: 291 | host = self.queue_in.get(True, 1) 292 | if host is None: 293 | QueueInWorkerExit = True 294 | break 295 | else: 296 | for payload in payloads: 297 | if self.monitor: 298 | sys.stderr.write('%40s\r%s\r' % (' ', host)) 299 | c = pycurl.Curl() 300 | self.c_init(c) 301 | c.host = host 302 | c.setopt(pycurl.URL, 'http://%s/%s' % (host, payload)) 303 | c.head = StringIO() 304 | c.setopt(pycurl.HEADERFUNCTION, c.head.write) 305 | c.body = StringIO() 306 | c.setopt(pycurl.WRITEFUNCTION, c.body.write) 307 | self.m.cpool.append(c) 308 | self.m.add_handle(c) 309 | except Queue.Empty: 310 | break 311 | 312 | num = len(self.m.cpool) 313 | if 0 == num: 314 | if QueueInWorkerExit: 315 | self.queue_out.put(None) 316 | break 317 | else: 318 | continue 319 | while True: 320 | ret, num_ret = self.m.perform() 321 | # print 'while in perform,num_ret:',num_ret 322 | if pycurl.E_CALL_MULTI_PERFORM != ret: 323 | break 324 | 325 | while True: 326 | num_ret, ok_list, error_list = self.m.info_read() 327 | #print 'while in get info,num_ret:',num_ret 328 | for c in ok_list: 329 | # head = AnyToSth(c.head.getvalue()) 330 | head = c.head.getvalue() 331 | is_plain = False 332 | if 'text/plain' in head: 333 | is_plain = True 334 | server = None 335 | # if '(error)' == head: 336 | # server = '(error)' 337 | # else: 338 | # try: 339 | # server = re.search('Server: ([^\\r\\n]+)', head, re.U).group(1) 340 | # server = AnyToSth(server, 'gbk') 341 | # except AttributeError: 342 | # server = None 343 | 344 | code = c.getinfo(pycurl.HTTP_CODE) 345 | # print 'code',code 346 | if 200 == code: 347 | buf = c.body.getvalue() 348 | effective_url = c.getinfo(pycurl.EFFECTIVE_URL) 349 | # print 'effective_url,buf',effective_url,buf 350 | if is_plain: 351 | if effective_url and effective_url.strip().endswith('config'): 352 | if 'repositoryformatversion' in buf: 353 | self.queue_out.put((c.host, code, server, effective_url)) 354 | print 'effective_url:',effective_url 355 | 356 | if effective_url and effective_url.strip().endswith('entries'): 357 | if 'dir' in buf: 358 | self.queue_out.put((c.host, code, server, effective_url)) 359 | print 'effective_url:v1.6:',effective_url 360 | elif re.match(r'^\d+.*', buf): 361 | self.queue_out.put((c.host, code, server, effective_url)) 362 | print 'effective_url:v1.7:',effective_url 363 | 364 | c.host = None 365 | c.head.close() 366 | c.head = None 367 | c.body.close() 368 | c.body = None 369 | self.m.remove_handle(c) 370 | self.m.cpool.remove(c) 371 | c.close() 372 | 373 | for c, errno, errmsg in error_list: 374 | c.host = None 375 | c.head.close() 376 | c.head = None 377 | c.body.close() 378 | c.body = None 379 | self.m.remove_handle(c) 380 | self.m.cpool.remove(c) 381 | c.close() 382 | 383 | if 0 == num_ret: 384 | break 385 | 386 | self.m.select(1.0) 387 | 388 | return 389 | 390 | 391 | QueueTmpWorkerExit = False 392 | QueueTmpNoneNum = 0 393 | QueueTmpRLock = RLock() 394 | 395 | class QueueTmpWorker(Thread): 396 | 397 | def __init__(self, queue_in, queue_out, num_fetch, httpheader = [], proxypool = [], noproxy = 'localhost,127.0.0.1', \ 398 | referer = '', followlocation = 1, maxredirs = 5, redir_protocols = 3, protocols = 3, connecttimeout = 30, timeout = 30, \ 399 | connect_max = 16, port = None): 400 | super(QueueTmpWorker, self).__init__() 401 | self.queue_in = queue_in 402 | self.queue_out = queue_out 403 | self.num_fetch = num_fetch 404 | self.httpheader = httpheader 405 | self.proxypool = proxypool 406 | self.noproxy = noproxy 407 | self.referer = referer 408 | self.followlocation = followlocation 409 | self.maxredirs = maxredirs 410 | self.redir_protocols = redir_protocols 411 | self.protocols = protocols 412 | self.connecttimeout = connecttimeout 413 | self.timeout = timeout 414 | self.connect_max = connect_max 415 | self.port = port 416 | self.m = pycurl.CurlMulti() 417 | self.m.cpool = [] 418 | self.setDaemon(True) 419 | 420 | def __del__(self): 421 | if self.m is not None: 422 | for c in self.m.cpool: 423 | if c.data is not None: 424 | c.data = None 425 | if c.head is not None: 426 | c.head.close() 427 | c.head = None 428 | if c.body is not None: 429 | c.body.close() 430 | c.body = None 431 | self.m.remove_handle(c) 432 | self.m.cpool.remove(c) 433 | c.close() 434 | 435 | self.m.cpool = [] 436 | self.m.close() 437 | self.m = None 438 | return 439 | 440 | def c_init(self, c): 441 | c.setopt(pycurl.HTTPHEADER, self.httpheader) 442 | if len(self.proxypool) > 0: 443 | c.setopt(pycurl.PROXY, random.choice(self.proxypool)) 444 | c.setopt(pycurl.NOPROXY, self.noproxy) 445 | if self.referer is None: 446 | c.setopt(pycurl.AUTOREFERER, 0) 447 | elif '' == self.referer: 448 | c.setopt(pycurl.AUTOREFERER, 1) 449 | else: 450 | c.setopt(pycurl.REFERER, self.referer) 451 | c.setopt(pycurl.FOLLOWLOCATION, self.followlocation) 452 | c.setopt(pycurl.MAXREDIRS, self.maxredirs) 453 | # c.setopt(pycurl.REDIR_PROTOCOLS, self.redir_protocols) 454 | # c.setopt(pycurl.PROTOCOLS, self.protocols) 455 | c.setopt(pycurl.NOSIGNAL, 1) 456 | c.setopt(pycurl.CONNECTTIMEOUT, self.connecttimeout) 457 | c.setopt(pycurl.TIMEOUT, self.timeout) 458 | c.setopt(pycurl.ENCODING, '') 459 | if self.port is not None: 460 | c.setopt(pycurl.PORT, self.port) 461 | return 462 | 463 | def run(self): 464 | global QueueTmpWorkerExit 465 | global QueueTmpNoneNum 466 | random.seed() 467 | while True: 468 | num = len(self.m.cpool) 469 | if not QueueTmpWorkerExit and num < self.connect_max: 470 | num = self.connect_max - num 471 | for i in xrange(num): 472 | try: 473 | data = self.queue_in.get(True, 1) 474 | if data is None: 475 | QueueTmpRLock.acquire() 476 | QueueTmpNoneNum += 1 477 | QueueTmpRLock.release() 478 | if self.num_fetch == QueueTmpNoneNum: 479 | QueueTmpWorkerExit = True 480 | break 481 | else: 482 | c = pycurl.Curl() 483 | self.c_init(c) 484 | c.data = data 485 | c.setopt(pycurl.URL, 'http://%s/' % data[0]) 486 | c.head = StringIO() 487 | c.setopt(pycurl.HEADERFUNCTION, c.head.write) 488 | c.body = StringIO() 489 | c.setopt(pycurl.WRITEFUNCTION, c.body.write) 490 | c.setopt(pycurl.CUSTOMREQUEST, 'HEAD') 491 | c.setopt(pycurl.NOBODY, True) 492 | self.m.cpool.append(c) 493 | self.m.add_handle(c) 494 | except Queue.Empty: 495 | break 496 | 497 | num = len(self.m.cpool) 498 | if 0 == num: 499 | if QueueTmpWorkerExit: 500 | self.queue_out.put(None) 501 | break 502 | else: 503 | continue 504 | while True: 505 | ret, num_ret = self.m.perform() 506 | if pycurl.E_CALL_MULTI_PERFORM != ret: 507 | break 508 | 509 | while True: 510 | num_ret, ok_list, error_list = self.m.info_read() 511 | for c in ok_list: 512 | code = c.getinfo(pycurl.HTTP_CODE) 513 | if 401 == code: 514 | head = AnyToSth(c.head.getvalue()) 515 | if '(error)' == head: 516 | model = '(error)' 517 | else: 518 | try: 519 | model = re.search('WWW-Authenticate: Basic realm="(.+?)"', c.head.getvalue(), re.U).group(1) 520 | model = AnyToSth(model, 'gbk') 521 | except AttributeError: 522 | model = None 523 | 524 | else: 525 | model = '(%u)' % code 526 | self.queue_out.put((c.data[0], 527 | c.data[1], 528 | c.data[2], 529 | c.data[3], 530 | model)) 531 | c.data = None 532 | c.head.close() 533 | c.head = None 534 | c.body.close() 535 | c.body = None 536 | self.m.remove_handle(c) 537 | self.m.cpool.remove(c) 538 | c.close() 539 | 540 | for c, errno, errmsg in error_list: 541 | model = '(%u:%s)' % (errno, errmsg) 542 | self.queue_out.put((c.data[0], 543 | c.data[1], 544 | c.data[2], 545 | c.data[3], 546 | model)) 547 | c.data = None 548 | c.head.close() 549 | c.head = None 550 | c.body.close() 551 | c.body = None 552 | self.m.remove_handle(c) 553 | self.m.cpool.remove(c) 554 | c.close() 555 | 556 | if 0 == num_ret: 557 | break 558 | 559 | self.m.select(1.0) 560 | 561 | return 562 | 563 | 564 | class QueueOutWorker(Thread): 565 | 566 | def __init__(self, queue_in, num_fetch, quiet = False): 567 | super(QueueOutWorker, self).__init__() 568 | self.queue_in = queue_in 569 | self.num_fetch = num_fetch 570 | self.quiet = quiet 571 | self.setDaemon(True) 572 | 573 | def __del__(self): 574 | pass 575 | 576 | def run(self): 577 | num_none = 0 578 | while True: 579 | data = self.queue_in.get() 580 | if data is None: 581 | num_none += 1 582 | if self.num_fetch == num_none: 583 | break 584 | else: 585 | host = data[0] 586 | code = data[1] 587 | server = data[2] 588 | effective_url = data[3] 589 | print 'go to hack it:', host, code, server,effective_url 590 | return 591 | 592 | import argparse 593 | from optparse import OptionParser 594 | def main1(): 595 | parser = argparse.ArgumentParser(description='git and svn scan') 596 | parser = OptionParser() 597 | parser.add_option('-f','--file',dest='file_path',help='读取文件',default=None) 598 | parser.add_option('-t','--time',dest='time_second',help='等待时间',default=30) 599 | parser.add_option('-n','--thread_num',dest='thread_num',help='线程数',default=8) 600 | parser.add_option('-b','--begin',dest='ip_begin',help='开始ip',default=None) 601 | parser.add_option('-e','--end',dest='ip_end',help='结束ip',default=None) 602 | options, args = parser.parse_args() 603 | file_path = options.file_path 604 | second = options.time_second 605 | connectnum = 80 606 | threadnum = int(options.thread_num) 607 | begin = options.ip_begin 608 | end = options.ip_end 609 | monitor = False 610 | quiet = False 611 | port = None 612 | head = None 613 | proxypool = [] 614 | try: 615 | with open('/root/tool/proxy.txt', 'r') as fd_proxy: 616 | for line in fd_proxy: 617 | proxy = 'http://%s' %(line.strip()) 618 | print 'proxy',proxy 619 | proxypool.append(proxy) 620 | except Exception,e: 621 | print 'can not open file proxy.txt' 622 | # proxypool = [] 623 | 624 | httpheader = ['User-Agent: Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2;\ 625 | .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)',\ 626 | 'Accept: image/jpeg, application/x-ms-application, image/gif, application/xaml+xml, image/pjpeg, application/x-ms-xbap,\ 627 | application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*'] 628 | 629 | 630 | queue_in = Queue.Queue(4096) 631 | #queue_tmp = Queue.Queue(1024) 632 | queue_out = Queue.Queue(4096) 633 | for i in xrange(threadnum): 634 | QueueInWorker(queue_in, queue_out, httpheader=httpheader, connect_max=connectnum, connecttimeout=second, timeout=second, monitor=monitor, port=port, proxypool=proxypool).start() 635 | 636 | qoworker = QueueOutWorker(queue_out, threadnum, quiet) 637 | qoworker.start() 638 | count = 0 639 | if file_path: 640 | with open(file_path, 'r') as fd: 641 | for line in fd: 642 | line = line.strip() 643 | if line and line.split(): 644 | ip = line.split()[0].strip() 645 | ip_port = ip + ":80" 646 | queue_in.put(ip_port) 647 | count = count + 1 648 | if count %10000 == 0: 649 | print 'count',count 650 | #print 'ip_port',ip_port 651 | elif begin and end: 652 | for ip in itertools.islice(itertools.count(begin), end-begin): 653 | tail = ip&255 654 | if not tail or 255 == tail: 655 | continue 656 | ip = socket.inet_ntoa(itos32(socket.ntohl(ip))) 657 | queue_in.put(ip+":80") 658 | 659 | qoworker.join() 660 | 661 | 662 | 663 | if '__main__' == __name__: 664 | try: 665 | # main(os.path.basename(sys.argv[0]), sys.argv[1:]) 666 | main1() 667 | except KeyboardInterrupt: 668 | pass 669 | --------------------------------------------------------------------------------