├── README.md
├── Remove_Duplicates.py
├── phone_search.py
└── see_what.py


/README.md:
--------------------------------------------------------------------------------
1 | # PythonTo-repeat-the-text-Bigdata
2 | Remove_Duplicates.py   : Python脚本实现千万级文本数据快速去重
3 | 
4 | see_what.py   :仅查看大文本内部数据样式
5 | 
6 | pthone_search.py   :定向检测是否存在特定phone泄露数据
7 | 
8 | 


--------------------------------------------------------------------------------
/Remove_Duplicates.py:
--------------------------------------------------------------------------------
 1 | #coding=utf-8
 2 | 
 3 | import sys, re, os
 4 | 
 5 | def getDictList(dict):
 6 | 
 7 |     regx = '''[\w\~`\!\@\#\$\%\^\&\*\(\)\_\-\+\=\[\]\{\}\:\;\,\.\/\<\>\?]+'''
 8 | 
 9 |     with open(dict) as f:
10 | 
11 |         data = f.read()
12 | 
13 |         return re.findall(regx, data)
14 | 
15 | def rmdp(dictList):
16 | 
17 |     return list(set(dictList))
18 | 
19 | def fileSave(dictRmdp, out):
20 | 
21 |     with open(out, 'a') as f:
22 | 
23 |         for line in dictRmdp:
24 | 
25 |             f.write(line + '\n')
26 | 
27 | def main():#用法是在命令行中执行Remove_Duplicates.py test.txt result.txt
28 | 
29 |     try:
30 | 
31 |         dict = sys.argv[1].strip()
32 | 
33 |         out = sys.argv[2].strip()
34 | 
35 |     except Exception, e:
36 | 
37 |         print 'error:', e
38 | 
39 |         me = os.path.basename(__file__)
40 | 
41 |         print 'usage: %s <input> <output>' %me #被去重文本
42 | 
43 |         print 'example: %s dict.txt dict_rmdp.txt' %me #去重后结果文本
44 | 
45 |         exit()
46 | 
47 |     dictList = getDictList(dict)
48 | 
49 |     dictRmdp = rmdp(dictList)
50 | 
51 |     fileSave(dictRmdp, out)
52 | 
53 | if __name__ == '__main__':
54 | 
55 |     main()
56 | 


--------------------------------------------------------------------------------
/phone_search.py:
--------------------------------------------------------------------------------
 1 | #coding=utf8
 2 | import sys
 3 | import re
 4 | import string
 5 | import time
 6 | import datetime
 7 | ''' 定向威胁分析 '''
 8 | 
 9 | def search(File):
10 |         
11 |         f=open(File,'r')
12 |         f1=open(r'result.txt','a')
13 |         num=0
14 |         success_num=0  #计数
15 |         flag=0  #标记
16 |         print '>>>Start search file:'+File+'...'
17 |         f1.write('>>>文件标记:'+File)
18 |         while 1:
19 |                 lines = f.readlines(100000)
20 |                 if not lines:
21 |                         break
22 |                 for line in lines:
23 |                         num=num+1
24 |                         linex=re.search(r'(18812341234)|(13512341234)|(18612341234)|(15612341234)|(18712341234)|\
25 |                                         (18912341234)|(15812341234)',line) #被搜查号码
26 |                         if linex:
27 |                             f1.write('    发现目标行:'+line)
28 |                             print '发现目标行:'+line
29 |                             success_num=success_num+1 #标记成功的个数
30 |                         else:
31 |                             continue
32 |                         if success_num==7:#被搜查号码个数
33 |                                 flag=1
34 |                 if flag==1:#标记，如果全部找出，直接结束
35 |                     break
36 |         f.close()
37 |         f1.write('>>>时间标记:%s \n\n'% datetime.datetime.now())
38 |         f1.close()
39 | 
40 |         print '>>>报告主人，该文件全部搜查完毕!!总共搜索行数：'+str(num)+',请检查result.txt \n<<<End search file：'+File+' \n'
41 | 
42 | 
43 | if __name__=='__main__':
44 |         start=time.clock() #The start time
45 |         file=['test.txt','test2.txt']######这里是扫描的txt数据文件，是list数组形式
46 |         for i in file:
47 |                 search(i)
48 |         end=time.clock() #The end time
49 |         i = datetime.datetime.now()
50 |         print "###########程序总耗时 : %.03f seconds" %(end-start)
51 |         print "###########当前的日期和时间是 %s" % i
52 |         
53 |         
54 |     
55 | 


--------------------------------------------------------------------------------
/see_what.py:
--------------------------------------------------------------------------------
 1 | #coding=utf8
 2 | import sys
 3 | import re
 4 | import string
 5 | import time
 6 | import datetime
 7 | '''By 凌晨G度'''
 8 | ''' 亿级手机相关密码泄露事件，定向威胁分析--文本查看 '''
 9 | 
10 | def search():
11 |         File='filename.txt'
12 |         f=open(File,'r')
13 |         lines = f.readlines(10000) #大批量会内存限制，仅读取前10000行
14 |         for line in lines[1:100]: #仅输出展示1-100行数据
15 |             print line
16 |         f.close()
17 | 
18 | if __name__=='__main__':
19 |         search()
20 |         
21 |     
22 | 


--------------------------------------------------------------------------------