├── a.txt
├── bug.txt
├── data.csv
├── get_weibo.py
├── get_weibo.pyc
├── network_graph.png
├── network_graph.py
├── post_encode.py
├── post_encode.pyc
├── readme.md
├── test.txt
├── time_graph.png
├── time_graph.py
├── weibo_login.py
├── weibo_login.pyc
└── weibo_main.py


/a.txt:
--------------------------------------------------------------------------------
1 | heheeh


--------------------------------------------------------------------------------
/bug.txt:
--------------------------------------------------------------------------------
1 | dddddddd


--------------------------------------------------------------------------------
/get_weibo.py:
--------------------------------------------------------------------------------
  1 | #coding:utf-8
  2 | import urllib
  3 | import urllib2
  4 | import re
  5 | import os
  6 | import time
  7 | import random
  8 | import json
  9 | #使用beautifulsoup对HTML页面进行解析
 10 | from bs4 import BeautifulSoup
 11 | import sys
 12 | reload(sys)
 13 | sys.setdefaultencoding('utf8')
 14 | #此函数获取单个页面的转发数据
 15 | def get_forward(html, origin_html):
 16 | 	#获取返回的json
 17 | 	get_json = json.load(html)
 18 | 	#获取返回json中的转发总数、
 19 | 	total_forward = get_json['data']['count']
 20 | 	#获取返回json中的评论页面总数
 21 | 	total_page = get_json['data']['page']['totalpage']
 22 | 	#获取返回json中的当前页面
 23 | 	current_page = get_json['data']['page']['pagenum']
 24 | 	#获取返回json中的html
 25 | 	forward_html = get_json['data']['html']
 26 | 	soup = BeautifulSoup(forward_html, "lxml")
 27 | 	#获取网页中转发的页面
 28 | 	div_forward = soup.find_all(attrs={'action-type' : 'feed_list_item'})
 29 | 	#一度转发uid
 30 | 	uid = []
 31 | 	#原微博的发出者
 32 | 	origin_uid_temp = get_origin_weibo(origin_html)
 33 | 	#一度转发时间
 34 | 	time = []
 35 | 	#二度转发原uid
 36 | 	origin_uid2 = []
 37 | 	uid2 = []
 38 | 	#二度转发时间
 39 | 	time2 = []
 40 | 	for i in range(len(div_forward)):
 41 | 		a = str(div_forward[i].find(attrs={'node-type' : 'text'}))
 42 | 		#b = a.find(re.compile("//<a(.*)</a>"))
 43 | 		p = re.compile('//(<a.*</a>):')
 44 | 		#如果有多个转发
 45 | 		if(p.search(a)):
 46 | 			temp_uid2 = []
 47 | 			#the end of forward user
 48 | 			end_uid = "name=" + div_forward[i].find(attrs={'node-type' : 'name'}).get_text()
 49 | 			soup = BeautifulSoup(p.search(a).group(1), "lxml")
 50 | 			temp = soup.find_all('a')
 51 | 			temp_time = div_forward[i].find(attrs={'node-type' : 'feed_list_item_date'}).get('title')
 52 | 			j = len(temp)-1
 53 | 			while (j >=0):
 54 | 				if(temp[j].get('usercard')):
 55 | 					temp_uid2.append(temp[j].get('usercard').encode('utf-8'))
 56 | 				j -=1
 57 | 
 58 | 			temp_uid2.append(end_uid)
 59 | 			if (temp_uid2[0] != origin_uid_temp):
 60 | 				temp_uid2.insert(0, origin_uid_temp)
 61 | 			for i in range(0, len(temp_uid2)-1):
 62 | 				origin_uid2.append(temp_uid2[i])
 63 | 				time2.append(temp_time)
 64 | 			for i in range(1, len(temp_uid2)):
 65 | 				uid2.append(temp_uid2[i])
 66 | 			continue
 67 | 		uid.append("name=" + div_forward[i].find(attrs={'node-type' : 'name'}).get_text())
 68 | 		time.append(div_forward[i].find(attrs={'node-type' : 'feed_list_item_date'}).get('title'))
 69 | 	return (uid, origin_uid2, uid2, time, time2, total_forward, total_page, current_page)
 70 | #此函数获取原微博的uid
 71 | def get_origin_weibo(original_html):
 72 | 	#将获取到的html源码分行，因为新浪微博将网页进行了压缩
 73 | 	decoded_html = original_html.encode("utf-8").replace("\\", "")
 74 | 	soup = BeautifulSoup(decoded_html, "lxml")
 75 | 	div_origin = soup.find(attrs={'name' : 'keywords'})
 76 | 	origin_uid = div_origin.get('content')
 77 | 	origin_uid = "name=" + origin_uid[0:-14]
 78 | 	return origin_uid
 79 | ##########################################
 80 | #后面的函数与分析无关，后面的函数是获取微博的具体内容的。
 81 | def decode_html(original_html):
 82 | 	#将获取到的html源码分行，因为新浪微博将网页进行了压缩
 83 | 	lines = original_html.splitlines()
 84 | 	for line in lines:
 85 | 		#以<script>STK && STK.pageletM && STK.pageletM.view({"pid":"pl_weibo_direct开头的是所有微博内容
 86 | 		#如果报错的话，说明被识别成机器人了，然后手动将最后输出的URL用浏览器打开，输入验证码
 87 | 		#一般爬取30几页就会被识别为爬虫，但是这个程序也有用，因为新浪微博搜索最多显示50页
 88 | 		if line.startswith('<script>STK && STK.pageletM && STK.pageletM.view({"pid":"pl_weibo_direct","js":["apps'):
 89 | 			#print "not caught"
 90 | 			n = line.find('"html":"')
 91 | 			if n > 0:
 92 | 				decoded_html = line[n + 8: ].encode("utf-8").decode('unicode_escape').encode("utf-8").replace("\\", "")
 93 | 			#else:
 94 | 			#	print "caught"
 95 | 	# print "@@@@@"
 96 | 	#print decoded_html
 97 | 	return decoded_html
 98 | 
 99 | #get_details()用于获得每条味浓的具体信息，如果看不懂的话可以去看看源码的HTML结构
100 | def get_details(decoded_html):
101 | 	# html = decode_html(decoded_html)
102 | 	soup=BeautifulSoup(decoded_html,"lxml")
103 | 	#得到作者、作者链接、微博正文
104 | 	div_content=soup.find_all(attrs={'class': 'content clearfix'})
105 | 	#得到发微博时间
106 | 	div_time=soup.find_all(attrs={'class':'feed_from W_textb'})
107 | 	#将用户名称，用户主页地址、微博正文、发微博时间初始化
108 | 	nick_name=[]
109 | 	nickname_href=[]
110 | 	content_text=[]
111 | 	time=[]
112 | 	#print get_content[0]
113 | 	for i in range(len(div_content)):
114 | 		#查找a标签
115 | 		a_tag=div_content[i].find('a')
116 | 		nick_name.append(a_tag.get('nick-name'))
117 | 		nickname_href.append(a_tag.get('href'))
118 | 		#查找p标签
119 | 		p_tag=div_content[i].find('p')
120 | 		content_text.append(p_tag.get_text())
121 | 	#得到发微博时间
122 | 	for j in range(len(div_time)):
123 | 		a_time=div_time[j].find('a')
124 | 		time.append(a_time.get('title'))
125 | 	# print content_text
126 | 	# print nickname_href
127 | 	return (nick_name,nickname_href,content_text,time)
128 | #get_number_info()用于获得转发、评论、赞的数据
129 | def get_number_info(decoded_html):#一次性得到所有数字
130 | 	# html = decoded_html(decoded_html)
131 | 	soup=BeautifulSoup(decoded_html,"lxml")
132 | 	#获取标签内容
133 | 	#查找
134 | 
135 | 	get=soup.find_all(attrs={'class': 'feed_action_info feed_action_row4'})
136 | 	# print get[1]
137 | 	#收集转发、评论、赞的数据
138 | 	get_number_info=[]
139 | 	for i in range(len(get)):
140 | 		#转发
141 | 		forward=get[i].find(attrs={'action-type':'feed_list_forward'})
142 | 		forward_em=forward.find_all('em')
143 | 		#判断数据是否为0，
144 | 		if (len(forward_em[0].get_text())==0):
145 | 			temp_forward="0"
146 | 			get_number_info.append(temp_forward)
147 | 		else:
148 | 			temp_forward=forward_em[0].get_text()
149 | 			get_number_info.append(temp_forward)
150 | 		#评论
151 | 		comment=get[i].find(attrs={'action-type':'feed_list_comment'})
152 | 		if bool(comment.find_all('em')):
153 | 			comment_em=comment.find_all('em')
154 | 			temp_comment=comment_em[0].get_text()
155 | 			get_number_info.append(temp_comment)
156 | 		else:
157 | 			temp_comment="0"
158 | 			get_number_info.append(temp_comment)
159 | 		#赞
160 | 		like=get[i].find(attrs={'action-type':'feed_list_like'})
161 | 		like_em=like.find_all('em')
162 | 		if (len(like_em[0].get_text())==0):
163 | 			temp_like="0"
164 | 			get_number_info.append(temp_like)
165 | 		else:
166 | 			temp_like=like_em[0].get_text()
167 | 			get_number_info.append(temp_like)
168 | 	return get_number_info
169 | #write_all_info()将所有数据写入文本
170 | def write_all_info(original_html):
171 | 	get_decoded_html = decode_html(original_html)
172 | 	(nick_name_list, nick_name_href_list, content_text_list, time_list) = get_details(get_decoded_html)
173 | 	number_info_list = get_number_info(get_decoded_html)
174 | 	#文件保存路径
175 | 	path='/home/ruansongsong/work/ruansongsong/' 
176 | 	isExists=os.path.exists(path)
177 | 	if not isExists:
178 | 	    os.makedirs(path)
179 | 	temp=0
180 | 	for i in range(len(nick_name_list)):
181 | 		write_all_list=open(path+"weibo.txt",'a')
182 | 		write_all_list.writelines("微博用户名称："+nick_name_list[i]+"\n"+"微博链接："+nick_name_href_list[i]+"\n"+"正文:"+"\n"+content_text_list[i]+"\n"+"发微博时间："+time_list[i]+"\n")
183 | 		j=0
184 | 		#由于我是将转发、评论、赞的数据储存到一个list中的，所以每一个微博正文要写入3个数据。
185 | 		while (j!=3):
186 | 			write_all_list.writelines("==="+number_info_list[temp]+"==="+"\n")
187 | 			j+=1
188 | 			temp+=1
189 | 		write_all_list.close()


--------------------------------------------------------------------------------
/get_weibo.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/IACJ/weibo_forward_analysis/4a9d9205a1e422e72a718ff58370d57c3f0948b4/get_weibo.pyc


--------------------------------------------------------------------------------
/network_graph.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/IACJ/weibo_forward_analysis/4a9d9205a1e422e72a718ff58370d57c3f0948b4/network_graph.png


--------------------------------------------------------------------------------
/network_graph.py:
--------------------------------------------------------------------------------
 1 | #coding:utf-8
 2 | import matplotlib.pyplot as plt
 3 | import networkx as nx 
 4 | import numpy
 5 | import numpy as np
 6 | with open('data.csv', 'r') as f:
 7 |     rt_time = []
 8 |     for line in f:
 9 |         time= line.strip().split(',')[-1]
10 |         # print time
11 |         day = time[1:5] + time[6:8] + time[9:11]
12 |         # print day
13 |        
14 |         #hms= time[9:17].replace(':', '')
15 |         hm = time[12:17].replace(':','')
16 |         time = int(day + hm)
17 | 
18 |         rt_time.append(time)
19 |  #计算转发时间的先后顺序
20 | array = numpy.array(rt_time)
21 | order = array.argsort()
22 | ranks = order.argsort()
23 | G = nx.Graph()
24 | with open('data.csv', 'r') as f:
25 |     for position, line in enumerate(f):
26 |         uid, rtuid= line.strip().split(',')[:-1]
27 |         G.add_edge(uid, rtuid, time = ranks[position])
28 | edges,colors = zip(*nx.get_edge_attributes(G,'time').items())
29 | #degree=G.degree()#计算节点的出度
30 | #degree=G.degree()#计算节点的出度
31 | degree = G.degree(G)
32 | closenesss = nx.closeness_centrality(G)
33 | betweenness = nx.betweenness_centrality(G)
34 | print G.number_of_edges() # 1547
35 | # print g.diameter(G) # 2
36 | path_length = nx.all_pairs_shortest_path_length(G)
37 | pos=nx.spring_layout(G) #设置网络的布局
38 | #fig = plt.figure(figsize=(80, 60),facecolor='white')
39 | #pos=nx.spring_layout(G) #设置网络的布局
40 | 
41 | fig = plt.figure(figsize=(20, 10),facecolor='white')
42 | nx.draw(G, pos, nodelist = degree.keys(),
43 |         node_size = [v*2 for v in degree.values()], node_color = 'orange',         
44 |         node_shape = 'o', cmap=plt.cm.gray,
45 |         edgelist = edges, edge_color = 'gray', width = 0.5,
46 |         with_labels = False, arrows = False)
47 | plt.title("A sina weibo social network")
48 | plt.show()


--------------------------------------------------------------------------------
/post_encode.py:
--------------------------------------------------------------------------------
 1 | #coding:utf8
 2 | import urllib
 3 | import rsa
 4 | import base64
 5 | import binascii
 6 | import re
 7 | #此函数将用户名用base64加密
 8 | def encrypt_user_name(original_user_name):
 9 | 	#将用户名url编码
10 | 	user_name_temp = urllib.quote(original_user_name)
11 | 	#将编码后的用户名用base64加密
12 | 	encrypted_user_name = base64.encodestring(user_name_temp)[: -1]
13 | 	# print encrypted_user_name
14 | 	return encrypted_user_name
15 | #此函数将用户密码加密，需要原始密码，servertime时间戳 ，nonce一次性随机码 ，pubkey公钥
16 | def encrypt_user_password(original_user_password, server_time, nonce , pubkey):
17 | 	#将pubkey转换为整型数字
18 | 	rsa_public_key = int(pubkey, 16)
19 | 	#创建公钥
20 | 	key = rsa.PublicKey(rsa_public_key, 65537)#1001?
21 | 	#加入servertime , nonce , password
22 | 	#js中的加密方式，可查看微博返回的js文件
23 | 	message = str(server_time) + '\t' +str(nonce) + '\n' + str(original_user_password)
24 | 	#将连接后的字符串加密
25 | 	encrypted_user_password = rsa.encrypt(message, key)
26 | 	#将加密后的信息转化为16进制
27 | 	encrypted_user_password = binascii.b2a_hex(encrypted_user_password)
28 | 	return encrypted_user_password
29 | #此函数将加密后的信息构成post数据
30 | def post_encode(user_name, user_password, server_time, nonce, pubkey, rsakv, pin_code = 0):
31 | 	#调用encrypt_user_name函数
32 | 	get_encrypt_user_name = encrypt_user_name(user_name)
33 | 	#调用encrypt_user_password函数
34 | 	get_encrypt_user_password = encrypt_user_password(user_password, server_time, nonce, pubkey)
35 | 	#构建post数据
36 | 	post_data = {
37 | 			'su': get_encrypt_user_name,
38 | 			'servertime': server_time,
39 | 			'nonce': nonce,
40 | 			'sp': get_encrypt_user_password,
41 | 			'rsakv': rsakv, 
42 | 			'encoding':'UTF-8',
43 | 			'entry':'weibo',
44 | 			'from':'',
45 | 			'gateway':'1',
46 | 			'prelt':'88',
47 | 			'pwencode':'rsa2',
48 | 			'returntype':'META',
49 | 			'savestate':'0',
50 | 			'service':'miniblog',
51 | 			'url':'http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack',
52 | 			'useticket':'1',
53 | 			'vsnf':'1 '
54 | 	}
55 | 	if(pin_code):
56 | 		post_data['door'] = pin_code
57 | 	#将post数据编码
58 | 	get_post_data = urllib.urlencode(post_data)	
59 | 	return get_post_data


--------------------------------------------------------------------------------
/post_encode.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/IACJ/weibo_forward_analysis/4a9d9205a1e422e72a718ff58370d57c3f0948b4/post_encode.pyc


--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
 1 | #基于Python的新浪微博社交网络分析
 2 | >具体内容请看代码注释
 3 | 
 4 | ##网络图
 5 | <img src="http://ww2.sinaimg.cn/large/680bc840jw1eyw2j1a8njj21kw0u60xh.jpg"/>
 6 | 
 7 | #时间图
 8 | <img src="http://ww1.sinaimg.cn/large/680bc840jw1eyw2lnu177j20mk0h0764.jpg"/>
 9 | 
10 | ##项目步骤
11 | 1. 爬取某个具体微博的转发数据；
12 | 2. 将爬下来的转发数据用图进行可视化。
13 | =======
14 | 2. 将爬下来的转发数据用图进行可视化。
15 | 
16 | ##基本模块
17 | 主要分了模拟登陆模块（weibo_login.py、post_encode.py）、解析网页模块（get_weibo.py）、绘制图像模块（network_graph.py、time_graph.py）。
18 | 
19 | ##文件说明
20 | 1. data.csv : 爬取下来的转发数据；
21 | 2. weibo_login.py、post_encode.py : 模拟登陆模块；
22 | 3. get_weibo.py ：解析网页模块；
23 | 4. network_graph.py ： 绘制网络图；
24 | 5. time_graph.py ： 绘制时间图；
25 | 6. network_graph.png : 网络图；
26 | 7. time_graph.png : 时间图。
27 | 
28 | ##参考博客
29 | 1. http://www.cnblogs.com/houkai/p/3488468.html；
30 | 2. http://computational-thinking.farbox.com/blog/2014-08-03-study-osn-using-python#toc_6。
31 | 


--------------------------------------------------------------------------------
/test.txt:
--------------------------------------------------------------------------------
1 | a branch from the master


--------------------------------------------------------------------------------
/time_graph.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/IACJ/weibo_forward_analysis/4a9d9205a1e422e72a718ff58370d57c3f0948b4/time_graph.png


--------------------------------------------------------------------------------
/time_graph.py:
--------------------------------------------------------------------------------
 1 | #coding:utf-8
 2 | #绘制转发时间扩散、增长图
 3 | import matplotlib.pyplot as plt
 4 | import networkx as nx 
 5 | from datetime import datetime
 6 | import numpy as np
 7 | with open('data.csv', 'r') as f:
 8 | 	rt_time = []
 9 | 	for line in f:
10 | 	    time= line.strip().split(',')[-1]
11 | 	    time = time[1:-1]
12 | 	    rt_time.append(time)
13 | day = [(str(i)[0:10]+'-'+str(i)[11:13]) for i in rt_time]
14 | day = [datetime.strptime(d, '%Y-%m-%d-%H') for d in day]
15 | day_weibo = datetime.strptime('2015-12-07-10', '%Y-%m-%d-%H') #源微博发出时间
16 | hours = [(i-day_weibo).total_seconds()/3600 for i in day]
17 | values, base = np.histogram(hours, bins = 40)
18 | cumulative = np.cumsum(values)
19 | plt.subplot(1, 2, 1)
20 | plt.plot(base[:-1], cumulative, c = 'red')
21 | plt.title('Cumulative Diffusion')
22 | plt.ylabel('Number of Retweets')
23 | plt.xlabel('Hours')
24 | plt.subplot(1, 2, 2 )
25 | plt.plot(base[:-1], values, c = 'orange')
26 | plt.title('Hourly Diffusion')
27 | plt.xlabel('Hours')
28 | plt.show()


--------------------------------------------------------------------------------
/weibo_login.py:
--------------------------------------------------------------------------------
  1 | #coding:utf-8
  2 | import urllib
  3 | import urllib2
  4 | import re
  5 | import json
  6 | import cookielib
  7 | import random
  8 | import time
  9 | import Image
 10 | import post_encode
 11 | import get_weibo
 12 | class WeiboLogin:
 13 | 	"""docstring for WeiboLogin"""
 14 | 	#初始化
 15 | 	def __init__(self, user_name, user_password):
 16 | 		self.user_name = user_name
 17 | 		self.user_password = user_password
 18 |  		su = post_encode.encrypt_user_name(user_name)
 19 | 		# ?entry=weibo
 20 | 		# &callback=sinaSSOController.preloginCallBack
 21 | 		# &su=MTg1MzU0Nzc5MDE%3D
 22 | 		# &rsakt=mod
 23 | 		# &checkpin=1
 24 | 		# &client=ssologin.js(v1.4.18)
 25 | 		# &_=1448083786752
 26 | 		curr_time = str(int(time.time()))
 27 | 		self.pre_login_url = "http://login.sina.com.cn/sso/prelogin.php?entry=weibo&checkpin=1&callback=sinaSSOController.preloginCallBack&rsakt=mod&client=ssologin.js(v1.4.18)" + "&su=" + su +"&_=" + curr_time
 28 | 		self.login_url = "http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.11)"
 29 | 		self.headers = {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:42.0) Gecko/20100101 Firefox/42.0'}
 30 | 	#cookie非常重要
 31 | 	def enableCookies(self):
 32 |             		#获取一个保存cookies的对象
 33 | 	            cookie= cookielib.CookieJar()
 34 | 	            #将一个保存cookies对象和一个HTTP的cookie的处理器绑定
 35 | 	            cookie_support = urllib2.HTTPCookieProcessor(cookie)
 36 | 	            #创建一个opener,设置一个handler用于处理http的url打开
 37 | 	            opener = urllib2.build_opener(cookie_support, urllib2.HTTPHandler)
 38 | 	            #安装opener，此后调用urlopen()时会使用安装过的opener对象
 39 | 	            urllib2.install_opener(opener)
 40 | 	#此函数获取服务器返回的必要的登录数据
 41 | 	def get_server_data(self):
 42 | 		server_data = urllib2.urlopen(self.pre_login_url).read()
 43 | 		#匹配返回的server_data
 44 | 		p = re.compile('\((.*)\)')
 45 | 		get_data = p.search(server_data).group(1)
 46 | 		print p.search(server_data).group(1)		
 47 | 		get_json = json.loads(get_data)
 48 | 		server_time = get_json['servertime']
 49 | 		nonce = get_json['nonce']
 50 | 		pubkey = get_json['pubkey']
 51 | 		rsakv = get_json['rsakv']
 52 | 		pcid = get_json['pcid']
 53 | 		show_pin = 0
 54 | 		show_pin = get_json['showpin']
 55 | 		# print show_pin
 56 | 		# try:
 57 | 		# 	show_pin = get_json['showpin']
 58 | 		# except:
 59 | 		# 	print "无验证码"
 60 | 		return (server_time, nonce, pubkey, rsakv, show_pin, pcid)
 61 | 	#此函数获取验证码
 62 | 	def get_pin(self, pcid):
 63 | 		pin_code_url = "http://login.sina.com.cn/cgi/pin.php"
 64 | 		#产生随机数
 65 | 		#pcid = "xd-01ee4d114e1c59964396d92937c2de294e20"
 66 | 		url_random = str(int(random.random() * 100000000))
 67 | 		pin_code_url = pin_code_url + "?r=" + url_random + "&s=0" + "&p=" + pcid
 68 | 		pin_code = urllib2.urlopen(pin_code_url).read()
 69 | 		f = open("/home/ruansongsong/work/code", "wb")
 70 | 		f.write(pin_code)
 71 | 		f.close()
 72 | 		#显示验证码图片
 73 | 		image = Image.open("/home/ruansongsong/work/code")
 74 | 		image.show()
 75 | 		#输入验证码
 76 | 		input_code = raw_input()
 77 | 		return input_code		        		
 78 | 	#返回登陆成功的url
 79 | 	def get_location_replace(self, get_html):
 80 | 		p = re.compile('location\.replace\([\'"](.*?)[\'"]\)')
 81 | 		get_login_url = p.search(get_html).group(1)
 82 | 		return get_login_url
 83 | 	#此函数执行登录
 84 | 	def login(self):
 85 | 		#调用cookie函数
 86 | 		self.enableCookies()
 87 | 		#调用get_server_data函数
 88 | 		(server_time, nonce, pubkey, rsakv, show_pin, pcid) = self.get_server_data()
 89 | 		if (show_pin):
 90 | 			get_input_code = self.get_pin(pcid)
 91 | 			#调用pos_encode函数
 92 | 			post_data = post_encode.post_encode(self.user_name, self.user_password, server_time, nonce, pubkey, rsakv, get_input_code)
 93 | 		else:
 94 | 			post_data = post_encode.post_encode(self.user_name, self.user_password, server_time, nonce, pubkey, rsakv)
 95 | 		#向服务器发送请求
 96 | 		req = urllib2.Request(self.login_url, post_data, self.headers)
 97 | 		get_html = urllib2.urlopen(req).read()
 98 | 		#调用get_location_url获取返回的地址
 99 | 		try:
100 | 			get_login_url = self.get_location_replace(get_html)
101 | 			print "登陆成功后的地址："
102 | 			print get_login_url
103 | 			urllib2.urlopen(get_login_url)
104 | 			
105 | 		except:
106 | 			print "登录失败"
107 | 			return False
108 | 		return True


--------------------------------------------------------------------------------
/weibo_login.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/IACJ/weibo_forward_analysis/4a9d9205a1e422e72a718ff58370d57c3f0948b4/weibo_login.pyc


--------------------------------------------------------------------------------
/weibo_main.py:
--------------------------------------------------------------------------------
 1 | #coding:utf-8
 2 | import urllib2
 3 | import post_encode
 4 | import time
 5 | import os
 6 | from weibo_login import WeiboLogin
 7 | import get_weibo
 8 | if __name__ == '__main__':
 9 | 	#Login = WeiboLogin('17089368196', 'tttt5555')
10 | 	Login = WeiboLogin('用户名', '密码')
11 | 	if Login.login() == True:
12 | 		print "登录成功"
13 | 	rnd = long((time.time())*1000)
14 | 	#可以根据page来循环以便达到爬取多页的目的
15 | 	init_url = "http://weibo.com/aj/v6/mblog/info/big?ajwvr=6&id=3917356680052180&max_id=3917762218921722"
16 | 	url = init_url + "&page=1&_rnd=" + str(rnd)
17 | 	html = urllib2.urlopen(url)
18 | 	#调用解析html内容的函数
19 | 	forward_html = urllib2.urlopen("http://weibo.com/1336593085/D7hwE0dzC?type=repost#_rnd1449550776472").read()
20 | 	#print forward_html
21 | 	origin_uid = get_weibo.get_origin_weibo(forward_html)
22 | 	(uid, origin_uid2_no, uid2_no, time_no, time_no2, total_forward, total_page, current_page) = get_weibo.get_forward(html, forward_html)
23 | 	#循环抓取多页
24 | 	print total_page
25 | 	uid = []
26 | 	origin_uid2 = []
27 | 	uid2 = []
28 | 	forward_time =[]
29 | 	forward_time2 = []
30 | 	count = 0
31 | 	for i in range(1,total_page):
32 | 		rnd = long((time.time())*1000)
33 | 		#可以根据page来循环以便达到爬取多页的目的
34 | 		url = init_url + "&page=" + str(i) + "&__rnd=" + str(rnd)
35 | 		html = urllib2.urlopen(url)
36 | 		(uid_temp, origin_uid2_temp, uid2_temp, time_temp, time_temp2, total_forward, total_page, current_page) = get_weibo.get_forward(html , forward_html)
37 | 		print "current page"
38 | 		print current_page
39 | 		#mid += mid_temp
40 | 		uid += uid_temp
41 | 		count  += len(uid_temp)
42 | 		origin_uid2 += origin_uid2_temp
43 | 		uid2 += uid2_temp
44 | 		forward_time += time_temp
45 | 		forward_time2 += time_temp2
46 | 	print len(uid)
47 | 	with open("data.csv", "w") as data:
48 | 		for i in range(0, len(uid)):
49 | 			print>>data, "%s,%s,'%s'" %(origin_uid, uid[i], forward_time[i])
50 | 	print total_forward
51 | 	with open("data.csv", "a") as data:
52 | 		for i in range(0, len(origin_uid2)):
53 | 			print >>data, "%s,%s,'%s'" %(origin_uid2[i], uid2[i], forward_time2[i])
54 | 	for i in range(0, len(origin_uid2)):
55 | 		print origin_uid2[i] + "->" +uid2[i]


--------------------------------------------------------------------------------