', methods=['POST'])
29 | def process(service_name):
30 | try:
31 | print service_name
32 | jsonData = json.loads(request.data)
33 | print request.data
34 | action = jsonData["action"]
35 | isAllow = False
36 | for item in utils.config.ALLOW_SERVICE.split(','):
37 | if service_name == item:
38 | isAllow = True
39 | if service_name is None or service_name == '' or not isAllow:
40 | return 'No such Service'
41 |
42 | result = ''
43 |
44 | if action == 'ACTION_SERVICE_CREATE':
45 | result = service.serverService.action_service_create(service_name, jsonData)
46 | elif action == 'ACTION_SERVICE_UPDATE':
47 | result = service.serverService.action_service_update(service_name, jsonData)
48 | elif action == 'ACTION_CLIENT_SERVICE_CREATE':
49 | result = service.serverService.action_client_service_create(service_name, jsonData)
50 | elif action == 'ACTION_SERVICE_FOLLOWED_CHANGE':
51 | result = service.serverService.action_service_followed_change(service_name, jsonData)
52 | elif action == 'ACTION_SERVICE_DING':
53 | result = service.serverService.action_service_ding(action, jsonData)
54 |
55 |
56 | except Exception, e:
57 | logging.error(e)
58 | print traceback.format_exc()
59 | result.code = "1001"
60 | result.message = "unknow Exception"
61 | resp = Response(result.get_json())
62 | return resp
63 |
64 | resp = Response()
65 | resp.data = result.get_json()
66 | resp.mimetype = "application/json"
67 |
68 | return resp
69 |
70 |
71 |
72 |
73 |
74 | xxx = ''
75 |
76 | schedudler = Scheduler(daemonic=False)
77 |
78 | schedudler.add_interval_job(func=plugin.lofter.lofterTask.task, seconds=60)
79 | schedudler.add_interval_job(func=plugin.toutiao.toutiaoTask.toutiao_gallery_task, seconds=60)
80 | schedudler.add_interval_job(func=plugin.toutiao.toutiaoTask.toutiao_news_task, seconds=60)
81 | # schedudler.add_interval_job(func=plugin.lofter.lofterTask.task(''),seconds=20)
82 |
83 |
84 |
85 | if __name__ == '__main__':
86 | schedudler.start()
87 | app.run(port=8888)
88 |
89 |
90 |
--------------------------------------------------------------------------------
/plugin/toutiao/toutiaoTask.py:
--------------------------------------------------------------------------------
1 | import traceback
2 | from flask import logging
3 | import model.message
4 | from plugin.toutiao import toutiaoUtils
5 | import utils.timeHelper
6 | import utils.Queue
7 |
8 |
9 | __author__ = 'chenyueling'
10 |
11 | import utils.redisFactory
12 |
13 | mark_news = 'toutiao_news'
14 | mark_gallery = 'toutiao_gallery'
15 |
16 |
17 | def toutiao_news_task():
18 | redis = utils.redisFactory.getRedis()
19 | set = redis.keys(mark_news + '*')
20 | print set
21 | try:
22 | for cid in set:
23 |
24 | if not utils.timeHelper.time_area(8, 20, 0, 10, 50, 0) and not utils.timeHelper.time_area(18, 20, 0, 18,
25 | 50, 0):
26 | redis.hset(cid, 'notify', 'false')
27 | continue
28 | else:
29 | status = redis.hmget(cid, 'notify')[0]
30 | if status != None and status == 'true':
31 | print status
32 | continue
33 | else:
34 | redis.hset(cid, 'notify', 'true')
35 |
36 | c_id_real = cid.split(':')[1]
37 | sid = redis.hmget(cid, 'sid')[0]
38 |
39 | article = toutiaoUtils.get_toutiao_news()
40 | api_secret = redis.get('api_secret' + ':' + sid)
41 | link = article[toutiaoUtils.LINK]
42 | title = article[toutiaoUtils.TITLE]
43 | summary = article[toutiaoUtils.SUMMARY]
44 | pushData = model.message.ArticleMessage(link, title, summary)
45 | cache_link = redis.hmget(cid, 'cache_link')
46 | print cache_link[0]
47 | print link
48 | print cid
49 | if (cache_link[0] == link):
50 | continue
51 | else:
52 | redis.hmset(cid, {'cache_link': link})
53 | try:
54 | Q = utils.Queue.Q
55 | print pushData.get_json()
56 | Q.enqueue('utils.push_task.push_article', pushData.get_json(), sid, c_id_real, api_secret)
57 | except Exception, e:
58 | print e
59 | except Exception, e:
60 | # logging.error(e)
61 | print traceback.format_exc()
62 |
63 |
64 | def toutiao_gallery_task():
65 | redis = utils.redisFactory.getRedis()
66 | set = redis.keys(mark_gallery + '*')
67 | print set
68 | for cid in set:
69 |
70 | if not utils.timeHelper.time_area(8, 30, 0, 10, 50, 0) and not utils.timeHelper.time_area(18, 30, 0, 18, 50, 0):
71 | redis.hset(cid, 'notify', 'false')
72 | continue
73 | else:
74 | status = redis.hmget(cid, 'notify')[0]
75 | if status != None and status == 'true':
76 | continue
77 | else:
78 | redis.hset(cid, 'notify', 'true')
79 |
80 | c_id_real = cid.split(':')[1]
81 | sid = redis.hmget(cid, 'sid')[0]
82 |
83 | article = toutiaoUtils.get_toutiao_gallery()
84 | api_secret = redis.get('api_secret' + ':' + sid)
85 | link = article[toutiaoUtils.LINK]
86 | title = article[toutiaoUtils.TITLE]
87 | pushData = model.message.ArticleMessage(link, title)
88 |
89 | cache_link = redis.hmget(cid, 'cache_link')
90 |
91 | if (cache_link[0] == link):
92 | continue
93 | else:
94 | redis.hmset(cid, {'cache_link': link})
95 | try:
96 | Q = utils.Queue.Q
97 | print pushData.get_json()
98 | Q.enqueue('utils.push_task.push_article', pushData.get_json(), sid, c_id_real, api_secret)
99 | except Exception, e:
100 | print e
101 |
102 |
103 |
--------------------------------------------------------------------------------
/service/serverService.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | # coding=utf-8
3 | __author__ = 'chenyueling'
4 |
5 | import json
6 | import utils.redisFactory
7 |
8 |
9 |
10 |
11 | # {
12 | # "s_data": "{\"date\":\"1\",\"status\":\"下雨\"}",
13 | # "action": "ACTION_SERVICE_CREATE",
14 | # "title": "明天下雨提醒",
15 | # "api_secret": "09ltfmd29f0our68q5bjij2x",
16 | # "sid": "5857",
17 | # "c_data": [
18 | # {
19 | # "data": "{\"location\":\"珠海\"}",
20 | # "cid": "6f782524-22d4-48fc-b0f6-ab79d49bf178"
21 | # },
22 | # {
23 | # "data": "{\"location\":\"深圳\"}",
24 | # "cid": "c388eb26-c9f2-41ae-93c5-fdcc0ff8d26a"
25 | # }
26 | # ]
27 | #}
28 | #
29 | #Service创建事件
30 | def action_service_create(service_name, jsonObj):
31 | r = utils.redisFactory.getRedis()
32 |
33 | title = jsonObj['title']
34 |
35 | api_secret = jsonObj['api_secret']
36 |
37 | sid = jsonObj['sid']
38 |
39 | r.set('title' + ':' + sid, title)
40 |
41 | r.set('api_secret' + ':' + sid, api_secret)
42 |
43 | #
44 | s_dict = {}
45 |
46 | if jsonObj.has_key('s_data'):
47 | s_dataStr = jsonObj['s_data']
48 | s_data_dict = json.loads(s_dataStr)
49 | s_data_dict.update({'sid': sid})
50 | s_dict.update(s_data_dict)
51 | s_store_dict = r.hgetall('s_data' + ':' + sid)
52 | #合并出最新
53 | s_dict.update(s_store_dict)
54 | #只要是服务端传来的s_data不为空都要存储,如果服务
55 | if s_data_dict.__len__() != 0:
56 | r.hmset('s_data' + ':' + sid, s_dict)
57 |
58 | if jsonObj.has_key('c_data'):
59 | c_data_list_dict = jsonObj['c_data']
60 | #c_data_list_dict = json.loads(c_data)
61 | for item in c_data_list_dict:
62 | #s_data = jsonObj['s_data']
63 | data = item['data']
64 | item.pop('data')
65 | data_dict = json.loads(data)
66 | item.update(s_dict)
67 | item.update(data_dict)
68 | item.update({'sid': sid})
69 | r.hmset(service_name + ':' + item['cid'], item)
70 |
71 | result = Result('Success', '3000');
72 |
73 | return result
74 |
75 |
76 | #用户互动事件
77 | def action_service_ding(service_name, jsonObj):
78 | result = Result('Success', '3000');
79 | return result
80 |
81 |
82 | #ClientService关注数变更事件
83 | def action_service_followed_change(service_name, jsonObj):
84 | print service_name
85 | result = Result('Success', '3000');
86 | return result
87 |
88 |
89 | # {
90 | # "data": "{\"date\":\"1\",\"status\":\"下雨\"}",
91 | # "action": "ACTION_CLIENT_SERVICE_CREATE",
92 | # "sid": "0556",
93 | # "cid": "bc6e20b4-a0f0-494f-8564-dc2490432e4e"
94 | # }
95 | #ClientService创建事件
96 | def action_client_service_create(service_name, jsonObj):
97 | print jsonObj
98 | print service_name
99 | r = utils.redisFactory.getRedis()
100 | sid = jsonObj['sid']
101 | c_id_real = jsonObj['cid']
102 | s_data_dict = r.hgetall('s_data' + ':' + sid)
103 | data = jsonObj['data']
104 | data_dict = json.loads(data)
105 | data_dict.update({'cid':c_id_real})
106 | data_dict.update({'sid':sid})
107 | s_data_dict.update(data_dict)
108 | if s_data_dict.__len__() == 0:
109 | pass
110 | else:
111 | r.hmset(service_name + ':' + c_id_real, s_data_dict)
112 |
113 | result = Result('Success', '3000');
114 | return result
115 |
116 |
117 | #Service更新事件
118 | def action_service_update(service_name, jsonObj):
119 | r = utils.redisFactory.getRedis()
120 |
121 | title = jsonObj['title']
122 |
123 | api_secret = jsonObj['api_secret']
124 |
125 | sid = jsonObj['sid']
126 |
127 | r.set('title' + ':' + sid, title)
128 |
129 | r.set('api_secret' + ':' + sid, api_secret)
130 |
131 | #
132 | s_dict = {}
133 |
134 | if jsonObj.has_key('s_data'):
135 | s_dataStr = jsonObj['s_data']
136 | s_data_dict = json.loads(s_dataStr)
137 | s_data_dict.update({'sid': sid})
138 | s_dict.update(s_data_dict)
139 | s_store_dict = r.hgetall('s_data' + ':' + sid)
140 | #合并出最新
141 | s_dict.update(s_store_dict)
142 | #只要是服务端传来的s_data不为空都要存储,如果服务
143 | if s_data_dict.__len__() != 0:
144 | r.hmset('s_data' + ':' + sid, s_dict)
145 |
146 | if jsonObj.has_key('c_data'):
147 | c_data_list_dict = jsonObj['c_data']
148 | #c_data_list_dict = json.loads(c_data)
149 | for item in c_data_list_dict:
150 | s_data = jsonObj['s_data']
151 | item.update(s_dict)
152 | r.hmset(service_name + ':' + item['cid'], item)
153 |
154 | result = Result('Success', '3000');
155 | return result
156 |
157 |
158 | class Result(object):
159 | def __init__(self, message, code):
160 | self.message = message
161 | self.code = code
162 |
163 | def get_json(self):
164 | return json.dumps(self.__dict__)
165 |
--------------------------------------------------------------------------------
/plugin/lofter/net.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | # coding=utf-8
3 | from BeautifulSoup import BeautifulSoup
4 | import urlparse
5 | import urllib
6 | import urllib2
7 | import cookielib
8 | import string
9 | import re
10 | import logging
11 | import traceback
12 | import re
13 | import redis
14 | # 登录的主页面
15 | hosturl = 'http://www.lofter.com' #//自己填写
16 | #post数据接收和处理的页面（我们要向这个页面发送我们构造的Post数据）
17 | posturl = 'https://reg.163.com/logins.jsp' #//从数据包中分析出，处理post请求的url
18 |
19 | #设置一个cookie处理器，它负责从服务器下载cookie到本地，并且在发送请求时带上本地的cookie
20 | cj = cookielib.LWPCookieJar()
21 | cookie_support = urllib2.HTTPCookieProcessor(cj)
22 | opener = urllib2.build_opener(cookie_support, urllib2.HTTPHandler)
23 | urllib2.install_opener(opener)
24 |
25 | #打开登录主页面（他的目的是从页面下载cookie，这样我们在再送post数据时就有cookie了，否则发送不成功）
26 | h = urllib2.urlopen(hosturl)
27 |
28 | #构造header，一般header至少要包含一下两项。这两项是从抓到的包里分析得出的。
29 | headers = {
30 | #'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
31 | #'Accept-Encoding':'gzip,deflate',
32 | #'Accept-Language':'zh-CN,zh;q=0.8,en;q=0.6',
33 | #'Cache-Control':'max-age=0',
34 | #'Connection':'keep-alive',
35 | #'Content-Length':'158',
36 | #'Content-Type':'application/x-www-form-urlencoded',
37 | #'Cookie':'_ntes_nuid=6df1d205baf5a8c60240be009e6f7c53; SID=8d664ab7-5d72-485b-8867-f1eae40553bc; JSESSIONID=adeNFouIl2x2_95sVRKLu; vjuids=1a6cae3bb.14964af2570.0.511d79e; vjlast=1414733440.1414733440.30; _ntes_nnid=4bdf630e8dd1d2ee865be1b985835395,1414733440382; ne_analysis_trace_id=1414733442305; pver_n_f_l_n3=a; PopPushData=1414726245; vinfo_n_f_l_n3=b072541d42cbf206.1.0.1414733442314.0.1414733450494; s_n_f_l_n3=b072541d42cbf2061414733442317; T_INFO=990A52703BFDA09B5C96530F747A8FD5; P_INFO=chen_yueling@163.com|1414733979|2|lofter|00&99|US&1414733949&lofter#US&null#10#0#0|137274&0|mail163&study&blog&lofter¬e|chen_yueling@163.com',
38 | #'Host':'reg.163.com',
39 | 'Origin': 'http://www.lofter.com',
40 | 'Referer': 'http://www.lofter.com/',
41 | 'User-Agent': 'Mozilla/5.0 (X11; Linux i686 (x86_64)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36'
42 | }
43 | #构造Post数据，他也是从抓大的包里分析得出的。
44 | postData = {
45 | 'password': 'rio332137628',
46 | 'type': '1',
47 | 'url': 'http://www.lofter.com/logingate.do',
48 | 'product': 'lofter',
49 | 'savelogin': '1',
50 | #'username':'chenyueling163@gmail.com',
51 | 'username': 'chen_yueling@163.com',
52 | 'domains': 'www.lofter.com'
53 | }
54 |
55 | postDataDwr = {
56 | 'callCount': '1',
57 | 'scriptSessionId': '${scriptSessionId}187',
58 | 'httpSessionId':'',
59 | 'c0-scriptName':'TagBean',
60 | 'c0-methodName': 'search',
61 | 'c0-id': '0',
62 | 'c0-param0': 'string:%E5%BE%AE%E8%B7%9D',
63 | 'c0-param1': 'number:0',
64 | 'c0-param2': 'string:',
65 | 'c0-param3': 'string:excellent',
66 | 'c0-param4': 'boolean:false',
67 | 'c0-param5': 'number:0',
68 | 'c0-param6': 'number:20',
69 | 'c0-param7': 'number:0',
70 | 'c0-param8': 'number:1414935540006',
71 | 'batchId': '370619',
72 | }
73 |
74 |
75 | #需要给Post数据编码
76 | postData = urllib.urlencode(postData)
77 |
78 | postDataDwr = urllib.urlencode(postDataDwr)
79 |
80 |
81 | #通过urllib2提供的request方法来向指定Url发送我们构造的数据，并完成登录过程
82 | request = urllib2.Request(posturl, postData, headers)
83 | print request
84 | try:
85 | response = urllib2.urlopen(request)
86 | print response.geturl()
87 | print response.getcode()
88 | text = response.read()
89 | #print text
90 | #response = urllib2.urlopen('http://www.lofter.com')
91 | #text = response.read()
92 | #f = file('test.html', 'w')
93 | #f.write(text)
94 | soup = BeautifulSoup(text)
95 |
96 | script = soup.find("a")
97 |
98 | #print script['href']
99 |
100 | response = urllib2.urlopen(script['href'])
101 |
102 | html = response.read()
103 |
104 | #print(html)
105 | f = file('html.html', 'w')
106 | f.write(html)
107 | f.close()
108 |
109 | #urllib2.urlopen('http://www.lofter.com/tag/%E6%A4%8D%E7%89%A9')
110 |
111 | h = {
112 | 'Accept': '* / *',
113 | 'Accept - Encoding':'gzip, deflate',
114 | 'Accept - Language':'zh - CN, zh;q = 0.8, en;q = 0.6',
115 | 'Cache - Control':'max - age = 0',
116 | 'Connection':'keep - alive',
117 | 'Content - Length':'178',
118 | 'Content - Type':'text / plain',
119 | 'Host':'www.lofter.com',
120 | 'Origin':'http://www.lofter.com',
121 | 'Referer':'http://www.lofter.com/tag/%E6%A4%8D%E7%89%A9',
122 | 'User-Agent': 'Mozilla/5.0 (X11; Linux i686 (x86_64)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36'
123 | }
124 |
125 | request = urllib2.Request('http://www.lofter.com/dwr/call/plaincall/TagBean.search.dwr', postDataDwr, h)
126 |
127 |
128 |
129 | #print request.get_header('Cookie')
130 |
131 | response = urllib2.urlopen(request)
132 |
133 | dwr = response.read()
134 | print dwr
135 | print type(dwr)
136 | prog = re.compile(r'http://.*?.lofter.com/post/.*?_.*?"') #"http://yuchunxie.lofter.com/post/c881f_2da737d"
137 | list = prog.findall(dwr)
138 | print list
139 | print len(list)
140 | title_re = re.compile(r'noticeLinkTitle=.*?;')
141 |
142 |
143 |
144 | list_title = title_re.findall(dwr)
145 |
146 | print list
147 | print len(list)
148 |
149 | i = 0
150 | len = len(list_title)
151 | for item in list:
152 | print item.replace('"','')
153 |
154 | if(i < len):
155 | print list_title[i].replace('noticeLinkTitle="','').replace('"','').decode('unicode_escape')
156 | i = i + 1
157 |
158 |
159 | r = redis.Redis(host='localhost', port=6379, db=0)
160 |
161 | #print text
162 | except Exception, e:
163 | print traceback.format_exc()
164 |
165 | #text = response.read()
166 |
--------------------------------------------------------------------------------
/plugin/lofter/lofterUtils.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | # coding=utf-8
3 | __author__ = 'chenyueling'
4 |
5 | from BeautifulSoup import BeautifulSoup
6 | import urlparse
7 | import urllib
8 | import urllib2
9 | import cookielib
10 | import string
11 | import re
12 | import logging
13 | import traceback
14 | import re
15 | import redis
16 |
17 | TITLE = 'title'
18 | LINK = 'link'
19 |
20 | hosturl = 'http://www.lofter.com' # //自己填写
21 | # post数据接收和处理的页面（我们要向这个页面发送我们构造的Post数据）
22 | posturl = 'https://reg.163.com/logins.jsp' # //从数据包中分析出，处理post请求的url
23 |
24 | # 设置一个cookie处理器，它负责从服务器下载cookie到本地，并且在发送请求时带上本地的cookie
25 | cj = cookielib.LWPCookieJar()
26 | cookie_support = urllib2.HTTPCookieProcessor(cj)
27 | opener = urllib2.build_opener(cookie_support, urllib2.HTTPHandler)
28 | urllib2.install_opener(opener)
29 |
30 | # 打开登录主页面（他的目的是从页面下载cookie，这样我们在再送post数据时就有cookie了，否则发送不成功）
31 | h = urllib2.urlopen(hosturl)
32 |
33 | # 构造header，一般header至少要包含一下两项。这两项是从抓到的包里分析得出的。
34 | headers = {
35 | #'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
36 | #'Accept-Encoding':'gzip,deflate',
37 | #'Accept-Language':'zh-CN,zh;q=0.8,en;q=0.6',
38 | #'Cache-Control':'max-age=0',
39 | #'Connection':'keep-alive',
40 | #'Content-Length':'158',
41 | #'Content-Type':'application/x-www-form-urlencoded',
42 | #'Cookie':'_ntes_nuid=6df1d205baf5a8c60240be009e6f7c53; SID=8d664ab7-5d72-485b-8867-f1eae40553bc; JSESSIONID=adeNFouIl2x2_95sVRKLu; vjuids=1a6cae3bb.14964af2570.0.511d79e; vjlast=1414733440.1414733440.30; _ntes_nnid=4bdf630e8dd1d2ee865be1b985835395,1414733440382; ne_analysis_trace_id=1414733442305; pver_n_f_l_n3=a; PopPushData=1414726245; vinfo_n_f_l_n3=b072541d42cbf206.1.0.1414733442314.0.1414733450494; s_n_f_l_n3=b072541d42cbf2061414733442317; T_INFO=990A52703BFDA09B5C96530F747A8FD5; P_INFO=chen_yueling@163.com|1414733979|2|lofter|00&99|US&1414733949&lofter#US&null#10#0#0|137274&0|mail163&study&blog&lofter¬e|chen_yueling@163.com',
43 | #'Host':'reg.163.com',
44 | 'Origin': 'http://www.lofter.com',
45 | 'Referer': 'http://www.lofter.com/',
46 | 'User-Agent': 'Mozilla/5.0 (X11; Linux i686 (x86_64)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36'
47 | }
48 | #构造Post数据，他也是从抓大的包里分析得出的。
49 | postData = {
50 | 'password': '***********',
51 | 'type': '1',
52 | 'url': 'http://www.lofter.com/logingate.do',
53 | 'product': 'lofter',
54 | 'savelogin': '1',
55 | #'username':'chenyueling163@gmail.com',
56 | 'username': 'chen_yueling@163.com',
57 | 'domains': 'www.lofter.com'
58 | }
59 |
60 |
61 |
62 |
63 | #需要给Post数据编码
64 | postData = urllib.urlencode(postData)
65 |
66 |
67 | def login():
68 | #通过urllib2提供的request方法来向指定Url发送我们构造的数据，并完成登录过程
69 | request = urllib2.Request(posturl, postData, headers)
70 | try:
71 | response = urllib2.urlopen(request)
72 | print response.geturl()
73 | print response.getcode()
74 | text = response.read()
75 | #print text
76 | #response = urllib2.urlopen('http://www.lofter.com')
77 | #text = response.read()
78 | #f = file('test.html', 'w')
79 | #f.write(text)
80 | soup = BeautifulSoup(text)
81 |
82 | script = soup.find("a")
83 |
84 | #print script['href']
85 |
86 | response = urllib2.urlopen(script['href'])
87 |
88 |
89 | #html = response.read()
90 |
91 | if response.getcode() == 200:
92 | print 'Login Success'
93 | else:
94 | print 'Login Error' + response.getcode()
95 |
96 |
97 |
98 |
99 | #print(html)
100 | #f = file('html.html', 'w')
101 | #f.write(html)
102 | #f.close()
103 | except Exception, e:
104 | print traceback.format_exc()
105 |
106 |
107 | def get_article_byTag(tag):
108 | #login()
109 | tag = urllib.urlencode({'': tag}).replace('=', '')
110 | h = {
111 | 'Accept': '* / *',
112 | 'Accept - Encoding': 'gzip, deflate',
113 | 'Accept - Language': 'zh - CN, zh;q = 0.8, en;q = 0.6',
114 | 'Cache - Control': 'max - age = 0',
115 | 'Connection': 'keep - alive',
116 | 'Content - Type': 'text / plain',
117 | 'Host': 'www.lofter.com',
118 | 'Origin': 'http://www.lofter.com',
119 | 'Referer': 'http://www.lofter.com/tag/%s' % (tag),
120 | 'User-Agent': 'Mozilla/5.0 (X11; Linux i686 (x86_64)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36'
121 | }
122 |
123 | postDataDwr = {
124 | 'callCount': '1',
125 | 'scriptSessionId': '${scriptSessionId}187',
126 | 'httpSessionId': '',
127 | 'c0-scriptName': 'TagBean',
128 | 'c0-methodName': 'search',
129 | 'c0-id': '0',
130 | 'c0-param0': 'string:%s' % (tag),
131 | 'c0-param1': 'number:0',
132 | 'c0-param2': 'string:',
133 | 'c0-param3': 'string:new',
134 | 'c0-param4': 'boolean:false',
135 | 'c0-param5': 'number:0',
136 | 'c0-param6': 'number:20',
137 | 'c0-param7': 'number:0',
138 | 'c0-param8': 'number:0',
139 | 'batchId': '370619',
140 | }
141 | postDataDwr = urllib.urlencode(postDataDwr)
142 |
143 | request = urllib2.Request('http://www.lofter.com/dwr/call/plaincall/TagBean.search.dwr', postDataDwr, h)
144 |
145 |
146 |
147 | #print request.get_header('Cookie')
148 |
149 | response = urllib2.urlopen(request)
150 |
151 | dwr = response.read()
152 | #dwr data
153 | print dwr
154 | #print type(dwr)
155 | #reg match
156 | prog = re.compile(r'http://.*?.lofter.com/post/.*?_.*?"') #"http://yuchunxie.lofter.com/post/c881f_2da737d"
157 | list = prog.findall(dwr)
158 | #print list
159 |
160 | title_re = re.compile(r'noticeLinkTitle=.*?;')
161 |
162 | list_title = title_re.findall(dwr)
163 |
164 | #print list
165 |
166 | i = 0
167 | article_list = []
168 |
169 | for item in list:
170 | article = {}
171 | #print item.replace('"', '')
172 | article = {LINK: item.replace('"', '')}
173 | if (i < list_title.__len__()):
174 | #print list_title[i].replace('noticeLinkTitle="', '').replace('"', '').decode('unicode_escape')
175 | article.update({
176 | TITLE: list_title[i].replace('noticeLinkTitle="', '').replace('"', '').replace(';', '').decode(
177 | 'unicode_escape')})
178 | print article
179 | article_list.append(article)
180 | #print
181 | i = i + 1
182 | return article_list
183 |
184 | #test
185 | print urllib.urlencode({'': '原画'}).replace('=', '')
186 | get_article_byTag(urllib.urlencode({'': '原画'}).replace('=', ''))
187 |
--------------------------------------------------------------------------------
/html.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | LOFTER
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
19 |
20 |
21 |
22 |
23 |
24 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
LOFTER for ipad —— 记录生活，发现同好~~
35 |
点击下载
36 |
关闭
37 |
38 |
39 |
42 |
43 |
44 |
45 |
46 | - 首页5
47 | - 浏览
48 | - ART
49 | - instagram备份NEW
50 | -
51 | 更多
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
69 |
70 |
71 |
405 |
406 |
407 |
408 |
410 |
411 |
412 |
413 |
414 | 
- 文字
416 | - 图片
417 | - 音乐
418 | - 视频
419 |
420 |
421 |
422 |
423 |
424 |
425 |
426 |
427 |
428 |
459 |
460 |
461 |
462 |
473 |
474 |
475 |
476 |
477 |
478 |
479 |
480 |
481 |
482 |
490 |
491 |
492 |
493 |
494 |
495 |
496 |
497 |
613 |
614 |
646 |
647 |
657 |
658 |
676 |
677 |
686 |
718 |
732 |
734 |
735 |
744 |
745 |
756 |
757 |
768 |
769 |
811 |
812 |
829 |
907 |
908 |
930 |
931 |
952 |
953 |
991 |
993 |
1021 |
1035 |
1046 |
1065 |
1077 |
1080 |
1082 |
1113 |
1121 |
1122 |
1123 |
1124 |
1125 |
1198 |
1199 |
1200 |
1201 |
--------------------------------------------------------------------------------
95 | 
109 | 
116 | 
123 | 
130 | 
137 | 
179 | 
186 | 
193 | 
200 | 
207 | 
214 | 
221 | 
228 | 
235 | 
242 | 
249 | 
256 | 
263 | 
277 | 
284 | 
291 | 
298 | 
305 | 
312 | 
319 | 
326 | 
333 | 
340 | 
347 | 
354 | 
361 | 
368 | 
375 | 
389 | 
396 |