(.{0,10})\s", re.S)
56 | items = re.findall(pattern, page)
57 | # print page.decode("UTF-8")
58 | fans = int(items[0].replace(',',''))
59 | print fans
60 | return fans
61 |
62 |
63 | def get_by_selenium():
64 | username = 'Insta360VRVideoCamera'
65 | url = 'https://www.facebook.com/plugins/fan.php?id=' + username
66 | cap = webdriver.DesiredCapabilities.PHANTOMJS
67 | cap["phantomjs.page.settings.resourceTimeout"] = 1000
68 | cap["phantomjs.page.settings.loadImages"] = False
69 | cap["phantomjs.page.settings.localToRemoteUrlAccessEnabled"] = True
70 | cap["userAgent"] = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0"
71 | cap["XSSAuditingEnabled"] = True
72 | driver = webdriver.PhantomJS(desired_capabilities=cap,
73 | service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any',
74 | '--web-security=true'])
75 | # driver = webdriver.Chrome()
76 | driver.get(url)
77 | wait = WebDriverWait(driver, 20)
78 | # print driver.page_source
79 | try:
80 | string = wait.until(lambda x: x.find_elements_by_class_name('_1drq')[0].text)
81 | except TimeoutException:
82 | string = 0
83 | pattern = re.compile("\d", re.S)
84 | items = re.findall(pattern, string)
85 | temp = ''
86 | for item in items:
87 | temp += item
88 | fans = int(temp)
89 | print fans
90 | driver.quit()
91 | return fans
92 |
93 | if __name__ == "__main__":
94 | # get_by_request()
95 | # get_by_selenium()
96 | get_by_api()
97 |
--------------------------------------------------------------------------------
/fans_crawler/instagram_crawler.py:
--------------------------------------------------------------------------------
1 | # -*- coding: UTF-8 -*-
2 | import urllib2
3 | import json
4 | import urllib
5 |
6 |
7 | def get_by_api():
8 | user_id = ''
9 | access_token = ''
10 | url = 'https://api.instagram.com/v1/users/' + user_id + '/?access_token=' + access_token
11 | # oauth = OAuth()
12 | request = urllib2.Request(url = url)
13 | response = urllib2.urlopen(request)
14 | page = response.read()
15 | print page
16 | data = json.loads(page, encoding="utf-8")
17 | fans = data['data']['counts']['followed_by']
18 | print fans
19 | return fans
20 |
21 |
22 | def OAuth():
23 | url = 'https://www.instagram.com/oauth/authorize/?client_id=a84f3a3ec8c44dfbbe9d2e3f07dc9c97&redirect_uri=http://www.baidu.com&response_type=token'
24 | request = urllib2.Request(url = url)
25 | response = urllib2.urlopen(request)
26 | redirect_url = response.geturl()
27 | request = urllib2.Request(url=redirect_url)
28 | response = urllib2.urlopen(request)
29 | redirect_url = response.geturl()
30 | print redirect_url
31 | # data = json.loads(page, encoding="utf-8")
32 |
33 |
34 | def get_by_request():
35 | username = 'insta360official'
36 | url = 'https://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20html%20where%20url%3D%22https%3A%2F%2Fwww.instagram.com%2F' + username + '%2F%22%20and%20xpath%3D%22%2Fhtml%2Fbody%2Fscript%5B1%5D%22&format=json'
37 | headers = {}
38 | headers['Host'] = 'query.yahooapis.com'
39 | headers['Connection'] = 'keep-alive'
40 | headers['Origin'] = 'https://livecounts.net'
41 | headers['Pragma'] = 'no-cache'
42 | headers['Referer'] = 'https://livecounts.net/instagram/cielni'
43 | headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36'
44 | request = urllib2.Request(url=url, headers=headers)
45 | response = urllib2.urlopen(request)
46 | page = response.read()
47 | print page
48 | jsonData = json.loads(page, encoding="utf-8")
49 | content = jsonData['query']['results']['script']['content']
50 | print content
51 | content = content[21:-1]
52 | print content
53 | content = json.loads(content, encoding="utf-8")
54 | fans = content['entry_data']['ProfilePage'][0]['user']['followed_by']['count']
55 | print fans
56 | if __name__ == "__main__":
57 | # OAuth()
58 | # get_by_request()
59 | get_by_api()
60 |
--------------------------------------------------------------------------------
/fans_crawler/main.py:
--------------------------------------------------------------------------------
1 | import datetime
2 | import json
3 |
4 | from fb_crawler import get_by_api as get_fb_fans
5 | from weibo_crawler import get_by_request as get_sina_fans
6 | from twitter_crawler import get_by_request as get_twitter_fans
7 | from youtube_crawler import get_by_api as get_youtube_fans
8 | from youku_crawler import get_by_api as get_youku_fans
9 | from weixin_crawler import get_by_api as get_weixin_fans
10 | from instagram_crawler import get_by_request as get_instagram_fans
11 |
12 | def main():
13 | platform = ['facebook', 'weibo', 'twitter', 'youtube', 'youku', 'weixin', 'instagram']
14 | result = []
15 | for i in platform:
16 | fans = 0
17 |
18 | if i == 'facebook':
19 | fans = get_fb_fans()
20 | elif i == 'weibo':
21 | fans = get_sina_fans()
22 | elif i == 'twitter':
23 | fans = get_twitter_fans()
24 | elif i == 'youtube':
25 | fans = get_youtube_fans()
26 | elif i == 'youku':
27 | fans = get_youku_fans()
28 | elif i == 'weixin':
29 | fans = get_weixin_fans()
30 | elif i == 'instagram':
31 | fans = get_instagram_fans()
32 |
33 | today = datetime.datetime.now().strftime('%Y-%m-%d')
34 | temp = {'platform': i, 'fans': fans, 'date': today}
35 | result.append(temp)
36 | jsonResult = json.dumps(result)
37 | print jsonResult
38 | return jsonResult
39 |
40 |
41 | if __name__ == "__main__":
42 | main()
43 |
--------------------------------------------------------------------------------
/fans_crawler/twitter_crawler.py:
--------------------------------------------------------------------------------
1 | # -*- coding: UTF-8 -*-
2 | import urllib2
3 | import json
4 | import ssl
5 | import urllib
6 | from functools import wraps
7 |
8 |
9 | def get_by_request():
10 | username = 'insta360'
11 | url = 'https://cdn.syndication.twimg.com/widgets/followbutton/info.json?screen_names=' + username
12 | # headers = {}
13 | # headers['Host'] = 'www.facebook.com'
14 | # headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0'
15 | # headers['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
16 | # headers['Connection'] = 'keep-alive'
17 | # headers['Upgrade-Insecure-Requests'] = '1'
18 | request = urllib2.Request(url = url)
19 | response = urllib2.urlopen(request)
20 | page = response.read()
21 | result = json.loads(page, encoding="utf-8")
22 | fans = result[0]['followers_count']
23 | print fans
24 | return fans
25 |
26 |
27 | def get_by_api():
28 | username = 'insta360'
29 | url = 'https://api.twitter.com/1.1/users/show.json?include_entities=fasle&screen_name=' + username
30 | oauth = OAuth()
31 | headers = {}
32 | headers['Host'] = 'api.twitter.com'
33 | headers['X-Target-URI'] = 'https://api.twitter.com'
34 | headers['Content-Type'] = 'application/x-www-form-urlencoded'
35 | headers['Connection'] = 'keep-alive'
36 | headers['Authorization'] = oauth
37 | request = urllib2.Request(url = url, headers = headers)
38 | response = urllib2.urlopen(request)
39 | page = response.read()
40 | print page
41 | data = json.loads(page, encoding="utf-8")
42 | fans = data['followers_count']
43 | print fans
44 | return fans
45 |
46 |
47 | def OAuth():
48 | ssl.wrap_socket = sslwrap(ssl.wrap_socket)
49 | url = 'https://api.twitter.com/oauth2/token'
50 | value = {}
51 | value['grant_type'] = 'client_credentials'
52 | value['client_id'] = ''
53 | value['client_secret'] = ''
54 | data = urllib.urlencode(value)
55 | request = urllib2.Request(url = url, data = data)
56 | response = urllib2.urlopen(request)
57 | page = response.read()
58 | data = json.loads(page, encoding="utf-8")
59 | result = data['token_type'] + ' ' + data['access_token']
60 | return result
61 |
62 |
63 | def sslwrap(func):
64 | @wraps(func)
65 | def bar(*args, **kw):
66 | kw['ssl_version'] = ssl._PROTOCOL_NAMES
67 | return func(*args, **kw)
68 | return bar
69 |
70 |
71 | if __name__ == "__main__":
72 | get_by_request()
73 | # get_by_api()
74 |
--------------------------------------------------------------------------------
/fans_crawler/weibo_crawler.py:
--------------------------------------------------------------------------------
1 | # -*- coding: UTF-8 -*-
2 | import re
3 | import urllib2
4 | from selenium import webdriver
5 | from selenium.common.exceptions import TimeoutException
6 | from selenium.webdriver.support.ui import WebDriverWait
7 |
8 | def get_by_request():
9 | username = 'insta360'
10 | url = 'http://weibo.cn/'+ username
11 | headers = {}
12 | headers['Host'] = 'weibo.cn'
13 | headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0'
14 | # headers['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
15 | headers['Cookie'] = '_T_WM=d2e28a98d3031cf98e282a29740b5f24'
16 | # headers['Connection'] = 'keep-alive'
17 | # headers['Upgrade-Insecure-Requests'] = '1'
18 | request = urllib2.Request(url = url, headers=headers)
19 | response = urllib2.urlopen(request)
20 | page = response.read()
21 | pattern = re.compile("\[(.{0,10})\]
<\\\\\/h2>", re.S)
22 | items = re.findall(pattern, page)
23 | # print page.decode("UTF-8")
24 | fans = int(items[1])
25 | print fans
26 | return fans
27 |
28 |
29 | def get_by_selenium():
30 | username = 'insta360'
31 | url = 'http://weibo.com/'+ username + '?is_all=1'
32 | # cap = webdriver.DesiredCapabilities.PHANTOMJS
33 | # cap["phantomjs.page.settings.resourceTimeout"] = 1000
34 | # cap["phantomjs.page.settings.loadImages"] = False
35 | # cap["phantomjs.page.settings.localToRemoteUrlAccessEnabled"] = True
36 | # cap["userAgent"] = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0"
37 | # cap["XSSAuditingEnabled"] = True
38 | # cap["host"] = 'weibo.com'
39 | # cap["cookie"] = 'UOR=www.umeng.com,widget.weibo.com,www.insta360.com; SINAGLOBAL=6982249232630.452.1472299450582; ULV=1475028466086:3:2:2:8231266012653.427.1475028466020:1474966940284; SUB=_2AkMgtrrUf8NhqwJRmP0czWrmZY53wgjEieLBAH7sJRMxHRl-yT83qm8AtRCo0NEVwCee4iQkVabYZqZ8gEhMng..; SUBP=0033WrSXqPxfM72-Ws9jqgMF55529P9D9WWT6ckK7WZ-8GkEahm6SKw1; TC-Page-G0=0cd4658437f38175b9211f1336161d7d; _s_tentry=-; Apache=8231266012653.427.1475028466020'
40 | # driver = webdriver.PhantomJS(desired_capabilities=cap,
41 | # service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any',
42 | # '--web-security=true'])
43 | driver = webdriver.Chrome()
44 | driver.get(url)
45 | wait = WebDriverWait(driver, 20)
46 | # print driver.page_source
47 | try:
48 | result = int(
49 | wait.until(lambda x: x.find_element_by_xpath('//*[@id="Pl_Core_T8CustomTriColumn__3"]/div/div/div/table/tbody/tr/td[2]/strong').text))
50 | except TimeoutException:
51 | result = 0
52 | print result
53 | driver.quit()
54 | return result
55 |
56 | if __name__ == "__main__":
57 | get_by_request()
58 | # get_by_selenium()
--------------------------------------------------------------------------------
/fans_crawler/weixin_crawler.py:
--------------------------------------------------------------------------------
1 | # -*- coding: UTF-8 -*-
2 | import urllib2
3 | import json
4 | import ssl
5 | import urllib
6 |
7 |
8 | def get_by_api():
9 | url = 'https://api.weixin.qq.com/cgi-bin/user/get'
10 | token = get_token()
11 | value = {}
12 | value['access_token'] = token
13 | value['next_openid'] = ''
14 | data = urllib.urlencode(value)
15 | request = urllib2.Request(url = url, data = data)
16 | response = urllib2.urlopen(request)
17 | page = response.read()
18 | print page
19 | data = json.loads(page, encoding="utf-8")
20 | fans = 0
21 | try:
22 | fans = data['total']
23 | except KeyError:
24 | pass
25 | print fans
26 | return fans
27 |
28 |
29 | def get_token():
30 | url = 'https://api.weixin.qq.com/cgi-bin/token'
31 | value = {}
32 | value['grant_type'] = 'client_credential'
33 | value['appid'] = ''
34 | value['secret'] = ''
35 | data = urllib.urlencode(value)
36 | request = urllib2.Request(url = url, data = data)
37 | response = urllib2.urlopen(request)
38 | page = response.read()
39 | print page
40 | data = json.loads(page, encoding="utf-8")
41 | result = ''
42 | try:
43 | result = data['access_token']
44 | except KeyError:
45 | pass
46 | return result
47 |
48 | if __name__ == "__main__":
49 | get_by_api()
50 |
--------------------------------------------------------------------------------
/fans_crawler/youku_crawler.py:
--------------------------------------------------------------------------------
1 | # -*- coding: UTF-8 -*-
2 | import urllib2
3 | import re
4 | import json
5 | from selenium import webdriver
6 | from selenium.common.exceptions import TimeoutException
7 | from selenium.webdriver.support.ui import WebDriverWait
8 |
9 | def get_by_request():
10 | url = 'http://i.youku.com/i/UMjk1ODg3NDgwOA=='
11 | headers = {}
12 | headers['Host'] = 'i.youku.com'
13 | headers['Referer'] = 'http://www.insta360.com/'
14 | headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0'
15 | # headers['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
16 | # headers['Connection'] = 'keep-alive'
17 | headers['Upgrade-Insecure-Requests'] = '1'
18 | request = urllib2.Request(url = url, headers=headers)
19 | response = urllib2.urlopen(request)
20 | page = response.read()
21 | # print page
22 | pattern = re.compile("", re.S)
23 | items = re.findall(pattern, page)
24 | fans = int(items[0])
25 | print fans
26 | return fans
27 |
28 | def get_by_api():
29 | url = 'https://openapi.youku.com/v2/users/friendship/followers.json?client_id=&user_id='
30 | request = urllib2.Request(url = url)
31 | response = urllib2.urlopen(request)
32 | page = response.read()
33 | result = json.loads(page, encoding="utf-8")
34 | fans = int(result['total'])
35 | print fans
36 | return fans
37 |
38 | def get_by_selenium():
39 | url = 'http://i.youku.com/i/UMjk1ODg3NDgwOA=='
40 | cap = webdriver.DesiredCapabilities.PHANTOMJS
41 | cap["phantomjs.page.settings.resourceTimeout"] = 1000
42 | cap["phantomjs.page.settings.loadImages"] = False
43 | cap["phantomjs.page.settings.localToRemoteUrlAccessEnabled"] = True
44 | cap["userAgent"] = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0"
45 | cap["XSSAuditingEnabled"] = True
46 | driver = webdriver.PhantomJS(desired_capabilities=cap,
47 | service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any',
48 | '--web-security=true'])
49 | # driver = webdriver.Chrome()
50 | driver.get(url)
51 | wait = WebDriverWait(driver, 20)
52 | print driver.page_source
53 | try:
54 | fans = int(wait.until(lambda x: x.find_elements_by_class_name('snum')[0].find_element_by_xpath('em').text))
55 | except TimeoutException:
56 | fans = 0
57 | print fans
58 | driver.quit()
59 | return fans
60 |
61 | if __name__ == "__main__":
62 | get_by_api()
63 | # get_by_selenium()
64 |
--------------------------------------------------------------------------------
/fans_crawler/youtube_crawler.py:
--------------------------------------------------------------------------------
1 | # -*- coding: UTF-8 -*-
2 | import urllib2
3 | import re
4 | import json
5 | from selenium import webdriver
6 | from selenium.common.exceptions import TimeoutException
7 | from selenium.webdriver.support.ui import WebDriverWait
8 |
9 | def get_by_request():
10 | url = 'https://www.youtube.com/channel/UC3qWcF49rv8VMZO7Vg6kj5w'
11 | headers = {}
12 | headers['Host'] = 'www.youtube.com'
13 | headers['Referer'] = 'http://www.insta360.com/'
14 | # headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0'
15 | # headers['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
16 | # headers['Connection'] = 'keep-alive'
17 | # headers['Upgrade-Insecure-Requests'] = '1'
18 | request = urllib2.Request(url = url, headers=headers)
19 | response = urllib2.urlopen(request)
20 | page = response.read()
21 | # print page
22 | pattern = re.compile("subscribers\">(.*)