)',source,re.S) 20 | return blocks 21 | 22 | def getnum(commentsnum): 23 | i = 0 24 | j = 0 25 | for char in commentsnum: 26 | if char == '(': 27 | break 28 | i = i + 1 29 | 30 | for char in commentsnum: 31 | if char == ')': 32 | break 33 | j = j + 1 34 | si = int(i) + 1 35 | ei = int(j) 36 | return commentsnum[si:ei] 37 | 38 | 39 | # 页面解析 40 | def jiexi(html): 41 | blocks = getblock(html) 42 | for block in blocks: 43 | selector = etree.HTML(block) 44 | poi = selector.xpath('//h3')[0] 45 | poi = poi.xpath('string(.)') 46 | # poi = poi.replace('\n','') 47 | 48 | elements = poi.split('-') 49 | leixin = elements[0] 50 | leixin = leixin.replace(' ', '') 51 | leixin = leixin.replace('\n', '') 52 | 53 | temp = elements[1].split() 54 | 55 | if len(temp) == 0: 56 | continue 57 | shouzimu = str(temp[0]) 58 | 59 | if len(temp)==1 and not shouzimu.isalpha(): 60 | zhongwen = temp[0] 61 | yingwen = '' 62 | elif shouzimu.isalpha(): 63 | yingwen = '' 64 | for i in range(0, len(temp)): 65 | yingwen = yingwen + ' ' + temp[i] 66 | zhongwen = '' 67 | else: 68 | zhongwen = temp[0] 69 | yingwen = '' 70 | for i in range(1, len(temp)): 71 | yingwen = yingwen + ' ' + temp[i] 72 | 73 | # 判断数据库是否已经存在该POI记录，决定是插入数据还是更新数据。 74 | sqli1 = "select * from " + db + "." + tb + " where poi_ch_name = " + "'%s'" % (zhongwen) 75 | sqli2 = "select * from " + db + "." + tb + " where poi_en_name = " + "'%s'" % (yingwen) 76 | try: 77 | r1 = cur.execute(sqli1) 78 | r2 = cur.execute(sqli2) 79 | except: 80 | pass 81 | else: 82 | # 国家、城市 83 | poiinfos = selector.xpath('//li/a/text()') 84 | country_city = poiinfos[0] 85 | country_city = country_city.split('-') 86 | country = country_city[0] 87 | city = country_city[1] 88 | 89 | # 查询城市id 90 | sqli1 = 'select region_id from map.map_region where region_ch_name = '+"'%s'"%(city) 91 | cur.execute(sqli1) 92 | id = cur.fetchmany(1) 93 | 94 | if len(id)==0: 95 | region_id = '' 96 | else: 97 | region_id = id[0][0] 98 | 99 | 100 | # 评论数 101 | pinglunshu = getnum(poiinfos[1]) 102 | # 相关游记数 103 | relatedyoujishu = getnum(poiinfos[2]) 104 | 105 | # tag_id 106 | if leixin == '美食': 107 | tag_id = 1 108 | elif leixin == '酒店': 109 | tag_id = 2 110 | elif leixin == '景点': 111 | tag_id = 3 112 | elif leixin == '购物': 113 | tag_id =4 114 | elif leixin == '娱乐': 115 | tag_id = 5 116 | elif leixin == '交通': 117 | tag_id = 6 118 | else: 119 | tag_id = '' 120 | 121 | 122 | # 详情链接 123 | link = selector.xpath('//h3/a/@href')[0] 124 | print '详情链接',link 125 | try: 126 | subhtml = getsource(link) 127 | subselector = etree.HTML(subhtml) 128 | except: 129 | pass 130 | else: 131 | if leixin == '景点': 132 | quguoshoucang = subselector.xpath('//span[@class="pa-num"]/text()') 133 | # 去过数 134 | quguonum = quguoshoucang[1] 135 | # 收藏数 136 | shoucangshu = quguoshoucang[0] 137 | # 评分 138 | poi_score = '' 139 | # 排名 140 | poi_rank = '' 141 | # 地址 142 | poi_address = '' 143 | poi_telephone_tag = subselector.xpath('//dl[@class="intro"]/dd/span/text()') 144 | if '电话' in poi_telephone_tag: 145 | for ti,tele_tag in enumerate(poi_telephone_tag): 146 | if tele_tag == '电话': 147 | telei = str(ti+1) 148 | xpath_tele = '//dl[@class="intro"]/dd['+telei+']/p/text()' 149 | poi_telephone = subselector.xpath(xpath_tele)[0] 150 | else: 151 | poi_telephone ='' 152 | 153 | else: 154 | quguonum = '' 155 | shoucangshu = '' 156 | # 评分 157 | poi_score = subselector.xpath('//span[@class="score-info"]/em/text()') 158 | if not poi_score: 159 | poi_score = '' 160 | else: 161 | poi_score = poi_score[0] 162 | # 等级 163 | poi_rank = subselector.xpath('//div[@class="ranking"]/em/text()') 164 | if not poi_rank: 165 | poi_rank = '' 166 | else: 167 | poi_rank = poi_rank[0][3:] 168 | # 地址，电话 169 | box_info = subselector.xpath('//div[@class="m-box m-info"]/ul[@class="clearfix"]/li/text()') 170 | # print '++++++++',len(box_info) 171 | if len(box_info)>=4: 172 | poi_address = subselector.xpath('//div[@class="m-box m-info"]/ul[@class="clearfix"]/li[1]/text()')[1].strip() 173 | for chari,addchar in enumerate(poi_address): 174 | if addchar == "：": 175 | tempi = chari 176 | poi_address = poi_address[tempi+1:] 177 | poi_telephone = subselector.xpath('//div[@class="m-box m-info"]/ul[@class="clearfix"]/li[2]/text()')[1].strip() 178 | else: 179 | poi_telephone ='' 180 | poi_address = '' 181 | 182 | # print '国家:' + country, '城市：' + city, '中文：' + zhongwen, '英文：' + yingwen, '城市id' + str(region_id), '类型id：' + str(tag_id), '类型:' + leixin, \ 183 | # '评论数' + str(pinglunshu), '相关游记数' + str(relatedyoujishu), '去过数' + str(quguonum), '收藏数' + str(shoucangshu), '评分' + str(poi_score), \ 184 | # '排名' + str(poi_rank), '电话' + str(poi_telephone), '地址' + poi_address 185 | 186 | if r1 or r2: 187 | print '已经存在记录，更新数据... ...' 188 | pass 189 | else: 190 | print '新增POI... ...' 191 | sqli = "INSERT INTO " + db + "." + tb + "(poi_ch_name,poi_en_name,poi_loc_name,poi_region_id,poi_tag_id,poi_score,poi_rank,poi_address,poi_telephone,visited_count,comments_count,collection_count,source_website)" + " VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" 192 | # print sqli 193 | 194 | cur.execute(sqli,(zhongwen,yingwen,yingwen,region_id,tag_id,poi_score,poi_rank,poi_address,poi_telephone,quguonum,pinglunshu,shoucangshu,source)) 195 | conn.commit() 196 | print '----------------------------------------' 197 | if __name__ == '__main__': 198 | # tag 199 | taglist = {'美食':1,'酒店':2,'景点':3,'购物':4,'娱乐':5,'交通':6} 200 | 201 | # 设置白名单，过滤国家 202 | chengshibaimingdan = ['美国'] 203 | # 来源 204 | source = '蚂蜂窝' 205 | 206 | db = 'map' 207 | # 数据表 208 | tb = 'map_poi_0' 209 | area = 'usa' 210 | 211 | if not os.path.exists('zhuaqu'): 212 | os.mkdir('zhuaqu') 213 | if not os.path.exists(os.path.join('zhuaqu',tb)): 214 | os.mkdir(os.path.join('zhuaqu',tb)) 215 | jilu = open(os.path.join('zhuaqu',tb,'jilu.txt'),'a') 216 | # 连接数据库 217 | try: 218 | conn = MySQLdb.connect(host='127.0.0.1', user='root', passwd='123456', port=3306, charset='utf8') 219 | cur = conn.cursor() 220 | cur.execute('set interactive_timeout=96*3600') 221 | conn.select_db(db) 222 | except MySQLdb.Error,e: 223 | print "Mysql Error %d: %s" % (e.args[0], e.args[1]) 224 | # 抽取待抓取的城市列表 225 | sqli0 = "select region_ch_name from map.map_region" 226 | num_city = cur.execute(sqli0) 227 | cities = cur.fetchmany(num_city) 228 | 229 | for city in cities: 230 | city = city[0] 231 | city = city.strip() 232 | if city in chengshibaimingdan: 233 | pass 234 | else: 235 | url0 = 'http://www.mafengwo.cn/group/s.php?q=%s&p=1&t=poi&kt=1'%(city) 236 | print url0 237 | html0 = getsource(url0) 238 | if not html0: 239 | pass 240 | else: 241 | #获取爬取的页数 242 | selector0 = etree.HTML(html0) 243 | # temp0 = selector0.xpath('//div[@class="m-pagination"]/span[@class="count"]/text()') 244 | poinum = selector0.xpath('//p[@class="ser-result-primary"]/text()')[0] 245 | poinum = poinum.strip() 246 | for k,zifu in enumerate(poinum): 247 | if poinum[k-1]=='结'and poinum[k]=='果': 248 | si = k 249 | if poinum[k] =='条': 250 | ei = k 251 | try: 252 | poinum = int(poinum[si+1:ei]) 253 | except: 254 | pass 255 | else: 256 | print 'poi数量:',poinum 257 | 258 | if poinum < 750: 259 | yema = poinum / 15 260 | else: 261 | yema = 50 262 | 263 | if yema == 0: 264 | print '----------%s 1页---------'%(city) 265 | url = 'http://www.mafengwo.cn/group/s.php?q=%s&p=%d&t=poi' % (city, 1) 266 | html = getsource(url) 267 | # jilu.write(html) 268 | jiexi(html) 269 | else: 270 | for i in range(yema+1): 271 | url = 'http://www.mafengwo.cn/group/s.php?q=%s&p=%d&t=poi'%(city,i) 272 | html = getsource(url) 273 | # jilu.write(html) 274 | jiexi(html) 275 | 276 | cur.close() 277 | conn.close() 278 | jilu.close() 279 | print 'finished' -------------------------------------------------------------------------------- /poispider/proxyIP.py: -------------------------------------------------------------------------------- 1 | #coding=utf-8 2 | 3 | proxy_list = [ 4 | {'http': "http://59.53.67.215:80"}, 5 | {'http': "http://60.161.14.77:8001"}, 6 | {'http': "http://61.144.14.68:80"}, 7 | {'http': "http://61.144.68.180:9999"}, 8 | {'http': "http://61.164.108.84:8844"}, 9 | {'http': "http://61.166.55.153:11808"} 10 | ] -------------------------------------------------------------------------------- /poispider/qyer.py: -------------------------------------------------------------------------------- 1 | #encoding=utf-8 2 | from lxml import etree 3 | import requests 4 | import os 5 | import sys 6 | import MySQLdb 7 | import re 8 | reload(sys) 9 | sys.setdefaultencoding("utf-8") 10 | 11 | #用来获取网页源代码 12 | def getsource(url): 13 | html = requests.get(url) 14 | html.encoding = 'utf-8' 15 | return html.text 16 | #获取信息块 17 | def getcountryblock(source): 18 | blocks = re.findall('(

)',source,re.S) 19 | return blocks 20 | 21 | def getpoiblock(source): 22 | blocks = re.findall('(
)',source,re.S) 23 | return blocks 24 | # 对匹配为空进行处理 25 | def pankong(poi_xx): 26 | if len(poi_xx)==0: 27 | poi_xx = '' 28 | else: 29 | poi_xx = poi_xx[0] 30 | return poi_xx 31 | 32 | if name == 'main': 33 | countries = ['cuba'] 34 | 35 | for country in countries: 36 | starturl = 'http://place.qyer.com/'+country+'/citylist-0-0-1/' 37 | db = 'map' 38 | # 数据表 39 | tb = 'map_poi' 40 | # 希望跳过抓取的城市 41 | hulvcities = range(1,1049) 42 | hulvcities.append(987) 43 | hulvcities.append(915) 44 | 45 | # 连接数据库 46 | try: 47 | # conn = MySQLdb.connect(host='127.0.0.1', user='root', passwd='123456', port=3306, charset='utf8') 48 | conn = MySQLdb.connect(host='172.22.185.130', user='root', passwd='123456', port=3306, charset='utf8') 49 | cur = conn.cursor() 50 | cur.execute('set interactive_timeout=963600') 51 | conn.select_db(db) 52 | except MySQLdb.Error, e: 53 | print "Mysql Error %d: %s" % (e.args[0], e.args[1]) 54 | temphtml = getsource(starturl) 55 | selector = etree.HTML(temphtml) 56 | pagenums = selector.xpath('//div[@class="ui_page"]/a/@data-page') 57 | # 需爬取得页数 58 | pagenum = pagenums[len(pagenums) - 2] 59 | 60 | for i in range(1, int(pagenum) + 1): 61 | url = 'http://place.qyer.com/'+country+'/citylist-0-0-%d' % (i) 62 | print url 63 | html = getsource(url) 64 | selector2 = etree.HTML(html) 65 | blocks = getcountryblock(html) 66 | for j, block in enumerate(blocks): 67 | selector1 = etree.HTML(block) 68 | city = selector1.xpath('//a/text()')[0].strip() 69 | cityenglishname = selector1.xpath('//span/text()')[0].strip() 70 | 71 | # 对应的城市id 72 | sqli1 = "select region_id from " + db + ".map_region" + " where region_ch_name = " + "'%s'" % (city) 73 | num_result = cur.execute(sqli1) 74 | if not num_result: 75 | region_id = '' 76 | else: 77 | region_id = cur.fetchmany(1) 78 | region_id = region_id[0][0] 79 | 80 | # 过滤已经抓取完成的城市 81 | if region_id in hulvcities: 82 | print city, '已经完成抓取，跳过' 83 | pass 84 | else: 85 | # 城市主页 86 | sub_url = selector1.xpath('//a/@href')[0] 87 | sighturl = sub_url+'sight/' 88 | foodurl = sub_url+'food/' 89 | shoppingurl = sub_url+'shopping/' 90 | city_urls = [sighturl,foodurl,shoppingurl] 91 | for ci,cityurl in enumerate(city_urls): 92 | sub_html = getsource(cityurl) 93 | sub_selector = etree.HTML(sub_html) 94 | # 爬取的页数 95 | poiyeshu = sub_selector.xpath('//div[@class="ui_page"]/a/@data-page') 96 | try: 97 | poiyeshu = poiyeshu[-2] 98 | except: 99 | pass 100 | else: 101 | for ye in range(1,int(poiyeshu)+1): 102 | dangqianurl = cityurl+"?page=%s"%(ye) 103 | print '列表页url',dangqianurl 104 | dangqianhtml = getsource(dangqianurl) 105 | poiblocks = getpoiblock(dangqianhtml) 106 | print '当前页的poi数',len(poiblocks) 107 | for poiblock in poiblocks: 108 | dangqianselector = etree.HTML(poiblock) 109 | 110 | # 中文、英文、本地名称 111 | name0 = dangqianselector.xpath('//h3[@class="title fontYaHei"]/a/text()')[0].strip() 112 | name1 = dangqianselector.xpath('//h3[@class="title fontYaHei"]/a/span/text()') 113 | if len(name1)==0: 114 | name1 = '' 115 | else: 116 | name1 = name1[0].strip() 117 | 118 | shouzimu = name0[0].encode('utf-8') 119 | if shouzimu.isalpha(): 120 | poi_en_name = name0 121 | poi_ch_name = '' 122 | else: 123 | poi_en_name = name1 124 | poi_ch_name = name0 125 | poi_loc_name = poi_en_name 126 | 127 | # 类别id 128 | if ci == 0: 129 | tag_id = 3 130 | elif ci == 1: 131 | tag_id =1 132 | elif ci == 2: 133 | tag_id = 4 134 | # 评分 135 | poi_score = dangqianselector.xpath('//span[@class="grade"]/text()') 136 | if len(poi_score)==0: 137 | poi_score='' 138 | else: 139 | poi_score = poi_score[0] 140 | 141 | # 排名 142 | poi_rank = dangqianselector.xpath('//em[@class="rank orange"]/text()') 143 | poi_rank = pankong(poi_rank) 144 | newstr = '' 145 | for sr in poi_rank: 146 | if sr.isdigit(): 147 | newstr = newstr + sr 148 | poi_rank = newstr 149 | 150 | # 详情页url 151 | xiangqingurl = dangqianselector.xpath('//h3[@class="title fontYaHei"]/a/@href')[0] 152 | print '详情页',xiangqingurl 153 | try: 154 | xiangqinghtml = getsource(xiangqingurl) 155 | except: 156 | pass 157 | else: 158 | xiangqingselector = etree.HTML(xiangqinghtml) 159 | poi_tips_biaoti = xiangqingselector.xpath('//div[@class="poiDet-main"]/ul[@class="poiDet-tips"]/li/span/text()') 160 | biaotilist = [] 161 | for tipi,biaoti in enumerate(poi_tips_biaoti): 162 | biaoti = biaoti.strip() 163 | biaotilist.append(biaoti) 164 | 165 | for ti,biaoti in enumerate(biaotilist): 166 | if biaoti =='': 167 | biaotilist.pop(ti) 168 | for bi,biaoti in enumerate(biaotilist): 169 | if biaoti == '地址：': 170 | addi = bi + 1 171 | if biaoti == '电话：': 172 | telei = bi + 1 173 | if '地址：'in biaotilist: 174 | # 地址 175 | xpath_str_add = "//ul[@class='poiDet-tips']/li["+str(addi)+"]/div/p/text()" 176 | poi_address = xiangqingselector.xpath(xpath_str_add) 177 | poi_address = pankong(poi_address) 178 | else: 179 | poi_address = '' 180 | 181 | if '电话：'in biaotilist: 182 | # 电话 183 | xpath_str_tele = "//ul[@class='poiDet-tips']/li[" + str(telei) + "]/div/p/text()" 184 | poi_telephone = xiangqingselector.xpath(xpath_str_tele) 185 | poi_telephone = pankong(poi_telephone) 186 | if not poi_telephone: 187 | poi_telephone = '' 188 | else: 189 | poi_telephone = '' 190 | # 评论数 191 | pinglunshu = dangqianselector.xpath('//div[@class="info"]/span[@class="dping"]/a/text()') 192 | if len(pinglunshu)==0: 193 | pinglunshu ='' 194 | else: 195 | pinglunshu = pinglunshu[0] 196 | 197 | comments_count = pinglunshu.strip() 198 | newstr1 = '' 199 | for sr1 in comments_count: 200 | if sr1.isdigit(): 201 | newstr1 = newstr1 + sr1 202 | comments_count = newstr1 203 | # 来源 204 | source = 'qyer' 205 | sqli = "INSERT INTO " + db + "." + tb + "(poi_ch_name,poi_en_name,poi_loc_name,poi_region_id,poi_tag_id,poi_score,poi_rank,poi_address,poi_telephone,comments_count,source_website)" + " VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" 206 | 207 | # 判断数据库是否已经存在城市数据，决定是插入数据还是更新数据。 208 | sqli1 = "select from " + db + "." + tb + " where poi_ch_name = " + "'%s'" % (poi_ch_name) 209 | sqli2 = "select * from " + db + "." + tb + " where poi_en_name = " + "'%s'" % (poi_en_name) 210 | # print '中文查询',sqli1 211 | # print '英文查询',sqli2 212 | try: 213 | r1 = cur.execute(sqli1) 214 | r2 = cur.execute(sqli2) 215 | except: 216 | pass 217 | if poi_ch_name =='': 218 | r1 = 0 219 | if poi_en_name == '': 220 | r2 = 0 221 | print '查询结果：','中文',r1,'英文',r2 222 | if r1 or r2: 223 | print '已经存在记录，迭代数据 ... ...' 224 | pass 225 | else: 226 | print '插入新POI... ...' 227 | cur.execute(sqli,(poi_ch_name, poi_en_name, poi_loc_name, region_id, tag_id,poi_score,poi_rank,poi_address,poi_telephone,comments_count,source)) 228 | conn.commit() 229 | print '------------------------------------------------' 230 | cur.close() 231 | conn.close() 232 | print '------------finished--------------' 233 | -------------------------------------------------------------------------------- /poispider/qyer_one_city.py: -------------------------------------------------------------------------------- 1 | #encoding=utf-8 2 | from lxml import etree 3 | import requests 4 | import os 5 | import sys 6 | import MySQLdb 7 | import re 8 | reload(sys) 9 | sys.setdefaultencoding("utf-8") 10 | 11 | #用来获取网页源代码 12 | def getsource(url): 13 | html = requests.get(url) 14 | html.encoding = 'utf-8' 15 | return html.text 16 | #获取信息块 17 | def getcountryblock(source): 18 | blocks = re.findall('(
)',source,re.S) 19 | return blocks 20 | 21 | def getpoiblock(source): 22 | blocks = re.findall('(
)',source,re.S) 23 | return blocks 24 | # 对匹配为空进行处理 25 | def pankong(poi_xx): 26 | if len(poi_xx)==0: 27 | poi_xx = '' 28 | else: 29 | poi_xx = poi_xx[0] 30 | return poi_xx 31 | 32 | if name == 'main': 33 | country = 'singapore' 34 | city = '新加坡' 35 | starturl = 'http://place.qyer.com/singapore/' 36 | 37 | db = 'map' 38 | # 数据表 39 | tb = 'map_poi' 40 | # 希望跳过抓取的城市 41 | hulvcities = range(1,826) 42 | 43 | # 连接数据库 44 | try: 45 | # conn = MySQLdb.connect(host='127.0.0.1', user='root', passwd='123456', port=3306, charset='utf8') 46 | conn = MySQLdb.connect(host='172.22.185.78', user='root', passwd='123456', port=3306, charset='utf8') 47 | cur = conn.cursor() 48 | cur.execute('set interactive_timeout=963600') 49 | conn.select_db(db) 50 | except MySQLdb.Error, e: 51 | print "Mysql Error %d: %s" % (e.args[0], e.args[1]) 52 | 53 | 54 | # 对应的城市id 55 | sqli1 = "select region_id from " + db + ".map_region" + " where region_ch_name = " + "'%s'" % (city) 56 | num_result = cur.execute(sqli1) 57 | if not num_result: 58 | region_id = '' 59 | else: 60 | region_id = cur.fetchmany(1) 61 | region_id = region_id[0][0] 62 | 63 | # 过滤已经抓取完成的城市 64 | if region_id in hulvcities: 65 | print city, '已经完成抓取，跳过' 66 | pass 67 | else: 68 | # 城市主页 69 | sighturl = starturl+'sight/' 70 | foodurl = starturl+'food/' 71 | shoppingurl = starturl+'shopping/' 72 | city_urls = [sighturl,foodurl,shoppingurl] 73 | for ci,cityurl in enumerate(city_urls): 74 | print cityurl 75 | sub_html = getsource(cityurl) 76 | sub_selector = etree.HTML(sub_html) 77 | # 爬取的页数 78 | poiyeshu = sub_selector.xpath('//div[@class="ui_page"]/a/@data-page') 79 | try: 80 | poiyeshu = poiyeshu[-2] 81 | except: 82 | pass 83 | else: 84 | for ye in range(1,int(poiyeshu)+1): 85 | dangqianurl = cityurl+"?page=%s"%(ye) 86 | print '列表页url',dangqianurl 87 | dangqianhtml = getsource(dangqianurl) 88 | poiblocks = getpoiblock(dangqianhtml) 89 | print '当前页的poi数',len(poiblocks) 90 | for poiblock in poiblocks: 91 | dangqianselector = etree.HTML(poiblock) 92 | 93 | # 中文、英文、本地名称 94 | name0 = dangqianselector.xpath('//h3[@class="title fontYaHei"]/a/text()')[0].strip() 95 | name1 = dangqianselector.xpath('//h3[@class="title fontYaHei"]/a/span/text()') 96 | if len(name1)==0: 97 | name1 = '' 98 | else: 99 | name1 = name1[0].strip() 100 | 101 | shouzimu = name0[0].encode('utf-8') 102 | if shouzimu.isalpha(): 103 | poi_en_name = name0 104 | poi_ch_name = '' 105 | else: 106 | poi_en_name = name1 107 | poi_ch_name = name0 108 | poi_loc_name = poi_en_name 109 | 110 | # 类别id 111 | if ci == 0: 112 | tag_id = 3 113 | elif ci == 1: 114 | tag_id =1 115 | elif ci == 2: 116 | tag_id = 4 117 | # 评分 118 | poi_score = dangqianselector.xpath('//span[@class="grade"]/text()') 119 | if len(poi_score)==0: 120 | poi_score='' 121 | else: 122 | poi_score = poi_score[0] 123 | 124 | # 排名 125 | poi_rank = dangqianselector.xpath('//em[@class="rank orange"]/text()') 126 | poi_rank = pankong(poi_rank) 127 | newstr = '' 128 | for sr in poi_rank: 129 | if sr.isdigit(): 130 | newstr = newstr + sr 131 | poi_rank = newstr 132 | 133 | # 详情页url 134 | xiangqingurl = dangqianselector.xpath('//h3[@class="title fontYaHei"]/a/@href')[0] 135 | print '详情页',xiangqingurl 136 | try: 137 | xiangqinghtml = getsource(xiangqingurl) 138 | except: 139 | pass 140 | else: 141 | xiangqingselector = etree.HTML(xiangqinghtml) 142 | poi_tips_biaoti = xiangqingselector.xpath('//div[@class="poiDet-main"]/ul[@class="poiDet-tips"]/li/span/text()') 143 | biaotilist = [] 144 | for tipi,biaoti in enumerate(poi_tips_biaoti): 145 | biaoti = biaoti.strip() 146 | biaotilist.append(biaoti) 147 | 148 | for ti,biaoti in enumerate(biaotilist): 149 | if biaoti =='': 150 | biaotilist.pop(ti) 151 | for bi,biaoti in enumerate(biaotilist): 152 | if biaoti == '地址：': 153 | addi = bi + 1 154 | if biaoti == '电话：': 155 | telei = bi + 1 156 | if '地址：'in biaotilist: 157 | # 地址 158 | xpath_str_add = "//ul[@class='poiDet-tips']/li["+str(addi)+"]/div/p/text()" 159 | poi_address = xiangqingselector.xpath(xpath_str_add) 160 | poi_address = pankong(poi_address) 161 | else: 162 | poi_address = '' 163 | 164 | if '电话：'in biaotilist: 165 | # 电话 166 | xpath_str_tele = "//ul[@class='poiDet-tips']/li[" + str(telei) + "]/div/p/text()" 167 | poi_telephone = xiangqingselector.xpath(xpath_str_tele) 168 | poi_telephone = pankong(poi_telephone) 169 | if not poi_telephone: 170 | poi_telephone = '' 171 | else: 172 | poi_telephone = '' 173 | # 评论数 174 | pinglunshu = dangqianselector.xpath('//div[@class="info"]/span[@class="dping"]/a/text()') 175 | if len(pinglunshu)==0: 176 | pinglunshu ='' 177 | else: 178 | pinglunshu = pinglunshu[0] 179 | 180 | comments_count = pinglunshu.strip() 181 | newstr1 = '' 182 | for sr1 in comments_count: 183 | if sr1.isdigit(): 184 | newstr1 = newstr1 + sr1 185 | comments_count = newstr1 186 | # 来源 187 | source = 'qyer' 188 | sqli = "INSERT INTO " + db + "." + tb + "(poi_ch_name,poi_en_name,poi_loc_name,poi_region_id,poi_tag_id,poi_score,poi_rank,poi_address,poi_telephone,comments_count,source_website)" + " VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" 189 | 190 | # 判断数据库是否已经存在城市数据，决定是插入数据还是更新数据。 191 | sqli1 = "select from " + db + "." + tb + " where poi_ch_name = " + "'%s'" % (poi_ch_name) 192 | sqli2 = "select * from " + db + "." + tb + " where poi_en_name = " + "'%s'" % (poi_en_name) 193 | # print '中文查询',sqli1 194 | # print '英文查询',sqli2 195 | try: 196 | r1 = cur.execute(sqli1) 197 | r2 = cur.execute(sqli2) 198 | except: 199 | pass 200 | if poi_ch_name =='': 201 | r1 = 0 202 | if poi_en_name == '': 203 | r2 = 0 204 | print '查询结果：','中文',r1,'英文',r2 205 | if r1 or r2: 206 | print '已经存在记录，迭代数据 ... ...' 207 | pass 208 | else: 209 | print '插入新POI... ...' 210 | cur.execute(sqli,(poi_ch_name, poi_en_name, poi_loc_name, region_id, tag_id,poi_score,poi_rank,poi_address,poi_telephone,comments_count,source)) 211 | conn.commit() 212 | print '------------------------------------------------' 213 | cur.close() 214 | conn.close() 215 | print '------------finished--------------' 216 | -------------------------------------------------------------------------------- /poispider/tripadvisor.py: -------------------------------------------------------------------------------- 1 | #encoding=utf-8 2 | from lxml import etree 3 | import requests 4 | import os 5 | import sys 6 | import MySQLdb 7 | import re 8 | import random 9 | import time 10 | reload(sys) 11 | sys.setdefaultencoding("utf-8") 12 | 13 | def qushuzi(str): 14 | num = '' 15 | if not str: 16 | return num 17 | else: 18 | for sr in str: 19 | if sr.isdigit(): 20 | num = num+sr 21 | return num 22 | #用来获取网页源代码 23 | def getsource(url): 24 | headlist = [{'User-Agent': 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.'}, 25 | {'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50'}, 26 | {'User-Agent': 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0;'}, 27 | {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36'}, 28 | {'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)'}, 29 | {'User-Agent': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)'}, 30 | {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1'}, 31 | {'User-Agent': 'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11'}, 32 | {'User-Agent': 'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11'}, 33 | {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11'}, 34 | {'User-Agent': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)'}, 35 | {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36'}] 36 | num = len(headlist)-1 37 | html = requests.get(url, headers=headlist[random.randint(0, num)]) 38 | html.encoding = 'utf-8' 39 | return html.text 40 | #获取信息块 41 | def getcityblock(source): 42 | blocks = re.findall('()',source,re.S) 48 | elif ci == 1: 49 | blocks = re.findall('(
.?
)', source, re.S) 50 | elif ci == 2: 51 | blocks = re.findall('(
.?
)',source,re.S) 52 | else: 53 | blocks = [] 54 | return blocks 55 | # 对匹配为空进行处理 56 | def pankong(poi_xx): 57 | if len(poi_xx)==0: 58 | poi_xx = '' 59 | else: 60 | poi_xx = poi_xx[0] 61 | return poi_xx 62 | def tiaoguo(selector): 63 | guolvcibiao = ['市'] 64 | city = selector.xpath('//li[@class="cityName tabItem dropDown hvrIE6"]/span/span/text()') 65 | city = pankong(city) 66 | # 对应的城市id 67 | sqli = "select region_id from " + db + ".map_region" + " where region_ch_name = " + "'%s'" % (city) 68 | num_result = cur.execute(sqli) 69 | if not num_result: 70 | region_id = '' 71 | else: 72 | region_id = cur.fetchmany(1) 73 | region_id = region_id[0][0] 74 | # 过滤已经抓取完成的城市 75 | if region_id in hulvcities: 76 | return city,region_id,True 77 | else: 78 | return city,region_id,False 79 | def yemianjiexi(block,ci): 80 | xiangqingselector = '' 81 | poi_ch_name = '' 82 | poi_en_name = '' 83 | poi_loc_name = '' 84 | poi_telephone ='' 85 | poi_address ='' 86 | poi_rank='' 87 | comments_count='' 88 | 89 | selector = etree.HTML(block) 90 | # 详情页信息获取异常标记 91 | tiaoguopoi = 0 92 | 93 | if ci == 0: 94 | # 详情页url 95 | 96 | xiangqingurl = selector.xpath('//div[@class="listing_title"]/a/@href')[0] 97 | xiangqingurl = starturl + xiangqingurl 98 | # print '详情页', xiangqingurl 99 | try: 100 | xiangqinghtml = getsource(xiangqingurl) 101 | except: 102 | tiaoguopoi = 1 103 | return tiaoguopoi, xiangqingselector, poi_ch_name, poi_en_name, poi_loc_name, poi_telephone, poi_address, poi_rank, comments_count 104 | else: 105 | time.sleep(tingliu) 106 | xiangqingselector = etree.HTML(xiangqinghtml) 107 | 108 | # 名称 109 | name0 = selector.xpath('//div[@class="listing_title"]/a/text()') 110 | name0 = pankong(name0) 111 | 112 | name1 = xiangqingselector.xpath('//span[@class="altHead"]/text()') 113 | name1 = pankong(name1) 114 | shouzimu0 = name0[0].encode('utf-8') 115 | if not name1: 116 | if shouzimu0.isalpha(): 117 | poi_en_name = name0 118 | poi_ch_name = '' 119 | else: 120 | poi_en_name = '' 121 | poi_ch_name = name0 122 | else: 123 | if shouzimu0.isalpha(): 124 | poi_en_name = name0 125 | poi_ch_name = name1 126 | else: 127 | poi_en_name = name1 128 | poi_ch_name = name0 129 | 130 | poi_loc_name = poi_en_name 131 | 132 | comments_count = xiangqingselector.xpath('//a[@class="more taLnk"]/@content') 133 | comments_count = pankong(comments_count) 134 | poi_rank = xiangqingselector.xpath('//b[@class="rank"]/text()') 135 | poi_rank = pankong(poi_rank) 136 | if poi_rank: 137 | poi_rank = qushuzi(poi_rank) 138 | # 酒店类的电话获取还未完成，后期优化 139 | poi_telephone = '' 140 | # 地址 141 | street_address = xiangqingselector.xpath('//span[@class="street-address"]/text()') 142 | street_address = pankong(street_address) 143 | 144 | extended_address = xiangqingselector.xpath('//span[@class="extended-address"]/text()') 145 | extended_address = pankong(extended_address) 146 | 147 | addresslocality = xiangqingselector.xpath('//span[@property="addressLocality"]/text()') 148 | addresslocality = pankong(addresslocality) 149 | 150 | addressregion = xiangqingselector.xpath('//span[@property="addressRegion"]/text()') 151 | addressregion = pankong(addressregion) 152 | 153 | postcode = xiangqingselector.xpath('//span[@property="postalCode"]/text()') 154 | postcode = pankong(postcode) 155 | 156 | elif ci == 1: 157 | # 详情页url 158 | xiangqingurl = selector.xpath('//div[@class="property_title"]/a/@href')[0] 159 | xiangqingurl = starturl + xiangqingurl 160 | # print '详情页', xiangqingurl 161 | try: 162 | xiangqinghtml = getsource(xiangqingurl) 163 | except: 164 | tiaoguopoi = 1 165 | return tiaoguopoi, xiangqingselector, poi_ch_name, poi_en_name, poi_loc_name, poi_telephone, poi_address, poi_rank, comments_count 166 | else: 167 | time.sleep(tingliu) 168 | xiangqingselector = etree.HTML(xiangqinghtml) 169 | # 名称 170 | name0 = selector.xpath('//a[@target="_blank"]/text()') 171 | name0 = pankong(name0).strip() 172 | if not name0: 173 | name0 = xiangqingselector.xpath('//h1[@id="HEADING"]/text()') 174 | name0 = pankong(name0).strip() 175 | name1 = xiangqingselector.xpath('//span[@class="altHead"]/text()') 176 | name1 = pankong(name1) 177 | shouzimu0 = name0[0].encode('utf-8') 178 | if not name1: 179 | if shouzimu0.isalpha(): 180 | poi_en_name = name0 181 | poi_ch_name = '' 182 | else: 183 | poi_en_name = '' 184 | poi_ch_name = name0 185 | else: 186 | if shouzimu0.isalpha(): 187 | poi_en_name = name0 188 | poi_ch_name = name1 189 | else: 190 | poi_en_name = name1 191 | poi_ch_name = name0 192 | poi_loc_name = poi_en_name 193 | 194 | comments_count = xiangqingselector.xpath('//a[@class="more"]/@content') 195 | comments_count = pankong(comments_count) 196 | poi_rank = xiangqingselector.xpath('//b[@class="rank_text wrap"]/span/text()') 197 | poi_rank = pankong(poi_rank) 198 | if poi_rank: 199 | poi_rank = qushuzi(poi_rank) 200 | poi_telephone = xiangqingselector.xpath('//div[@class="phoneNumber"]/text()') 201 | poi_telephone = pankong(poi_telephone) 202 | if poi_telephone: 203 | for i,ch in enumerate(poi_telephone): 204 | if ch.isdigit(): 205 | tempi = i 206 | break 207 | poi_telephone = poi_telephone[tempi:] 208 | 209 | # 地址 210 | street_address = xiangqingselector.xpath('//span[@class="street-address"]/text()') 211 | street_address = pankong(street_address) 212 | 213 | extended_address = xiangqingselector.xpath('//span[@class="extended-address"]/text()') 214 | extended_address = pankong(extended_address) 215 | 216 | addresslocality = xiangqingselector.xpath('//span[@property="addressLocality"]/text()') 217 | addresslocality = pankong(addresslocality) 218 | 219 | addressregion = xiangqingselector.xpath('//span[@property="addressRegion"]/text()') 220 | addressregion = pankong(addressregion) 221 | 222 | postcode = xiangqingselector.xpath('//span[@property="postalCode"]/text()') 223 | postcode = pankong(postcode) 224 | 225 | elif ci == 2: 226 | # 详情页url 227 | xiangqingurl = selector.xpath('//a[@class="property_title"]/@href')[0] 228 | xiangqingurl = starturl + xiangqingurl 229 | # print '详情页', xiangqingurl 230 | try: 231 | xiangqinghtml = getsource(xiangqingurl) 232 | except: 233 | tiaoguopoi = 1 234 | return tiaoguopoi,xiangqingselector,poi_ch_name,poi_en_name,poi_loc_name,poi_telephone,poi_address,poi_rank,comments_count 235 | else: 236 | time.sleep(tingliu) 237 | xiangqingselector = etree.HTML(xiangqinghtml) 238 | 239 | # 名称 240 | poi_ch_name = '' 241 | poi_en_name = selector.xpath('//a[@target="_blank"]/text()') 242 | poi_en_name = pankong(poi_en_name) 243 | poi_en_name = poi_en_name.strip() 244 | poi_loc_name = poi_en_name 245 | 246 | comments_count = xiangqingselector.xpath('//a[@class="more"]/@content') 247 | comments_count = pankong(comments_count) 248 | poi_rank = xiangqingselector.xpath('//b[@class="rank_text wrap"]/span/text()') 249 | poi_rank = pankong(poi_rank) 250 | if poi_rank: 251 | poi_rank = qushuzi(poi_rank) 252 | poi_telephone = xiangqingselector.xpath('//div[@class="fl phoneNumber"]/text()') 253 | poi_telephone = pankong(poi_telephone) 254 | 255 | # 地址 256 | street_address = xiangqingselector.xpath('//span[@class="street-address"]/text()') 257 | street_address = pankong(street_address) 258 | 259 | extended_address = xiangqingselector.xpath('//span[@class="extended-address"]/text()') 260 | extended_address = pankong(extended_address) 261 | 262 | addresslocality = xiangqingselector.xpath('//span[@property="addressLocality"]/text()') 263 | addresslocality = pankong(addresslocality) 264 | 265 | addressregion = xiangqingselector.xpath('//span[@property="addressRegion"]/text()') 266 | addressregion = pankong(addressregion) 267 | 268 | postcode = xiangqingselector.xpath('//span[@property="postalCode"]/text()') 269 | postcode = pankong(postcode) 270 | 271 | if extended_address: 272 | poi_address = street_address + ',' + extended_address+','+addresslocality + ',' + addressregion + postcode 273 | else: 274 | poi_address = street_address + ',' + addresslocality + ',' + addressregion + postcode 275 | 276 | if not poi_telephone: 277 | poi_telephone = '' 278 | else: 279 | pass 280 | return tiaoguopoi,xiangqingselector,poi_ch_name,poi_en_name,poi_loc_name,poi_telephone,poi_address,poi_rank,comments_count 281 | 282 | if name == 'main': 283 | starturl = 'http://www.tripadvisor.cn' 284 | tingliu = 2 285 | db = 'map' 286 | # 数据表 287 | tb = 'map_poi' 288 | # 希望跳过抓取的城市 289 | hulvcities = range(1,2) 290 | # 跳过多少个城市 291 | num_tiao = 3 292 | # 来源 293 | source = 'tripadvisor' 294 | # 连接数据库 295 | try: 296 | # conn = MySQLdb.connect(host='127.0.0.1', user='root', passwd='123456', port=3306, charset='utf8') 297 | conn = MySQLdb.connect(host='172.22.185.130', user='root', passwd='123456', port=3306, charset='utf8') 298 | cur = conn.cursor() 299 | cur.execute('set interactive_timeout=963600') 300 | conn.select_db(db) 301 | except MySQLdb.Error, e: 302 | print "Mysql Error %d: %s" % (e.args[0], e.args[1]) 303 | 304 | for i in range(2,51): 305 | # 城市列表页 306 | # url = 'http://www.tripadvisor.cn/TourismChildrenAjax?geo=191&offset=%d&desktop=true'% (i) 307 | url = 'http://www.tripadvisor.cn/TourismChildrenAjax?geo=150768&offset=%d&desktop=true' % (i) 308 | print '城市列表页：',url 309 | html = getsource(url) 310 | time.sleep(tingliu) 311 | blocks = getcityblock(html)# 设置当前列表页跳过前面多少个城市 312 | 313 | for j, block in enumerate(blocks): 314 | if num_tiao == 0: 315 | pass 316 | elif num_tiao>j: 317 | continue 318 | selector1 = etree.HTML(block) 319 | 320 | # 城市主页 321 | sub_url = selector1.xpath('//a/@href')[0] 322 | zhuye_url = starturl+sub_url 323 | print '城市主页：',zhuye_url 324 | zhuye_html = getsource(zhuye_url) 325 | time.sleep(tingliu) 326 | zhuye_selector = etree.HTML(zhuye_html) 327 | poi_region,region_id, tiaoma = tiaoguo(zhuye_selector) 328 | if tiaoma: 329 | pass 330 | else: 331 | hotel_url = zhuye_selector.xpath('//li[@class="hotels twoLines"]/a/@href')[0] 332 | jingdian_url = zhuye_selector.xpath('//li[@class="attractions twoLines"]/a/@href')[0] 333 | canting_url = zhuye_selector.xpath('//li[@class="restaurants twoLines"]/a/@href')[0] 334 | city_urls = [hotel_url,jingdian_url,canting_url] 335 | for ci,cityurl in enumerate(city_urls): 336 | city_url = starturl+cityurl 337 | print '行业url',city_url 338 | if ci ==0: 339 | tag_id = 2 340 | if ci == 1: 341 | tag_id = 3 342 | if ci == 2: 343 | tag_id = 1 344 | dizhiyuanshu = city_url.split('-') 345 | sub_html = getsource(city_url) 346 | time.sleep(tingliu) 347 | sub_selector = etree.HTML(sub_html) 348 | # 爬取的页数 349 | poiyeshu = sub_selector.xpath('//a[@class="pageNum last taLnk"]/@data-page-number') 350 | if len(poiyeshu) == 0: 351 | poiyeshu = 1 352 | else: 353 | poiyeshu = poiyeshu[0] 354 | 355 | for ye in range(1,int(poiyeshu)+1): 356 | index = 30(ye-1) 357 | if index != 0: 358 | if ci == 0: 359 | dangqianurl = dizhiyuanshu[0]+'-'+dizhiyuanshu[1]+'-'+'oa'+str(index)+'-'+dizhiyuanshu[2]+'-'+dizhiyuanshu[3] 360 | if ci == 1: 361 | dangqianhtml = dizhiyuanshu[0]+'-'+dizhiyuanshu[1]+'-'+dizhiyuanshu[2]+'-'+'oa'+str(index)+'-'+dizhiyuanshu[3] 362 | if ci == 2: 363 | dangqianhtml = dizhiyuanshu[0]+'-'+dizhiyuanshu[1]+'-'+'oa'+str(index)+'-'+dizhiyuanshu[2] 364 | else: 365 | dangqianurl = city_url 366 | print '列表页url',dangqianurl 367 | dangqianhtml = getsource(dangqianurl) 368 | time.sleep(tingliu) 369 | poiblocks = getpoiblock(dangqianhtml,ci) 370 | print '当前页的poi数',len(poiblocks) 371 | for poiblock in poiblocks: 372 | try: 373 | tiaoguopoi,xiangqingselector,poi_ch_name, poi_en_name, poi_loc_name, poi_telephone, poi_address, poi_rank,comments_count = yemianjiexi(poiblock,ci) 374 | except: 375 | pass 376 | else: 377 | # tiaoguopoi为poi详情页获取阶段，异常获取标志。若获取异常，tiaoguopoi为1，跳过该poi 378 | if tiaoguopoi: 379 | pass 380 | else: 381 | sqli = "INSERT INTO " + db + "." + tb + "(poi_ch_name,poi_en_name,poi_loc_name,poi_region_id,poi_region,poi_tag_id,poi_rank,poi_address,poi_telephone,comments_count,source_website)" + " VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" 382 | 383 | # 判断数据库是否已经存在城市数据，决定是插入数据还是更新数据。 384 | sqli1 = "select * from " + db + "." + tb + " where poi_ch_name = " + "'%s'" % (poi_ch_name) 385 | sqli2 = "select * from " + db + "." + tb + " where poi_en_name = " + "'%s'" % (poi_en_name) 386 | 387 | try: 388 | r1 = cur.execute(sqli1) 389 | r2 = cur.execute(sqli2) 390 | except: 391 | pass 392 | 393 | if not poi_ch_name : 394 | r1 = 0 395 | if not poi_en_name : 396 | r2 = 0 397 | # print '中文：',poi_ch_name,'英文：',poi_en_name 398 | # print '查询结果：','中文',r1,'英文',r2 399 | 400 | if r1 or r2: 401 | print '已经存在记录，迭代数据 ... ...' 402 | pass 403 | else: 404 | print '插入新POI... ...' 405 | print '中文：' + poi_ch_name,'英文：' + poi_en_name, '本地语言名称:' + poi_en_name, '城市id:' + str(region_id),'城市:'+poi_region, '类型：' + str(tag_id), '评论数:' + str(comments_count), '排名:' + str(poi_rank), '地址:' + str(poi_address), '电话:' + str(poi_telephone) 406 | cur.execute(sqli,(poi_ch_name, poi_en_name, poi_loc_name, region_id,poi_region, tag_id,poi_rank,poi_address,poi_telephone,comments_count,source)) 407 | conn.commit() 408 | print '------------------------------------------------' 409 | cur.close() 410 | conn.close() 411 | print '------------finished--------------' 412 | -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | #encoding=utf-8 2 | from lxml import etree 3 | import requests 4 | import os 5 | import sys 6 | import MySQLdb 7 | import re 8 | import random 9 | import time 10 | reload(sys) 11 | sys.setdefaultencoding("utf-8") 12 | 13 | def qushuzi(str): 14 | num = '' 15 | if not str: 16 | return num 17 | else: 18 | for sr in str: 19 | if sr.isdigit(): 20 | num = num+sr 21 | return num 22 | #用来获取网页源代码 23 | def getsource(url): 24 | headlist = [{'User-Agent': 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.'}, 25 | {'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50'}, 26 | {'User-Agent': 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0;'}, 27 | {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36'}, 28 | {'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)'}, 29 | {'User-Agent': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)'}, 30 | {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1'}, 31 | {'User-Agent': 'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11'}, 32 | {'User-Agent': 'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11'}, 33 | {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11'}, 34 | {'User-Agent': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)'}, 35 | {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36'}] 36 | num = len(headlist)-1 37 | html = requests.get(url, headers=headlist[random.randint(0, num)]) 38 | html.encoding = 'utf-8' 39 | return html.text 40 | #获取信息块 41 | def getcityblock(source): 42 | blocks = re.findall('()',source,re.S) 48 | elif ci == 1: 49 | blocks = re.findall('(
.?
)', source, re.S) 50 | elif ci == 2: 51 | blocks = re.findall('(
.?
)',source,re.S) 52 | else: 53 | blocks = [] 54 | return blocks 55 | # 对匹配为空进行处理 56 | def pankong(poi_xx): 57 | if len(poi_xx)==0: 58 | poi_xx = '' 59 | else: 60 | poi_xx = poi_xx[0] 61 | return poi_xx 62 | def tiaoguo(selector): 63 | guolvcibiao = ['市'] 64 | city = selector.xpath('//li[@class="cityName tabItem dropDown hvrIE6"]/span/span/text()') 65 | city = pankong(city) 66 | # 对应的城市id 67 | sqli = "select region_id from " + db + ".map_region" + " where region_ch_name = " + "'%s'" % (city) 68 | num_result = cur.execute(sqli) 69 | if not num_result: 70 | region_id = '' 71 | else: 72 | region_id = cur.fetchmany(1) 73 | region_id = region_id[0][0] 74 | # 过滤已经抓取完成的城市 75 | if region_id in hulvcities: 76 | return city,region_id,True 77 | else: 78 | return city,region_id,False 79 | def yemianjiexi(block,ci): 80 | selector = etree.HTML(block) 81 | 82 | if ci == 0: 83 | # 详情页url 84 | 85 | xiangqingurl = selector.xpath('//div[@class="listing_title"]/a/@href')[0] 86 | xiangqingurl = starturl + xiangqingurl 87 | print '详情页', xiangqingurl 88 | xiangqinghtml = getsource(xiangqingurl) 89 | time.sleep(tingliu) 90 | xiangqingselector = etree.HTML(xiangqinghtml) 91 | 92 | # 名称 93 | name0 = selector.xpath('//div[@class="listing_title"]/a/text()') 94 | name0 = pankong(name0) 95 | 96 | name1 = xiangqingselector.xpath('//span[@class="altHead"]/text()') 97 | name1 = pankong(name1) 98 | shouzimu0 = name0[0].encode('utf-8') 99 | if not name1: 100 | if shouzimu0.isalpha(): 101 | poi_en_name = name0 102 | poi_ch_name = '' 103 | else: 104 | poi_en_name = '' 105 | poi_ch_name = name0 106 | else: 107 | if shouzimu0.isalpha(): 108 | poi_en_name = name0 109 | poi_ch_name = name1 110 | else: 111 | poi_en_name = name1 112 | poi_ch_name = name0 113 | 114 | poi_loc_name = poi_en_name 115 | 116 | comments_count = xiangqingselector.xpath('//a[@class="more taLnk"]/@content') 117 | comments_count = pankong(comments_count) 118 | poi_rank = xiangqingselector.xpath('//b[@class="rank"]/text()') 119 | poi_rank = pankong(poi_rank) 120 | if poi_rank: 121 | poi_rank = qushuzi(poi_rank) 122 | poi_telephone = xiangqingselector.xpath('//div[@class="fl"]') 123 | poi_telephone = pankong(poi_telephone) 124 | # 地址 125 | street_address = xiangqingselector.xpath('//span[@class="street-address"]/text()') 126 | street_address = pankong(street_address) 127 | 128 | extended_address = xiangqingselector.xpath('//span[@class="extended-address"]/text()') 129 | extended_address = pankong(extended_address) 130 | 131 | addresslocality = xiangqingselector.xpath('//span[@property="addressLocality"]/text()') 132 | addresslocality = pankong(addresslocality) 133 | 134 | addressregion = xiangqingselector.xpath('//span[@property="addressRegion"]/text()') 135 | addressregion = pankong(addressregion) 136 | 137 | postcode = xiangqingselector.xpath('//span[@property="postalCode"]/text()') 138 | postcode = pankong(postcode) 139 | 140 | elif ci == 1: 141 | # 详情页url 142 | xiangqingurl = selector.xpath('//div[@class="property_title"]/a/@href')[0] 143 | xiangqingurl = starturl + xiangqingurl 144 | print '详情页', xiangqingurl 145 | xiangqinghtml = getsource(xiangqingurl) 146 | time.sleep(tingliu) 147 | xiangqingselector = etree.HTML(xiangqinghtml) 148 | # 名称 149 | name0 = selector.xpath('//a[@target="_blank"]/text()') 150 | name0 = pankong(name0).strip() 151 | if not name0: 152 | name0 = xiangqingselector.xpath('//h1[@id="HEADING"]/text()') 153 | name0 = pankong(name0).strip() 154 | name1 = xiangqingselector.xpath('//span[@class="altHead"]/text()') 155 | name1 = pankong(name1) 156 | shouzimu0 = name0[0].encode('utf-8') 157 | if not name1: 158 | if shouzimu0.isalpha(): 159 | poi_en_name = name0 160 | poi_ch_name = '' 161 | else: 162 | poi_en_name = '' 163 | poi_ch_name = name0 164 | else: 165 | if shouzimu0.isalpha(): 166 | poi_en_name = name0 167 | poi_ch_name = name1 168 | else: 169 | poi_en_name = name1 170 | poi_ch_name = name0 171 | poi_loc_name = poi_en_name 172 | 173 | comments_count = xiangqingselector.xpath('//a[@class="more"]/@content') 174 | comments_count = pankong(comments_count) 175 | poi_rank = xiangqingselector.xpath('//b[@class="rank_text wrap"]/span/text()') 176 | poi_rank = pankong(poi_rank) 177 | if poi_rank: 178 | poi_rank = qushuzi(poi_rank) 179 | poi_telephone = xiangqingselector.xpath('//div[@class="phoneNumber"]/text()') 180 | poi_telephone = pankong(poi_telephone) 181 | if poi_telephone: 182 | for i,ch in enumerate(poi_telephone): 183 | if ch.isdigit(): 184 | tempi = i 185 | break 186 | poi_telephone = poi_telephone[tempi:] 187 | 188 | # 地址 189 | street_address = xiangqingselector.xpath('//span[@class="street-address"]/text()') 190 | street_address = pankong(street_address) 191 | 192 | extended_address = xiangqingselector.xpath('//span[@class="extended-address"]/text()') 193 | extended_address = pankong(extended_address) 194 | 195 | addresslocality = xiangqingselector.xpath('//span[@property="addressLocality"]/text()') 196 | addresslocality = pankong(addresslocality) 197 | 198 | addressregion = xiangqingselector.xpath('//span[@property="addressRegion"]/text()') 199 | addressregion = pankong(addressregion) 200 | 201 | postcode = xiangqingselector.xpath('//span[@property="postalCode"]/text()') 202 | postcode = pankong(postcode) 203 | 204 | elif ci == 2: 205 | # 详情页url 206 | xiangqingurl = selector.xpath('//a[@class="property_title"]/@href')[0] 207 | xiangqingurl = starturl + xiangqingurl 208 | print '详情页', xiangqingurl 209 | xiangqinghtml = getsource(xiangqingurl) 210 | time.sleep(tingliu) 211 | xiangqingselector = etree.HTML(xiangqinghtml) 212 | 213 | # 名称 214 | poi_ch_name = '' 215 | poi_en_name = selector.xpath('//a[@target="_blank"]/text()') 216 | poi_en_name = pankong(poi_en_name) 217 | poi_en_name = poi_en_name.strip() 218 | poi_loc_name = poi_en_name 219 | 220 | comments_count = xiangqingselector.xpath('//a[@class="more"]/@content') 221 | comments_count = pankong(comments_count) 222 | poi_rank = xiangqingselector.xpath('//b[@class="rank_text wrap"]/span/text()') 223 | poi_rank = pankong(poi_rank) 224 | if poi_rank: 225 | poi_rank = qushuzi(poi_rank) 226 | poi_telephone = xiangqingselector.xpath('//div[@class="fl phoneNumber"]/text()') 227 | poi_telephone = pankong(poi_telephone) 228 | 229 | # 地址 230 | street_address = xiangqingselector.xpath('//span[@class="street-address"]/text()') 231 | street_address = pankong(street_address) 232 | 233 | extended_address = xiangqingselector.xpath('//span[@class="extended-address"]/text()') 234 | extended_address = pankong(extended_address) 235 | 236 | addresslocality = xiangqingselector.xpath('//span[@property="addressLocality"]/text()') 237 | addresslocality = pankong(addresslocality) 238 | 239 | addressregion = xiangqingselector.xpath('//span[@property="addressRegion"]/text()') 240 | addressregion = pankong(addressregion) 241 | 242 | postcode = xiangqingselector.xpath('//span[@property="postalCode"]/text()') 243 | postcode = pankong(postcode) 244 | 245 | if extended_address: 246 | poi_address = street_address + ',' + extended_address+','+addresslocality + ',' + addressregion + postcode 247 | else: 248 | poi_address = street_address + ',' + addresslocality + ',' + addressregion + postcode 249 | 250 | if not poi_telephone: 251 | poi_telephone = '' 252 | else: 253 | pass 254 | return xiangqingselector,poi_ch_name,poi_en_name,poi_loc_name,poi_telephone,poi_address,poi_rank,comments_count 255 | 256 | if name == 'main': 257 | 258 | starturl = 'http://www.tripadvisor.cn' 259 | tingliu = 3 260 | db = 'map' 261 | # 数据表 262 | tb = 'map_poi4' 263 | # 希望跳过抓取的城市 264 | hulvcities = range(1,2) 265 | # 来源 266 | source = 'tripadvisor' 267 | # 连接数据库 268 | try: 269 | conn = MySQLdb.connect(host='127.0.0.1', user='root', passwd='123456', port=3306, charset='utf8') 270 | # conn = MySQLdb.connect(host='172.22.185.78', user='root', passwd='123456', port=3306, charset='utf8') 271 | cur = conn.cursor() 272 | cur.execute('set interactive_timeout=963600') 273 | conn.select_db(db) 274 | except MySQLdb.Error, e: 275 | print "Mysql Error %d: %s" % (e.args[0], e.args[1]) 276 | 277 | for i in range(4,51): 278 | url = 'http://www.tripadvisor.cn/TourismChildrenAjax?geo=191&offset=%d&desktop=true'% (i) 279 | print '城市列表页：',url 280 | html = getsource(url) 281 | time.sleep(tingliu) 282 | blocks = getcityblock(html) 283 | for j, block in enumerate(blocks): 284 | selector1 = etree.HTML(block) 285 | 286 | # 城市主页 287 | sub_url = selector1.xpath('//a/@href')[0] 288 | zhuye_url = starturl+sub_url 289 | print '城市主页：',zhuye_url 290 | zhuye_html = getsource(zhuye_url) 291 | time.sleep(tingliu) 292 | zhuye_selector = etree.HTML(zhuye_html) 293 | poi_region,region_id, tiaoma = tiaoguo(zhuye_selector) 294 | if tiaoma: 295 | pass 296 | else: 297 | # hotel_url = zhuye_selector.xpath('//li[@class="hotels twoLines"]/a/@href')[0] 298 | # jingdian_url = zhuye_selector.xpath('//li[@class="attractions twoLines"]/a/@href')[0] 299 | canting_url = zhuye_selector.xpath('//li[@class="restaurants twoLines"]/a/@href')[0] 300 | # city_urls = [hotel_url,jingdian_url,canting_url] 301 | city_urls = [canting_url] 302 | for ci,cityurl in enumerate(city_urls): 303 | city_url = starturl+cityurl 304 | print '行业url',city_url 305 | if ci ==0: 306 | tag_id = 2 307 | if ci == 1: 308 | tag_id = 3 309 | if ci == 2: 310 | tag_id = 1 311 | ci = 2 312 | dizhiyuanshu = city_url.split('-') 313 | sub_html = getsource(city_url) 314 | time.sleep(tingliu) 315 | sub_selector = etree.HTML(sub_html) 316 | # 爬取的页数 317 | poiyeshu = sub_selector.xpath('//a[@class="pageNum last taLnk"]/@data-page-number') 318 | if len(poiyeshu) == 0: 319 | poiyeshu = 1 320 | else: 321 | poiyeshu = poiyeshu[0] 322 | 323 | for ye in range(1,int(poiyeshu)+1): 324 | index = 30(ye-1) 325 | if index != 0: 326 | if ci == 0: 327 | dangqianurl = dizhiyuanshu[0]+'-'+dizhiyuanshu[1]+'-'+'oa'+str(index)+'-'+dizhiyuanshu[2]+'-'+dizhiyuanshu[3] 328 | if ci == 1: 329 | dangqianhtml = dizhiyuanshu[0]+'-'+dizhiyuanshu[1]+'-'+dizhiyuanshu[2]+'-'+'oa'+str(index)+'-'+dizhiyuanshu[3] 330 | if ci == 2: 331 | dangqianhtml = dizhiyuanshu[0]+'-'+dizhiyuanshu[1]+'-'+'oa'+str(index)+'-'+dizhiyuanshu[2] 332 | else: 333 | dangqianurl = city_url 334 | print '列表页url',dangqianurl 335 | dangqianhtml = getsource(dangqianurl) 336 | time.sleep(tingliu) 337 | poiblocks = getpoiblock(dangqianhtml,ci) 338 | print '当前页的poi数',len(poiblocks) 339 | for poiblock in poiblocks: 340 | 341 | xiangqingselector,poi_ch_name, poi_en_name, poi_loc_name, poi_telephone, poi_address, poi_rank,comments_count = yemianjiexi(poiblock,ci) 342 | 343 | sqli = "INSERT INTO " + db + "." + tb + "(poi_ch_name,poi_en_name,poi_loc_name,poi_region_id,poi_region,poi_tag_id,poi_rank,poi_address,poi_telephone,comments_count,source_website)" + " VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" 344 | 345 | # 判断数据库是否已经存在城市数据，决定是插入数据还是更新数据。 346 | sqli1 = "select * from " + db + "." + tb + " where poi_ch_name = " + "'%s'" % (poi_ch_name) 347 | sqli2 = "select * from " + db + "." + tb + " where poi_en_name = " + "'%s'" % (poi_en_name) 348 | 349 | try: 350 | r1 = cur.execute(sqli1) 351 | r2 = cur.execute(sqli2) 352 | except: 353 | pass 354 | 355 | if not poi_ch_name : 356 | r1 = 0 357 | if not poi_en_name : 358 | r2 = 0 359 | print '中文：',poi_ch_name,'英文：',poi_en_name 360 | print '查询结果：','中文',r1,'英文',r2 361 | 362 | if r1 or r2: 363 | print '已经存在记录，迭代数据 ... ...' 364 | pass 365 | else: 366 | print '插入新POI... ...' 367 | print '中文：' + poi_ch_name,'英文：' + poi_en_name, '本地语言名称:' + poi_en_name, '城市id:' + str(region_id),'城市:'+poi_region, '类型：' + str(tag_id), '评论数:' + str(comments_count), '排名:' + str(poi_rank), '地址:' + poi_address, '电话:' + poi_telephone 368 | cur.execute(sqli,(poi_ch_name, poi_en_name, poi_loc_name, region_id,poi_region, tag_id,poi_rank,poi_address,poi_telephone,comments_count,source)) 369 | conn.commit() 370 | print '------------------------------------------------' 371 | cur.close() 372 | conn.close() 373 | print '------------finished--------------' 374 | -------------------------------------------------------------------------------- /tst.py: -------------------------------------------------------------------------------- 1 | #coding=utf-8 2 | import random 3 | import re 4 | import sys 5 | import urllib2 6 | 7 | # import MySQLdb 8 | import requests 9 | # from lxml import etree 10 | import socket 11 | # import proxyIP 12 | 13 | reload(sys) 14 | sys.setdefaultencoding("utf-8") 15 | 16 | #getsource用来获取网页源代码 17 | def getsource(url): 18 | headers = { 19 | 'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36', 20 | # 详情页 cookie 21 | # 'Cookie':'JSESSIONID=EE4F457A78D5ADB9B416BA9BD1260691; BIGipServerESI-WEB-APP-8080=748214282.36895.0000; PSSID="A1-wS2nsbT8yAaiz92yMmlx2BpnjzUUjx2BdSTY-18x2dyfQZ1yAMOOmix2BC4bxxs5fRAx3Dx3DS0YEN7J3xxOrhIYxxYDlHQyQx3Dx3D-9vvmzcndpRgQCGPd1c2qPQx3Dx3D-wx2BJQh9GKVmtdJw3700KssQx3Dx3D"; CUSTOMER="WUHAN UNIV"; E_GROUP_NAME="IC2 Platform"; esi.isLocalStorageCleared=true; Type=documents; Show=null; esi.docsActiveTab=0; esi.Type=; esi.FilterValue=; esi.GroupBy=; esi.FilterBy=; esi.Show=; esi.authorsList=; esi.frontList=; esi.fieldsList=; esi.instList=; esi.journalList=; esi.terriList=; esi.titleList=', 22 | # 列表页 cookie 23 | 'Cookie' : 'Type=documents; Show=null; esi.docsActiveTab=0; esi.Type=; esi.FilterValue=; esi.GroupBy=; esi.FilterBy=; esi.Show=; esi.authorsList=; esi.frontList=; esi.fieldsList=; esi.instList=; esi.journalList=; esi.terriList=; esi.titleList=; clientId=CID1242b7d4b50d1e0ed3ac323a908e08bd; esi.isLocalStorageCleared=true; qq%5Flogin%5Fstate=54D44A36B7E6D2165B2090C42F8CB27E; JSESSIONID=811CB477BFE1C370AB4A99C711A0C0E1; _pk_ref.2.a2d0=%5B%22%22%2C%22%22%2C1480485842%2C%22http%3A%2F%2Fwww.lib.whu.edu.cn%2Fdc%2Furlto_proxy.asp%3Fid%3D638%26url%3Dhttp%3A%2F%2Firas.lib.whu.edu.cn%3A8080%2Fgo%3Fid%3DESI%26source_id%3DWHU03752%26u%3D%26title%3DESI%C2%A3%C2%A8Essential%20Science%20Indicators%C2%A3%C2%A9%22%5D; _pk_id.2.a2d0=b5d6c9744eade5ad.1480474056.2.1480485842.1480485842.; _pk_ses.2.a2d0=' 24 | } 25 | socket.setdefaulttimeout(60) 26 | 27 | html = requests.get(url,headers = headers) 28 | html.encoding = 'utf-8' 29 | return html.text 30 | 31 | if name == 'main': 32 | # 详情页 url 33 | # url = 'http://gateway.webofknowledge.com/gateway/Gateway.cgi?GWVersion=2&SrcAuth=TSMetrics&SrcApp=TSM_TEST&DestApp=WOS_CPL&DestLinkType=FullRecord&KeyUT=ISI:000262948300002' 34 | # 列表页 url 35 | url = 'http://iras.lib.whu.edu.cn:8080/rwt/ESI/https/MW3XTLUJN3SXT7DFPMYHI4DQNW3X85UTMW4YI3LTPMYGG55N/IndicatorsDataAction.action?_dc=1480468834254&type=documents&author=&researchField=CLINICAL%20MEDICINE&institution=&journal=&territory=&article_UT=&researchFront=&articleTitle=&docType=Top&year=&page=1&start=10&limit=10&sort=%5B%7B%22property%22%3A%22citations%22%2C%22direction%22%3A%22DESC%22%7D%5D' 36 | result = getsource(url) 37 | print result 38 | 39 | -------------------------------------------------------------------------------- /webofscience.py: -------------------------------------------------------------------------------- 1 | #coding=utf-8 2 | import random 3 | import re 4 | import sys 5 | import urllib2 6 | 7 | # import MySQLdb 8 | import requests 9 | # from lxml import etree 10 | import socket 11 | # import proxyIP 12 | 13 | reload(sys) 14 | sys.setdefaultencoding("utf-8") 15 | 16 | #getsource用来获取网页源代码 17 | def getsource(url): 18 | headers = { 19 | 'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36', 20 | # 详情页 cookie 21 | # 'Cookie':'JSESSIONID=EE4F457A78D5ADB9B416BA9BD1260691; BIGipServerESI-WEB-APP-8080=748214282.36895.0000; PSSID="A1-wS2nsbT8yAaiz92yMmlx2BpnjzUUjx2BdSTY-18x2dyfQZ1yAMOOmix2BC4bxxs5fRAx3Dx3DS0YEN7J3xxOrhIYxxYDlHQyQx3Dx3D-9vvmzcndpRgQCGPd1c2qPQx3Dx3D-wx2BJQh9GKVmtdJw3700KssQx3Dx3D"; CUSTOMER="WUHAN UNIV"; E_GROUP_NAME="IC2 Platform"; esi.isLocalStorageCleared=true; Type=documents; Show=null; esi.docsActiveTab=0; esi.Type=; esi.FilterValue=; esi.GroupBy=; esi.FilterBy=; esi.Show=; esi.authorsList=; esi.frontList=; esi.fieldsList=; esi.instList=; esi.journalList=; esi.terriList=; esi.titleList=', 22 | # 列表页 cookie 23 | 'Cookie' : 'Type=documents; Show=null; esi.docsActiveTab=0; esi.Type=; esi.FilterValue=; esi.GroupBy=; esi.FilterBy=; esi.Show=; esi.authorsList=; esi.frontList=; esi.fieldsList=; esi.instList=; esi.journalList=; esi.terriList=; esi.titleList=; clientId=CID1242b7d4b50d1e0ed3ac323a908e08bd; esi.isLocalStorageCleared=true; qq%5Flogin%5Fstate=54D44A36B7E6D2165B2090C42F8CB27E; JSESSIONID=811CB477BFE1C370AB4A99C711A0C0E1; _pk_ref.2.a2d0=%5B%22%22%2C%22%22%2C1480485842%2C%22http%3A%2F%2Fwww.lib.whu.edu.cn%2Fdc%2Furlto_proxy.asp%3Fid%3D638%26url%3Dhttp%3A%2F%2Firas.lib.whu.edu.cn%3A8080%2Fgo%3Fid%3DESI%26source_id%3DWHU03752%26u%3D%26title%3DESI%C2%A3%C2%A8Essential%20Science%20Indicators%C2%A3%C2%A9%22%5D; _pk_id.2.a2d0=b5d6c9744eade5ad.1480474056.2.1480485842.1480485842.; _pk_ses.2.a2d0=' 24 | } 25 | socket.setdefaulttimeout(60) 26 | 27 | html = requests.get(url,headers = headers) 28 | html.encoding = 'utf-8' 29 | return html.text 30 | 31 | if name == 'main': 32 | # 详情页 url 33 | # url = 'http://gateway.webofknowledge.com/gateway/Gateway.cgi?GWVersion=2&SrcAuth=TSMetrics&SrcApp=TSM_TEST&DestApp=WOS_CPL&DestLinkType=FullRecord&KeyUT=ISI:000262948300002' 34 | # 列表页 url 35 | url = 'http://iras.lib.whu.edu.cn:8080/rwt/ESI/https/MW3XTLUJN3SXT7DFPMYHI4DQNW3X85UTMW4YI3LTPMYGG55N/IndicatorsDataAction.action?_dc=1480468834254&type=documents&author=&researchField=CLINICAL%20MEDICINE&institution=&journal=&territory=&article_UT=&researchFront=&articleTitle=&docType=Top&year=&page=1&start=10&limit=10&sort=%5B%7B%22property%22%3A%22citations%22%2C%22direction%22%3A%22DESC%22%7D%5D' 36 | result = getsource(url) 37 | print result 38 | 39 | --------------------------------------------------------------------------------