{{title}}
3 |{{summary}}
4 |No results found
"); 64 | } 65 | 66 | if(jQuery){ 67 | /* 68 | * We currently only automatically hide bootstrap models. This 69 | * requires jQuery to work. 70 | */ 71 | jQuery('#mkdocs_search_modal a').click(function(){ 72 | jQuery('#mkdocs_search_modal').modal('hide'); 73 | }) 74 | } 75 | 76 | }; 77 | 78 | var search_input = document.getElementById('mkdocs-search-query'); 79 | 80 | var term = getSearchTerm(); 81 | if (term){ 82 | search_input.value = term; 83 | search(); 84 | } 85 | 86 | search_input.addEventListener("keyup", search); 87 | 88 | }); 89 | -------------------------------------------------------------------------------- /ugc.aggregator.esri/docs/ugc.aggregator/site/css/theme_extra.css: -------------------------------------------------------------------------------- 1 | /* 2 | * Tweak the overal size to better match RTD. 3 | */ 4 | body { 5 | font-size: 90%; 6 | } 7 | 8 | h3, h4, h5, h6 { 9 | color: #2980b9; 10 | font-weight: 300 11 | } 12 | 13 | /* 14 | * Sphinx doesn't have support for section dividers like we do in 15 | * MkDocs, this styles the section titles in the nav 16 | * 17 | * https://github.com/mkdocs/mkdocs/issues/175 18 | */ 19 | .wy-menu-vertical span { 20 | line-height: 18px; 21 | padding: 0.4045em 1.618em; 22 | display: block; 23 | position: relative; 24 | font-size: 90%; 25 | color: #838383; 26 | } 27 | 28 | .wy-menu-vertical .subnav a { 29 | padding: 0.4045em 2.427em; 30 | } 31 | 32 | /* 33 | * Long navigations run off the bottom of the screen as the nav 34 | * area doesn't scroll. 35 | * 36 | * https://github.com/mkdocs/mkdocs/pull/202 37 | */ 38 | .wy-nav-side { 39 | height: 100%; 40 | overflow-y: auto; 41 | } 42 | 43 | /* 44 | * readthedocs theme hides nav items when the window height is 45 | * too small to contain them. 46 | * 47 | * https://github.com/mkdocs/mkdocs/issues/#348 48 | */ 49 | .wy-menu-vertical ul { 50 | margin-bottom: 2em; 51 | } 52 | 53 | /* 54 | * Fix wrapping in the code highlighting 55 | * 56 | * https://github.com/mkdocs/mkdocs/issues/233 57 | */ 58 | code { 59 | white-space: pre; 60 | } 61 | 62 | /* 63 | * Wrap inline code samples otherwise they shoot of the side and 64 | * can't be read at all. 65 | * 66 | * https://github.com/mkdocs/mkdocs/issues/313 67 | */ 68 | p code { 69 | word-wrap: break-word; 70 | } 71 | 72 | /* 73 | * The CSS classes from highlight.js seem to clash with the 74 | * ReadTheDocs theme causing some code to be incorrectly made 75 | * bold and italic. 76 | * 77 | * https://github.com/mkdocs/mkdocs/issues/411 78 | */ 79 | code.cs, code.c { 80 | font-weight: inherit; 81 | font-style: inherit; 82 | } 83 | 84 | /* 85 | * Fix some issues with the theme and non-highlighted code 86 | * samples. Without and highlighting styles attached the 87 | * formatting is broken. 88 | * 89 | * https://github.com/mkdocs/mkdocs/issues/319 90 | */ 91 | .no-highlight { 92 | display: block; 93 | padding: 0.5em; 94 | color: #333; 95 | } 96 | 97 | 98 | /* 99 | * Additions specific to the search functionality provided by MkDocs 100 | */ 101 | 102 | #mkdocs-search-results article h3 103 | { 104 | margin-top: 23px; 105 | border-top: 1px solid #E1E4E5; 106 | padding-top: 24px; 107 | } 108 | 109 | #mkdocs-search-results article:first-child h3 { 110 | border-top: none; 111 | } 112 | 113 | #mkdocs-search-query{ 114 | width: 100%; 115 | border-radius: 50px; 116 | padding: 6px 12px; 117 | border-color: #D1D4D5; 118 | } 119 | 120 | .wy-menu-vertical li ul { 121 | display: inherit; 122 | } 123 | 124 | .wy-menu-vertical li ul.subnav ul.subnav{ 125 | padding-left: 1em; 126 | } 127 | -------------------------------------------------------------------------------- /ugc.hotel.web.esri/html/quality-testing.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 |
50 |
59 |
60 |
61 |
--------------------------------------------------------------------------------
/ugc.aggregator.esri/src/main/python/service/nlp/Bayes.py:
--------------------------------------------------------------------------------
1 | # -*- coding:utf-8 -*-
2 | __author__ = 'DreamCathcer，LiuYang'
3 |
4 |
5 | import sys
6 | import gzip
7 | import marshal
8 | from math import log, exp
9 |
10 | class Bayes(object):
11 |
12 | def __init__(self):
13 | self.d = {}
14 | self.total = 0
15 |
16 | def save(self, fname, iszip=True):
17 | d = {}
18 | d['total'] = self.total
19 | d['d'] = {}
20 | for k, v in self.d.items():
21 | d['d'][k] = v.__dict__
22 | if sys.version_info[0] == 3:
23 | fname = fname + '.3'
24 | if not iszip:
25 | marshal.dump(d, open(fname, 'wb'))
26 | else:
27 | f = gzip.open(fname, 'wb')
28 | f.write(marshal.dumps(d))
29 | f.close()
30 |
31 | def load(self, fname, iszip=True):
32 | if sys.version_info[0] == 3:
33 | fname = fname + '.3'
34 | if not iszip:
35 | d = marshal.load(open(fname, 'rb'))
36 | else:
37 | try:
38 | f = gzip.open(fname, 'rb')
39 | d = marshal.loads(f.read())
40 | except IOError:
41 | f = open(fname, 'rb')
42 | d = marshal.loads(f.read())
43 | f.close()
44 | self.total = d['total']
45 | self.d = {}
46 | for k, v in d['d'].items():
47 | self.d[k] = AddOneProb()
48 | self.d[k].__dict__ = v
49 |
50 | def train(self, data):
51 | for d in data:
52 | c = d[1]
53 | if c not in self.d:
54 | self.d[c] = AddOneProb()
55 | for word in d[0]:
56 | self.d[c].add(word, 1)
57 | self.total = sum(map(lambda x: self.d[x].getsum(), self.d.keys()))
58 |
59 | def classify(self, x):
60 | tmp = {}
61 | for k in self.d:
62 | tmp[k] = log(self.d[k].getsum()) - log(self.total)
63 | for word in x:
64 | tmp[k] += log(self.d[k].freq(word))
65 | ret, prob = 0, 0
66 | for k in self.d:
67 | now = 0
68 | try:
69 | for otherk in self.d:
70 | now += exp(tmp[otherk]-tmp[k])
71 | now = 1/now
72 | except OverflowError:
73 | now = 0
74 | if now > prob:
75 | ret, prob = k, now
76 | return (ret, prob)
77 |
78 |
79 |
80 |
81 | class BaseProb(object):
82 |
83 | def __init__(self):
84 | self.d = {}
85 | self.total = 0.0
86 | self.none = 0
87 |
88 | def exists(self, key):
89 | return key in self.d
90 |
91 | def getsum(self):
92 | return self.total
93 |
94 | def get(self, key):
95 | if not self.exists(key):
96 | return False, self.none
97 | return True, self.d[key]
98 |
99 | def freq(self, key):
100 | return float(self.get(key)[1])/self.total
101 |
102 | def samples(self):
103 | return self.d.keys()
104 |
105 | class AddOneProb(BaseProb):
106 |
107 | def __init__(self):
108 | self.d = {}
109 | self.total = 0.0
110 | self.none = 1
111 |
112 | def add(self, key, value):
113 | self.total += value
114 | if not self.exists(key):
115 | self.d[key] = 1
116 | self.total += 1
117 | self.d[key] += value
--------------------------------------------------------------------------------
/ugc.aggregator.esri/src/main/scripts/Hotel/HotelSentimentProcessor.py:
--------------------------------------------------------------------------------
1 | # -*- coding:utf-8 -*-
2 | import re
3 |
4 | __author__ = 'DreamCathcer'
5 |
6 |
7 | import traceback
8 | import json
9 | from thulac import thulac
10 |
11 | from setting import local_hotel_setting
12 | from service.nlp.HotelNLP import HotelNLP
13 | from dao.hotel.TuniuDao import TuniuDAO
14 | from dao.hotel.HotelDAO import HotelDAO
15 |
16 | dao_setting = local_hotel_setting
17 |
18 |
19 | class HotelSentimentProcessor(object):
20 |
21 | def __init__(self):
22 | self.hotel_dao = HotelDAO(dao_setting["host"], dao_setting["db"], dao_setting["user"], dao_setting["password"])
23 | self.dao = TuniuDAO(dao_setting["host"], dao_setting["db"], dao_setting["user"], dao_setting["password"])
24 |
25 | def set_sentiment_and_viewpoint(self):
26 | self.hotelnlp = HotelNLP()
27 | self.thu = thulac("")
28 | comm_list = self.hotel_dao.get_remarks()
29 | print len(comm_list)
30 | sentiment_comm_list = []
31 | i = 0
32 | for comm in comm_list:
33 | if comm[8] is None or comm[9] is None:
34 | sentiment_value = None
35 | viewpoint = None
36 | remark = re.sub(u"\@",u"",comm[2])
37 | try:
38 | sentiment_value = self.hotelnlp.sentiment(remark.encode("utf-8"))
39 | sentiment_value = round(sentiment_value*1000)/1000
40 | print sentiment_value
41 | except:
42 | print comm[2]
43 | traceback.print_exc()
44 | try:
45 | viewpoint = self.hotelnlp.viewpoint(remark.encode("utf-8"),decoding="utf-8")
46 | viewpoint = json.dumps(viewpoint, ensure_ascii=False)
47 | except:
48 | print remark
49 | traceback.print_exc()
50 | comm = {"guid":comm[0], "senti_value":sentiment_value, "viewpoint":viewpoint}
51 | sentiment_comm_list.append(comm)
52 | if len(sentiment_comm_list)==10000:
53 | i+=1
54 | print "update %d time"%i
55 | self.hotel_dao.update_remarks(sentiment_comm_list)
56 | sentiment_comm_list = []
57 |
58 |
59 | def count_word_frq(self):
60 | self.thu = thulac("-input cs.txt")
61 | comm_list = self.hotel_dao.get_remarks()
62 | sentiment_comm_list = []
63 | i = 0
64 | for comm in comm_list:
65 | a_dict = {}
66 | try:
67 | cut_comm = map(lambda x: x.split("_"), self.thu.cut(comm[2].encode("utf-8")))
68 | except:
69 | cut_comm = []
70 | print comm[2]
71 | traceback.print_exc()
72 | for word in cut_comm:
73 | if word[1].decode("utf-8") == "a":
74 | if word[0].decode("utf-8") not in a_dict:
75 | a_dict[word[0].decode("utf-8")] = 1
76 | else:
77 | a_dict[word[0].decode("utf-8")] += 1
78 | comm = {"guid":comm[0], "word_freq":json.dumps(a_dict, ensure_ascii=False)}
79 | sentiment_comm_list.append(comm)
80 | if len(sentiment_comm_list)==10000:
81 | i+=1
82 | print "update %d time"%i
83 | self.hotel_dao.update_hotel_comm_word_freq(sentiment_comm_list)
84 | sentiment_comm_list = []
85 |
86 | if __name__ == "__main__":
87 | HotelSentimentProcessor().count_word_frq()
--------------------------------------------------------------------------------
/ugc.aggregator.esri/src/main/python/dao/weibo/WeiboDAO.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | __author__ = 'LiuYang'
3 |
4 | import MySQLdb
5 | import uuid
6 | import traceback
7 |
8 | from dao.SuperDAO import SuperDAO
9 |
10 |
11 | class WeiboDAO(SuperDAO):
12 |
13 | def __init__(self, host, db, user, password):
14 | SuperDAO.__init__(self, host, db, user, password)
15 |
16 | # 存储微博id
17 | def saveWeiboID(self,weiboIDSet,userID,pageNum):
18 | db = MySQLdb.connect(self.host,self.user,self.password,self.db,charset='utf8')
19 | cursor = db.cursor()
20 | for weiboID in weiboIDSet:
21 | cursor.execute("insert into weibo_id(guid,userID,weiboID,pageNum)values(%s,%s,%s,%s)" ,(uuid.uuid1(),userID,weiboID,pageNum))
22 | db.commit()
23 | cursor.close()
24 | db.close()
25 |
26 | # 存储微博评论
27 | def saveWeiboComment(self,items):
28 | db = MySQLdb.connect(self.host,self.user,self.password,self.db,charset='utf8')
29 | cursor = db.cursor()
30 | for item in items:
31 | try:
32 | cursor.execute("insert into weibo_comment(guid,userID,weiboID,pageNum,commPeople,commentText,commentTime,crawlTime,likeNum)values(%s,%s,%s,%s,%s,%s,%s,%s,%s)" ,(uuid.uuid1(),item["userID"],item["weiboID"],item["pageNum"],item["commPeople"],item["commentText"],item["commentTime"],item["crawlTime"],item["likeNum"]))
33 | except:
34 | continue
35 | db.commit()
36 | cursor.close()
37 | db.close()
38 |
39 | # 获取api账号数量
40 | def countweiboaccountnumber(self):
41 | db = MySQLdb.connect(self.host,self.user,self.password,self.db,charset='utf8')
42 | cursor = db.cursor()
43 | cursor.execute("select count(*) from api_account")
44 | data = cursor.fetchone()
45 | cursor.close()
46 | db.close()
47 | return data
48 |
49 | #从mysql中获取微博账号
50 | def get_weibo_accounts(self):
51 | weibo_accounts = None
52 | db = MySQLdb.connect(self.host,self.user,self.password,self.db,charset='utf8')
53 | cursor = db.cursor()
54 | try:
55 | cursor.execute("select * from api_account")
56 | weibo_accounts = cursor.fetchall()
57 | except:
58 | print traceback.print_exc()
59 | db.commit()
60 | cursor.close()
61 | db.close()
62 | return weibo_accounts
63 |
64 | def saveWeibo_ByAPI(self,weiboid,text,lat,lon,title,userid,location,userdecription,gender,created_at,fax,localcity,formatted):
65 | db = MySQLdb.connect(self.host,self.user,self.password,self.db,charset='utf8')
66 | cursor = db.cursor()
67 | cursor.execute("insert into weibo_content(weiboid,text,lat,lon,title,userid,location,userdescription,gender,created_at,fax,locality,formatted)values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
68 | ,(weiboid,text,lat,lon,title,userid,location,userdecription,gender,created_at,fax,localcity,formatted))
69 | db.commit()
70 | cursor.close()
71 | db.close()
72 |
73 | '''
74 | 获取地址
75 | '''
76 | def get_location(self, city):
77 | db = MySQLdb.connect(self.host, self.user, self.password, self.db, charset='utf8')
78 | cursor = db.cursor()
79 | location = None
80 | try:
81 | cursor.execute("select * from city_location where city='%s'"%city)
82 | location = cursor.fetchone()
83 | except Exception, e:
84 | print e
85 | db.commit()
86 | cursor.close()
87 | db.close()
88 | return location
89 |
90 | def save_location(self, location):
91 | self.save_record("city_location", location)
--------------------------------------------------------------------------------
/ugc.hotel.web.esri/css/messages.css:
--------------------------------------------------------------------------------
1 |
2 | /*--------------------���------------------------*/
3 | .map-div {background-color:white;width:78%;height:100%;position:fixed;top:53px;left:0;}
4 | nav {z-index:2; background-color:#7A6868;width:59px;height:180px;right:8px;position:absolute;margin:140px 10px;cursor:pointer}
5 | .floor-info{right:25px;margin-top:75px;position:absolute;font-size:30px;font-weight:bold;z-index:2;}
6 | .query-info{right:20px;margin-top:20px;position:absolute;z-index:2;}
7 | .legend-info{z-index:2;position:absolute;bottom:70px;left:10px;padding:10px;border:2px solid #F1C40F;/*border-radius:4px*/}
8 | .button-one {height:60px;background-color:#EBEBEC}
9 | .button-two {height:60px;background-color:#C4C4C7}
10 | .button-three {height:60px;background-color:#D7D7DC}
11 | .button-img {width:22px;height:15px;margin:20px 18px}
12 |
13 | /*--------------------ͼ��------------------------*/
14 | .legend-red{line-height:50px;height:50px;width:75px;border-radius:4px;background-color:#F3859B;margin-bottom:3px;color:#FFF;text-align:center;}
15 | .legend-yellow{line-height:50px;height:50px;width:75px;border-radius:4px;background-color:#FCDE89;margin-bottom:3px;color:#FFF;text-align:center}
16 | .legend-green{line-height:50px;height:50px;width:75px;border-radius:4px;background-color:#BBFF8B;margin-bottom:3px;color:#FFF;text-align:center}
17 | .legend-gray{line-height:50px;height:50px;width:75px;border-radius:4px;background-color:#D7D7DC;margin-bottom:3px;color:#FFF;text-align:center}
18 |
19 | /*--------------------�����б���Ϣ------------------------*/
20 | .button-danger {background-color:red;color:#fff;font-family:'Microsoft YaHei';font-size:14px;position:relative;right:25px;padding:5px 10px;float:right;margin-top:5px;display:block}
21 | .button-require {background-color:#D7D7DC;color:#fff;font-family:'Microsoft YaHei';font-size:14px;position:relative;right:25px;padding:5px 10px;float:right;margin-top:5px;display:block}
22 | .button-warning {background-color:#ffd800;color:#fff;font-family:'Microsoft YaHei';font-size:14px;position:relative;right:25px;padding:5px 10px;float:right;margin-top:5px;display:block}
23 | .button-default {background-color:#29C1BF;color:#fff;font-family:'Microsoft YaHei';font-size:14px;position:relative;right:25px;padding:5px 10px;float:right;margin-top:5px;display:block}
24 |
25 | /*--------------------��ֵ����------------------------*/
26 | .sliderdiv{z-index:3;position:absolute; right:50px;bottom:50px;}
27 |
28 | /*--------------------�Ҳ�------------------------*/
29 | .right {background-color:#4A5A69;width:22%;height:100%;top:53px;left:78%;position:fixed;}
30 | .right-top {background-color:white;width:100%;height:95%;border-left:1px solid #B6CFD3}
31 | .right-top-title{background-color:#B6CFD3;text-align:center;}
32 | .right-mess {overflow:auto;height:90%;}
33 | .right-list-div{border-bottom:1px solid #808080;height:45px ;}
34 | .right-list-div:hover{cursor:pointer;background-color:gray;border-color:#fff;color:white}
35 | .list-roomnum {top:7px;left:25px;position:relative;padding:10px 0px;}
36 | .list-roomnum:hover{cursor:pointer;background-color:gray;border-color:#fff;color:white}
37 | .room-button {background: transparent;border-bottom:1px solid #ffd800;color:#1ABC9C;padding:5px 20px;font-family:'Microsoft YaHei';position:relative;top:5px;display:block}
38 | /*.room-button:hover {cursor:pointer;background-color:white;border-color:#fff;color:#ffd800}*/
39 |
40 | /*-----------------------�Ҳ��·�---------------------------*/
41 | .right-bottom {background-color:#C4CDE4;width:100%;height:40%;border-left:2px solid #B6CFD3}
42 |
43 | /*--------------------ģ̬����------------------------*/
44 | #remarkbody {overflow:auto;height:400px;}
45 | /*--------------------��ͼ------------------------*/
46 | #floormap {padding: 0;margin: 0;height: 100%;width: 100%;}
--------------------------------------------------------------------------------
/ugc.aggregator.esri/src/main/python/service/map/baidu/APIService.py:
--------------------------------------------------------------------------------
1 | # -*- coding:utf-8 -*-
2 | __author__ = 'DreamCathcer'
3 |
4 | import logging
5 |
6 | from util.http.UniversalSDK import APIClient
7 |
8 | logger = logging.getLogger('ugc')
9 |
10 |
11 | class BaiduMapAPIService(object):
12 | def __init__(self, ak):
13 | self.baiduClient = APIClient("http://api.map.baidu.com")
14 | self.__ak = ak
15 |
16 | '''
17 | 正向地理编码geocoding
18 | 文档：http://lbsyun.baidu.com/index.php?title=webapi/guide/webservice-geocoding
19 | Rest地址：http://api.map.baidu.com/geocoder/v2/
20 | 类型：get
21 | '''
22 | def doGeocoding(self, addressText,city=None):
23 | if city==None:
24 | data = self.baiduClient.geocoder.v2.addtrail("/").get(ak=self.__ak, output="json", address=addressText)
25 | else:
26 | data = self.baiduClient.geocoder.v2.addtrail("/").get(ak=self.__ak, output="json", address=addressText, city=city)
27 | return data
28 |
29 | def reverseGeocodingBatch(self, locationList):
30 | '''
31 | 逆向地理编码批量处理入口
32 | 地址：http://api.map.baidu.com/geocoder/v2/
33 | 类型：get
34 | '''
35 | respList = self.reverseGeocodingBatchHandler(locationList=locationList, respList=[], errorList=[])
36 | return respList
37 |
38 | def reverseGeocodingBatchHandler(self, locationList, respList, errorList):
39 | '''
40 | 逆向地理编码批量处理，处理timeout
41 | 地址：http://api.map.baidu.com/geocoder/v2/
42 | 类型：get
43 | '''
44 | for i in range(0, len(locationList), 1):
45 | location = locationList[i]
46 | resp = self.reverseGeocoding(location=location)
47 | if resp is not None and resp["status"] == 0:
48 | respList.append(resp)
49 | else:
50 | logging.debug("current token: %s " % self.__ak)
51 | logging.debug( resp)
52 | logging.debug("at point:%s",str(location))
53 | if len(errorList) > 0:
54 | # http请求异常重新处理
55 | logging.debug("http exception ,rehandle size : " + str(len(errorList)))
56 | self.reverseGeocodingBatchHandler(locationList=errorList, respList=respList, errorList=[])
57 | return respList
58 |
59 | def reverseGeocoding(self, location, coordtype='bd09ll', output="json", pois='0'):
60 | '''
61 | 逆向地理编码request
62 | 地址：http://api.map.baidu.com/geocoder/v2/
63 | 类型：get
64 | coordtype，默认bd09ll，坐标的类型，目前支持的坐标类型包括：bd09ll（百度经纬度坐标）、bd09mc（百度米制坐标）、gcj02ll（国测局经纬度坐标）、wgs84ll（ GPS经纬度）
65 | '''
66 | resp = self.baiduClient.geocoder.v2.addtrail("/").get(ak=self.__ak, output="json",pois=2, location=location)
67 | return resp
68 |
69 | def placeSearchBatch(self, query, bounds, pageNumber="0"):
70 | '''
71 | Place地名批量查询，处理timeout
72 | 地址：http://api.map.baidu.com/geocoder/v2/
73 | 类型：get
74 | '''
75 | resp = self.placeSearch(query=query, bounds=bounds, pageNumber=pageNumber)
76 | if resp is None:
77 | logging.debug("http exception ,rehandle...")
78 | reHandleResp = self.placeSearch(query=query, bounds=bounds, pageNumber=pageNumber)
79 | while reHandleResp is None:
80 | reHandleResp = self.placeSearch(query=query, bounds=bounds, pageNumber=pageNumber)
81 | return reHandleResp
82 | else:
83 | # TODO 为什么返回None
84 | return resp
85 |
86 | # coord_type(坐标类型)，1（wgs84ll），2（gcj02ll），3（bd09ll），4（bd09mc）
87 | def placeSearch(self, query, bounds, output="json", pageSize="20", pageNumber="0", coord_type="1", scope=2):
88 | '''
89 | Place地名
90 | 地址：http://api.map.baidu.com/place/v2/search
91 | 类型：get
92 | '''
93 | data = self.baiduClient.place.v2.search.get(ak=self.__ak, query=query, bounds=bounds, output=output,
94 | coord_type=coord_type, page_size=pageSize, page_num=pageNumber,
95 | scope=scope)
96 | return data
97 |
--------------------------------------------------------------------------------
/ugc.aggregator.esri/.idea/inspectionProfiles/Project_Default.xml:
--------------------------------------------------------------------------------
1 |
51 |
56 |
52 |
55 |
53 |
54 |
57 |
58 |
35 |
36 |
37 |
68 |
69 |
70 |
71 |
72 |
76 |
77 |
78 |
126 |
127 |
128 |
129 |
79 |
124 |
125 |
80 |
91 |
123 |
92 |
109 |
121 |
122 |
93 |
94 |
95 |
108 | Search Results
96 | 97 | 101 | 102 |
103 | Sorry, page not found.
104 |
105 |
106 |
107 |
130 |
131 |
132 |
133 |
134 |
135 |
136 |
137 |
138 |
139 |
--------------------------------------------------------------------------------
/ugc.hotel.web.esri/css/demo.css:
--------------------------------------------------------------------------------
1 | /* General Demo Style */
2 | @import url(http://fonts.googleapis.com/css?family=Lato:300,400,700);
3 |
4 | html { height: 100%; }
5 |
6 | body {
7 | font-family: 'Lato', Calibri, Arial, sans-serif;
8 | background: #ddd url(../images/bg.jpg) repeat top left;
9 | font-weight: 300;
10 | font-size: 15px;
11 | color: #333;
12 | -webkit-font-smoothing: antialiased;
13 | overflow-y: scroll;
14 | overflow-x: hidden;
15 | }
16 |
17 | a {
18 | color: #555;
19 | text-decoration: none;
20 | }
21 |
22 | .container {
23 | width: 100%;
24 | position: relative;
25 | }
26 |
27 | .clr {
28 | clear: both;
29 | padding: 0;
30 | height: 0;
31 | margin: 0;
32 | }
33 |
34 | .main {
35 | width: 90%;
36 | margin: 0 auto;
37 | position: relative;
38 | }
39 |
40 | .container > header {
41 | margin: 10px;
42 | padding: 20px 10px 10px 10px;
43 | position: relative;
44 | display: block;
45 | text-shadow: 1px 1px 1px rgba(0,0,0,0.2);
46 | text-align: center;
47 | }
48 |
49 | .container > header h1 {
50 | font-size: 30px;
51 | line-height: 38px;
52 | margin: 0;
53 | position: relative;
54 | font-weight: 300;
55 | color: #666;
56 | text-shadow: 0 1px 1px rgba(255,255,255,0.6);
57 | }
58 |
59 | .container > header h2 {
60 | font-size: 14px;
61 | font-weight: 300;
62 | margin: 0;
63 | padding: 15px 0 5px 0;
64 | color: #666;
65 | font-family: Cambria, Georgia, serif;
66 | font-style: italic;
67 | text-shadow: 0 1px 1px rgba(255,255,255,0.6);
68 | }
69 |
70 | /* Header Style */
71 | .codrops-top {
72 | line-height: 24px;
73 | font-size: 11px;
74 | background: #fff;
75 | background: rgba(255, 255, 255, 0.8);
76 | text-transform: uppercase;
77 | z-index: 9999;
78 | position: relative;
79 | font-family: Cambria, Georgia, serif;
80 | box-shadow: 1px 0px 2px rgba(0,0,0,0.2);
81 | }
82 |
83 | /* Clearfix hack by Nicolas Gallagher: http://nicolasgallagher.com/micro-clearfix-hack/ */
84 |
85 | .codrops-top:before,
86 | .codrops-top:after {
87 | content: " "; /* 1 */
88 | display: table; /* 2 */
89 | }
90 |
91 | .codrops-top:after {
92 | clear: both
93 | }
94 |
95 | .codrops-top a {
96 | padding: 0px 10px;
97 | letter-spacing: 1px;
98 | color: #333;
99 | display: inline-block;
100 | }
101 |
102 | .codrops-top a:hover {
103 | background: rgba(255,255,255,0.6);
104 | }
105 |
106 | .codrops-top span.right {
107 | float: right;
108 | }
109 |
110 | .codrops-top span.right a {
111 | float: left;
112 | display: block;
113 | }
114 |
115 | /* Demo Buttons Style */
116 | .codrops-demos {
117 | text-align:center;
118 | display: block;
119 | line-height: 30px;
120 | padding: 5px 0px;
121 | }
122 |
123 | .codrops-demos a {
124 | display: inline-block;
125 | margin: 0px 4px;
126 | padding: 0px 6px;
127 | color: #8c8c8c;
128 | line-height: 20px;
129 | font-size: 12px;
130 | font-weight: 700;
131 | text-shadow: 1px 1px 1px #fff;
132 | border: 1px solid #fff;
133 | background: #ffffff; /* Old browsers */
134 | background: -moz-linear-gradient(top, #ffffff 0%, #f6f6f6 47%, #ededed 100%); /* FF3.6+ */
135 | background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#ffffff), color-stop(47%,#f6f6f6), color-stop(100%,#ededed)); /* Chrome,Safari4+ */
136 | background: -webkit-linear-gradient(top, #ffffff 0%,#f6f6f6 47%,#ededed 100%); /* Chrome10+,Safari5.1+ */
137 | background: -o-linear-gradient(top, #ffffff 0%,#f6f6f6 47%,#ededed 100%); /* Opera 11.10+ */
138 | background: -ms-linear-gradient(top, #ffffff 0%,#f6f6f6 47%,#ededed 100%); /* IE10+ */
139 | background: linear-gradient(top, #ffffff 0%,#f6f6f6 47%,#ededed 100%); /* W3C */
140 | box-shadow: 0 1px 1px rgba(0, 0, 0, 0.1);
141 | }
142 |
143 | .codrops-demos a:hover {
144 | color: #333;
145 | background: #fff;
146 | }
147 |
148 | .codrops-demos a:active {
149 | background: #fff;
150 | }
151 |
152 | .codrops-demos a.current-demo,
153 | .codrops-demos a.current-demo:hover {
154 | background: #555;
155 | border-color: #555;
156 | color: #ddd;
157 | text-shadow: 0 1px 1px rgba(0,0,0,0.5);
158 | box-shadow: 0 1px 0 rgba(255,255,255,0.2), inset 0 1px 1px rgba(0,0,0,0.5);
159 | }
160 |
161 | .support-note span{
162 | color: #ac375d;
163 | font-size: 16px;
164 | display: none;
165 | font-weight: bold;
166 | text-align: center;
167 | padding: 5px 0;
168 | }
--------------------------------------------------------------------------------
/ugc.hotel.web.esri/js/application.js:
--------------------------------------------------------------------------------
1 | // Some general UI pack related JS
2 | // Extend JS String with repeat method
3 | String.prototype.repeat = function (num) {
4 | return new Array(Math.round(num) + 1).join(this);
5 | };
6 |
7 | (function ($) {
8 |
9 | // Add segments to a slider
10 | $.fn.addSliderSegments = function () {
11 | return this.each(function () {
12 | var $this = $(this),
13 | option = $this.slider('option'),
14 | amount = (option.max - option.min)/option.step,
15 | orientation = option.orientation;
16 | if ( 'vertical' === orientation ) {
17 | var output = '', i;
18 | console.log(amount);
19 | for (i = 1; i <= amount - 1; i++) {
20 | output += '';
21 | }
22 | $this.prepend(output);
23 | } else {
24 | var segmentGap = 100 / (amount) + '%';
25 | var segment = '';
26 | $this.prepend(segment.repeat(amount - 1));
27 | }
28 | });
29 | };
30 |
31 | $(function () {
32 |
33 | // Todo list
34 | $('.todo').on('click', 'li', function () {
35 | $(this).toggleClass('todo-done');
36 | });
37 |
38 | // Custom Selects
39 | if ($('[data-toggle="select"]').length) {
40 | $('[data-toggle="select"]').select2();
41 | }
42 |
43 | // Checkboxes and Radio buttons
44 | $('[data-toggle="checkbox"]').radiocheck();
45 | $('[data-toggle="radio"]').radiocheck();
46 |
47 | // Tooltips
48 | $('[data-toggle=tooltip]').tooltip('show');
49 |
50 | // jQuery UI Sliders
51 | var $slider = $('#slider');
52 | if ($slider.length > 0) {
53 | $slider.slider({
54 | min: 0,
55 | max: 10,
56 | values: [1,4],
57 | orientation: 'horizontal',
58 | range: true
59 | }).addSliderSegments($slider.slider('option').max, 'vertical');
60 | }
61 |
62 | var $verticalSlider = $('#vertical-slider');
63 | if ($verticalSlider.length) {
64 | $verticalSlider.slider({
65 | min: 0,
66 | max: 10,
67 | values: [4,5],
68 | orientation: 'horizontal',
69 | range: 'min'
70 | }).addSliderSegments($verticalSlider.slider('option').max, 'vertical');
71 | }
72 |
73 |
74 |
75 | // Focus state for append/prepend inputs
76 | $('.input-group').on('focus', '.form-control', function () {
77 | $(this).closest('.input-group, .form-group').addClass('focus');
78 | }).on('blur', '.form-control', function () {
79 | $(this).closest('.input-group, .form-group').removeClass('focus');
80 | });
81 |
82 | // Make pagination demo work
83 | $('.pagination').on('click', 'a', function () {
84 | $(this).parent().siblings('li').removeClass('active').end().addClass('active');
85 | });
86 |
87 | $('.btn-group').on('click', 'a', function () {
88 | $(this).siblings().removeClass('active').end().addClass('active');
89 | });
90 |
91 | // Disable link clicks to prevent page scrolling
92 | $(document).on('click', 'a[href="#fakelink"]', function (e) {
93 | e.preventDefault();
94 | });
95 |
96 | // Switches
97 | if ($('[data-toggle="switch"]').length) {
98 | $('[data-toggle="switch"]').bootstrapSwitch();
99 | }
100 |
101 | // Typeahead
102 | if ($('#typeahead-demo-01').length) {
103 | var states = new Bloodhound({
104 | datumTokenizer: function (d) { return Bloodhound.tokenizers.whitespace(d.word); },
105 | queryTokenizer: Bloodhound.tokenizers.whitespace,
106 | limit: 4,
107 | local: [
108 | { word: 'Alabama' },
109 | { word: 'Alaska' },
110 | { word: 'Arizona' },
111 | { word: 'Arkansas' },
112 | { word: 'California' },
113 | { word: 'Colorado' }
114 | ]
115 | });
116 |
117 | states.initialize();
118 |
119 | $('#typeahead-demo-01').typeahead(null, {
120 | name: 'states',
121 | displayKey: 'word',
122 | source: states.ttAdapter()
123 | });
124 | }
125 |
126 | // make code pretty
127 | window.prettyPrint && prettyPrint();
128 |
129 | });
130 |
131 | })(jQuery);
132 |
--------------------------------------------------------------------------------
/ugc.aggregator.esri/docs/ugc.aggregator/site/about/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
37 |
38 |
39 |
74 |
75 |
76 |
77 |
78 |
82 |
83 |
84 |
130 |
131 |
132 |
133 |
85 |
128 |
129 |
86 |
99 |
127 |
100 |
106 |
125 |
126 |
101 |
102 |
103 |
104 |
105 |
134 |
135 |
136 |
137 | « Previous
138 |
139 |
140 |
141 |
142 |
143 |
144 |
145 |
--------------------------------------------------------------------------------
/ugc.aggregator.esri/src/main/python/service/hotel/SuperHotelService.py:
--------------------------------------------------------------------------------
1 | # -*- coding:utf-8 -*-
2 | __author__ = 'DreamCathcer，LiuYang'
3 |
4 | from selenium import webdriver
5 | from selenium.webdriver.common.keys import Keys
6 | import time
7 | import traceback
8 |
9 |
10 | class HotelService(object):
11 |
12 | def __init__(self):
13 | self.driver = webdriver.Chrome()
14 |
15 | self.if_crawl_hotel_info = True
16 |
17 | self.if_crawl_hotel_comment = True
18 |
19 | self.if_crawl_hotel_price = True
20 |
21 | self._city = None
22 |
23 | '''
24 | 打开页面
25 | '''
26 | def openPage(self,url):
27 | self.driver.get(url)
28 | # 将界面最大化
29 | self.driver.maximize_window()
30 |
31 | '''
32 | 等待加载
33 | '''
34 | def wait(self,timeout):
35 | self.driver.implicitly_wait(timeout)
36 |
37 | '''
38 | 关闭驱动
39 | '''
40 | def closeDriver(self):
41 | self.driver.close()
42 |
43 | '''
44 | 遍历酒店信息列表页，爬取酒店详情页链接
45 | 抓取成功返回True 失败返回False
46 | '''
47 | def crawlListPage(self):
48 | pass
49 |
50 | '''
51 | 保存爬取的酒店列表页数据
52 | '''
53 | def saveListPageInfo(self):
54 | pass
55 |
56 | def set_city(self, city):
57 | self._city = city
58 |
59 | '''
60 | 抓取酒店信息
61 | '''
62 | def crawlHotelInfo(self,target):
63 | pass
64 |
65 | '''
66 | 保存抓取的酒店信息
67 | '''
68 | def saveHotelInfo(self):
69 | pass
70 |
71 | '''
72 | 获取酒店列表页数据
73 | '''
74 | def getListPageInfo(self):
75 | pass
76 |
77 | def scroll_and_click_by_partial_link_text(self, text, from_bottom=False):
78 | if from_bottom:
79 | # 跳到页尾
80 | self.driver.find_element_by_tag_name("body").send_keys(Keys.END)
81 | key = Keys.ARROW_UP
82 | else:
83 | # 跳到页头
84 | self.driver.find_element_by_tag_name("body").send_keys(Keys.HOME)
85 | key = Keys.ARROW_DOWN
86 | x = 0
87 | while 1:
88 | x += 1
89 | if x%500 == 0:
90 | self.driver.refresh()
91 | time.sleep(2)
92 | if from_bottom:
93 | self.driver.find_element_by_tag_name("body").send_keys(Keys.END)
94 | else:
95 | self.driver.find_element_by_tag_name("body").send_keys(Keys.HOME)
96 | if x == 1501:
97 | print u"点击评论类型出错" + self.driver.current_url
98 | break
99 | self.driver.find_element_by_tag_name("body").send_keys(key)
100 | try:
101 | self.driver.find_element_by_partial_link_text(text).click()
102 | break
103 | except:
104 | continue
105 |
106 | def scroll_and_click_by_xpath(self, text, from_bottom=False, refresh_if_failed=True, sleep_time=0):
107 | if from_bottom:
108 | # 跳到页尾
109 | self.driver.find_element_by_tag_name("body").send_keys(Keys.END)
110 | key = Keys.ARROW_UP
111 | else:
112 | # 跳到页头
113 | self.driver.find_element_by_tag_name("body").send_keys(Keys.HOME)
114 | key = Keys.ARROW_DOWN
115 | time.sleep(sleep_time)
116 | x = 0
117 | while 1:
118 | x += 1
119 | if x%500 == 0:
120 | # 刷新整个页面
121 | if refresh_if_failed:
122 | self.driver.refresh()
123 | time.sleep(sleep_time)
124 | if from_bottom:
125 | self.driver.find_element_by_tag_name("body").send_keys(Keys.END)
126 |
127 | else:
128 | self.driver.find_element_by_tag_name("body").send_keys(Keys.HOME)
129 | time.sleep(sleep_time)
130 | if x == 1501:
131 | print u"点击评论类型出错" + self.driver.current_url
132 | break
133 | self.driver.find_element_by_tag_name("body").send_keys(key)
134 | try:
135 | self.driver.find_element_by_xpath(text).click()
136 | break
137 | except:
138 | # print text
139 | continue
140 |
141 | '''
142 | 设置爬取内容
143 | '''
144 | def set_crawl_content(self,if_crawl_hotel_info, if_crawl_hotel_comment, if_crawl_hotel_price):
145 | self.if_crawl_hotel_info = if_crawl_hotel_info
146 |
147 | self.if_crawl_hotel_comment = if_crawl_hotel_comment
148 |
149 | self.if_crawl_hotel_price = if_crawl_hotel_price
--------------------------------------------------------------------------------
/ugc.hotel.web.esri/js/review-monitor/main.js:
--------------------------------------------------------------------------------
1 | (function () {
2 |
3 | var user = JSON.parse(sessionStorage.user);
4 |
5 | var location = JSON.parse(sessionStorage.location);
6 |
7 | //初始化页面
8 | (function () {
9 | // navbar菜单账号名
10 | if (user["user_name"] != null)
11 | document.getElementById("user_name").innerHTML = location["hotel_name"] + document.getElementById("user_name").innerHTML;
12 | })();
13 |
14 | window.reviewMonitor = function(ota) {
15 | // initial 页脚和页面评论
16 | var commentsData = requestComments(location['hotel_name'], 1, ota);
17 | // 如果成功返回数据
18 | if (commentsData != null) {
19 | document.getElementById("page_list").innerHTML = generateFooterHtml(location['hotel_name'], 1, commentsData["pageNum"], ota);
20 | document.getElementById("comment_list").innerHTML = generateCommentsHtml(commentsData["comments_info"]);
21 | }
22 | };
23 |
24 | /**
25 | * 请求酒店的评论
26 | * @param hotelName String 酒店名
27 | * @param text String 待查询文本
28 | * @param page Int 页下标
29 | * @param ota String OTA名称
30 | */
31 | function requestComments(hotelName, page, ota) {
32 | var commentsData = null;
33 | var paramStr = "?hotel_name=" + hotelName + "&page=" + page + "&ota=" + ota;
34 | $.ajax({
35 | type: "get",
36 | async: false,
37 | url: domain + getComments + paramStr,
38 | dataType: "json",
39 | timeout: 5000,
40 | success: function (result) {
41 | commentsData = result;
42 | },
43 | error: function (errorMsg) {
44 | console.log(errorMsg);
45 | alert("你输入的值有误,请输入完整参数或者重试");
46 | }
47 | });
48 | return commentsData;
49 | }
50 |
51 | /**
52 | * 根据页数生成切页列表
53 | * @param hoteName String 酒店名
54 | * @param text Stirng 查询文本
55 | * @param origin int 起始页数
56 | * @param pageNum int 总页数
57 | */
58 | function generateFooterHtml(hotelName, origin, pageNum, ota) {
59 | var pagination = "";
60 | for (var i = origin; i <= pageNum; i++) {
61 | if (i < origin + 10) {
62 | // 对前10页做处理
63 | if (i == origin) {
64 | pagination += '
50 |
93 |
94 |
95 |
--------------------------------------------------------------------------------
/ugc.hotel.web.esri/html/public-opinion-monitor.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
51 |
90 |
52 |
89 |
53 |
88 |
54 |
60 |
61 | 
56 |
62 |
68 |
69 |
70 | 
64 |
71 |
77 |
78 |
79 | 
73 |
80 |
86 |
87 | 
82 |
91 |
92 |
50 |
93 |
94 |
95 |
--------------------------------------------------------------------------------
/ugc.aggregator.esri/src/main/python/service/map/baidu/CoordinateTransferService.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | import json
3 | import requests
4 | import math
5 |
6 | key = 'your key here' # 这里填写你的百度开放平台的key
7 | x_pi = 3.14159265358979324 * 3000.0 / 180.0
8 | pi = 3.1415926535897932384626 # π
9 | a = 6378245.0 # 长半轴
10 | ee = 0.00669342162296594323 # 扁率
11 |
12 |
13 | def geocode(address):
14 | """
15 | 利用百度geocoding服务解析地址获取位置坐标
16 | :param address:需要解析的地址
17 | :return:
18 | """
19 | geocoding = {'s': 'rsv3',
20 | 'key': key,
21 | 'city': '全国',
22 | 'address': address}
23 | res = requests.get(
24 | "http://restapi.amap.com/v3/geocode/geo", params=geocoding)
25 | if res.status_code == 200:
26 | json = res.json()
27 | status = json.get('status')
28 | count = json.get('count')
29 | if status == '1' and int(count) >= 1:
30 | geocodes = json.get('geocodes')[0]
31 | lng = float(geocodes.get('location').split(',')[0])
32 | lat = float(geocodes.get('location').split(',')[1])
33 | return [lng, lat]
34 | else:
35 | return None
36 | else:
37 | return None
38 |
39 |
40 | def gcj02tobd09(lng, lat):
41 | """
42 | 火星坐标系(GCJ-02)转百度坐标系(BD-09)
43 | 谷歌、高德——>百度
44 | :param lng:火星坐标经度
45 | :param lat:火星坐标纬度
46 | :return:
47 | """
48 | z = math.sqrt(lng * lng + lat * lat) + 0.00002 * math.sin(lat * x_pi)
49 | theta = math.atan2(lat, lng) + 0.000003 * math.cos(lng * x_pi)
50 | bd_lng = z * math.cos(theta) + 0.0065
51 | bd_lat = z * math.sin(theta) + 0.006
52 | return [bd_lng, bd_lat]
53 |
54 |
55 | def bd09togcj02(bd_lon, bd_lat):
56 | """
57 | 百度坐标系(BD-09)转火星坐标系(GCJ-02)
58 | 百度——>谷歌、高德
59 | :param bd_lat:百度坐标纬度
60 | :param bd_lon:百度坐标经度
61 | :return:转换后的坐标列表形式
62 | """
63 | x = bd_lon - 0.0065
64 | y = bd_lat - 0.006
65 | z = math.sqrt(x * x + y * y) - 0.00002 * math.sin(y * x_pi)
66 | theta = math.atan2(y, x) - 0.000003 * math.cos(x * x_pi)
67 | gg_lng = z * math.cos(theta)
68 | gg_lat = z * math.sin(theta)
69 | return [gg_lng, gg_lat]
70 |
71 |
72 | def wgs84togcj02(lng, lat):
73 | """
74 | WGS84转GCJ02(火星坐标系)
75 | :param lng:WGS84坐标系的经度
76 | :param lat:WGS84坐标系的纬度
77 | :return:
78 | """
79 | if out_of_china(lng, lat): # 判断是否在国内
80 | return lng, lat
81 | dlat = transformlat(lng - 105.0, lat - 35.0)
82 | dlng = transformlng(lng - 105.0, lat - 35.0)
83 | radlat = lat / 180.0 * pi
84 | magic = math.sin(radlat)
85 | magic = 1 - ee * magic * magic
86 | sqrtmagic = math.sqrt(magic)
87 | dlat = (dlat * 180.0) / ((a * (1 - ee)) / (magic * sqrtmagic) * pi)
88 | dlng = (dlng * 180.0) / (a / sqrtmagic * math.cos(radlat) * pi)
89 | mglat = lat + dlat
90 | mglng = lng + dlng
91 | return [mglng, mglat]
92 |
93 |
94 | def gcj02towgs84(lng, lat):
95 | """
96 | GCJ02(火星坐标系)转GPS84
97 | :param lng:火星坐标系的经度
98 | :param lat:火星坐标系纬度
99 | :return:
100 | """
101 | if out_of_china(lng, lat):
102 | return lng, lat
103 | dlat = transformlat(lng - 105.0, lat - 35.0)
104 | dlng = transformlng(lng - 105.0, lat - 35.0)
105 | radlat = lat / 180.0 * pi
106 | magic = math.sin(radlat)
107 | magic = 1 - ee * magic * magic
108 | sqrtmagic = math.sqrt(magic)
109 | dlat = (dlat * 180.0) / ((a * (1 - ee)) / (magic * sqrtmagic) * pi)
110 | dlng = (dlng * 180.0) / (a / sqrtmagic * math.cos(radlat) * pi)
111 | mglat = lat + dlat
112 | mglng = lng + dlng
113 | return [lng * 2 - mglng, lat * 2 - mglat]
114 |
115 |
116 | def transformlat(lng, lat):
117 | ret = -100.0 + 2.0 * lng + 3.0 * lat + 0.2 * lat * lat + \
118 | 0.1 * lng * lat + 0.2 * math.sqrt(math.fabs(lng))
119 | ret += (20.0 * math.sin(6.0 * lng * pi) + 20.0 *
120 | math.sin(2.0 * lng * pi)) * 2.0 / 3.0
121 | ret += (20.0 * math.sin(lat * pi) + 40.0 *
122 | math.sin(lat / 3.0 * pi)) * 2.0 / 3.0
123 | ret += (160.0 * math.sin(lat / 12.0 * pi) + 320 *
124 | math.sin(lat * pi / 30.0)) * 2.0 / 3.0
125 | return ret
126 |
127 |
128 | def transformlng(lng, lat):
129 | ret = 300.0 + lng + 2.0 * lat + 0.1 * lng * lng + \
130 | 0.1 * lng * lat + 0.1 * math.sqrt(math.fabs(lng))
131 | ret += (20.0 * math.sin(6.0 * lng * pi) + 20.0 *
132 | math.sin(2.0 * lng * pi)) * 2.0 / 3.0
133 | ret += (20.0 * math.sin(lng * pi) + 40.0 *
134 | math.sin(lng / 3.0 * pi)) * 2.0 / 3.0
135 | ret += (150.0 * math.sin(lng / 12.0 * pi) + 300.0 *
136 | math.sin(lng / 30.0 * pi)) * 2.0 / 3.0
137 | return ret
138 |
139 |
140 | def out_of_china(lng, lat):
141 | """
142 | 判断是否在国内，不在国内不做偏移
143 | :param lng:
144 | :param lat:
145 | :return:
146 | """
147 | if lng < 72.004 or lng > 137.8347:
148 | return True
149 | if lat < 0.8293 or lat > 55.8271:
150 | return True
151 | return False
152 |
153 |
154 | if __name__ == '__main__':
155 | lng = 128.543
156 | lat = 37.065
157 | result1 = gcj02tobd09(lng, lat)
158 | result2 = bd09togcj02(lng, lat)
159 | result3 = wgs84togcj02(lng, lat)
160 | result4 = gcj02towgs84(lng, lat)
161 | result5 = geocode('北京市朝阳区朝阳公园')
162 | print result1, result2, result3, result4, result5
163 |
--------------------------------------------------------------------------------
/ugc.aggregator.esri/src/main/scripts/Hotel/TuniuCatcher.py:
--------------------------------------------------------------------------------
1 | # -*- coding:utf-8 -*-
2 | __author__ = 'DreamCathcer'
3 |
4 |
5 | import time
6 | import uuid
7 |
8 | from service.hotel.TuniuAPIClient import TuniuAPIClient
9 | from dao.hotel.HotelDAO import HotelDAO
10 | from setting import local_hotel_setting
11 | from util.geo import CoordTransor
12 |
13 | # 配置数据库
14 | dao_setting = local_hotel_setting
15 |
16 |
17 | class TuniuCatcher(object):
18 |
19 | def __init__(self):
20 | self._city = None
21 | self.__ota_info = "途牛"
22 | self.tuniu_api_client = TuniuAPIClient()
23 | self.hotel_dao = HotelDAO(dao_setting["host"], dao_setting["db"], dao_setting["user"], dao_setting["password"])
24 |
25 | def setCity(self, city):
26 | self._city = city
27 |
28 | def getHotelList(self, city_code):
29 | if self._city == None:
30 | print "未设置城市,请先使用setCity方法"
31 | return
32 | hotel_list = []
33 | page_index = 1
34 | page_amount =10000
35 | while page_index <= page_amount - 1:
36 | try:
37 | page_data = self.tuniu_api_client.get_hotel_list(page_index, city_code)
38 | # 接口返回的酒店数不稳定，所以爬取页数以最小数为准
39 | if page_amount > page_data["data"]["total"] / 20:
40 | page_amount = page_data["data"]["total"] / 20
41 | print "page_amount=%d"%page_amount
42 | hotel_list.extend(page_data["data"]["list"])
43 | print "Page_%d Success"%page_index
44 | time.sleep(5)
45 | page_index += 1
46 | except:
47 | print "Page_%d Fail"%page_index
48 | continue
49 | return hotel_list
50 |
51 | def saveHolteList(self, hotel_list):
52 | old_location_info = self.hotel_dao.get_locations(self._city)
53 | old_baseinfo = list(self.hotel_dao.get_baseinfo(self._city, self.__ota_info))
54 | # 将基础数据中的if_overtime先假设为都已过时
55 | for i in range(0, len(old_baseinfo)):
56 | old_baseinfo[i] = list(old_baseinfo[i])
57 | old_baseinfo[i][5] = 1
58 | new_locations = []
59 | new_baseinfo = []
60 | update_baseinfo = []
61 | # 遍历将要保存的数据
62 | for item in hotel_list:
63 | location_id = None
64 | # 首先检查该酒店是否已经保存在location表中
65 | for location in old_location_info:
66 | if item["name"] == location[3]:
67 | location_id = location[0]
68 | break
69 | # 如果没有则插入一条新的记录到location表中
70 | if location_id is None:
71 | location_id = uuid.uuid1()
72 | trans_location = CoordTransor.gcj02towgs84(lng=float(item["pos"]["lng"]), lat=float(item["pos"]["lat"]))
73 | new_locations.append({
74 | "guid": location_id,
75 | "x": trans_location[1],
76 | "y": trans_location[0],
77 | "hotel_name": item["name"],
78 | "city": self._city,
79 | "address": item["address"]
80 | })
81 | # 根据location的id号到baseinfo表中查询
82 | # 如果已经存于表中，则更新该条数据
83 | # 如果没有，则插入一条新的数据
84 | if_exist = False
85 | for baseinfo in old_baseinfo:
86 | if location_id == baseinfo[2]:
87 | if_exist = True
88 | baseinfo[1] = item["url"]
89 | baseinfo[4] = item["remarkCount"]
90 | baseinfo[5] = 0
91 | baseinfo[6] = int(item["remarkCount"]) - int(baseinfo[4]) if int(item["remarkCount"]) - int(baseinfo[4]) > 0 else 0
92 | baseinfo[7] = item["snapshot"]
93 | baseinfo[8] = item["id"]
94 | break
95 | if not if_exist:
96 | new_baseinfo.append({
97 | "guid": uuid.uuid1(),
98 | "url": item["url"],
99 | "location_id": location_id,
100 | "OTA": self.__ota_info,
101 | "comm_num": item["remarkCount"],
102 | "if_overtime": 0,
103 | "incre_num": item["remarkCount"],
104 | "img": item["snapshot"],
105 | "id_in_ota": item["id"]
106 | })
107 | for baseinfo in old_baseinfo:
108 | update_baseinfo.append({
109 | "guid": baseinfo[0],
110 | "url": baseinfo[1],
111 | "location_id": baseinfo[2],
112 | "OTA": baseinfo[3],
113 | "comm_num": baseinfo[4],
114 | "if_overtime": baseinfo[5],
115 | "incre_num": baseinfo[6],
116 | "img": baseinfo[7],
117 | "id_in_ota": baseinfo[8]
118 | })
119 | print len(new_locations), len(new_baseinfo), len(update_baseinfo)
120 | self.hotel_dao.save_locations(new_locations)
121 | self.hotel_dao.save_baseinfo(new_baseinfo)
122 | self.hotel_dao.update_baseinfo(update_baseinfo)
123 |
124 |
125 |
126 | if __name__ == "__main__":
127 | starttime = time.time()
128 | tuniu_catcher = TuniuCatcher()
129 | tuniu_catcher.setCity("南京")
130 | hotel_list = tuniu_catcher.getHotelList(1602)
131 | tuniu_catcher.saveHolteList(hotel_list)
132 | endtime = time.time()
133 | print endtime-starttime
--------------------------------------------------------------------------------
/ugc.aggregator.esri/src/main/scripts/GeocodingService.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf-8 -*-
3 | import math
4 | from time import time, strftime, localtime
5 | import os
6 | import shutil
7 |
8 | from util.geo.GeoUtil import GeoUtil
9 | from util.common.CollectionUtil import CollectionUtil
10 | from util.io.FileUtil import FileUtil
11 |
12 | import gevent
13 | import gevent.monkey
14 |
15 | gevent.monkey.patch_socket()
16 |
17 | import logging
18 | import logging.config
19 | from setting import baidu_map_uadb_setting
20 |
21 | logging.config.fileConfig(FileUtil().getLogConfigPath())
22 | logger = logging.getLogger("ugc")
23 |
24 | from service.map.baidu.SnatcherService import BaiduMapSnatcherService
25 |
26 | # 数据库配置
27 | dao_setting = baidu_map_uadb_setting
28 |
29 | def frange2(x, y, step):
30 | while x < y:
31 | yield x
32 | x += step
33 | if x >= y:
34 | x = y
35 | yield x
36 |
37 |
38 | # 每个Token正向编码100万，企业号300万
39 | goodAkList = [你的ak ]
40 |
41 | path = "c:/data/point_cache/"
42 | path_bak = "c:/data/point_cache_bak/" + strftime("%Y-%m-%d %H-%M-%S", localtime(time()))
43 | if os.path.exists(path_bak) == False:
44 | os.makedirs(path_bak)
45 |
46 |
47 | # 调用百度GeocodingAPI爬取数据
48 | class GeocodingService(object):
49 |
50 | # 地址节表名
51 | addressNodeTableName = 'AddressNode_Xuzhou'
52 | # poi点表名
53 | placeTableName = 'Place_Xuzhou'
54 |
55 | def fetchAddressNodeByPoints(self, index, points):
56 | # 循环Token，
57 | if index >= len(goodAkList):
58 | token = goodAkList[-1]
59 | else:
60 | token = goodAkList[index]
61 |
62 | logger.info('current index %s,points %s' % (index, str(len(points))))
63 | snatcherService = BaiduMapSnatcherService(dao_setting["host"], dao_setting["db"], dao_setting["user"], dao_setting["password"], token)
64 | snatcherService.fetchAddressNode(index, points, self.addressNodeTableName,placeTableName=self.placeTableName)
65 | logger.debug('Process %s done' % index)
66 | # TODO multiprocessing多进程实现，此处代码最后一个process不执行fetchAddressNode内的代码
67 |
68 | def run(self):
69 | pointList = []
70 | limitSize = 950000
71 | # TODO 从文件缓存读取limitSize个points，待测试
72 | fileNames = os.listdir(path)
73 | # 从point_cache读取大约limitSize*len(goodAkList)个点，并将读取的文件移至point_cache_bak文件夹
74 | for fileName in fileNames:
75 | print len(pointList)
76 | if len(pointList) < (limitSize) * len(goodAkList):
77 | file = path + fileName
78 | myList = FileUtil().readFileToObj(file)
79 | logger.debug("read file %s,size %s" % (file, len(myList)))
80 | pointList.extend(myList)
81 |
82 | print "cut file %s to %s" % (file, path_bak)
83 | shutil.move(file, path_bak + "")
84 | else:
85 | break
86 | # 将所有点按goodAkList的数目分桶装载
87 | chunkPoints = CollectionUtil().chunksByAverage(pointList,len(goodAkList))
88 | threads = []
89 | threadSize = len(chunkPoints)
90 | logger.debug('thread size ...%s ' % threadSize)
91 | # 6个线程分发桶中的点数据
92 | for i in xrange(0, threadSize, 1):
93 | threadChunkPointsList = CollectionUtil().chunksBySize(chunkPoints[i], limitSize / 6)
94 | for j in xrange(0, len(threadChunkPointsList), 1):
95 | index = str(i) + "_" + str(j)
96 | logger.debug('current thread ...%s ' % index)
97 | threads.append(gevent.spawn(self.fetchAddressNodeByPoints, i, threadChunkPointsList[j]))
98 | gevent.joinall(threads)
99 | # 将地址节表中的空字符串设置为null
100 | snatcherService = BaiduMapSnatcherService(dao_setting["host"], dao_setting["db"], dao_setting["user"], dao_setting["password"])
101 | snatcherService.setNullStrToNull(self.addressNodeTableName)
102 |
103 | def concurrentRequest(self):
104 | # 测试
105 | # bounds = [113.149662, 23.038528, 113.15175, 23.039123]
106 | # 桂城街道
107 | bounds = [113.129391, 22.98257, 113.261335, 23.072904]
108 | # 狮山镇
109 | # bounds = [113.092391, 23.132011, 113.123293, 23.167699]
110 | # 南京
111 | # bounds = [118.710042, 31.960759, 118.905082, 32.134843]
112 | # 获取区域内点集
113 | points = GeoUtil().getPointByBounds(bounds, 1000)
114 | start = int(math.ceil(len(points) / 2))
115 | # 点集拆分爬取
116 | # end = len(points)
117 | # points = points[start:end]
118 | points = points[0:start]
119 |
120 | logger.debug('points size %s' % len(points))
121 | # 点集合子集
122 | subPoints = CollectionUtil().chunksByAverage(points, len(goodAkList))
123 |
124 | threads = []
125 | processSize = len(subPoints)
126 | logger.debug('process size ...%s ,per process data size...%s' % (processSize, len(subPoints)))
127 | for index in range(0, processSize, 1):
128 | logger.debug('current process ...%s ' % index)
129 | threads.append(gevent.spawn(self.fetchAddressNodeByPoints, index, subPoints[index]))
130 | gevent.joinall(threads)
131 |
132 |
133 | if __name__ == '__main__':
134 | # python E:\PythonWorkspace\ugc\ugc.aggregator\src\main\scripts\GeocodingService.py
135 | ts = time()
136 | service = GeocodingService()
137 | # service.concurrentRequest()
138 | service.run()
139 |
140 | logger.debug('Took %s' % format(time() - ts))
141 |
--------------------------------------------------------------------------------
/ugc.aggregator.esri/src/main/scripts/Hotel/XiechengCatcher.py:
--------------------------------------------------------------------------------
1 | # -*- coding:utf-8 -*-
2 | __author__ = 'LiuYang'
3 |
4 |
5 | import time
6 | import uuid
7 |
8 | from service.hotel.XieChengAPIClient import XieChengAPIClient
9 | from dao.hotel.HotelDAO import HotelDAO
10 | from setting import local_hotel_setting
11 | from util.geo import CoordTransor
12 |
13 | # 配置数据库
14 | dao_setting = local_hotel_setting
15 |
16 |
17 | class XiechengCatcher(object):
18 |
19 | def __init__(self):
20 | self._city = None
21 | self.__ota_info = "携程"
22 | self.xiecheng_api_client = XieChengAPIClient()
23 | self.hotel_dao = HotelDAO(dao_setting["host"], dao_setting["db"], dao_setting["user"], dao_setting["password"])
24 |
25 | def setCity(self, city):
26 | self._city = city
27 |
28 | def getHotelList(self, cityId):
29 | if self._city == None:
30 | print "未设置城市,请先使用setCity方法"
31 | hotel_list = []
32 | page_index = 1
33 | page_amount =10000
34 | while page_index <= page_amount - 1:
35 | try:
36 | page_data = self.xiecheng_api_client.get_hotel_list(page_index, cityId, self._city)
37 | # 接口返回的酒店数不稳定，所以爬取页数以最小数为准
38 | if page_amount > page_data["hotelAmount"] / 25:
39 | page_amount = page_data["hotelAmount"] / 25
40 | print "page_amount=%d"%page_amount
41 | hotel_list.extend(page_data["hotelPositionJSON"])
42 | print "Page_%d Success"%page_index
43 | page_index += 1
44 | except:
45 | print "Page_%d Fail"%page_index
46 | continue
47 | return hotel_list
48 |
49 | def saveHolteList(self, hotel_list):
50 | old_location_info = self.hotel_dao.get_locations(self._city)
51 | old_baseinfo = list(self.hotel_dao.get_baseinfo(self._city, self.__ota_info))
52 | # 将基础数据中的if_overtime先假设为都已过时
53 | for i in range(0, len(old_baseinfo)):
54 | old_baseinfo[i] = list(old_baseinfo[i])
55 | old_baseinfo[i][5] = 1
56 | new_locations = []
57 | new_baseinfo = []
58 | update_baseinfo = []
59 | # 遍历将要保存的数据
60 | for item in hotel_list:
61 | location_id = None
62 | # 首先检查该酒店是否已经保存在location表中
63 | for location in old_location_info:
64 | if item["name"] == location[3]:
65 | location_id = location[0]
66 | break
67 | # 如果没有则插入一条新的记录到location表中
68 | if location_id is None:
69 | location_id = uuid.uuid1()
70 | trans_location = CoordTransor.bd09togcj02(bd_lon=float(item["lon"]), bd_lat=float(item["lat"]))
71 | trans_location = CoordTransor.gcj02towgs84(trans_location[1], trans_location[0])
72 | new_locations.append({
73 | "guid": location_id,
74 | "x": trans_location[1],
75 | "y": trans_location[0],
76 | "hotel_name": item["name"],
77 | "city": self._city,
78 | "address": item["address"]
79 | })
80 | # 根据location的id号到baseinfo表中查询
81 | # 如果已经存于表中，则更新该条数据
82 | # 如果没有，则插入一条新的数据
83 | if_exist = False
84 | for baseinfo in old_baseinfo:
85 | if location_id == baseinfo[2]:
86 | if_exist = True
87 | baseinfo[1] = item["url"]
88 | baseinfo[4] = item["dpcount"]
89 | baseinfo[5] = 0
90 | baseinfo[6] = int(item["dpcount"]) - int(baseinfo[4]) if int(item["dpcount"]) - int(baseinfo[4]) > 0 else 0
91 | baseinfo[7] = item["img"]
92 | baseinfo[8] = item["id"]
93 | break
94 | if not if_exist:
95 | new_baseinfo.append({
96 | "guid": uuid.uuid1(),
97 | "url": item["url"],
98 | "location_id": location_id,
99 | "OTA": self.__ota_info,
100 | "comm_num": item["dpcount"],
101 | "if_overtime": 0,
102 | "incre_num": item["dpcount"],
103 | "img": item["img"],
104 | "id_in_ota": item["id"]
105 | })
106 | for baseinfo in old_baseinfo:
107 | update_baseinfo.append({
108 | "guid": baseinfo[0],
109 | "url": baseinfo[1],
110 | "location_id": baseinfo[2],
111 | "OTA": baseinfo[3],
112 | "comm_num": baseinfo[4],
113 | "if_overtime": baseinfo[5],
114 | "incre_num": baseinfo[6],
115 | "img": baseinfo[7],
116 | "id_in_ota": baseinfo[8]
117 | })
118 | print len(new_locations), len(new_baseinfo), len(update_baseinfo)
119 | self.hotel_dao.save_locations(new_locations)
120 | self.hotel_dao.save_baseinfo(new_baseinfo)
121 | self.hotel_dao.update_baseinfo(update_baseinfo)
122 |
123 |
124 |
125 | if __name__ == "__main__":
126 | starttime = time.time()
127 | xiecheng_catcher = XiechengCatcher()
128 | xiecheng_catcher.setCity("南京")
129 | hotel_list = xiecheng_catcher.getHotelList(12)
130 | xiecheng_catcher.saveHolteList(hotel_list)
131 | endtime = time.time()
132 | print endtime-starttime
--------------------------------------------------------------------------------
/ugc.aggregator.esri/docs/ugc.aggregator/site/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
51 |
90 |
52 |
89 |
53 |
88 |
54 |
60 |
61 |
62 |
68 |
69 |
70 |
71 |
77 |
78 |
79 |
80 |
86 |
87 |
91 |
92 |
37 |
38 |
39 |
81 |
82 |
83 |
84 |
85 |
89 |
90 |
91 |
151 |
152 |
153 |
154 |
92 |
149 |
150 |
93 |
106 |
148 |
107 |
127 |
146 |
147 |
108 |
109 |
126 | Welcome to MkDocs
110 |For full documentation visit mkdocs.org.
111 |Commands
112 |-
113 |
mkdocs new [dir-name]- Create a new project.
114 | mkdocs serve- Start the live-reloading docs server.
115 | mkdocs build- Build the documentation site.
116 | mkdocs help- Print this help message.
117 |
Project layout
119 |mkdocs.yml # The configuration file.
120 | docs/
121 | index.md # The documentation homepage.
122 | ... # Other markdown pages, images and other files.
123 |
155 |
156 |
157 |
158 |
159 | Next »
160 |
161 |
162 |
163 |
164 |
165 |
166 |
167 |
171 |
--------------------------------------------------------------------------------
/ugc.aggregator.esri/src/main/python/dao/hotel/xiechengdao/xiecheng.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | __author__ = 'LiuYang'
3 | import MySQLdb
4 | import uuid
5 | import random
6 | import json
7 |
8 | from dao.SuperDAO import SuperDAO
9 |
10 |
11 | class xiechengDAO(SuperDAO):
12 |
13 | def __init__(self, host, db, user, password):
14 | SuperDAO.__init__(self, host, db, user, password)
15 |
16 | # 存储酒店基本信息
17 | def savehotelComment(self,items):
18 | db = MySQLdb.connect(self.host,self.user,self.password,self.db,charset='utf8')
19 | cursor = db.cursor()
20 | for item in items:
21 | try:
22 | cursor.execute("replace into hotelinfo(guid,city,title,price,score,recommend,area,havawifi,discussNum,common_facilities,activity_facilities,service_facilities,room_facilities)values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" ,(item["guid"],item["city"],item["title"],item["price"],item["score"],item["recommend"],item["area"],item["havawifi"],item["discussNum"],item["common_facilities"],item["activity_facilities"],item["service_facilities"],item["room_facilities"]))
23 | except Exception, e:
24 | print e
25 | db.commit()
26 | cursor.close()
27 | db.close()
28 |
29 |
30 | # 存储所有酒店的链接
31 | def savehotellink(self,listPageInfo):
32 | db = MySQLdb.connect(self.host,self.user,self.password,self.db,charset='utf8')
33 | cursor = db.cursor()
34 | for hotel in listPageInfo:
35 | try:
36 | id = uuid.uuid1()
37 | cursor.execute("replace into hotellianjie(guid,lianjie,city,comm_num)values(%s,%s,%s,%s)" ,(id,hotel["url"],hotel["city"],hotel["comm_num"]))
38 | except Exception,e:
39 | print hotel["url"]
40 | db.commit()
41 | cursor.close()
42 | db.close()
43 |
44 | # 取出周围设施数据
45 | def get_around_facilities_data(self):
46 | db = MySQLdb.connect(self.host,self.user,self.password,self.db,charset='utf8')
47 | cursor = db.cursor()
48 | cursor.execute("SELECT facilities_lntandlang FROM around_facilities_distance")
49 | data = []
50 | rows = cursor.fetchall()
51 | db.commit()
52 | cursor.close()
53 | db.close()
54 | for i in rows:
55 | data.append(json.loads(i[0]))
56 | return data
57 |
58 |
59 |
60 | # 取出周围设施中最远距离的设施和距离
61 | def get_max_distance_data(self):
62 | db = MySQLdb.connect(self.host,self.user,self.password,self.db,charset='utf8')
63 | cursor = db.cursor()
64 | cursor.execute("SELECT maxdistance FROM around_facilities_distance")
65 | data = []
66 | rows = cursor.fetchall()
67 |
68 | for i in rows:
69 | data.append(json.loads(i[0]))
70 | return data
71 | db.commit()
72 | cursor.close()
73 | db.close()
74 |
75 | # 从数据库中读取链接数据
76 | def _return(self):
77 | db = MySQLdb.connect(self.host,self.user,self.password,self.db,charset='utf8')
78 | cursor = db.cursor()
79 |
80 | cursor.execute("SELECT * FROM hotellianjie")
81 |
82 | rows = cursor.fetchall()
83 | return rows
84 |
85 | db.commit()
86 | cursor.close()
87 | db.close()
88 |
89 |
90 | # 从数据库中读取评论数据
91 | def _returncommentinfo(self):
92 | db = MySQLdb.connect(self.host,self.user,self.password,self.db,charset='utf8')
93 | cursor = db.cursor()
94 |
95 | cursor.execute("SELECT * FROM hotelcommentinfo")
96 |
97 | rows = cursor.fetchall()
98 | db.commit()
99 | cursor.close()
100 | db.close()
101 | return rows
102 |
103 | # 存储酒店评论信息
104 | def savehotelCommentinfo(self,items):
105 | db = MySQLdb.connect(self.host,self.user,self.password,self.db,charset='utf8')
106 | cursor = db.cursor()
107 | for item in items:
108 |
109 | try:
110 | cursor.execute("insert into hotelcommentinfo(hotelname,username,commentscore,intime,tourstyle,praisenum,commenttime,comment)values(%s,%s,%s,%s,%s,%s,%s,%s)" ,(item["title"],item["username"],item["commentscore"],item["intime"],item["tourstyle"],item["praisenum"],item["commenttime"],item["comment"]))
111 | except :
112 | print item
113 | db.commit()
114 | cursor.close()
115 | db.close()
116 |
117 | # 存储酒店评论信息(含好感度)
118 | def savehotelCommentinfosenti(self,items):
119 | db = MySQLdb.connect(self.host,self.user,self.password,self.db,charset='utf8')
120 | cursor = db.cursor()
121 | for item in items:
122 |
123 | try:
124 | cursor.execute("insert into xiechengcomment(hotelname,username,commentscore,intime,tourstyle,praisenum,comment,senti_value,viewpoint)values(%s,%s,%s,%s,%s,%s,%s,%s,%s)" ,(item["hotelname"],item["username"],item["commentscore"],item["intime"],item["tourstyle"],item["praisenum"],item["comment"],item["senti_value"],item["viewpoint"]))
125 | except Exception,e:
126 | print e
127 | db.commit()
128 | cursor.close()
129 | db.close()
130 |
131 |
132 | # 从数据库中读取评论数据
133 | def _returncomment(self):
134 | db = MySQLdb.connect(self.host,self.user,self.password,self.db,charset='utf8')
135 | cursor = db.cursor()
136 | cursor.execute("SELECT * FROM hotelcommentinfo")
137 | rows = cursor.fetchall()
138 | db.commit()
139 | cursor.close()
140 | db.close()
141 | return rows
142 |
143 | def get_comments(self):
144 | return self.get_records("xiechengcomment")
--------------------------------------------------------------------------------
/ugc.aggregator.esri/src/main/python/util/http/UniversalSDK.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding:utf-8 -*-
3 | __author__ = 'DreamCatcher'
4 | __version__ = '1.0.0'
5 |
6 | import json,urllib2,urllib,gzip,collections
7 | from util.common.Decorators import retry
8 | import logging
9 | logger = logging.getLogger('ugc')
10 |
11 | try:
12 | from cStringIO import StringIO
13 | except ImportError:
14 | from StringIO import StringIO
15 |
16 |
17 | class APIClient(object):
18 | domain = ""
19 |
20 | def __init__(self,domain):
21 | self.domain = domain
22 |
23 | def __getattr__(self,attr):
24 | return _Callable('%s/%s'%(self.domain,attr))
25 |
26 | class _Callable(object):
27 | def __init__(self,client):
28 | self.client = client
29 | self.header = None
30 |
31 | def __getattr__(self,attr):
32 | @retry((urllib2.URLError,ValueError), tries=10, delay=1, backoff=2)
33 | def execute(**kw):
34 | params = '%s'%(_encode_params(**kw))
35 | if len(params)!=0:
36 | http_url = '%s?%s'%(self.client,params) if self.method=='get' else self.client
37 | else:
38 | http_url = self.client
39 | http_body = None if self.method == 'get' else params
40 | # logging.info(http_url)
41 | req = urllib2.Request(http_url,data=http_body)
42 | req.add_header('Accept-Encoding', 'gzip')
43 | if self.header is not None:
44 | for key in self.header:
45 | req.add_header(key,self.header[key])
46 | try:
47 | resp = urllib2.urlopen(req,timeout=200)
48 | body = _read_body(resp)
49 | r = _parse_json(body)
50 | return r
51 | except Exception as e:
52 | logging.error(e)
53 | pass
54 | def execute_by_dict(dict):
55 | params = '%s'%(_encode_params_by_dict(dict))
56 | if len(params)!=0:
57 | http_url = '%s?%s'%(self.client,params) if self.method=='get' else self.client
58 | else:
59 | http_url = self.client
60 | http_body = None if self.method == 'get' else params
61 |
62 | req = urllib2.Request(http_url,data=http_body)
63 | req.add_header('Accept-Encoding', 'gzip')
64 | if self.header is not None:
65 | for key in self.header:
66 | req.add_header(key,self.header[key])
67 |
68 | try:
69 | resp = urllib2.urlopen(req,timeout=200)
70 | body = _read_body(resp)
71 | r = _parse_json(body)
72 | return r
73 | except Exception as e:
74 | logging.error(e)
75 | pass
76 | # 添加尾巴
77 | def add_trail(trail):
78 | return _Callable('%s%s'%(self.client,trail))
79 |
80 | def add_header(header):
81 | self.header = header
82 | return _Callable('%s'%self.client)
83 |
84 | if attr == 'get':
85 | self.method = 'get'
86 | return execute
87 | if attr == 'get_by_dict':
88 | self.method = 'get'
89 | return execute_by_dict
90 | if attr == 'post':
91 | self.method = 'post'
92 | return execute
93 | if attr == 'addtrail':
94 | return add_trail
95 | if attr == 'addheader':
96 | return add_header
97 | return _Callable('%s/%s'%(self.client,attr))
98 |
99 |
100 | def _parse_json(s):
101 | ' parse str into JsonDict '
102 |
103 | def _obj_hook(pairs):
104 | ' convert json object to python object '
105 | o = JsonDict()
106 | for k, v in pairs.iteritems():
107 | o[str(k)] = v
108 | return o
109 | return json.loads(s, object_hook=_obj_hook)
110 |
111 | class JsonDict(dict):
112 | ' general json object that allows attributes to be bound to and also behaves like a dict '
113 |
114 | def __getattr__(self, attr):
115 | try:
116 | return self[attr]
117 | except KeyError:
118 | raise AttributeError(r"'JsonDict' object has no attribute '%s'" % attr)
119 |
120 | def __setattr__(self, attr, value):
121 | self[attr] = value
122 |
123 | def _encode_params(**kw):
124 | '''
125 | do url-encode parameters
126 |
127 | >>> _encode_params(a=1, b='R&D')
128 | 'a=1&b=R%26D'
129 | >>> _encode_params(a=u'\u4e2d\u6587', b=['A', 'B', 123])
130 | 'a=%E4%B8%AD%E6%96%87&b=A&b=B&b=123'
131 | '''
132 | args = []
133 | for k, v in kw.iteritems():
134 | if isinstance(v, basestring):
135 | qv = v.encode('utf-8') if isinstance(v, unicode) else v
136 | args.append('%s=%s' % (k, urllib.quote(qv)))
137 | elif isinstance(v, collections.Iterable):
138 | for i in v:
139 | qv = i.encode('utf-8') if isinstance(i, unicode) else str(i)
140 | args.append('%s=%s' % (k, urllib.quote(qv)))
141 | else:
142 | qv = str(v)
143 | args.append('%s=%s' % (k, urllib.quote(qv)))
144 | return '&'.join(args)
145 |
146 | def _encode_params_by_dict(dict):
147 | args = []
148 | for k, v in dict.iteritems():
149 | if isinstance(v, basestring):
150 | qv = v.encode('utf-8') if isinstance(v, unicode) else v
151 | args.append('%s=%s' % (k, urllib.quote(qv)))
152 | elif isinstance(v, collections.Iterable):
153 | for i in v:
154 | qv = i.encode('utf-8') if isinstance(i, unicode) else str(i)
155 | args.append('%s=%s' % (k, urllib.quote(qv)))
156 | else:
157 | qv = str(v)
158 | args.append('%s=%s' % (k, urllib.quote(qv)))
159 | return '&'.join(args)
160 | def _read_body(obj):
161 | using_gzip = obj.headers.get('Content-Encoding', '')=='gzip'
162 | body = obj.read()
163 | if using_gzip:
164 | gzipper = gzip.GzipFile(fileobj=StringIO(body))
165 | fcontent = gzipper.read()
166 | gzipper.close()
167 | return fcontent
168 | return body
169 |
--------------------------------------------------------------------------------
/ugc.hotel.web.esri/html/review-monitor.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
48 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
--------------------------------------------------------------------------------
/ugc.hotel.web.esri/html/quality.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
49 |
92 |
50 |
91 |
51 |
60 |
69 |
70 |
79 |
80 |
89 |
90 |
93 |
102 |
94 |
97 | -
95 |
98 |
101 | -
99 |
50 |
51 |
89 |
90 |
91 |
52 |
57 |
58 |
59 |
53 |
54 |
56 | Loading....
55 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
74 |
75 |
76 |
87 |
88 |
92 |
106 |
107 |
108 |
130 |
131 |
133 |
134 |
135 |
136 |
137 |
138 |
139 |
140 |
141 |
142 |
143 |
144 |
145 |
146 |
147 |
148 |
--------------------------------------------------------------------------------
93 |
104 |
105 | Room Message List
94 |
95 |
96 |
97 |
101 |
102 |
103 |