();
19 |
20 | public void execute(Tuple input, BasicOutputCollector collector) {
21 | String website = input.getStringByField("website");
22 | long price = Long.valueOf(input.getStringByField("price"));
23 | long totalPrice = price;
24 | if (website_price.containsKey(website)) {
25 | totalPrice += website_price.get(website);
26 | }
27 | website_price.put(website, totalPrice);
28 | collector.emit(new Values(website, String.valueOf(totalPrice)));
29 | }
30 |
31 | public void declareOutputFields(OutputFieldsDeclarer declarer) {
32 | declarer.declare(new Fields("website", "price"));
33 | }
34 | }
35 |
36 |
--------------------------------------------------------------------------------
/storm/storm_realtime_process/src/main/java/bigdata/storm/WebsiteStoreMapper.java:
--------------------------------------------------------------------------------
1 | package bigdata.storm;
2 |
3 | import org.apache.storm.redis.common.mapper.RedisDataTypeDescription;
4 | import org.apache.storm.redis.common.mapper.RedisStoreMapper;
5 | import org.apache.storm.tuple.ITuple;
6 |
7 | /**
8 | * Created by qianxi.zhang on 11/26/16.
9 | */
10 | public class WebsiteStoreMapper implements RedisStoreMapper {
11 | private RedisDataTypeDescription description;
12 | private final String hashKey = "website";
13 |
14 | public WebsiteStoreMapper() {
15 | description =
16 | new RedisDataTypeDescription(RedisDataTypeDescription.RedisDataType.HASH, hashKey);
17 | }
18 |
19 | public RedisDataTypeDescription getDataTypeDescription() {
20 | return description;
21 | }
22 |
23 | public String getKeyFromTuple(ITuple iTuple) {
24 | return iTuple.getStringByField("website");
25 | }
26 |
27 | public String getValueFromTuple(ITuple iTuple) {
28 | return iTuple.getStringByField("price");
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/visualization/command/start-web.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | python /home/bigdata/visualization/py-echarts/main.py
4 |
--------------------------------------------------------------------------------
/visualization/py-echarts/main.py:
--------------------------------------------------------------------------------
1 | # -*- coding:utf-8 -*-
2 |
3 | from flask import Flask, render_template
4 | import json
5 | from models import Chart
6 | from query_presto import Presto_Query
7 | from query_redis import Redis_Query
8 |
9 | app = Flask(__name__)
10 |
11 | @app.route("/")
12 | def index():
13 | presto=Presto_Query()
14 | age_price_tuples=presto.query_age_price()
15 | age_dict=presto.getAgeDict(age_price_tuples)
16 | chart1 = Chart().pie("饼图", data=age_dict
17 | )
18 |
19 | tuples=presto.query_brand_price()
20 | keys=presto.getKeys(tuples)
21 | values=presto.getValues(tuples)
22 | chart2 = Chart() \
23 | .x_axis(data=keys) \
24 | .y_axis(formatter="{value}") \
25 | .bar(u"Brand Price", values, show_item_label=True)
26 |
27 | redis=Redis_Query()
28 | province_price=redis.query_province()
29 | china_province_price=redis.get_province_price(province_price)
30 | print china_province_price
31 | chart3= Chart()\
32 | .map(china_province_price)
33 |
34 | render = {
35 | "title": u"电商双十一大数据日志分析系统",
36 | "templates": [
37 | {"type": "chart", "title":u"不同年龄消费的情况", "option": json.dumps(chart1, indent=2)},
38 | {"type": "chart", "title":u"消费商品的情况", "option": json.dumps(chart2, indent=2)},
39 | {"type": "chart", "title":u"各省购买情况", "option": json.dumps(chart3, indent=2)}
40 | ]
41 | }
42 | return render_template("main.html", **render)
43 |
44 | if __name__ == "__main__":
45 | app.run(debug=True)
46 |
--------------------------------------------------------------------------------
/visualization/py-echarts/models.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | import json
4 |
5 | class Chart(dict):
6 | """
7 | 图表模板
8 | """
9 | def __init__(self):
10 | super(Chart, self).__init__()
11 | self["calculable"] = True
12 | self["tooltip"] = {"show": True}
13 | self["toolbox"] = {
14 | "show": True,
15 | "x": "left",
16 | "feature": {
17 | "dataView": {
18 | "show": True,
19 | "readOnly": False
20 | },
21 | "magicType": {
22 | "show": True,
23 | "type": ["line", "bar"]
24 | },
25 | "restore": {
26 | "show": True
27 | },
28 | "saveAsImage": {
29 | "show": True
30 | },
31 | "dataZoom": {
32 | "show": True,
33 | "title": {
34 | "dataZoom": u"区域缩放",
35 | "dataZoomReset": u"区域缩放后退"
36 | }
37 | }
38 | }
39 | }
40 | self["legend"] = {
41 | "show": True,
42 | "data": []
43 | }
44 | self["series"] = []
45 |
46 | def title(self, x="center", **kwargs):
47 | """
48 | 设置图表标题
49 | """
50 | self["title"].update({
51 | "x": x
52 | })
53 | self["title"].update(kwargs)
54 | return self
55 |
56 | def tooltip(self, show=True, trigger='axis', formatter=None, **kwargs):
57 | """
58 | 设置提示信息
59 | """
60 | self["tooltip"].update({
61 | "show": show,
62 | "trigger": trigger
63 | })
64 | if formatter is not None:
65 | self["tooltip"].update({"formatter": formatter})
66 | self["tooltip"].update(kwargs)
67 | return self
68 |
69 | def legend(self, show=True, data=None, orient='horizontal', **kwargs):
70 | """
71 | 设置图例
72 | `data`: [u"图例1", u"图例2", u"图例3"]
73 | `orient`: "vertical"|"horizontal"
74 | """
75 | data = [] if data is None else data
76 | self["legend"].update({
77 | "show": show,
78 | "data": data,
79 | "orient": orient
80 | })
81 | self["legend"].update(kwargs)
82 | return self
83 |
84 | def toolbox(self, show=True, x='left', **kwargs):
85 | """
86 | 设置工具箱
87 | """
88 | self["toolbox"].update({
89 | "show": show,
90 | "x": x
91 | })
92 | self["toolbox"].update(kwargs)
93 | return self
94 |
95 | def pie(self, name, data=None, radius="55%", center=None, auto_legend=True, **kwargs):
96 | """
97 | 添加一个饼图
98 | `data`: {u"名称": 100}, u"名称2": 200}
99 | """
100 | center = ["50%", "60%"] if center is None else center
101 | data = {} if data is None else data
102 | self["series"].append(self.__merge_dict({
103 | "type": "pie",
104 | "name": name,
105 | "radius": radius,
106 | "center": center,
107 | "data": [{"name": n, "value": v} for n, v in data.items()]
108 | }, kwargs))
109 | if auto_legend:
110 | legend_data = self["legend"]["data"]
111 | [legend_data.append(x) for x in data if x not in legend_data]
112 | return self
113 |
114 | def bar(self, name, data=None, auto_legend=True, y_axis_index=0, **kwargs):
115 | """
116 | 添加一个柱状图
117 | `data`: [10, 20, 30, 40]
118 | `auto_legend`: 自动生成图例
119 | """
120 | data = [] if data is None else data
121 | self["series"].append(self.__merge_dict({
122 | "type": "bar",
123 | "name": name,
124 | "data": data,
125 | "yAxisIndex": y_axis_index
126 | }, kwargs))
127 | if "yAxis" not in self:
128 | self.y_axis()
129 | if name not in self["legend"]["data"] and auto_legend:
130 | self["legend"]["data"].append(name)
131 | return self
132 |
133 | def line(self, name, data=None, mark_max_point=False, mark_min_point=False, show_item_label=False, auto_legend=True, y_axis_index=0, **kwargs):
134 | """
135 | 添加一个折线图
136 | `data`: [10, 20, 30, 40]
137 | """
138 | data = [] if data is None else data
139 | mark_point = []
140 | if mark_max_point:
141 | mark_point.append({"type": "max", "name": "最大值"})
142 | if mark_min_point:
143 | mark_point.append({"type": "min", "name": "最小值"})
144 | self["series"].append(self.__merge_dict({
145 | "type": "line",
146 | "name": name,
147 | "data": data,
148 | "markPoint": {
149 | "data":mark_point
150 | },
151 | "itemStyle": {
152 | "normal": {
153 | "label": {"show": show_item_label}
154 | }
155 | },
156 | "yAxisIndex": y_axis_index
157 | }, kwargs))
158 | if "yAxis" not in self:
159 | self.y_axis()
160 | if name not in self["legend"]["data"] and auto_legend:
161 | self["legend"]["data"].append(name)
162 | return self
163 |
164 | def x_axis(self, data=None, type_="category", name="", **kwargs):
165 | """
166 | 添加X轴
167 | """
168 | data = [] if data is None else data
169 | if "xAxis" not in self:
170 | self["xAxis"] = []
171 | self["xAxis"].append(self.__merge_dict({
172 | "type": type_,
173 | "name": name,
174 | "data": data
175 | }, kwargs))
176 | return self
177 |
178 | def y_axis(self, data=None, type_="value", name="", formatter=None, **kwargs):
179 | """
180 | 添加X轴
181 | """
182 | if "yAxis" not in self:
183 | self["yAxis"] = []
184 | self["yAxis"].append(self.__merge_dict({
185 | "type": type_,
186 | "name": name,
187 | }, {"axisLabel": {"formatter": formatter}} if formatter is not None else {}, kwargs))
188 | if data is not None:
189 | self["yAxis"] = data
190 | return self
191 | def map(self,data,**kwargs):
192 | self["legend"]={
193 | "orient":"vertical",
194 | "left": "left",
195 | "data":['price']
196 | }
197 | self["toolbox"]={
198 | "show": True,
199 | "orient": "vertical",
200 | "left": "right",
201 | "top": "center",
202 | "feature": {
203 | "mark":{"show":True},
204 | "dataView": {"show": True, "readOnly": False},
205 | "restore": {"show":True},
206 | "saveAsImage": {"show":True}
207 | }
208 | }
209 | #data={"name": '北京',"value": 10 }
210 | #data=json.dumps(data,ensure_ascii=False)
211 | self["series"]=[{
212 | "name":"price",
213 | "type": "map",
214 | "mapType": "china",
215 | "roam": False,
216 | "label": {
217 | "normal": {
218 | "show": True
219 | },
220 | "emphasis": {
221 | "show": True
222 | }
223 | },
224 | #"data": [data]
225 | "data": [{"name": n, "value": v} for n, v in data.items()]
226 | }]
227 | self["visualMap"]={
228 | "min": 0,
229 | "max": 2500,
230 | "left": 'left',
231 | "top": 'bottom',
232 | "text": ['高','低'],
233 | "calculable": True
234 | }
235 | self["tooltip"]={
236 | "trigger":"item"
237 | }
238 | # self["title"]={
239 | # "text": 'price',
240 | # "subtext": 'price',
241 | # "left": 'center'
242 | # }
243 | return self
244 |
245 |
246 | @staticmethod
247 | def __merge_dict(*args):
248 | """
249 | 合并多个字典并返回
250 | """
251 | return reduce(lambda x, y: dict(x.items() + y.items()), args)
252 |
253 |
254 | def main():
255 | c = Chart().tooltip()
256 | print json.dumps(c)
257 |
258 | if __name__ == "__main__":
259 | main()
260 |
--------------------------------------------------------------------------------
/visualization/py-echarts/models.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bigdataguide/hadooptraining/f374203eefeb3d6a0f4f64c9c0b841306a197349/visualization/py-echarts/models.pyc
--------------------------------------------------------------------------------
/visualization/py-echarts/query_presto.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf-8 -*-
3 |
4 | from pyhive import presto
5 |
6 | PRESTO_SERVER = {'host': 'bigdata', 'port': 8080, 'catalog': 'hive', 'schema': 'default'}
7 | BRAND_PRICE_QUERY="select brand,sum(price) as totalPrice from record join brand_dimension on record.bid=brand_dimension.bid group by brand_dimension.brand order by totalPrice desc limit 10"
8 |
9 | AGE_PRICE_QUERY="select cast((year(CURRENT_DATE)-year(birth)) as integer) as age,sum(price) as totalPrice from record join user_dimension on record.uid=user_dimension.uid group by cast((year(CURRENT_DATE)-year(birth)) as integer) order by totalPrice desc"
10 |
11 | class Presto_Query:
12 |
13 | def query_brand_price(self):
14 | conn = presto.connect(**PRESTO_SERVER)
15 | cursor = conn.cursor()
16 | cursor.execute(BRAND_PRICE_QUERY)
17 | tuples=cursor.fetchall()
18 | return tuples
19 |
20 | def getKeys(self,tuples):
21 | keys=[]
22 | for tuple in tuples:
23 | keys.append(tuple[0])
24 | return keys
25 |
26 | def getValues(self, tuples):
27 | values=[]
28 | for tuple in tuples:
29 | values.append(tuple[1])
30 | return values
31 |
32 | def query_age_price(self):
33 | conn = presto.connect(**PRESTO_SERVER)
34 | cursor = conn.cursor()
35 | cursor.execute(AGE_PRICE_QUERY)
36 | tuples=cursor.fetchall()
37 | return tuples
38 |
39 | def getAgeDict(self, tuples):
40 | dict={'<10':0L,'10~20':0L,'20~30':0L,'30~40':0L,'40~50':0L,'50~60':0L,'60~70':0L,'>70':0L}
41 | for tuple in tuples:
42 | age=int(tuple[0])
43 | price=long(tuple[1])
44 | age=age/10;
45 | if age<1:
46 | value=dict['<10']
47 | dict['<10']=value+price
48 | elif age>=1 and age<2:
49 | value=dict['10~20']
50 | dict['10~20']=value+price
51 | elif age>=2 and age<3:
52 | value=dict['20~30']
53 | dict['20~30']=value+price
54 | elif age>=3 and age<4:
55 | value=dict['30~40']
56 | dict['30~40']=value+price
57 | elif age>=4 and age<5:
58 | value=dict['40~50']
59 | dict['40~50']=value+price
60 | elif age>=5 and age<6:
61 | value=dict['50~60']
62 | dict['50~60']=value+price
63 | elif age>=6 and age<7:
64 | value=dict['60~70']
65 | dict['60~70']=value+price
66 | else:
67 | value=dict['>70']
68 | dict['>70']=value+price
69 | return dict
70 |
--------------------------------------------------------------------------------
/visualization/py-echarts/query_presto.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bigdataguide/hadooptraining/f374203eefeb3d6a0f4f64c9c0b841306a197349/visualization/py-echarts/query_presto.pyc
--------------------------------------------------------------------------------
/visualization/py-echarts/query_redis.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf-8 -*-
3 |
4 |
5 | import redis
6 |
7 | PROVINCE_MAP={"BeiJing":"北京","ShangHai":"上海","TianJin":"天津","ChongQing":"重庆","XiangGang":"香港","Aomen":"澳门","AnHui":"安徽","FuJian":"福建","GuangDong":"广东","GuangXi":"广西","GuiZhou":"贵州","GanSu":"甘肃","HaiNan":"海南","HeBei":"河北","HeNan":"河南","HeiLongJiang":"黑龙江","HuBei":"湖北","HuNan":"湖南","JiLin":"吉林","JiangSu":"江苏","JiangXi":"江西","LiaoNing":"辽宁","NeiMengGu":"内蒙古","NingXia":"宁夏","QingHai":"青海","ShanXi1":"山西","ShanXi3":"陕西","ShanDong":"山东","SiChuan":"四川","TaiWan":"台湾","XiZang":"西藏","XinJiang":"新疆","YunNan":"云南","ZheJiang":"浙江"}
8 |
9 | class Redis_Query:
10 |
11 | def query_province(self):
12 | r = redis.StrictRedis(host='127.0.0.1', port=6379)
13 | return r.hgetall('province')
14 |
15 | def get_province_price(self,dict):
16 | china_price={}
17 | for k,v in dict.items():
18 | if k in PROVINCE_MAP:
19 | new_key=PROVINCE_MAP[k]
20 | china_price[new_key]=v
21 | return china_price
22 |
--------------------------------------------------------------------------------
/visualization/py-echarts/query_redis.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bigdataguide/hadooptraining/f374203eefeb3d6a0f4f64c9c0b841306a197349/visualization/py-echarts/query_redis.pyc
--------------------------------------------------------------------------------
/visualization/py-echarts/templates/chart.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
52 |
--------------------------------------------------------------------------------
/visualization/py-echarts/templates/main.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | 电商双十一大数据日志分析系统
6 |
7 |
14 |
15 |
16 |
17 | {{title}}
18 | {% set i = 1 %}
19 | {% for template in templates %}
20 | {% if template.type == 'chart' %}
21 |
22 | {{template.title}}
23 |
24 |
25 |
31 |
45 |
46 | {% set i = i + 1 %}
47 | {% endif %}
48 | {% endfor %}
49 |
50 |
51 |
--------------------------------------------------------------------------------
/visualization/result/image-1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bigdataguide/hadooptraining/f374203eefeb3d6a0f4f64c9c0b841306a197349/visualization/result/image-1.png
--------------------------------------------------------------------------------
/visualization/result/image-2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bigdataguide/hadooptraining/f374203eefeb3d6a0f4f64c9c0b841306a197349/visualization/result/image-2.png
--------------------------------------------------------------------------------
/visualization/result/image-3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bigdataguide/hadooptraining/f374203eefeb3d6a0f4f64c9c0b841306a197349/visualization/result/image-3.png
--------------------------------------------------------------------------------