fuels;
29 | }
30 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/entity/car/CarMonthlySales.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.entity.car;
2 |
3 | import lombok.Data;
4 |
5 | /**
6 | * 每个月的汽车销售数量的比例
7 | * select month,round(summon/sumcount,2)as per
8 | * from (select month,count(number) as summon from cars where
9 | * month is not null group by month) as a,
10 | * (select count(number) as sumcount from cars) as b;
11 | */
12 | @Data
13 | public class CarMonthlySales {
14 | /**
15 | * 月份
16 | */
17 | private int month;
18 | /**
19 | * 比例
20 | */
21 | private double per;
22 | }
23 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/entity/car/CarOwnershipTypeBrand.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.entity.car;
2 |
3 | import lombok.Data;
4 |
5 | /**
6 | * 统计的车的所有权、 车辆类型和品牌的分布
7 | * select ower,mold,brand,count(number) as number from cars
8 | * group by ower,mold,brand;
9 | */
10 | @Data
11 | public class CarOwnershipTypeBrand {
12 | /**
13 | * 所有权
14 | */
15 | private String owner;
16 | /**
17 | * 车辆类型
18 | */
19 | private String mold;
20 | /**
21 | * 品牌
22 | */
23 | private String brand;
24 | /**
25 | * 分布数量
26 | */
27 | private int number;
28 | }
29 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/entity/car/CarSexRatio.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.entity.car;
2 |
3 | import lombok.Data;
4 |
5 | /**
6 | * 统计买车的男女比例
7 | * 以下SQL属于 hive 非等值连接， 需要设置hive为nonstrict模式 set hive.mapred.mode=nonstrict;
8 | *
9 | * select sex,round((sumsex/sumcount),2) as sexper from
10 | * (select sex,count(number) as sumsex from cars where sex!=''
11 | * group by sex) as a,
12 | * (select count(number) as sumcount from cars where sex !='') as b;
13 | */
14 | @Data
15 | public class CarSexRatio {
16 | /**
17 | * 性别
18 | */
19 | private String sex;
20 | /**
21 | * 性别比例
22 | */
23 | private double sexper;
24 | }
25 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/entity/car/CarUsageQuantity.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.entity.car;
2 |
3 | import lombok.Data;
4 |
5 | /**
6 | * 汽车用途和数量统计
7 | * select nature,count(number) as number from cars where nature!='' group by nature;
8 | */
9 | @Data
10 | public class CarUsageQuantity {
11 | /**
12 | * 使用性质
13 | */
14 | private String nature;
15 | /**
16 | * 数量
17 | */
18 | private Long number;
19 | }
20 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/entity/database/TableInfo.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.entity.database;
2 |
3 | import lombok.Data;
4 |
5 | /**
6 | * Hive 数据表结构信息
7 | */
8 | @Data
9 | public class TableInfo {
10 | /**
11 | * 字段名称
12 | */
13 | private String columnName;
14 | /**
15 | * 字段类型
16 | */
17 | private String columnType;
18 | /**
19 | * 字段描述
20 | */
21 | private String columnComment;
22 | }
23 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/entity/employee/Employee.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.entity.employee;
2 |
3 | import com.hadoop.hive.entity.BaseInfo;
4 | import lombok.Data;
5 |
6 | @Data
7 | public class Employee {
8 | private int id;
9 | private BaseInfo info;
10 | }
11 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/entity/employee/EmployeeComplexStructure.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.entity.employee;
2 |
3 | import lombok.Data;
4 |
5 | import java.util.List;
6 |
7 | /**
8 | * CREATE TABLE demo_database_enterprise.employee_complex_structure(name STRING,sa1ary FLOAT,subordinates ARRAY,deductions MAP,address STRUCT) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' COLLECTION ITEMS TERMINATED BY '_' MAP KEYS TERMINATED BY ':' LINES TERMINATED BY '\n';
9 | */
10 | @Data
11 | public class EmployeeComplexStructure {
12 | private String name;
13 | private Double sa1ary;
14 | private List subordinates;
15 |
16 | private String deductions;
17 | private String address;
18 | }
19 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/entity/log/EventLog.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.entity.log;
2 |
3 | import com.fasterxml.jackson.annotation.JsonFormat;
4 | import com.sun.istack.NotNull;
5 | import io.swagger.annotations.ApiModelProperty;
6 | import lombok.Data;
7 | import org.springframework.format.annotation.DateTimeFormat;
8 |
9 | import java.time.LocalDateTime;
10 |
11 | /**
12 | * 事件日志包
13 | */
14 | @Data
15 | public class EventLog {
16 | /**
17 | * 日志包Json字符串
18 | */
19 | @NotNull
20 | private String context;
21 | /**
22 | * 日志日期
23 | */
24 | @ApiModelProperty(value = "时间格式：yyyy-MM-dd HH:mm:ss", hidden = false, notes = "格式：yyyy-MM-dd HH:mm:ss", dataType = "String")
25 | @JsonFormat(shape = JsonFormat.Shape.STRING, pattern="yyyy-MM-dd HH:mm:ss")
26 | @DateTimeFormat(pattern="yyyy-MM-dd HH:mm:ss")
27 | private LocalDateTime dateTime;
28 | }
29 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/entity/log/StartLog.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.entity.log;
2 |
3 |
4 | import com.fasterxml.jackson.annotation.JsonFormat;
5 | import io.swagger.annotations.ApiModelProperty;
6 | import lombok.Data;
7 | import org.springframework.format.annotation.DateTimeFormat;
8 |
9 | import java.time.LocalDateTime;
10 |
11 | /**
12 | * 启动日志数据包
13 | */
14 | @Data
15 | public class StartLog {
16 | /**
17 | * 日志包Json字符串
18 | */
19 | private String context;
20 | /**
21 | * 日志日期
22 | */
23 | @ApiModelProperty(value = "时间，格式：yyyy-MM-dd HH:mm:ss", hidden = false, notes = "格式：yyyy-MM-dd HH:mm:ss", dataType = "String")
24 | @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd HH:mm:ss")
25 | @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss")
26 | private LocalDateTime dateTime;
27 | }
28 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/entity/student/Student.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.entity.student;
2 |
3 | import lombok.AllArgsConstructor;
4 | import lombok.Data;
5 | import lombok.NoArgsConstructor;
6 |
7 | @Data
8 | @NoArgsConstructor
9 | @AllArgsConstructor
10 | public class Student {
11 | private Long id;
12 | private String name;
13 | private Double score;
14 | private int age;
15 | }
16 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/entity/student/StudentHobby.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.entity.student;
2 |
3 | import lombok.AllArgsConstructor;
4 | import lombok.Data;
5 | import lombok.NoArgsConstructor;
6 |
7 | import java.util.List;
8 |
9 | @Data
10 | @NoArgsConstructor
11 | @AllArgsConstructor
12 | public class StudentHobby {
13 | private int id;
14 | private String name;
15 | private List hobby;
16 | private String add;
17 | }
18 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/entity/user/UserAccessTimes.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.entity.user;
2 |
3 | import lombok.Data;
4 |
5 | /**
6 | * 用户访问次数
7 | */
8 | @Data
9 | public class UserAccessTimes {
10 | /**
11 | * 用户名
12 | */
13 | private String username;
14 | /**
15 | * 月份
16 | */
17 | private String month;
18 | /**
19 | * 访问次数
20 | */
21 | private int counts;
22 | }
23 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/etl/job/JobBeanConfig.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.etl.job;
2 |
3 | import com.hadoop.hive.config.DataSourceProperties;
4 | import org.springframework.beans.factory.annotation.Autowired;
5 | import org.springframework.context.annotation.Bean;
6 | import org.springframework.context.annotation.Configuration;
7 |
8 | /**
9 | * 用于配置和注册带参数的Job实例到IOC中
10 | */
11 | @Configuration
12 | public class JobBeanConfig {
13 |
14 | @Autowired
15 | private DataSourceProperties dataSourceProperties;
16 |
17 | /**
18 | * 单词统计
19 | *
20 | * @return
21 | */
22 | @Bean("wordCountJob")
23 | public WordCountJob wordCountJob() {
24 | return new WordCountJob(
25 | WordCountJob.class.getName(),
26 | dataSourceProperties.getHdfs().get("url") + "/eric/hadoop_data/input/words.txt",
27 | dataSourceProperties.getHdfs().get("url") + "/eric/hadoop_data/output_words");
28 | }
29 |
30 | /**
31 | * 共同好友统计
32 | *
33 | * @return
34 | */
35 | @Bean("friendsJob")
36 | public FriendsJob friendsJob() {
37 | return new FriendsJob(
38 | FriendsJob.class.getName(),
39 | dataSourceProperties.getHdfs().get("url") + "/eric/hadoop_data/input/friends.txt",
40 | dataSourceProperties.getHdfs().get("url") + "/eric/hadoop_data/output_friends"
41 | );
42 | }
43 |
44 | /**
45 | * 统计最高温度
46 | *
47 | * @return
48 | */
49 | @Bean("weatherJob")
50 | public WeatherJob weatherJob() {
51 | return new WeatherJob(
52 | WeatherJob.class.getName(),
53 | dataSourceProperties.getHdfs().get("url") + "/eric/hadoop_data/input/weathers.txt",
54 | dataSourceProperties.getHdfs().get("url") + "/eric/hadoop_data/output_weathers"
55 | );
56 | }
57 | }
58 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/etl/mapper/FriendsMapper.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.etl.mapper;
2 |
3 | import lombok.extern.slf4j.Slf4j;
4 | import org.apache.hadoop.io.LongWritable;
5 | import org.apache.hadoop.io.Text;
6 | import org.apache.hadoop.mapreduce.Mapper;
7 | import org.springframework.stereotype.Component;
8 |
9 | import java.io.IOException;
10 |
11 | /**
12 | * 查找共同的好友
13 | */
14 | @Slf4j
15 | @Component
16 | public class FriendsMapper extends Mapper {
17 | private Text k = new Text();
18 | private Text v = new Text();
19 |
20 | /**
21 | * 读取 friends.txt 内容格式 A:B,C,D,F,E,O
22 | *
23 | * @param key
24 | * @param value
25 | * @param context
26 | * @throws IOException
27 | * @throws InterruptedException
28 | */
29 | @Override
30 | protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
31 | String line = value.toString().trim();
32 | // 根据冒号拆分
33 | String[] personFriends = line.split(":");
34 | // 第一个为用户
35 | String person = personFriends[0];
36 | // 第二个为好友
37 | String friends = personFriends[1];
38 | // 好友根据逗号拆分
39 | String[] friendsList = friends.split(",");
40 | log.info("----->好友数据信息,用户->{},好友->{}", person, friends);
41 | for (String friend : friendsList) {
42 | k.set(friend);
43 | v.set(person);
44 | context.write(k, v);
45 | }
46 | }
47 | }
48 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/etl/mapper/WeatherMapper.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.etl.mapper;
2 |
3 | import lombok.extern.slf4j.Slf4j;
4 | import org.apache.hadoop.io.LongWritable;
5 | import org.apache.hadoop.io.Text;
6 | import org.apache.hadoop.mapred.MapReduceBase;
7 | import org.apache.hadoop.mapred.Mapper;
8 | import org.apache.hadoop.mapred.OutputCollector;
9 | import org.apache.hadoop.mapred.Reporter;
10 | import org.springframework.stereotype.Component;
11 |
12 | import java.io.IOException;
13 |
14 | /**
15 | * 读取一年中某天的最高气温
16 | */
17 | @Slf4j
18 | @Component
19 | public class WeatherMapper extends MapReduceBase implements Mapper {
20 |
21 | private final Text word = new Text();
22 |
23 | @Override
24 | public void map(LongWritable key, Text value, OutputCollector output, Reporter reporter)
25 | throws IOException {
26 | // 打印输入样本: ==== Before Mapper: ==== 0,2018120715
27 | log.info("==== Before Mapper: ==== {},{}", key, value);
28 | String line = value.toString();
29 | // 截取年份
30 | String year = line.substring(0, 4);
31 | // 截取温度
32 | int temperature = Integer.parseInt(line.substring(8));
33 | word.set(year);
34 | output.collect(word, new LongWritable(temperature));
35 |
36 | // 打印输出样本: ==== After Mapper: ==== 2000, 15
37 | log.info("==== After Mapper: ==== {},{}", new Text(year), new LongWritable(temperature));
38 | }
39 |
40 | }
41 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/etl/reducer/FriendsReducer.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.etl.reducer;
2 |
3 | import lombok.extern.slf4j.Slf4j;
4 | import org.apache.hadoop.io.Text;
5 | import org.apache.hadoop.mapreduce.Reducer;
6 | import org.springframework.stereotype.Component;
7 |
8 | import java.io.IOException;
9 |
10 | /**
11 | * 获取共同好友
12 | */
13 | @Slf4j
14 | @Component
15 | public class FriendsReducer extends Reducer {
16 | private Text k = new Text();
17 | private Text v = new Text();
18 |
19 | /**
20 | * 读取 FriendsMapper 输出，内容格式 B A
21 | *
22 | * @param key
23 | * @param values
24 | * @param context
25 | * @throws IOException
26 | * @throws InterruptedException
27 | */
28 | @Override
29 | protected void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException {
30 | StringBuilder sb = new StringBuilder();
31 | // 循环好友
32 | for (Text person : values) {
33 | sb.append(person).append(",");
34 | }
35 |
36 | k.set(key);
37 | v.set(sb.toString());
38 | context.write(k, v);
39 | log.info("---->共同好友数据信息,用户->{},同时存在->{} 好友列表中！", key.toString(), sb.toString());
40 | }
41 | }
42 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/etl/reducer/WeatherReducer.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.etl.reducer;
2 |
3 | import lombok.extern.slf4j.Slf4j;
4 | import org.apache.hadoop.io.LongWritable;
5 | import org.apache.hadoop.io.Text;
6 | import org.apache.hadoop.mapred.MapReduceBase;
7 | import org.apache.hadoop.mapred.OutputCollector;
8 | import org.apache.hadoop.mapred.Reducer;
9 | import org.apache.hadoop.mapred.Reporter;
10 | import org.springframework.stereotype.Component;
11 |
12 | import java.io.IOException;
13 | import java.util.Iterator;
14 |
15 | /**
16 | * 统计一年天气最高温
17 | */
18 | @Slf4j
19 | @Component
20 | public class WeatherReducer extends MapReduceBase implements Reducer {
21 |
22 | @Override
23 | public void reduce(Text key, Iterator values, OutputCollector output,
24 | Reporter reporter) throws IOException {
25 | long maxValue = Integer.MIN_VALUE;
26 | StringBuilder sb = new StringBuilder();
27 | // 取values温度的最大值
28 | while (values.hasNext()) {
29 | long tmp = values.next().get();
30 | maxValue = Math.max(maxValue, tmp);
31 | sb.append(tmp).append(", ");
32 |
33 | }
34 | // 打印样本： ==== 全年气温列表 ==== 2020, 15, 23, 99, 12, 22
35 | log.info("==== 全年气温列表 ==== {}, {}", key, sb.toString());
36 | // 打印样本： ----年份：2020，最高温度：99 ----
37 | log.info("----年份：{}，最高温度：{} ----", key, maxValue);
38 | output.collect(key, new LongWritable(maxValue));
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/etl/reducer/WordCountReducer.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.etl.reducer;
2 |
3 | import lombok.extern.slf4j.Slf4j;
4 | import org.apache.hadoop.io.IntWritable;
5 | import org.apache.hadoop.io.Text;
6 | import org.apache.hadoop.mapreduce.Reducer;
7 | import org.springframework.stereotype.Component;
8 |
9 | import java.io.IOException;
10 |
11 | /**
12 | * 继承自Reducer类，负责重写父类Reducer中的reduce方法
13 | * 参数第一个Text表示reduce函数输入的键值对的键值类型
14 | * 参数第一个IntWritable表示reduce函数输入的键值对的值类型
15 | * 参数第二个Text表示reduce函数的键类型
16 | * 参数第二个IntWritable表示reduce函数输出键值对的值类型
17 | */
18 | @Slf4j
19 | @Component
20 | public class WordCountReducer extends Reducer {
21 | /**
22 | * reduce函数主要负责对map函数处理之后的中间结果进行最后处理，负责聚合统计，将map函数处理完的中间结果key-value键和值的列表
23 | * 针对每个key对应的value值列表集合中的数值，进行聚合累加计算。
24 | * 参数key是map函数处理完后输出的中间结果键值对的键值
25 | * values是map函数处理完后输出的中间结果值的列表
26 | * context上MapReduce框架的上下文对象，可以存放公共类型的数据，比如reduce函数处理完成的中间结果可以保存到context上下文对象中
27 | * 由上下文再写入HDFS中
28 | *
29 | * @param key
30 | * @param values
31 | * @param context
32 | * @throws IOException
33 | * @throws InterruptedException
34 | */
35 | @Override
36 | protected void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException {
37 | //初始化一个局部int型变量值为0，统计最终每个单词出现的次数
38 | int sum = 0;
39 |
40 | for (IntWritable v : values) {
41 | sum += v.get();
42 | log.info("******* 单词:{} 次数:{} *******", key, v.get());
43 | }
44 | //将reduce处理完的结果输出到HDFS文件系统中
45 | context.write(key, new IntWritable(sum));
46 |
47 | log.info("-----------单词:{} 总计次数:{}----------", key, sum);
48 | }
49 | }
50 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/repository/HiveBaseJDBCTemplate.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.repository;
2 |
3 | import lombok.extern.slf4j.Slf4j;
4 | import org.springframework.beans.factory.annotation.Autowired;
5 | import org.springframework.beans.factory.annotation.Qualifier;
6 | import org.springframework.jdbc.core.JdbcTemplate;
7 |
8 | import javax.sql.DataSource;
9 |
10 | /**
11 | * 注入hive数据源
12 | *
13 | * @author heyong
14 | * @Date 2020年09月01日
15 | */
16 | @Slf4j
17 | public class HiveBaseJDBCTemplate {
18 | private JdbcTemplate jdbcTemplate;
19 |
20 | public JdbcTemplate getJdbcTemplate() {
21 | return this.jdbcTemplate;
22 | }
23 |
24 | @Autowired
25 | public void setJdbcTemplate(@Qualifier("hiveDruidDataSource") DataSource dataSource) {
26 | this.jdbcTemplate = new JdbcTemplate(dataSource);
27 | }
28 |
29 | @Autowired
30 | @Qualifier("hiveJdbcDataSource")
31 | private org.apache.tomcat.jdbc.pool.DataSource jdbcDataSource;
32 |
33 | public org.apache.tomcat.jdbc.pool.DataSource getJdbcDataSource() {
34 | return this.jdbcDataSource;
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/repository/UserAccessTimesRepository.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.repository;
2 |
3 | import com.hadoop.hive.annotation.LogAspect;
4 | import com.hadoop.hive.entity.user.UserAccessTimes;
5 | import lombok.extern.slf4j.Slf4j;
6 | import org.springframework.jdbc.core.BeanPropertyRowMapper;
7 | import org.springframework.stereotype.Repository;
8 |
9 | import java.util.List;
10 |
11 | @Slf4j
12 | @Repository
13 | public class UserAccessTimesRepository extends HiveBaseJDBCTemplate {
14 | /**
15 | * 获取数据列表
16 | *
17 | * @param sql SQL指令必须要有列，且列名必须和对象中的属性保持一致，不可以select * from .....，否则BeanPropertyRowMapper无法映射到类对象中。
18 | * @return
19 | */
20 | @LogAspect(value = "getUserAccessTimesList")
21 | public List getUserAccessTimesList(String sql) {
22 | return this.getJdbcTemplate().query(sql, new BeanPropertyRowMapper<>(UserAccessTimes.class));
23 | }
24 | }
25 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/service/CarService.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.service;
2 |
3 | import com.hadoop.hive.entity.car.*;
4 |
5 | import java.util.List;
6 |
7 | /**
8 | * 汽车销售数据统计
9 | */
10 | public interface CarService {
11 | /**
12 | * 获取销售数据列表
13 | *
14 | * @return
15 | */
16 | List getCarList();
17 |
18 | /**
19 | * 统计各市、 区县的汽车销售的分布情况
20 | *
21 | * @return
22 | */
23 | List getCarCityCountyDistribution();
24 |
25 | /**
26 | * 汽车用途和数量统计
27 | *
28 | * @return
29 | */
30 | List getCarUsageQuantity();
31 |
32 | /**
33 | * 统计买车的男女比例
34 | *
35 | * @return
36 | */
37 | List getCarGenderRatio();
38 |
39 | /**
40 | * 通过不同类型（品牌） 车销售情况， 来统计发动机型号和燃料种类
41 | *
42 | * @return
43 | */
44 | List getCarModelFuel();
45 |
46 | /**
47 | * 每个月的汽车销售数量的比例
48 | *
49 | * @return
50 | */
51 | List getCarMonthlySales();
52 |
53 | /**
54 | * 统计的车的所有权、 车辆类型和品牌的分布
55 | *
56 | * @return
57 | */
58 | List getCarOwnershipTypeBrand();
59 | }
60 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/service/ETLService.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.service;
2 |
3 | public interface ETLService {
4 | /**
5 | * 执行统计语句
6 | *
7 | * @param sql
8 | */
9 | void execute(String sql);
10 | }
11 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/service/EmployeeService.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.service;
2 |
3 | import com.hadoop.hive.entity.employee.Employee;
4 | import com.hadoop.hive.entity.employee.EmployeeComplexStructure;
5 |
6 | import java.util.List;
7 |
8 | public interface EmployeeService {
9 | /**
10 | * 获取员工记录
11 | *
12 | * @return
13 | */
14 | List getListEmployee();
15 |
16 | /**
17 | * 获取复杂员工记录
18 | *
19 | * @return
20 | */
21 | List getListEmployeeComplexStructure();
22 |
23 | /**
24 | * 查询指定条件复杂员工记录
25 | *
26 | * @param name
27 | * @param city
28 | * @return
29 | */
30 | List getListEmployeeComplexStructureByParam(String name, String city);
31 | }
32 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/service/StudentService.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.service;
2 |
3 | import com.hadoop.hive.entity.student.Student;
4 | import com.hadoop.hive.entity.student.StudentHobby;
5 |
6 | import java.util.List;
7 |
8 | public interface StudentService {
9 | /**
10 | * 获取学生对象
11 | *
12 | * @param id
13 | * @return
14 | */
15 | Student getLimitOne(String id);
16 |
17 | /**
18 | * 获取学生对象列表
19 | *
20 | * @return
21 | */
22 | List getListForObject();
23 |
24 | /**
25 | * 获取学生爱好列表
26 | *
27 | * @return
28 | */
29 | List getListStudentHobby();
30 | }
31 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/service/UserAccessTimesService.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.service;
2 |
3 | import com.hadoop.hive.entity.user.UserAccessTimes;
4 |
5 | import java.util.List;
6 |
7 | public interface UserAccessTimesService {
8 |
9 | /**
10 | * 获取用户访问次数原始数据列表
11 | *
12 | * @return
13 | */
14 | List getUserAccessTimesList();
15 |
16 | /**
17 | * 按用户和月份统计访问次数
18 | *
19 | * @return
20 | */
21 | List getUserAccessTimeGroupByNameAndMonth();
22 | }
23 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/service/ads/StartLogADSService.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.service.ads;
2 |
3 | import com.hadoop.hive.service.ETLService;
4 |
5 | /**
6 | * ADS层 数据应用层，为各种报表提供可视化数据
7 | */
8 | public interface StartLogADSService extends ETLService {
9 | /**
10 | * 活跃设备数
11 | *
12 | * @param sql
13 | */
14 | void activeDevices(String sql);
15 |
16 | /**
17 | * 连续活跃设备数
18 | *
19 | * @param sql
20 | */
21 | void continuousActiveDevices(String sql);
22 |
23 | /**
24 | * 最近连续三周活跃用户数
25 | *
26 | * @param sql
27 | */
28 | void threeConsecutiveWeeks(String sql);
29 |
30 | /**
31 | * 每日用户留存情况
32 | *
33 | * @param sql
34 | */
35 | void dailyUserRetentionStatus(String sql);
36 |
37 | /**
38 | * 流失用户数
39 | *
40 | * @param sql
41 | */
42 | void lostUsers(String sql);
43 |
44 | /**
45 | * 每日新增设备信息数量
46 | *
47 | * @param sql
48 | */
49 | void newDeviceAddedDaily(String sql);
50 |
51 | /**
52 | * 沉默用户数
53 | *
54 | * @param sql
55 | */
56 | void numberOfSilentUsers(String sql);
57 |
58 | /**
59 | * 本周回流用户数
60 | *
61 | * @param sql
62 | */
63 | void returningUsers(String sql);
64 | }
65 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/service/dwd/EventLogDWDService.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.service.dwd;
2 |
3 | import com.hadoop.hive.service.ETLService;
4 |
5 | /**
6 | * DWD层 明细数据层，对ODS层进行数据清洗
7 | */
8 | public interface EventLogDWDService extends ETLService {
9 | /**
10 | * 从事件日志中抓取评论内容
11 | *
12 | * @param sql
13 | */
14 | void comment(String sql);
15 |
16 | /**
17 | * 从事件日志中抓取点赞内容
18 | *
19 | * @param sql
20 | */
21 | void praise(String sql);
22 |
23 | /**
24 | * 从事件日志中抓取活跃用户内容
25 | *
26 | * @param sql
27 | */
28 | void active(String sql);
29 | }
30 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/service/dwd/StartLogDWDService.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.service.dwd;
2 |
3 | import com.hadoop.hive.service.ETLService;
4 |
5 | /**
6 | * DWD层 明细数据层，对ODS层进行数据清洗
7 | */
8 | public interface StartLogDWDService extends ETLService {
9 |
10 | }
11 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/service/dwd/impl/StartLogDWDServiceImpl.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.service.dwd.impl;
2 |
3 | import com.hadoop.hive.repository.HiveRepository;
4 | import com.hadoop.hive.service.dwd.StartLogDWDService;
5 | import lombok.extern.slf4j.Slf4j;
6 | import org.springframework.beans.factory.annotation.Autowired;
7 | import org.springframework.stereotype.Service;
8 |
9 | import java.util.concurrent.ExecutorService;
10 | import java.util.concurrent.Executors;
11 |
12 | @Slf4j
13 | @Service("startLogDWDService")
14 | public class StartLogDWDServiceImpl implements StartLogDWDService {
15 | @Autowired
16 | private HiveRepository hiveRepository;
17 |
18 | /**
19 | * 执行统计语句
20 | *
21 | * @param sql
22 | */
23 | @Override
24 | public void execute(String sql) {
25 | try {
26 | log.info("--------------------开始执行 StartLog ODS To DWD Job作业----------------");
27 | hiveRepository.execute(sql);
28 | log.info("--------------------结束执行 StartLog ODS To DWD Job作业----------------");
29 | } catch (Exception e) {
30 | log.error("******执行作业异常->{}", e.getMessage());
31 | e.printStackTrace();
32 | }
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/service/dws/StartLogDWSService.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.service.dws;
2 |
3 | import com.hadoop.hive.service.ETLService;
4 |
5 | /**
6 | * DWS层 服务数据层，以DWD为基础按天进行轻度汇总
7 | */
8 | public interface StartLogDWSService extends ETLService {
9 | }
10 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/service/dws/impl/StartLogDWSServiceImpl.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.service.dws.impl;
2 |
3 | import com.hadoop.hive.repository.HiveRepository;
4 | import com.hadoop.hive.service.dws.StartLogDWSService;
5 | import lombok.extern.slf4j.Slf4j;
6 | import org.springframework.beans.factory.annotation.Autowired;
7 | import org.springframework.stereotype.Service;
8 |
9 | import java.util.concurrent.ExecutorService;
10 | import java.util.concurrent.Executors;
11 |
12 | @Slf4j
13 | @Service("startLogDWSService")
14 | public class StartLogDWSServiceImpl implements StartLogDWSService {
15 | @Autowired
16 | private HiveRepository hiveRepository;
17 |
18 | /**
19 | * 执行统计语句
20 | *
21 | * @param sql
22 | */
23 | @Override
24 | public void execute(String sql) {
25 | try {
26 | log.info("--------------------开始执行 Start Log DWD To DWS Job 作业----------------");
27 | hiveRepository.execute(sql);
28 | log.info("--------------------结束执行 Start Log DWD To DWS Job 作业----------------");
29 | } catch (Exception e) {
30 | log.error("******执行作业异常->{}", e.getMessage());
31 | e.printStackTrace();
32 | }
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/service/dwt/StartLogDWTService.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.service.dwt;
2 |
3 | import com.hadoop.hive.service.ETLService;
4 |
5 | /**
6 | * DWT层 数据主题层，以DWS层为基础按主题进行汇总
7 | */
8 | public interface StartLogDWTService extends ETLService {
9 | }
10 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/service/dwt/impl/StartLogDWTServiceImpl.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.service.dwt.impl;
2 |
3 | import com.hadoop.hive.repository.HiveRepository;
4 | import com.hadoop.hive.service.dwt.StartLogDWTService;
5 | import lombok.extern.slf4j.Slf4j;
6 | import org.springframework.beans.factory.annotation.Autowired;
7 | import org.springframework.stereotype.Service;
8 |
9 | import java.util.concurrent.ExecutorService;
10 | import java.util.concurrent.Executors;
11 |
12 | @Slf4j
13 | @Service("startLogDWTService")
14 | public class StartLogDWTServiceImpl implements StartLogDWTService {
15 | @Autowired
16 | private HiveRepository hiveRepository;
17 |
18 | /**
19 | * 执行统计语句
20 | *
21 | * @param sql
22 | */
23 | @Override
24 | public void execute(String sql) {
25 | try {
26 | log.info("--------------------开始执行 >Start Log DWS To DWT Job 作业----------------");
27 | hiveRepository.execute(sql);
28 | log.info("--------------------结束执行 >Start Log DWS To DWT Job 作业----------------");
29 | } catch (Exception e) {
30 | log.error("******执行作业异常->{}", e.getMessage());
31 | e.printStackTrace();
32 | }
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/service/impl/EmployeeServiceImpl.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.service.impl;
2 |
3 | import com.hadoop.hive.entity.employee.Employee;
4 | import com.hadoop.hive.entity.employee.EmployeeComplexStructure;
5 | import com.hadoop.hive.repository.EmployeeRepository;
6 | import com.hadoop.hive.service.EmployeeService;
7 | import org.springframework.beans.factory.annotation.Autowired;
8 | import org.springframework.stereotype.Service;
9 |
10 | import java.util.List;
11 |
12 | @Service
13 | public class EmployeeServiceImpl implements EmployeeService {
14 |
15 | @Autowired
16 | private EmployeeRepository employeeRepository;
17 |
18 | /**
19 | * 获取员工记录
20 | *
21 | * @return
22 | */
23 | @Override
24 | public List getListEmployee() {
25 | String sql = "select id,info from employee";
26 | return employeeRepository.getListEmployee(sql);
27 | }
28 |
29 | /**
30 | * 获取复杂员工记录
31 | *
32 | * @return
33 | */
34 | @Override
35 | public List getListEmployeeComplexStructure() {
36 | String sql = "select name,sa1ary,subordinates,deductions,address from employee_complex_structure";
37 | return employeeRepository.getListEmployeeComplexStructure(sql);
38 | }
39 |
40 | /**
41 | * 查询指定条件复杂员工记录
42 | *
43 | * @param name
44 | * @param city
45 | * @return
46 | */
47 | @Override
48 | public List getListEmployeeComplexStructureByParam(String name, String city) {
49 | String sql = "select name,sa1ary,subordinates,deductions,address from employee_complex_structure where 1=1 ";
50 | return employeeRepository.getListEmployeeComplexStructureByParam(sql, name, city);
51 | }
52 | }
53 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/service/impl/StudentServiceImpl.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.service.impl;
2 |
3 | import com.hadoop.hive.entity.student.Student;
4 | import com.hadoop.hive.entity.student.StudentHobby;
5 | import com.hadoop.hive.repository.StudentRepository;
6 | import com.hadoop.hive.service.StudentService;
7 | import org.springframework.beans.factory.annotation.Autowired;
8 | import org.springframework.stereotype.Service;
9 |
10 | import java.util.List;
11 |
12 | @Service
13 | public class StudentServiceImpl implements StudentService {
14 |
15 | @Autowired
16 | private StudentRepository studentRepository;
17 |
18 | /**
19 | * 获取学生对象
20 | *
21 | * @param id
22 | * @return
23 | */
24 | @Override
25 | public Student getLimitOne(String id) {
26 | String sql = "select id,name,score,age from student where id ='" + id + "'";
27 | return studentRepository.getLimitOne(sql);
28 | }
29 |
30 | /**
31 | * 获取学生对象列表
32 | *
33 | * @return
34 | */
35 | @Override
36 | public List getListForObject() {
37 | String sql = "select id,name,score,age from student";
38 | return studentRepository.getListForObject(sql);
39 | }
40 |
41 | /**
42 | * 获取学生爱好列表
43 | *
44 | * @return
45 | */
46 | @Override
47 | public List getListStudentHobby() {
48 | String sql = "select id,name,hobby,add from student";
49 | return studentRepository.getListStudentHobby(sql);
50 | }
51 | }
52 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/service/impl/UserAccessTimesServiceImpl.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.service.impl;
2 |
3 | import com.hadoop.hive.entity.user.UserAccessTimes;
4 | import com.hadoop.hive.repository.UserAccessTimesRepository;
5 | import com.hadoop.hive.service.UserAccessTimesService;
6 | import lombok.extern.slf4j.Slf4j;
7 | import org.springframework.beans.factory.annotation.Autowired;
8 | import org.springframework.stereotype.Service;
9 |
10 | import java.util.List;
11 |
12 | @Slf4j
13 | @Service
14 | public class UserAccessTimesServiceImpl implements UserAccessTimesService {
15 |
16 | @Autowired
17 | private UserAccessTimesRepository repository;
18 |
19 | /**
20 | * 获取用户访问次数原始数据列表
21 | *
22 | * @return
23 | */
24 | @Override
25 | public List getUserAccessTimesList() {
26 | String sql = "select username,month,counts from user_access_times";
27 | return repository.getUserAccessTimesList(sql);
28 | }
29 |
30 | /**
31 | * 按用户和月份统计访问次数
32 | *
33 | * @return
34 | */
35 | @Override
36 | public List getUserAccessTimeGroupByNameAndMonth() {
37 | String sql = "select username,substr(month,1,7) as month,sum(counts) from user_access_times group by username,substr(month,1,7)";
38 | return repository.getUserAccessTimesList(sql);
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/service/ods/ODSService.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.service.ods;
2 |
3 | /**
4 | * 日志ODS原始层服务
5 | */
6 | public interface ODSService {
7 | /**
8 | * 保存日志包原始数据
9 | *
10 | * @param t
11 | */
12 | void insert(T t);
13 | }
14 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/service/ods/impl/AbstractODSService.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.service.ods.impl;
2 |
3 | import com.hadoop.hive.repository.HiveRepository;
4 | import com.hadoop.hive.service.ods.ODSService;
5 | import lombok.extern.slf4j.Slf4j;
6 | import org.springframework.beans.factory.annotation.Autowired;
7 |
8 | import java.util.concurrent.ExecutorService;
9 | import java.util.concurrent.Executors;
10 |
11 | @Slf4j
12 | public abstract class AbstractODSService implements ODSService {
13 |
14 | @Autowired
15 | private HiveRepository hiveRepository;
16 |
17 | protected abstract String takeContext(T t);
18 |
19 | /**
20 | * 保存日志包原始数据
21 | *
22 | * @param t
23 | */
24 | @Override
25 | public void insert(T t) {
26 | ExecutorService cachedThreadPool = Executors.newCachedThreadPool();
27 | cachedThreadPool.execute(() -> {
28 | try {
29 | log.info("---------------接收到原始数据--------------");
30 | String sql = this.takeContext(t);
31 | hiveRepository.insertIntoTable(sql);
32 | log.info("---------------原始数据处理结束--------------");
33 | } catch (Exception e) {
34 | log.error("******执行原始数据作业异常->{}", e.getMessage());
35 | e.printStackTrace();
36 | }
37 | });
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/service/ods/impl/EventLogODSServiceImpl.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.service.ods.impl;
2 |
3 | import com.hadoop.hive.common.DateUtil;
4 | import com.hadoop.hive.entity.log.EventLog;
5 | import org.springframework.stereotype.Service;
6 |
7 | @Service
8 | public class EventLogODSServiceImpl extends AbstractODSService {
9 |
10 | @Override
11 | protected String takeContext(EventLog startLog) {
12 | return String.format("INSERT OVERWRITE TABLE ods_event_log PARTITION (dt = '%s')" +
13 | "SELECT '%s'",
14 | DateUtil.getLocalDateTime(startLog.getDateTime(), DateUtil.DEFAULT_DATE_FORMAT),
15 | DateUtil.getLocalDateTime(startLog.getDateTime(), DateUtil.DEFAULT_DATE_TIME_FORMAT) + "%" + startLog.getContext());
16 | }
17 | }
18 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/java/com/hadoop/hive/service/ods/impl/StartLogODSServiceImpl.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.hive.service.ods.impl;
2 |
3 | import com.hadoop.hive.common.DateUtil;
4 | import com.hadoop.hive.entity.log.StartLog;
5 | import org.springframework.stereotype.Service;
6 |
7 | /**
8 | * 启动日志原始数据
9 | */
10 | @Service
11 | public class StartLogODSServiceImpl extends AbstractODSService {
12 |
13 | @Override
14 | protected String takeContext(StartLog startLog) {
15 | return String.format("INSERT OVERWRITE TABLE ods_start_log PARTITION (dt = '%s')" +
16 | "SELECT '%s'", DateUtil.getLocalDateTime(startLog.getDateTime(), DateUtil.DEFAULT_DATE_FORMAT), startLog.getContext());
17 | }
18 | }
19 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/AzkabanJob-START-DWD-to-DWS.job:
--------------------------------------------------------------------------------
1 | type=command
2 |
3 | echo "--------------Start to Execute DWD To DWS For Start Log Action-----------------"
4 | echo "-----------API-> http://172.12.0.7:8089/api/v1/etl/dws/startLog--------"
5 |
6 | command=curl -X GET "http://172.12.0.7:8089/api/v1/etl/dws/startLog" -H "accept: */*"
7 |
8 | echo "--------------End to Execute DWD To DWS For Start Log Action-----------------"
9 |
10 | dependencies=AzkabanJob-START-ODS-to-DWD
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/AzkabanJob-START-DWS-to-DWT.job:
--------------------------------------------------------------------------------
1 | type=command
2 |
3 | echo "--------------Start to Execute DWS To DWT For Start Log Action-----------------"
4 | echo "-----------API-> http://172.12.0.7:8089/api/v1/etl/dwt/startLog--------"
5 |
6 | command=curl -X GET "http://172.12.0.7:8089/api/v1/etl/dwt/startLog" -H "accept: */*"
7 |
8 | echo "--------------End to Execute DWS To DWT For Start Log Action-----------------"
9 |
10 | dependencies=AzkabanJob-START-DWD-to-DWS
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/AzkabanJob-START-DWT-to-ADS-ActiveDevices.job:
--------------------------------------------------------------------------------
1 | type=command
2 |
3 | echo "--------------Start to Execute DWT To ADS For Active Devices Action-----------------"
4 | echo "-----------API-> http://172.12.0.7:8089/api/v1/etl/ads/activeDevices--------"
5 |
6 | command=curl -X GET "http://172.12.0.7:8089/api/v1/etl/ads/activeDevices" -H "accept: */*"
7 |
8 | echo "--------------End to Execute DWT To ADS For Active Devices Action-----------------"
9 |
10 | dependencies=AzkabanJob-START-DWS-to-DWT
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/AzkabanJob-START-DWT-to-ADS-ActiveUsersLastThreeConsecutiveWeeks.job:
--------------------------------------------------------------------------------
1 | type=command
2 |
3 | echo "--------------Start to Execute DWT To ADS For Active Users Last Three Consecutive Weeks Action-----------------"
4 | echo "-----------API-> http://172.12.0.7:8089/api/v1/etl/ads/threeConsecutiveWeeks--------"
5 |
6 | command=curl -X GET "http://172.12.0.7:8089/api/v1/etl/ads/threeConsecutiveWeeks" -H "accept: */*"
7 |
8 | echo "--------------End to Execute DWT To ADS For Active Users Last Three Consecutive Weeks Action-----------------"
9 |
10 | dependencies=AzkabanJob-START-DWS-to-DWT
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/AzkabanJob-START-DWT-to-ADS-ContinuousActiveDevices.job:
--------------------------------------------------------------------------------
1 | type=command
2 |
3 | echo "--------------Start to Execute DWT To ADS For Continuous Active Devices Action-----------------"
4 | echo "-----------API-> http://172.12.0.7:8089/api/v1/etl/ads/continuousActiveDevices--------"
5 |
6 | command=curl -X GET "http://172.12.0.7:8089/api/v1/etl/ads/continuousActiveDevices" -H "accept: */*"
7 |
8 | echo "--------------End to Execute DWT To ADS For Continuous Active Devices Action-----------------"
9 |
10 | dependencies=AzkabanJob-START-DWS-to-DWT
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/AzkabanJob-START-DWT-to-ADS-DailyUserRetentionStatus.job:
--------------------------------------------------------------------------------
1 | type=command
2 |
3 | echo "--------------Start to Execute DWT To ADS For Daily User Retention Status Action-----------------"
4 | echo "-----------API-> http://172.12.0.7:8089/api/v1/etl/ads/dailyUserRetentionStatus--------"
5 |
6 | command=curl -X GET "http://172.12.0.7:8089/api/v1/etl/ads/dailyUserRetentionStatus" -H "accept: */*"
7 |
8 | echo "--------------End to Execute DWT To ADS For Daily User Retention Status Action-----------------"
9 |
10 | dependencies=AzkabanJob-START-DWS-to-DWT
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/AzkabanJob-START-DWT-to-ADS-LostUsers.job:
--------------------------------------------------------------------------------
1 | type=command
2 |
3 | echo "--------------Start to Execute DWT To ADS For Lost Users Action-----------------"
4 | echo "-----------API-> http://172.12.0.7:8089/api/v1/etl/ads/lostUsers--------"
5 |
6 | command=curl -X GET "http://172.12.0.7:8089/api/v1/etl/ads/lostUsers" -H "accept: */*"
7 |
8 | echo "--------------End to Execute DWT To ADS For Lost Users Action-----------------"
9 |
10 | dependencies=AzkabanJob-START-DWS-to-DWT
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/AzkabanJob-START-DWT-to-ADS-NumberOfNewDeviceAddedDaily.job:
--------------------------------------------------------------------------------
1 | type=command
2 |
3 | echo "--------------Start to Execute DWT To ADS For Number Of New Device Added Daily Action-----------------"
4 | echo "-----------API-> http://172.12.0.7:8089/api/v1/etl/ads/newDeviceAddedDaily--------"
5 |
6 | command=curl -X GET "http://172.12.0.7:8089/api/v1/etl/ads/newDeviceAddedDaily" -H "accept: */*"
7 |
8 | echo "--------------End to Execute DWT To ADS For Number Of New Device Added Daily Action-----------------"
9 |
10 | dependencies=AzkabanJob-START-DWS-to-DWT
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/AzkabanJob-START-DWT-to-ADS-NumberOfSilentUsers.job:
--------------------------------------------------------------------------------
1 | type=command
2 |
3 | echo "--------------Start to Execute DWT To ADS For Number Of Silent Users Action-----------------"
4 | echo "-----------API-> http://172.12.0.7:8089/api/v1/etl/ads/numberOfSilentUsers--------"
5 |
6 | command=curl -X GET "http://172.12.0.7:8089/api/v1/etl/ads/numberOfSilentUsers" -H "accept: */*"
7 |
8 | echo "--------------End to Execute DWT To ADS For Number Of Silent Users Action-----------------"
9 |
10 | dependencies=AzkabanJob-START-DWS-to-DWT
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/AzkabanJob-START-DWT-to-ADS-ReturningUsers.job:
--------------------------------------------------------------------------------
1 | type=command
2 |
3 | echo "--------------Start to Execute DWT To ADS For Number Of Returning Users Action-----------------"
4 | echo "-----------API-> http://172.12.0.7:8089/api/v1/etl/ads/returningUsers--------"
5 |
6 | command=curl -X GET "http://172.12.0.7:8089/api/v1/etl/ads/returningUsers" -H "accept: */*"
7 |
8 | echo "--------------End to Execute DWT To ADS For Number Of Returning Users Action-----------------"
9 |
10 | dependencies=AzkabanJob-START-DWS-to-DWT
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/AzkabanJob-START-ODS-to-DWD.job:
--------------------------------------------------------------------------------
1 | type=command
2 |
3 | echo "--------------Start to Execute ODS To DWD For Start Log Action-----------------"
4 | echo "-----------API-> http://172.12.0.7:8089/api/v1/etl/dwd/startLog--------"
5 |
6 | command=curl -X GET "http://172.12.0.7:8089/api/v1/etl/dwd/startLog" -H "accept: */*"
7 |
8 | echo "--------------End to Execute ODS To DWD For Start Log Action-----------------"
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/AzkabanJob-Sqoop-Hive-to-MySQL.job:
--------------------------------------------------------------------------------
1 | type=command
2 |
3 | echo "--------------Start to Execute Sqoop Hive To MySQL Action-----------------"
4 |
5 | echo "sh /usr/local/sqoop-1.4.7/Sqoop-Hive-To-MySQL.sh"
6 |
7 | command:sh /usr/local/Sqoop-Hive-To-MySQL.sh
8 |
9 | echo "--------------End to Execute Sqoop Hive To MySQL Action-----------------"
10 |
11 | dependencies=AzkabanJob-START-DWT-to-ADS-ActiveDevices,AzkabanJob-START-DWT-to-ADS-ActiveUsersLastThreeConsecutiveWeeks,AzkabanJob-START-DWT-to-ADS-ContinuousActiveDevices,AzkabanJob-START-DWT-to-ADS-DailyUserRetentionStatus,AzkabanJob-START-DWT-to-ADS-LostUsers,AzkabanJob-START-DWT-to-ADS-NumberOfNewDeviceAddedDaily,AzkabanJob-START-DWT-to-ADS-NumberOfSilentUsers,AzkabanJob-START-DWT-to-ADS-ReturningUsers
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/Hive-DataBase-ACTIVE_BACKGROUND-DWD.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # 数据库名称
3 | APP=demo_database_gmall
4 |
5 | # hive目录
6 | hive=/usr/local/hive-2.3.7/bin/hive
7 |
8 | # 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天
9 | if [ -n "$1" ]; then
10 | do_date=$1
11 | else
12 | do_date=$(date -d "-1 day" +%F)
13 | fi
14 |
15 | sql="
16 | INSERT overwrite TABLE ${APP}.dwd_active_background_log PARTITION (dt = '$do_date') SELECT
17 | mid_id,
18 | user_id,
19 | version_code,
20 | version_name,
21 | lang,
22 | source,
23 | os,
24 | area,
25 | model,
26 | brand,
27 | sdk_version,
28 | gmail,
29 | height_width,
30 | app_time,
31 | network,
32 | lng,
33 | lat,
34 | get_json_object(event_json,'$.kv.active_source') active_source,
35 | server_time
36 | FROM
37 | ${APP}.dwd_base_event_log
38 | WHERE
39 | dt = '$do_date'
40 | AND event_name = 'active_background';"
41 |
42 | echo "===开始从事件明细中提取日期为 $do_date 的活跃用户数据==="
43 | $hive -e "$sql"
44 | echo "===从事件明细中提取日期为 $do_date 的活跃用户数据完成==="
45 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/Hive-DataBase-COMMENT-DWD.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # 数据库名称
3 | APP=demo_database_gmall
4 |
5 | # hive目录
6 | hive=/usr/local/hive-2.3.7/bin/hive
7 |
8 | # 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天
9 | if [ -n "$1" ]; then
10 | do_date=$1
11 | else
12 | do_date=$(date -d "-1 day" +%F)
13 | fi
14 |
15 | sql="
16 | INSERT overwrite TABLE ${APP}.dwd_comment_log PARTITION (dt = '$do_date') SELECT
17 | mid_id,
18 | user_id,
19 | version_code,
20 | version_name,
21 | lang,
22 | source,
23 | os,
24 | area,
25 | model,
26 | brand,
27 | sdk_version,
28 | gmail,
29 | height_width,
30 | app_time,
31 | network,
32 | lng,
33 | lat,
34 | get_json_object(event_json,'$.kv.comment_id') comment_id,
35 | get_json_object(event_json,'$.kv.userid') userid,
36 | get_json_object(event_json,'$.kv.p_comment_id') p_comment_id,
37 | get_json_object(event_json,'$.kv.content') content,
38 | get_json_object(event_json,'$.kv.addtime') addtime,
39 | get_json_object(event_json,'$.kv.other_id') other_id,
40 | get_json_object(event_json,'$.kv.praise_count') praise_count,
41 | get_json_object(event_json,'$.kv.reply_count') reply_count,
42 | server_time
43 | FROM
44 | ${APP}.dwd_base_event_log
45 | WHERE
46 | dt = '$do_date'
47 | AND event_name = 'comment';"
48 |
49 | echo "===开始从事件明细中提取日期为 $do_date 的评论数据==="
50 | $hive -e "$sql"
51 | echo "===从事件明细中提取日期为 $do_date 的评论数据完成==="
52 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/Hive-DataBase-EVENT-ODS-to-DWD.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # 数据库名称
3 | APP=demo_database_gmall
4 |
5 | # hive目录
6 | hive=/usr/local/hive-2.3.7/bin/hive
7 |
8 | # 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天
9 | if [ -n "$1" ]; then
10 | do_date=$1
11 | else
12 | do_date=$(date -d "-1 day" +%F)
13 | fi
14 |
15 | sql=" use demo_database_gmall;
16 | insert overwrite table ${APP}.dwd_base_event_log
17 | PARTITION(dt='$do_date')
18 | select
19 | base_analizer(line,'mid') as mid_id,
20 | base_analizer(line,'uid') as user_id,
21 | base_analizer(line,'vc') as version_code,
22 | base_analizer(line,'vn') as version_name,
23 | base_analizer(line,'l') as lang,
24 | base_analizer(line,'sr') as source,
25 | base_analizer(line,'os') as os,
26 | base_analizer(line,'ar') as area,
27 | base_analizer(line,'md') as model,
28 | base_analizer(line,'ba') as brand,
29 | base_analizer(line,'sv') as sdk_version,
30 | base_analizer(line,'g') as gmail,
31 | base_analizer(line,'hw') as height_width,
32 | base_analizer(line,'t') as app_time,
33 | base_analizer(line,'nw') as network,
34 | base_analizer(line,'ln') as lng,
35 | base_analizer(line,'la') as lat,
36 | event_name,
37 | event_json,
38 | base_analizer(line,'st') as server_time
39 | from ${APP}.ods_event_log lateral view flat_analizer(base_analizer(line,'et')) tem_flat as event_name,event_json
40 | where dt='$do_date'
41 | and base_analizer(line,'et')<>''; "
42 |
43 | echo "===开始清洗事件日期为 $do_date 的数据==="
44 | $hive -e "$sql"
45 | echo "===事件日期为 $do_date 的数据清洗完成==="
46 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/Hive-DataBase-EVENT_START-HDFS-to-ODS.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # 数据库名称
3 | APP=demo_database_gmall
4 |
5 | # hive目录
6 | hive=/usr/local/hive-2.3.7/bin/hive
7 |
8 | # [ -n 变量值 ] 判断变量的值，是否为空
9 | # 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天
10 | if [ -n "$1" ]; then
11 | do_date=$1
12 | else
13 | do_date=$(date -d "-1 day" +%F)
14 | fi
15 |
16 |
17 | sql="
18 | load data inpath '/eric/hive_data/$do_date/start_log.txt' overwrite into table ${APP}.ods_start_log partition(dt='$do_date');
19 | load data inpath '/eric/hive_data/$do_date/event_log.txt' overwrite into table ${APP}.ods_event_log partition(dt='$do_date'); "
20 |
21 | echo "===开始导入日志日期为 $do_date 的数据==="
22 |
23 | $hive -e "$sql"
24 |
25 | echo "===日志日期为 $do_date 的数据导入完成==="
26 |
27 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/Hive-DataBase-PRAISE-DWD.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # 数据库名称
3 | APP=demo_database_gmall
4 |
5 | # hive目录
6 | hive=/usr/local/hive-2.3.7/bin/hive
7 |
8 | # 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天
9 | if [ -n "$1" ]; then
10 | do_date=$1
11 | else
12 | do_date=$(date -d "-1 day" +%F)
13 | fi
14 |
15 | sql="
16 |
17 | INSERT overwrite TABLE ${APP}.dwd_praise_log PARTITION (dt = '$do_date') SELECT
18 | mid_id,
19 | user_id,
20 | version_code,
21 | version_name,
22 | lang,
23 | source,
24 | os,
25 | area,
26 | model,
27 | brand,
28 | sdk_version,
29 | gmail,
30 | height_width,
31 | app_time,
32 | network,
33 | lng,
34 | lat,
35 | get_json_object(event_json,'$.kv.id') id,
36 | get_json_object(event_json,'$.kv.userid') userid,
37 | get_json_object(event_json,'$.kv.target_id') target_id,
38 | get_json_object(event_json,'$.kv.type') type,
39 | get_json_object(event_json,'$.kv.add_time') add_time,
40 | server_time
41 | FROM
42 | ${APP}.dwd_base_event_log
43 | WHERE
44 | dt = '$do_date'
45 | AND event_name = 'praise';"
46 |
47 | echo "===开始从事件明细中提取日期为 $do_date 的点赞数据==="
48 | $hive -e "$sql"
49 | echo "===从事件明细中提取日期为 $do_date 的点赞数据完成==="
50 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/Hive-DataBase-START-DWD-to-DWS.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # 数据库名称
3 | APP=demo_database_gmall
4 |
5 | # hive目录
6 | hive=/usr/local/hive-2.3.7/bin/hive
7 |
8 | # 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天
9 | if [ -n "$1" ]; then
10 | do_date=$1
11 | else
12 | do_date=$(date -d "0 day" +%F)
13 | fi
14 |
15 | #每日设备行为
16 | sql="
17 | INSERT overwrite TABLE ${APP}.dws_uv_detail_daycount PARTITION (dt = '$do_date') SELECT
18 | mid_id,
19 | concat_ws('|', collect_set(user_id)) user_id,
20 | concat_ws(
21 | '|',
22 | collect_set (version_code)
23 | ) version_code,
24 | concat_ws(
25 | '|',
26 | collect_set (version_name)
27 | ) version_name,
28 | concat_ws('|', collect_set(lang)) lang,
29 | concat_ws('|', collect_set(source)) source,
30 | concat_ws('|', collect_set(os)) os,
31 | concat_ws('|', collect_set(area)) area,
32 | concat_ws('|', collect_set(model)) model,
33 | concat_ws('|', collect_set(brand)) brand,
34 | concat_ws(
35 | '|',
36 | collect_set (sdk_version)
37 | ) sdk_version,
38 | concat_ws('|', collect_set(gmail)) gmail,
39 | concat_ws(
40 | '|',
41 | collect_set (height_width)
42 | ) height_width,
43 | concat_ws('|', collect_set(app_time)) app_time,
44 | concat_ws('|', collect_set(network)) network,
45 | concat_ws('|', collect_set(lng)) lng,
46 | concat_ws('|', collect_set(lat)) lat,
47 | count(*) login_count
48 | FROM
49 | ${APP}.dwd_start_log
50 | WHERE
51 | dt = '$do_date'
52 | GROUP BY
53 | mid_id;"
54 |
55 | echo "===START to TAKE User Daily Device behavior DATA From $do_date ==="
56 | $hive -e "$sql"
57 | echo "===END to TAKE User Daily Device behavior DATA From $do_date ==="
58 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/Hive-DataBase-START-DWS-to-DWT.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # 数据库名称
3 | APP=demo_database_gmall
4 |
5 | # hive目录
6 | hive=/usr/local/hive-2.3.7/bin/hive
7 |
8 | # 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天
9 | if [ -n "$1" ]; then
10 | do_date=$1
11 | else
12 | do_date=$(date -d "0 day" +%F)
13 | fi
14 |
15 | #设备主题宽表
16 | sql="
17 | INSERT overwrite TABLE ${APP}.dwt_uv_topic SELECT
18 | nvl (new.mid_id, old.mid_id),
19 | nvl (new.user_id, old.user_id),
20 | nvl (
21 | new.version_code,
22 | old.version_code
23 | ),
24 | nvl (
25 | new.version_name,
26 | old.version_name
27 | ),
28 | nvl (new.lang, old.lang),
29 | nvl (new.source, old.source),
30 | nvl (new.os, old.os),
31 | nvl (new.area, old.area),
32 | nvl (new.model, old.model),
33 | nvl (new.brand, old.brand),
34 | nvl (
35 | new.sdk_version,
36 | old.sdk_version
37 | ),
38 | nvl (new.gmail, old.gmail),
39 | nvl (
40 | new.height_width,
41 | old.height_width
42 | ),
43 | nvl (new.app_time, old.app_time),
44 | nvl (new.network, old.network),
45 | nvl (new.lng, old.lng),
46 | nvl (new.lat, old.lat),
47 |
48 | IF (
49 | old.mid_id IS NULL,
50 | '$do_date',
51 | old.login_date_first
52 | ),
53 |
54 | IF (
55 | new.mid_id IS NOT NULL,
56 | '$do_date',
57 | old.login_date_last
58 | ),
59 |
60 | IF (
61 | new.mid_id IS NOT NULL,
62 | new.login_count,
63 | 0
64 | ),
65 | nvl (old.login_count, 0) +
66 | IF (new.login_count > 0, 1, 0)
67 | FROM
68 | (SELECT * FROM ${APP}.dwt_uv_topic) old
69 | FULL OUTER JOIN (
70 | SELECT
71 | *
72 | FROM
73 | ${APP}.dws_uv_detail_daycount
74 | WHERE
75 | dt = '$do_date'
76 | ) new ON old.mid_id = new.mid_id;"
77 |
78 | echo "===START to TAKE User Device subject wide table DATA From $do_date ==="
79 | $hive -e "$sql"
80 | echo "===END to TAKE User Device subject wide table DATA From $do_date ==="
81 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/Hive-DataBase-START-DWT-to-ADS-ActiveDevices.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # 数据库名称
3 | APP=demo_database_gmall
4 |
5 | # hive目录
6 | hive=/usr/local/hive-2.3.7/bin/hive
7 |
8 | # 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天
9 | if [ -n "$1" ]; then
10 | do_date=$1
11 | else
12 | do_date=$(date -d "-1 day" +%F)
13 | fi
14 |
15 | sql="
16 |
17 | INSERT INTO TABLE ${APP}.ads_uv_count SELECT
18 | '$do_date' dt,
19 | daycount.ct,
20 | wkcount.ct,
21 | mncount.ct,
22 |
23 | IF (
24 | date_add(
25 | next_day ('$do_date', 'MO') ,- 1
26 | ) = '$do_date',
27 | 'Y',
28 | 'N'
29 | ),
30 |
31 | IF (
32 | last_day('$do_date') = '$do_date',
33 | 'Y',
34 | 'N'
35 | )
36 | FROM
37 | (
38 | SELECT
39 | '$do_date' dt,
40 | count(*) ct
41 | FROM
42 | ${APP}.dwt_uv_topic
43 | WHERE
44 | login_date_last = '$do_date'
45 | ) daycount
46 | JOIN (
47 | SELECT
48 | '$do_date' dt,
49 | count(*) ct
50 | FROM
51 | ${APP}.dwt_uv_topic
52 | WHERE
53 | login_date_last >= date_add(
54 | next_day ('$do_date', 'MO') ,- 7
55 | )
56 | AND login_date_last <= date_add(
57 | next_day ('$do_date', 'MO') ,- 1
58 | )
59 | ) wkcount ON daycount.dt = wkcount.dt
60 | JOIN (
61 | SELECT
62 | '$do_date' dt,
63 | count(*) ct
64 | FROM
65 | ${APP}.dwt_uv_topic
66 | WHERE
67 | date_format(login_date_last, 'yyyy-MM') = date_format('$do_date', 'yyyy-MM')
68 | ) mncount ON daycount.dt = mncount.dt; "
69 |
70 | echo "===START TO TAKE Number of active devices DATA From $do_date ==="
71 | $hive -e "$sql"
72 | echo "===Finished TO TAKE Number of active devices DATA From $do_date ==="
73 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/Hive-DataBase-START-DWT-to-ADS-ActiveUsersLastThreeConsecutiveWeeks.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # 数据库名称
3 | APP=demo_database_gmall
4 |
5 | # hive目录
6 | hive=/usr/local/hive-2.3.7/bin/hive
7 |
8 | # 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天
9 | if [ -n "$1" ]; then
10 | do_date=$1
11 | else
12 | do_date=$(date -d "-1 day" +%F)
13 | fi
14 |
15 | sql="
16 | INSERT INTO TABLE ${APP}.ads_continuity_wk_count SELECT
17 | '$do_date',
18 | concat(
19 | date_add(
20 | next_day ('$do_date', 'MO') ,- 7 * 3
21 | ),
22 | '_',
23 | date_add(
24 | next_day ('$do_date', 'MO') ,- 1
25 | )
26 | ),
27 | count(*)
28 | FROM
29 | (
30 | SELECT
31 | mid_id
32 | FROM
33 | (
34 | SELECT mid_id
35 | FROM
36 | ${APP}.dws_uv_detail_daycount
37 | WHERE
38 | dt >= date_add(
39 | next_day ('$do_date', 'monday') ,- 7
40 | )
41 | AND dt <= date_add(
42 | next_day ('$do_date', 'monday') ,- 1
43 | )
44 | GROUP BY
45 | mid_id
46 | UNION ALL
47 | SELECT
48 | mid_id
49 | FROM
50 | ${APP}.dws_uv_detail_daycount
51 | WHERE
52 | dt >= date_add(
53 | next_day ('$do_date', 'monday') ,- 7 * 2
54 | )
55 | AND dt <= date_add(
56 | next_day ('$do_date', 'monday') ,- 7 - 1
57 | )
58 | GROUP BY
59 | mid_id
60 | UNION ALL
61 | SELECT
62 | mid_id
63 | FROM
64 | ${APP}.dws_uv_detail_daycount
65 | WHERE
66 | dt >= date_add(
67 | next_day ('$do_date', 'monday') ,- 7 * 3
68 | )
69 | AND dt <= date_add(
70 | next_day ('$do_date', 'monday') ,- 7 * 2 - 1
71 | )
72 | GROUP BY
73 | mid_id
74 | ) t1
75 | GROUP BY
76 | mid_id
77 | HAVING
78 | count(*) = 3
79 | ) t2;"
80 |
81 | echo "===START TO TAKE The Number of active users in the last three consecutive weeks DATA From $do_date ==="
82 | $hive -e "$sql"
83 | echo "===Finished TO TAKE The Number of active users in the last three consecutive weeks DATA From $do_date ==="
84 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/Hive-DataBase-START-DWT-to-ADS-ContinuousActiveDevices.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # 数据库名称
3 | APP=demo_database_gmall
4 |
5 | # hive目录
6 | hive=/usr/local/hive-2.3.7/bin/hive
7 |
8 | # 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天
9 | if [ -n "$1" ]; then
10 | do_date=$1
11 | else
12 | do_date=$(date -d "-1 day" +%F)
13 | fi
14 |
15 | sql="
16 | INSERT INTO TABLE ${APP}.ads_continuity_uv_count SELECT
17 | '$do_date',
18 | concat(
19 | date_add('$do_date' ,- 6),
20 | '_',
21 | '$do_date'
22 | ),
23 | count(*)
24 | FROM
25 | (
26 | SELECT
27 | mid_id
28 | FROM
29 | (
30 | SELECT
31 | mid_id
32 | FROM
33 | (
34 | SELECT
35 | mid_id,
36 | date_sub(dt, rank) date_dif
37 | FROM
38 | (
39 | SELECT
40 | mid_id,
41 | dt,
42 | rank () over (PARTITION BY mid_id ORDER BY dt) rank
43 | FROM
44 | ${APP}.dws_uv_detail_daycount
45 | WHERE
46 | dt >= date_add('$do_date' ,- 6)
47 | AND dt <= '$do_date'
48 | ) t1
49 | ) t2
50 | GROUP BY
51 | mid_id,
52 | date_dif
53 | HAVING
54 | count(*) >= 3
55 | ) t3
56 | GROUP BY
57 | mid_id
58 | ) t4;"
59 |
60 | echo "===START TO TAKE The Number of continuous active devices DATA From $do_date ==="
61 | $hive -e "$sql"
62 | echo "===Finished TO TAKE The Number of continuous active devices DATA From $do_date ==="
63 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/Hive-DataBase-START-DWT-to-ADS-LostUsers.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # 数据库名称
3 | APP=demo_database_gmall
4 |
5 | # hive目录
6 | hive=/usr/local/hive-2.3.7/bin/hive
7 |
8 | # 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天
9 | if [ -n "$1" ]; then
10 | do_date=$1
11 | else
12 | do_date=$(date -d "-1 day" +%F)
13 | fi
14 |
15 | sql="
16 | INSERT INTO TABLE ${APP}.ads_wastage_count SELECT
17 | '$do_date',
18 | count(*)
19 | FROM
20 | (
21 | SELECT
22 | mid_id
23 | FROM
24 | ${APP}.dwt_uv_topic
25 | WHERE
26 | login_date_last <= date_add('$do_date' ,- 7)
27 | GROUP BY
28 | mid_id
29 | ) t1;"
30 |
31 | echo "===START TO TAKE The Number of lost users DATA From $do_date ==="
32 | $hive -e "$sql"
33 | echo "===Finished TO TAKE The Number of lost users DATA From $do_date ==="
34 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/Hive-DataBase-START-DWT-to-ADS-NumberOfNewDeviceAddedDaily.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # 数据库名称
3 | APP=demo_database_gmall
4 |
5 | # hive目录
6 | hive=/usr/local/hive-2.3.7/bin/hive
7 |
8 | # 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天
9 | if [ -n "$1" ]; then
10 | do_date=$1
11 | else
12 | do_date=$(date -d "-1 day" +%F)
13 | fi
14 |
15 | sql="
16 |
17 | INSERT INTO TABLE ${APP}.ads_new_mid_count SELECT
18 | login_date_first,
19 | count(*)
20 | FROM
21 | ${APP}.dwt_uv_topic
22 | WHERE
23 | login_date_first = '$do_date'
24 | GROUP BY
25 | login_date_first;"
26 |
27 | echo "===START TO TAKE The number of new device information added daily DATA From $do_date ==="
28 | $hive -e "$sql"
29 | echo "===Finished TO TAKE The number of new device information added daily DATA From $do_date ==="
30 |
31 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/Hive-DataBase-START-DWT-to-ADS-NumberOfSilentUsers.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # 数据库名称
3 | APP=demo_database_gmall
4 |
5 | # hive目录
6 | hive=/usr/local/hive-2.3.7/bin/hive
7 |
8 | # 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天
9 | if [ -n "$1" ]; then
10 | do_date=$1
11 | else
12 | do_date=$(date -d "-1 day" +%F)
13 | fi
14 |
15 | sql="
16 |
17 | INSERT INTO TABLE ${APP}.ads_silent_count SELECT
18 | '$do_date',
19 | count(*)
20 | FROM
21 | ${APP}.dwt_uv_topic
22 | WHERE
23 | login_date_first = login_date_last
24 | AND login_date_last <= date_add('$do_date' ,- 7);"
25 |
26 | echo "===START TO TAKE The Number of silent users DATA From $do_date ==="
27 | $hive -e "$sql"
28 | echo "===Finished TO TAKE The Number of silent users DATA From $do_date ==="
29 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/Hive-DataBase-START-DWT-to-ADS-ReturningUsers.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # 数据库名称
3 | APP=demo_database_gmall
4 |
5 | # hive目录
6 | hive=/usr/local/hive-2.3.7/bin/hive
7 |
8 | # 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天
9 | if [ -n "$1" ]; then
10 | do_date=$1
11 | else
12 | do_date=$(date -d "-1 day" +%F)
13 | fi
14 |
15 | sql="
16 | INSERT INTO TABLE ${APP}.ads_back_count SELECT
17 | '$do_date',
18 | count(*)
19 | FROM
20 | (
21 | SELECT
22 | mid_id
23 | FROM
24 | ${APP}.dwt_uv_topic
25 | WHERE
26 | login_date_last >= date_add(
27 | next_day ('$do_date', 'MO') ,- 7
28 | )
29 | AND login_date_last <= date_add(
30 | next_day ('$do_date', 'MO') ,- 1
31 | )
32 | AND login_date_first < date_add(
33 | next_day ('$do_date', 'MO') ,- 7
34 | )
35 | ) current_wk
36 | LEFT JOIN (
37 | SELECT
38 | mid_id
39 | FROM
40 | ${APP}.dws_uv_detail_daycount
41 | WHERE
42 | dt >= date_add(
43 | next_day ('$do_date', 'MO') ,- 7 * 2
44 | )
45 | AND dt <= date_add(
46 | next_day ('$do_date', 'MO') ,- 7 - 1
47 | )
48 | GROUP BY
49 | mid_id
50 | ) last_wk ON current_wk.mid_id = last_wk.mid_id
51 | WHERE
52 | last_wk.mid_id IS NULL;
53 | "
54 |
55 | echo "===START TO TAKE The Number of returning users this week DATA From $do_date ==="
56 | $hive -e "$sql"
57 | echo "===Finished TO TAKE The Number of returning users this week DATA From $do_date ==="
58 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/Hive-DataBase-START-ODS-to-DWD.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # 数据库名称
3 | APP=demo_database_gmall
4 |
5 | # hive目录
6 | hive=/usr/local/hive-2.3.7/bin/hive
7 |
8 | # 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天
9 | if [ -n "$1" ]; then
10 | do_date=$1
11 | else
12 | do_date=$(date -d "-1 day" +%F)
13 | fi
14 |
15 | # get_json_object(string json_string, string path)
16 | #
17 | #说明：
18 | #第一个参数填写json对象变量，第二个参数使用$表示json变量标识，然后用 . 或 [] 读取对象或数组；如果输入的json字符串无效，那么返回NULL。
19 | #每次只能返回一个数据项。
20 |
21 | sql="
22 | set hive.exec.dynamic.partition.mode=nonstrict;
23 | INSERT OVERWRITE TABLE ${APP}.dwd_start_log
24 | PARTITION (dt='$do_date')
25 | select
26 | get_json_object(line,'$.mid') mid_id,
27 | get_json_object(line,'$.uid') user_id,
28 | get_json_object(line,'$.vc') version_code,
29 | get_json_object(line,'$.vn') version_name,
30 | get_json_object(line,'$.l') lang,
31 | get_json_object(line,'$.sr') source,
32 | get_json_object(line,'$.os') os,
33 | get_json_object(line,'$.ar') area,
34 | get_json_object(line,'$.md') model,
35 | get_json_object(line,'$.ba') brand,
36 | get_json_object(line,'$.sv') sdk_version,
37 | get_json_object(line,'$.g') gmail,
38 | get_json_object(line,'$.hw') height_width,
39 | get_json_object(line,'$.t') app_time,
40 | get_json_object(line,'$.nw') network,
41 | get_json_object(line,'$.ln') lng,
42 | get_json_object(line,'$.la') lat,
43 | get_json_object(line,'$.entry') entry,
44 | get_json_object(line,'$.open_ad_type') open_ad_type,
45 | get_json_object(line,'$.action') action,
46 | get_json_object(line,'$.loading_time') loading_time,
47 | get_json_object(line,'$.detail') detail,
48 | get_json_object(line,'$.extend1') extend1 from ${APP}.ods_start_log where dt='$do_date'; "
49 |
50 | echo "===开始清洗日志日期为 $do_date 的数据==="
51 | $hive -e "$sql"
52 | echo "===日志日期为 $do_date 的数据清洗完成==="
53 |
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/Start_Log.json:
--------------------------------------------------------------------------------
1 | {
2 | "mid": "M0001",
3 | "uid": "2020092409590001",
4 | "vc": "V 2.3.1",
5 | "vn": "Beta",
6 | "l": "CHN",
7 | "sr": "C",
8 | "os": "8.6",
9 | "ar": "shen zhen",
10 | "md": "Plus",
11 | "ba": "Apple",
12 | "sv": "V 1.5.6",
13 | "g": "heyong@gmail.com",
14 | "hw": "20*10",
15 | "t": "2020-10-20 09:35:00",
16 | "nw": "4G",
17 | "ln": "-63.8",
18 | "la": "-0.69999999",
19 | "entry": "App",
20 | "open_ad_type": "Web H5",
21 | "action": "login",
22 | "loading_time": "202009241038521",
23 | "detail": "用户->2020092409590001在设备->M0001启动了APP!",
24 | "extend1": {
25 | "data": "APP启动日志",
26 | "date": "2020-10-20"
27 | }
28 | }
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/application-dev.yml:
--------------------------------------------------------------------------------
1 | #日志配置文件
2 | logging:
3 | config: classpath:logback-spring.xml
4 |
5 | #服务器端口号
6 | server:
7 | port: 8089
8 |
9 | #配置hive数据源
10 | spring:
11 | datasource:
12 | hdfs: #hdfs数据源 对应的是/etc/hadoop目录下的core-site.xml文件中的配置
13 | name: fs.defaultFS
14 | url: hdfs://172.12.0.4:9000
15 | user: root
16 | hive: #hive数据源 一定要启动hiveserver2服务
17 | url: jdbc:hive2://172.12.0.4:10000/demo_database_gmall
18 | type: com.alibaba.druid.pool.DruidDataSource
19 | username: hive
20 | password: hive
21 | driver-class-name: org.apache.hive.jdbc.HiveDriver
22 | pool: #连接池统一配置，应用到所有的数据源
23 | initial-size: 1
24 | min-idle: 1
25 | max-idle: 5
26 | max-active: 50
27 | max-wait: 10000 # 配置获取连接等待超时的时间
28 | time-between-eviction-runs-millis: 10000 # 配置间隔多久才进行一次检测，检测需要关闭的空闲连接，单位是毫秒
29 | min-evictable-idle-time-millis: 300000 # 配置一个连接在池中最小生存的时间，单位是毫秒
30 | validation-query: select 'x'
31 | test-while-idle: true
32 | test-on-borrow: false
33 | test-on-return: false
34 | pool-prepared-statements: true # 打开PSCache，并且指定每个连接上PSCache的大小
35 | max-pool-prepared-statement-per-connection-size: 20
36 | filters: stat
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/application-pro.yml:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bys-eric-he/com-hadoop-bigdata-demo/390cc4369033261c1abac8867fd59ec3f7ac2c8b/com-hadoop-hive-demo/src/main/resources/application-pro.yml
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/application.yml:
--------------------------------------------------------------------------------
1 | spring:
2 | profiles:
3 | active: dev
--------------------------------------------------------------------------------
/com-hadoop-hive-demo/src/main/resources/azkaban-job.zip:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bys-eric-he/com-hadoop-bigdata-demo/390cc4369033261c1abac8867fd59ec3f7ac2c8b/com-hadoop-hive-demo/src/main/resources/azkaban-job.zip
--------------------------------------------------------------------------------
/com-hadoop-kafka-demo/src/main/java/com/hadoop/kafka/SpringBootHadoopKafkaApplication.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.kafka;
2 |
3 | import com.hadoop.kafka.common.KafkaUtils;
4 | import lombok.extern.slf4j.Slf4j;
5 | import org.mybatis.spring.annotation.MapperScan;
6 | import org.springframework.boot.SpringApplication;
7 | import org.springframework.boot.autoconfigure.SpringBootApplication;
8 | import org.springframework.context.ConfigurableApplicationContext;
9 |
10 | import java.util.List;
11 |
12 | @Slf4j
13 | @MapperScan(basePackages = "com.hadoop.kafka.mapper")
14 | @SpringBootApplication
15 | public class SpringBootHadoopKafkaApplication {
16 | public static void main(String[] args) {
17 | ConfigurableApplicationContext context = SpringApplication.run(SpringBootHadoopKafkaApplication.class, args);
18 | KafkaUtils kafkaUtils = context.getBean(KafkaUtils.class);
19 | List queryAllTopic = kafkaUtils.queryAllTopic();
20 | log.info("---->当前系统所有消息主题：{}", queryAllTopic.toString());
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/com-hadoop-kafka-demo/src/main/java/com/hadoop/kafka/common/Result.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.kafka.common;
2 |
3 |
4 | import lombok.Data;
5 |
6 | /**
7 | * API 统一返回结果
8 | *
9 | * @param
10 | */
11 | @Data
12 | public class Result {
13 |
14 | /**
15 | * 响应码
16 | */
17 | private Integer resCode;
18 |
19 | /**
20 | * 错误码
21 | */
22 | private String errCode;
23 |
24 | /**
25 | * 错误信息
26 | */
27 | private String errMsg;
28 |
29 | /**
30 | * 数据
31 | */
32 | private T data;
33 |
34 | public Result(Integer resCode, String errCode, String errMsg, T data) {
35 | this.resCode = resCode;
36 | this.errCode = errCode;
37 | this.errMsg = errMsg;
38 | this.data = data;
39 | }
40 | }
--------------------------------------------------------------------------------
/com-hadoop-kafka-demo/src/main/java/com/hadoop/kafka/common/SpringContextUtil.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.kafka.common;
2 |
3 | import org.springframework.context.ApplicationContext;
4 | import org.springframework.context.ApplicationContextAware;
5 | import org.springframework.stereotype.Component;
6 |
7 | /**
8 | * 使用上下文获取bean
9 | */
10 | @Component
11 | public class SpringContextUtil implements ApplicationContextAware {
12 | private static ApplicationContext applicationContext;
13 |
14 | /**
15 | * 获取上下文
16 | *
17 | * @return
18 | */
19 | public static ApplicationContext getApplicationContext() {
20 | return applicationContext;
21 | }
22 |
23 | /**
24 | * 设置上下文
25 | *
26 | * @param applicationContext
27 | */
28 | @Override
29 | public void setApplicationContext(ApplicationContext applicationContext) {
30 | SpringContextUtil.applicationContext = applicationContext;
31 | }
32 |
33 | /**
34 | * 通过名字获取上下文中的bean
35 | *
36 | * @param name
37 | * @return
38 | */
39 | public static Object getBean(String name) {
40 | return applicationContext.getBean(name);
41 | }
42 |
43 | /**
44 | * 通过类型获取上下文中的bean
45 | *
46 | * @param requiredType
47 | * @return
48 | */
49 | public static Object getBean(Class requiredType) {
50 | return applicationContext.getBean(requiredType);
51 | }
52 | }
53 |
--------------------------------------------------------------------------------
/com-hadoop-kafka-demo/src/main/java/com/hadoop/kafka/common/TopicConstant.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.kafka.common;
2 |
3 | /**
4 | * 常量类
5 | */
6 | public class TopicConstant {
7 | /**
8 | * 用户日志消息主题
9 | */
10 | public static String USER_LOG_TOPIC_MESSAGE = "kafka_topic_user_log_message";
11 |
12 | /**
13 | * 用户订单消息主题
14 | */
15 | public static String USER_ORDER_TOPIC_MESSAGE = "kafka_topic_user_order_message";
16 |
17 | /**
18 | * 用户日志流式处理主题
19 | */
20 | public static String USER_LOG_PROCESSOR_TOPIC_FROM = "kafka_topic_log_processor_from";
21 |
22 | /**
23 | * 用户日志流式处理主题
24 | */
25 | public static String USER_LOG_PROCESSOR_TOPIC_TO = "kafka_topic_log_processor_to";
26 | /**
27 | * 实时流处理输入主题
28 | */
29 | public static String KAFKA_STREAMS_PIPE_INPUT = "kafka_streams_pipe_input";
30 | /**
31 | * 实时流处理输出主题
32 | */
33 | public static String KAFKA_STREAMS_PIPE_OUTPUT = "kafka_streams_pipe_output";
34 |
35 | /**
36 | * 实时流单词分割字符串输入主题
37 | */
38 | public static String KAFKA_STREAMS_LINESPLIT_INPUT = "kafka_streams_linesplit_input";
39 |
40 | /**
41 | * 实时流单词分割字符串输出主题
42 | */
43 | public static String KAFKA_STREAMS_LINESPLIT_OUTPUT = "kafka_streams_linesplit_output";
44 | }
45 |
--------------------------------------------------------------------------------
/com-hadoop-kafka-demo/src/main/java/com/hadoop/kafka/config/PartitionStrategyConfig.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.kafka.config;
2 |
3 | import lombok.extern.slf4j.Slf4j;
4 | import org.apache.kafka.clients.producer.Partitioner;
5 | import org.apache.kafka.common.Cluster;
6 | import org.apache.kafka.common.PartitionInfo;
7 |
8 | import java.util.List;
9 | import java.util.Map;
10 | import java.util.Random;
11 |
12 | /**
13 | * 自定义分区策略
14 | * 在发送一条消息时,我们可以指定这个 Key,那么 Producer 会根据 Key 和 partition 机制来判断,
15 | * 当前这条消息应该发送并存储到哪个 partition 中（这个就跟分片机制类似）。
16 | * 我们可以根据需要进行扩展 Producer 的 partition 机制（默认算法是 hash 取 %）。
17 | */
18 |
19 | @Slf4j
20 | public class PartitionStrategyConfig implements Partitioner {
21 |
22 | @Override
23 | public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) {
24 | //获取当前 topic 有多少个分区（分区列表）
25 | List partitions = cluster.partitionsForTopic(topic);
26 | int partitionNum = 0;
27 |
28 | //Key 是可以传空值的
29 | if (key == null) {
30 | partitionNum = new Random().nextInt(partitions.size()); //随机
31 | } else {
32 | //取 %
33 | partitionNum = Math.abs((key.hashCode()) % partitions.size());
34 | }
35 | log.warn("自定义分区策略---->key：" + key + ",value：" + value + ",partitionNum：" + partitionNum + ",partitionSize: " + partitions.size());
36 | //发送到指定分区
37 | return partitionNum;
38 | }
39 |
40 | @Override
41 | public void close() {
42 |
43 | }
44 |
45 | @Override
46 | public void configure(Map configs) {
47 |
48 | }
49 | }
50 |
--------------------------------------------------------------------------------
/com-hadoop-kafka-demo/src/main/java/com/hadoop/kafka/config/Swagger2Config.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.kafka.config;
2 |
3 |
4 | import org.springframework.context.annotation.Bean;
5 | import org.springframework.context.annotation.Configuration;
6 | import org.springframework.web.bind.annotation.RestController;
7 | import springfox.documentation.builders.ApiInfoBuilder;
8 | import springfox.documentation.builders.PathSelectors;
9 | import springfox.documentation.builders.RequestHandlerSelectors;
10 | import springfox.documentation.service.ApiInfo;
11 | import springfox.documentation.service.Contact;
12 | import springfox.documentation.spi.DocumentationType;
13 | import springfox.documentation.spring.web.plugins.Docket;
14 | import springfox.documentation.swagger2.annotations.EnableSwagger2;
15 |
16 | import java.time.LocalDateTime;
17 | import java.util.Date;
18 |
19 | @Configuration
20 | @EnableSwagger2
21 | public class Swagger2Config {
22 |
23 | /**
24 | * Swagger组件注册
25 | */
26 | @Bean
27 | public Docket api() {
28 | return new Docket(DocumentationType.SWAGGER_2)
29 | .apiInfo(apiInfo())
30 | .select()
31 | .apis(RequestHandlerSelectors.withClassAnnotation(RestController.class))
32 | .paths(PathSelectors.any())
33 | .build()
34 | .directModelSubstitute(LocalDateTime.class, Date.class);
35 | }
36 |
37 | private ApiInfo apiInfo() {
38 | return new ApiInfoBuilder()
39 | .title("Spring Boot For Kafka Demo API V1.0")
40 | .description("Spring Boot Hadoop Kafka Application")
41 | .termsOfServiceUrl("http://localhost:8500/swagger-ui.html")
42 | .contact(new Contact("he yong", "", "heyong@9fstock.com"))
43 | .version("1.0")
44 | .build();
45 | }
46 | }
47 |
--------------------------------------------------------------------------------
/com-hadoop-kafka-demo/src/main/java/com/hadoop/kafka/consumer/AbstractBaseConsumer.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.kafka.consumer;
2 |
3 | import org.apache.kafka.clients.consumer.ConsumerRecord;
4 |
5 | public abstract class AbstractBaseConsumer {
6 | /**
7 | * 消费处理
8 | *
9 | * @param consumerRecord
10 | * @return
11 | */
12 | protected abstract void call(ConsumerRecord consumerRecord);
13 |
14 | public void action(ConsumerRecord consumerRecord) {
15 | this.call(consumerRecord);
16 | }
17 | }
18 |
--------------------------------------------------------------------------------
/com-hadoop-kafka-demo/src/main/java/com/hadoop/kafka/consumer/UserLogConsumer.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.kafka.consumer;
2 |
3 | import com.hadoop.kafka.handler.KafkaConsumerResultHandler;
4 | import lombok.extern.slf4j.Slf4j;
5 | import org.apache.kafka.clients.consumer.ConsumerRecord;
6 | import org.springframework.kafka.annotation.KafkaListener;
7 | import org.springframework.stereotype.Component;
8 |
9 | /**
10 | * 日志消费者
11 | */
12 | @Slf4j
13 | @Component
14 | public class UserLogConsumer extends AbstractBaseConsumer {
15 | /**
16 | * 订阅者
17 | * 对同一个topic,不同组中的订阅者都会收到消息,即一个topic对应多个consumer,
18 | * 同一组中的订阅者只有一个consumer能够收到消息,即一个topic对应一个consumer.
19 | *
20 | * @param consumerRecord
21 | */
22 | @KafkaListener(
23 | id = "user-log-consumer-one",
24 | groupId = "kafka_consumer_group_user_log",
25 | topics = "kafka_topic_user_log_message")
26 | public void consumer(ConsumerRecord consumerRecord) {
27 | super.action(consumerRecord);
28 | }
29 |
30 | /**
31 | * 消费处理
32 | *
33 | * @param consumerRecord
34 | * @return
35 | */
36 | @Override
37 | protected void call(ConsumerRecord consumerRecord) {
38 | KafkaConsumerResultHandler consumerData = new KafkaConsumerResultHandler(consumerRecord);
39 | try {
40 | String result = consumerData.call();
41 | log.info(result);
42 | } catch (Exception e) {
43 | e.printStackTrace();
44 | }
45 | }
46 | }
47 |
--------------------------------------------------------------------------------
/com-hadoop-kafka-demo/src/main/java/com/hadoop/kafka/container/ThreadMessageListenerContainer.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.kafka.container;
2 |
3 | import org.springframework.kafka.core.ConsumerFactory;
4 | import org.springframework.kafka.listener.ConcurrentMessageListenerContainer;
5 | import org.springframework.kafka.listener.ContainerProperties;
6 |
7 | /**
8 | * 继承消息监听容器
9 | */
10 | public class ThreadMessageListenerContainer extends ConcurrentMessageListenerContainer {
11 |
12 | public ThreadMessageListenerContainer(ConsumerFactory consumerFactory, ContainerProperties containerProperties) {
13 | super(consumerFactory, containerProperties);
14 | }
15 |
16 | public void startContainer() {
17 | super.doStart();
18 | }
19 | }
--------------------------------------------------------------------------------
/com-hadoop-kafka-demo/src/main/java/com/hadoop/kafka/controller/UserLogController.java:
--------------------------------------------------------------------------------
1 | package com.hadoop.kafka.controller;
2 |
3 | import com.hadoop.kafka.common.Result;
4 | import com.hadoop.kafka.common.ResultUtil;
5 | import com.hadoop.kafka.model.UserLog;
6 | import com.hadoop.kafka.producer.UserLogProducer;
7 | import io.swagger.annotations.Api;
8 | import io.swagger.annotations.ApiOperation;
9 | import org.springframework.beans.factory.annotation.Autowired;
10 | import org.springframework.web.bind.annotation.RequestMapping;
11 | import org.springframework.web.bind.annotation.RequestMethod;
12 | import org.springframework.web.bind.annotation.RestController;
13 |
14 | @Api(value = "/api/v1/user-log", tags = "用户日志API服务")
15 | @RestController
16 | @RequestMapping("/api/v1/user-log")
17 | public class UserLogController {
18 |
19 | @Autowired
20 | private UserLogProducer userLogProducer;
21 |
22 | @ApiOperation("发送日志到后端")
23 | @RequestMapping(path = "/send", method = RequestMethod.POST)
24 | public Result