├── .gitignore ├── CTCC_analysis ├── CTCC_analysis.iml ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── nil │ ├── converter │ ├── DimensionConverter.java │ └── DimensionConverterImpl.java │ ├── kv │ ├── base │ │ ├── BaseDimension.java │ │ └── BaseValue.java │ ├── key │ │ ├── ComDimension.java │ │ ├── ContactDimension.java │ │ └── DateDimension.java │ └── value │ │ └── CountDurationValue.java │ ├── mapper │ └── CountDurationMapper.java │ ├── outputformat │ └── MysqlOutputFormat.java │ ├── reducer │ └── CountDurationReducer.java │ ├── runner │ └── CountDurationRunner.java │ └── utils │ ├── JdbcInstance.java │ ├── JdbcUtil.java │ └── LRUCache.java ├── CTCC_consumer ├── CTCC_consumer.iml ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── nil │ │ ├── hbase │ │ ├── CalleeWriteObserver.java │ │ └── HBaseDAO.java │ │ ├── kafka │ │ └── HBaseConsumer.java │ │ └── utils │ │ ├── ConnectionInstance.java │ │ ├── HBaseUtil.java │ │ └── PropertiesUtil.java │ └── resources │ ├── core-site.xml │ ├── hbase-site.xml │ ├── hbase_consumer.properties │ ├── hdfs-site.xml │ └── log4j.properties ├── CTCC_producer ├── CTCC_producer.iml ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── nil │ ├── ProductDriver.java │ └── ProductLog.java ├── CTCC_web ├── CTCC_web.iml ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── nil │ │ ├── bean │ │ ├── CallLog.java │ │ └── QueryInfo.java │ │ ├── controller │ │ └── CallLogHandler.java │ │ └── dao │ │ └── CallLogDAO.java │ ├── resources │ ├── applicationContext.xml │ └── dbconfig.properties │ └── webapp │ ├── WEB-INF │ └── web.xml │ ├── index.jsp │ ├── js │ └── echarts.min.js │ └── jsp │ └── CallLogListEchart.jsp ├── Nil_ctcc.iml ├── ReadMe.md ├── doc └── db_telecom.sql ├── index.png └── pom.xml /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .idea 3 | target 4 | *.class 5 | Test -------------------------------------------------------------------------------- /CTCC_analysis/CTCC_analysis.iml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /CTCC_analysis/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | Nil_ctcc 7 | com.nil 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | CTCC_analysis 13 | 14 | CTCC_analysis 15 | 16 | http://www.example.com 17 | 18 | 19 | UTF-8 20 | 1.8 21 | 1.8 22 | 23 | 24 | 25 | 26 | junit 27 | junit 28 | 4.11 29 | test 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | maven-clean-plugin 38 | 3.0.0 39 | 40 | 41 | 42 | maven-resources-plugin 43 | 3.0.2 44 | 45 | 46 | maven-compiler-plugin 47 | 3.7.0 48 | 49 | 50 | maven-surefire-plugin 51 | 2.20.1 52 | 53 | 54 | maven-jar-plugin 55 | 3.0.2 56 | 57 | 58 | maven-install-plugin 59 | 2.5.2 60 | 61 | 62 | maven-deploy-plugin 63 | 2.8.2 64 | 65 | 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /CTCC_analysis/src/main/java/com/nil/converter/DimensionConverter.java: -------------------------------------------------------------------------------- 1 | package com.nil.converter; 2 | 3 | import com.nil.kv.base.BaseDimension; 4 | /** 5 | * @author lianyou 6 | * @version 1.0 7 | */ 8 | public interface DimensionConverter { 9 | /** 10 | * 基本维度定义 11 | * 12 | * @param dimension 13 | * @return 14 | */ 15 | int getDimensionID(BaseDimension dimension); 16 | } 17 | -------------------------------------------------------------------------------- /CTCC_analysis/src/main/java/com/nil/converter/DimensionConverterImpl.java: -------------------------------------------------------------------------------- 1 | package com.nil.converter; 2 | 3 | import com.nil.kv.base.BaseDimension; 4 | import com.nil.kv.key.ContactDimension; 5 | import com.nil.kv.key.DateDimension; 6 | import com.nil.utils.JdbcInstance; 7 | import com.nil.utils.JdbcUtil; 8 | import com.nil.utils.LRUCache; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.sql.Connection; 13 | import java.sql.PreparedStatement; 14 | import java.sql.ResultSet; 15 | import java.sql.SQLException; 16 | 17 | /** 18 | * 根据传入的维度数据，得到该数据对应的在表中的主键id 19 | * 20 | *

做内存缓存，LRUCache 分支 -- 缓存中有数据 -> 直接返回id -- 缓存中无数据 -> 21 | * 22 | *

查询Mysql 分支： 23 | * 24 | *

-- Mysql中有该条数据 -> 直接返回id -> 将本次读取到的id缓存到内存中 -- Mysql中没有该数据 -> 插入该条数据 -> 再次反查该数据，得到id并返回缓存到内存中 25 | * 26 | * @author lianyou 27 | * @version 1.0 28 | */ 29 | public class DimensionConverterImpl implements DimensionConverter { 30 | /** Logger 打印该类的日志,取代resources里的log4j.properties */ 31 | private static final Logger logger = LoggerFactory.getLogger(DimensionConverterImpl.class); 32 | /** 对象线程化 用于每个线程管理自己的JDBC连接器 */ 33 | private ThreadLocal threadLocalConnection = new ThreadLocal<>(); 34 | /** 构建内存缓存对象 */ 35 | private LRUCache lruCache = new LRUCache(3000); 36 | 37 | public DimensionConverterImpl() { 38 | // jvm关闭时，释放资源 39 | Runtime.getRuntime() 40 | .addShutdownHook(new Thread(() -> JdbcUtil.close(threadLocalConnection.get(), null, null))); 41 | } 42 | 43 | @Override 44 | public int getDimensionID(BaseDimension dimension) { 45 | // 1、根据传入的维度对象获取对应的主键id，先从LRUCache中获取 46 | // 时间维度：date_dimension_year_month_day, 10 47 | // 联系人维度：contact_dimension_telephone_name(到了电话号码就不会重复了), 12 48 | String cacheKey = genCacheKey(dimension); 49 | // 尝试获取缓存的id 50 | if (lruCache.containsKey(cacheKey)) { 51 | return lruCache.get(cacheKey); 52 | } 53 | 54 | // 没有得到缓存id，需要执行select操作 55 | // sqls包含了1组sql语句：查询和插入 56 | String[] sqls = null; 57 | if (dimension instanceof DateDimension) { 58 | sqls = getDateDimensionSQL(); 59 | } else if (dimension instanceof ContactDimension) { 60 | sqls = getContactDimensionSQL(); 61 | } else { 62 | throw new RuntimeException("没有匹配到对应维度信息."); 63 | } 64 | 65 | // 准备对Mysql表进行操作，先查询，有可能再插入 66 | Connection conn = this.getConnection(); 67 | int id = -1; 68 | synchronized (this) { 69 | id = execSQL(conn, sqls, dimension); 70 | } 71 | // 将刚查询到的id加入到缓存中 72 | lruCache.put(cacheKey, id); 73 | return id; 74 | } 75 | 76 | /** 77 | * 得到当前线程维护的Connection对象 78 | * 79 | * @return 80 | */ 81 | private Connection getConnection() { 82 | Connection conn = null; 83 | try { 84 | conn = threadLocalConnection.get(); 85 | if (conn == null || conn.isClosed()) { 86 | conn = JdbcInstance.getInstance(); 87 | threadLocalConnection.set(conn); 88 | } 89 | } catch (SQLException e) { 90 | e.printStackTrace(); 91 | } 92 | return conn; 93 | } 94 | 95 | /** 96 | * @param conn JDBC连接器 97 | * @param sqls 长度为2，第一个位置为查询语句，第二个位置为插入语句 98 | * @param dimension 对应维度所保存的数据 99 | * @return 100 | */ 101 | private int execSQL(Connection conn, String[] sqls, BaseDimension dimension) { 102 | PreparedStatement preparedStatement = null; 103 | ResultSet resultSet = null; 104 | try { 105 | // 1 106 | // 查询的preparedStatement 107 | preparedStatement = conn.prepareStatement(sqls[0]); 108 | // 根据不同的维度，封装不同的SQL语句 109 | setArguments(preparedStatement, dimension); 110 | // 执行查询 111 | resultSet = preparedStatement.executeQuery(); 112 | if (resultSet.next()) { 113 | int result = resultSet.getInt(1); 114 | // 释放资源 115 | JdbcUtil.close(null, preparedStatement, resultSet); 116 | return result; 117 | } 118 | // 释放资源 119 | JdbcUtil.close(null, preparedStatement, resultSet); 120 | 121 | // 2 122 | // 执行插入，封装插入的sql语句 123 | preparedStatement = conn.prepareStatement(sqls[1]); 124 | setArguments(preparedStatement, dimension); 125 | // 执行插入 126 | preparedStatement.executeUpdate(); 127 | // 释放资源 128 | JdbcUtil.close(null, preparedStatement, null); 129 | 130 | // 3 131 | // 查询的preparedStatement 132 | preparedStatement = conn.prepareStatement(sqls[0]); 133 | // 根据不同的维度，封装不同的SQL语句 134 | setArguments(preparedStatement, dimension); 135 | // 执行查询 136 | resultSet = preparedStatement.executeQuery(); 137 | if (resultSet.next()) { 138 | return resultSet.getInt(1); 139 | } 140 | } catch (SQLException e) { 141 | e.printStackTrace(); 142 | } finally { 143 | // 释放资源 144 | JdbcUtil.close(null, preparedStatement, resultSet); 145 | } 146 | return -1; 147 | } 148 | 149 | /** 150 | * 设置SQL语句的具体参数 151 | * 152 | * @param preparedStatement 153 | * @param dimension 154 | */ 155 | private void setArguments(PreparedStatement preparedStatement, BaseDimension dimension) { 156 | int i = 0; 157 | try { 158 | if (dimension instanceof DateDimension) { 159 | // 可以优化 160 | DateDimension dateDimension = (DateDimension) dimension; 161 | preparedStatement.setString(++i, dateDimension.getYear()); 162 | preparedStatement.setString(++i, dateDimension.getMonth()); 163 | preparedStatement.setString(++i, dateDimension.getDay()); 164 | } else if (dimension instanceof ContactDimension) { 165 | ContactDimension contactDimension = (ContactDimension) dimension; 166 | preparedStatement.setString(++i, contactDimension.getTelephone()); 167 | preparedStatement.setString(++i, contactDimension.getName()); 168 | } 169 | } catch (SQLException e) { 170 | e.printStackTrace(); 171 | } 172 | } 173 | 174 | /** 175 | * 返回联系人表的查询和插入语句 176 | * 177 | * @return 178 | */ 179 | private String[] getContactDimensionSQL() { 180 | String query = 181 | "SELECT `id` FROM `tb_contacts` WHERE `telephone` = ? AND `name` = ? ORDER BY `id`;"; 182 | String insert = "INSERT INTO `tb_contacts` (`telephone`, `name`) VALUES (?, ?);"; 183 | return new String[] {query, insert}; 184 | } 185 | 186 | /** 187 | * 返回时间表的查询和插入语句 188 | * 189 | * @return 190 | */ 191 | private String[] getDateDimensionSQL() { 192 | String query = 193 | "SELECT `id` FROM `tb_dimension_date` WHERE `year` = ? AND `month` = ? AND `day` = ? ORDER BY `id`;"; 194 | String insert = "INSERT INTO `tb_dimension_date` (`year`, `month`, `day`) VALUES (?, ?, ?);"; 195 | return new String[] {query, insert}; 196 | } 197 | 198 | /** 199 | * 根据维度信息得到维度对应的缓存键 200 | * 201 | * @param dimension 202 | * @return 203 | */ 204 | private String genCacheKey(BaseDimension dimension) { 205 | StringBuilder sb = new StringBuilder(); 206 | if (dimension instanceof DateDimension) { 207 | DateDimension dateDimension = (DateDimension) dimension; 208 | sb.append("date_dimension") 209 | .append(dateDimension.getYear()) 210 | .append(dateDimension.getMonth()) 211 | .append(dateDimension.getDay()); 212 | } else if (dimension instanceof ContactDimension) { 213 | ContactDimension contactDimension = (ContactDimension) dimension; 214 | sb.append("contact_dimension").append(contactDimension.getTelephone()); 215 | } 216 | return sb.toString(); 217 | } 218 | } 219 | -------------------------------------------------------------------------------- /CTCC_analysis/src/main/java/com/nil/kv/base/BaseDimension.java: -------------------------------------------------------------------------------- 1 | package com.nil.kv.base; 2 | 3 | import org.apache.hadoop.io.WritableComparable; 4 | 5 | import java.io.DataInput; 6 | import java.io.DataOutput; 7 | import java.io.IOException; 8 | /** 9 | * @author lianyou 10 | * @version 1.0 11 | */ 12 | public abstract class BaseDimension implements WritableComparable { 13 | /** 14 | * 比较 15 | * 16 | * @param o 17 | * @return 18 | */ 19 | @Override 20 | public abstract int compareTo(BaseDimension o); 21 | 22 | /** 23 | * 写数据 24 | * 25 | * @param out 26 | * @throws IOException 27 | */ 28 | @Override 29 | public abstract void write(DataOutput out) throws IOException; 30 | 31 | /** 32 | * 读数据 33 | * 34 | * @param in 35 | * @throws IOException 36 | */ 37 | @Override 38 | public abstract void readFields(DataInput in) throws IOException; 39 | } 40 | -------------------------------------------------------------------------------- /CTCC_analysis/src/main/java/com/nil/kv/base/BaseValue.java: -------------------------------------------------------------------------------- 1 | package com.nil.kv.base; 2 | 3 | import org.apache.hadoop.io.Writable; 4 | /** 5 | * @author lianyou 6 | * @version 1.0 7 | */ 8 | public abstract class BaseValue implements Writable {} 9 | -------------------------------------------------------------------------------- /CTCC_analysis/src/main/java/com/nil/kv/key/ComDimension.java: -------------------------------------------------------------------------------- 1 | package com.nil.kv.key; 2 | 3 | import com.nil.kv.base.BaseDimension; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Getter; 6 | import lombok.NoArgsConstructor; 7 | import lombok.Setter; 8 | 9 | import java.io.DataInput; 10 | import java.io.DataOutput; 11 | import java.io.IOException; 12 | /** 13 | * @author lianyou 14 | * @version 1.0 15 | */ 16 | @AllArgsConstructor 17 | @NoArgsConstructor 18 | @Setter 19 | @Getter 20 | public class ComDimension extends BaseDimension { 21 | /** 注:不要用抽象类或者接口当做类型来使用 */ 22 | private ContactDimension contactDimension = new ContactDimension(); 23 | 24 | private DateDimension dateDimension = new DateDimension(); 25 | 26 | @Override 27 | public int compareTo(BaseDimension o) { 28 | ComDimension anotherComDimension = (ComDimension) o; 29 | // 先年月日 30 | int result = this.dateDimension.compareTo(anotherComDimension.dateDimension); 31 | if (result != 0) return result; 32 | // 比较电话号 33 | result = this.contactDimension.compareTo(anotherComDimension.contactDimension); 34 | return result; 35 | } 36 | 37 | @Override 38 | public void write(DataOutput out) throws IOException { 39 | contactDimension.write(out); 40 | dateDimension.write(out); 41 | } 42 | 43 | @Override 44 | public void readFields(DataInput in) throws IOException { 45 | contactDimension.readFields(in); 46 | dateDimension.readFields(in); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /CTCC_analysis/src/main/java/com/nil/kv/key/ContactDimension.java: -------------------------------------------------------------------------------- 1 | package com.nil.kv.key; 2 | 3 | import com.nil.kv.base.BaseDimension; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Getter; 6 | import lombok.NoArgsConstructor; 7 | import lombok.Setter; 8 | 9 | import java.io.DataInput; 10 | import java.io.DataOutput; 11 | import java.io.IOException; 12 | /** 13 | * @author lianyou 14 | * @version 1.0 15 | */ 16 | @AllArgsConstructor 17 | @NoArgsConstructor 18 | @Setter 19 | @Getter 20 | public class ContactDimension extends BaseDimension { 21 | 22 | private String telephone; 23 | private String name; 24 | 25 | @Override 26 | public boolean equals(Object o) { 27 | if (this == o) { 28 | return true; 29 | } 30 | if (o == null || getClass() != o.getClass()) { 31 | return false; 32 | } 33 | 34 | ContactDimension that = (ContactDimension) o; 35 | 36 | if (telephone != null ? !telephone.equals(that.telephone) : that.telephone != null) { 37 | return false; 38 | } 39 | 40 | return name != null ? name.equals(that.name) : that.name == null; 41 | } 42 | 43 | @Override 44 | public int hashCode() { 45 | int result = telephone != null ? telephone.hashCode() : 0; 46 | result = 31 * result + (name != null ? name.hashCode() : 0); 47 | return result; 48 | } 49 | 50 | @Override 51 | public int compareTo(BaseDimension o) { 52 | ContactDimension anotherContactDimension = (ContactDimension) o; 53 | 54 | int result = this.name.compareTo(anotherContactDimension.name); 55 | if (result != 0) { 56 | return result; 57 | } 58 | 59 | result = this.telephone.compareTo(anotherContactDimension.telephone); 60 | return result; 61 | } 62 | 63 | @Override 64 | public void write(DataOutput out) throws IOException { 65 | out.writeUTF(this.telephone); 66 | out.writeUTF(this.name); 67 | } 68 | 69 | @Override 70 | public void readFields(DataInput in) throws IOException { 71 | this.telephone = in.readUTF(); 72 | this.name = in.readUTF(); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /CTCC_analysis/src/main/java/com/nil/kv/key/DateDimension.java: -------------------------------------------------------------------------------- 1 | package com.nil.kv.key; 2 | 3 | import com.nil.kv.base.BaseDimension; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Getter; 6 | import lombok.NoArgsConstructor; 7 | import lombok.Setter; 8 | 9 | import java.io.DataInput; 10 | import java.io.DataOutput; 11 | import java.io.IOException; 12 | /** 13 | * @author lianyou 14 | * @version 1.0 15 | */ 16 | @AllArgsConstructor 17 | @NoArgsConstructor 18 | @Setter 19 | @Getter 20 | public class DateDimension extends BaseDimension { 21 | private String year; 22 | private String month; 23 | private String day; 24 | 25 | @Override 26 | public boolean equals(Object o) { 27 | if (this == o) { 28 | return true; 29 | } 30 | if (o == null || getClass() != o.getClass()) { 31 | return false; 32 | } 33 | 34 | DateDimension that = (DateDimension) o; 35 | 36 | if (year != null ? !year.equals(that.year) : that.year != null) { 37 | return false; 38 | } 39 | if (month != null ? !month.equals(that.month) : that.month != null) { 40 | return false; 41 | } 42 | return day != null ? day.equals(that.day) : that.day == null; 43 | } 44 | 45 | @Override 46 | public int hashCode() { 47 | int result = year != null ? year.hashCode() : 0; 48 | result = 31 * result + (month != null ? month.hashCode() : 0); 49 | result = 31 * result + (day != null ? day.hashCode() : 0); 50 | return result; 51 | } 52 | 53 | @Override 54 | public int compareTo(BaseDimension o) { 55 | DateDimension anotherDateDimension = (DateDimension) o; 56 | int result = this.year.compareTo(anotherDateDimension.year); 57 | if (result != 0) return result; 58 | 59 | result = this.month.compareTo(anotherDateDimension.month); 60 | if (result != 0) return result; 61 | result = this.day.compareTo(anotherDateDimension.day); 62 | return result; 63 | } 64 | 65 | @Override 66 | public void write(DataOutput out) throws IOException { 67 | out.writeUTF(this.year); 68 | out.writeUTF(this.month); 69 | out.writeUTF(this.day); 70 | } 71 | 72 | @Override 73 | public void readFields(DataInput in) throws IOException { 74 | this.year = in.readUTF(); 75 | this.month = in.readUTF(); 76 | this.day = in.readUTF(); 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /CTCC_analysis/src/main/java/com/nil/kv/value/CountDurationValue.java: -------------------------------------------------------------------------------- 1 | package com.nil.kv.value; 2 | 3 | import com.nil.kv.base.BaseValue; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Getter; 6 | import lombok.NoArgsConstructor; 7 | import lombok.Setter; 8 | 9 | import java.io.DataInput; 10 | import java.io.DataOutput; 11 | import java.io.IOException; 12 | /** 13 | * @author lianyou 14 | * @version 1.0 15 | */ 16 | @AllArgsConstructor 17 | @NoArgsConstructor 18 | @Setter 19 | @Getter 20 | public class CountDurationValue extends BaseValue { 21 | private String callSum; 22 | private String callDurationSum; 23 | 24 | @Override 25 | public void write(DataOutput out) throws IOException { 26 | out.writeUTF(callSum); 27 | out.writeUTF(callDurationSum); 28 | } 29 | 30 | @Override 31 | public void readFields(DataInput in) throws IOException { 32 | this.callSum = in.readUTF(); 33 | this.callDurationSum = in.readUTF(); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /CTCC_analysis/src/main/java/com/nil/mapper/CountDurationMapper.java: -------------------------------------------------------------------------------- 1 | package com.nil.mapper; 2 | 3 | import com.nil.kv.key.ComDimension; 4 | import com.nil.kv.key.ContactDimension; 5 | import com.nil.kv.key.DateDimension; 6 | import org.apache.hadoop.hbase.client.Result; 7 | import org.apache.hadoop.hbase.io.ImmutableBytesWritable; 8 | import org.apache.hadoop.hbase.mapreduce.TableMapper; 9 | import org.apache.hadoop.hbase.util.Bytes; 10 | import org.apache.hadoop.io.Text; 11 | 12 | import java.io.IOException; 13 | import java.util.HashMap; 14 | import java.util.Map; 15 | /** 16 | * @author lianyou 17 | * @version 1.0 18 | */ 19 | public class CountDurationMapper extends TableMapper { 20 | /** 联系人和时间 */ 21 | private ComDimension comDimension = new ComDimension(); 22 | /** 通话时长 */ 23 | private Text durationText = new Text(); 24 | 25 | private Map phoneNameMap = null; 26 | 27 | @Override 28 | protected void setup(Context context) throws IOException, InterruptedException { 29 | super.setup(context); 30 | phoneNameMap = new HashMap<>(); 31 | phoneNameMap.put("17078388295", "李雁"); 32 | phoneNameMap.put("13980337439", "卫艺"); 33 | phoneNameMap.put("14575535933", "仰莉"); 34 | phoneNameMap.put("19902496992", "陶欣悦"); 35 | phoneNameMap.put("18549641558", "施梅梅"); 36 | phoneNameMap.put("17005930322", "金虹霖"); 37 | phoneNameMap.put("18468618874", "魏明艳"); 38 | phoneNameMap.put("18576581848", "华贞"); 39 | phoneNameMap.put("15978226424", "华啟倩"); 40 | phoneNameMap.put("15542823911", "仲采绿"); 41 | phoneNameMap.put("17526304161", "卫丹"); 42 | phoneNameMap.put("15422018558", "戚丽红"); 43 | phoneNameMap.put("17269452013", "何翠柔"); 44 | phoneNameMap.put("17764278604", "钱溶艳"); 45 | phoneNameMap.put("15711910344", "钱琳"); 46 | phoneNameMap.put("15714728273", "缪静欣"); 47 | phoneNameMap.put("16061028454", "焦秋菊"); 48 | phoneNameMap.put("16264433631", "吕访琴"); 49 | phoneNameMap.put("17601615878", "沈丹"); 50 | phoneNameMap.put("15897468949", "褚美丽"); 51 | } 52 | 53 | @Override 54 | protected void map(ImmutableBytesWritable key, Result value, Context context) 55 | throws IOException, InterruptedException { 56 | // 05_19902496992_20180312154840_15542823911_1_1288 57 | String rowKey = Bytes.toString(key.get()); 58 | String[] splits = rowKey.split("_"); 59 | if (splits[4].equals("0")) { 60 | return; 61 | } 62 | 63 | // 以下数据全部是主叫数据，但是也包含了被叫电话的数据 64 | String caller = splits[1]; 65 | String callee = splits[3]; 66 | String buildTime = splits[2]; 67 | String duration = splits[5]; 68 | durationText.set(duration); 69 | 70 | String year = buildTime.substring(0, 4); 71 | String month = buildTime.substring(4, 6); 72 | String day = buildTime.substring(6, 8); 73 | 74 | // 组装ComDimension 75 | // 组装DateDimension 76 | //// 05_19902496992_20170312154840_15542823911_1_1288 77 | DateDimension yearDimension = new DateDimension(year, "-1", "-1"); 78 | DateDimension monthDimension = new DateDimension(year, month, "-1"); 79 | DateDimension dayDimension = new DateDimension(year, month, day); 80 | 81 | // 组装ContactDimension 82 | ContactDimension callerContactDimension = 83 | new ContactDimension(caller, phoneNameMap.get(caller)); 84 | 85 | // 开始聚合主叫数据 86 | comDimension.setContactDimension(callerContactDimension); 87 | // 年 88 | comDimension.setDateDimension(yearDimension); 89 | context.write(comDimension, durationText); 90 | // 月 91 | comDimension.setDateDimension(monthDimension); 92 | context.write(comDimension, durationText); 93 | // 日 94 | comDimension.setDateDimension(dayDimension); 95 | context.write(comDimension, durationText); 96 | 97 | // 开始聚合被叫数据 98 | ContactDimension calleeContactDimension = 99 | new ContactDimension(callee, phoneNameMap.get(callee)); 100 | comDimension.setContactDimension(calleeContactDimension); 101 | // 年 102 | comDimension.setDateDimension(yearDimension); 103 | context.write(comDimension, durationText); 104 | // 月 105 | comDimension.setDateDimension(monthDimension); 106 | context.write(comDimension, durationText); 107 | // 日 108 | comDimension.setDateDimension(dayDimension); 109 | context.write(comDimension, durationText); 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /CTCC_analysis/src/main/java/com/nil/outputformat/MysqlOutputFormat.java: -------------------------------------------------------------------------------- 1 | package com.nil.outputformat; 2 | 3 | import com.nil.converter.DimensionConverterImpl; 4 | import com.nil.kv.key.ComDimension; 5 | import com.nil.kv.value.CountDurationValue; 6 | import com.nil.utils.JdbcInstance; 7 | import com.nil.utils.JdbcUtil; 8 | import org.apache.hadoop.fs.Path; 9 | import org.apache.hadoop.mapreduce.JobContext; 10 | import org.apache.hadoop.mapreduce.OutputCommitter; 11 | import org.apache.hadoop.mapreduce.OutputFormat; 12 | import org.apache.hadoop.mapreduce.RecordWriter; 13 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 14 | import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; 15 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 16 | 17 | import java.io.IOException; 18 | import java.sql.Connection; 19 | import java.sql.PreparedStatement; 20 | import java.sql.SQLException; 21 | /** 22 | * @author lianyou 23 | * @version 1.0 24 | */ 25 | public class MysqlOutputFormat extends OutputFormat { 26 | private OutputCommitter committer = null; 27 | 28 | @Override 29 | public RecordWriter getRecordWriter(TaskAttemptContext context) 30 | throws IOException, InterruptedException { 31 | // 初始化JDBC连接器对象 32 | Connection conn = null; 33 | conn = JdbcInstance.getInstance(); 34 | try { 35 | conn.setAutoCommit(false); 36 | } catch (SQLException e) { 37 | throw new RuntimeException(e.getMessage()); 38 | } 39 | return new MysqlRecordWriter(conn); 40 | } 41 | 42 | @Override 43 | public void checkOutputSpecs(JobContext context) throws IOException, InterruptedException { 44 | // 输出校检 45 | } 46 | 47 | @Override 48 | public OutputCommitter getOutputCommitter(TaskAttemptContext context) 49 | throws IOException, InterruptedException { 50 | // 此方法点击Outputformat按ctrl + F10 的源码，复制getOutputCommitter，在FlieOutputFormat 51 | if (committer == null) { 52 | String name = context.getConfiguration().get(FileOutputFormat.OUTDIR); 53 | Path outputPath = name == null ? null : new Path(name); 54 | committer = new FileOutputCommitter(outputPath, context); 55 | } 56 | return committer; 57 | } 58 | 59 | static class MysqlRecordWriter extends RecordWriter { 60 | private DimensionConverterImpl dci = new DimensionConverterImpl(); 61 | private Connection conn = null; 62 | private PreparedStatement preparedStatement = null; 63 | private String insertSQL = null; 64 | private int count = 0; 65 | private final int BATCH_SIZE = 500; 66 | 67 | public MysqlRecordWriter(Connection conn) { 68 | this.conn = conn; 69 | } 70 | 71 | @Override 72 | public void write(ComDimension key, CountDurationValue value) 73 | throws IOException, InterruptedException { 74 | try { 75 | // tb_call 76 | // id_date_contact, id_date_dimension, id_contact, call_sum, call_duration_sum 77 | 78 | // year month day 79 | int idDateDimension = dci.getDimensionID(key.getDateDimension()); 80 | // telephone name 81 | int idContactDimension = dci.getDimensionID(key.getContactDimension()); 82 | 83 | String idDateContact = idDateDimension + "_" + idContactDimension; 84 | 85 | int callSum = Integer.valueOf(value.getCallSum()); 86 | int callDurationSum = Integer.valueOf(value.getCallDurationSum()); 87 | 88 | if (insertSQL == null) { 89 | insertSQL = 90 | "INSERT INTO `tb_call` (`id_date_contact`, `id_date_dimension`, `id_contact`, `call_sum`, `call_duration_sum`) VALUES (?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE `id_date_contact` = ?;"; 91 | } 92 | 93 | if (preparedStatement == null) { 94 | preparedStatement = conn.prepareStatement(insertSQL); 95 | } 96 | // 本次SQL 97 | int i = 0; 98 | preparedStatement.setString(++i, idDateContact); 99 | preparedStatement.setInt(++i, idDateDimension); 100 | preparedStatement.setInt(++i, idContactDimension); 101 | preparedStatement.setInt(++i, callSum); 102 | preparedStatement.setInt(++i, callDurationSum); 103 | // 无则插入，有则更新的判断依据 104 | preparedStatement.setString(++i, idDateContact); 105 | preparedStatement.addBatch(); 106 | count++; 107 | if (count >= BATCH_SIZE) { 108 | preparedStatement.executeBatch(); 109 | conn.commit(); 110 | count = 0; 111 | preparedStatement.clearBatch(); 112 | } 113 | } catch (SQLException e) { 114 | e.printStackTrace(); 115 | } 116 | } 117 | 118 | @Override 119 | public void close(TaskAttemptContext context) throws IOException, InterruptedException { 120 | try { 121 | if (preparedStatement != null) { 122 | preparedStatement.executeBatch(); 123 | this.conn.commit(); 124 | } 125 | } catch (SQLException e) { 126 | e.printStackTrace(); 127 | } finally { 128 | JdbcUtil.close(conn, preparedStatement, null); 129 | } 130 | } 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /CTCC_analysis/src/main/java/com/nil/reducer/CountDurationReducer.java: -------------------------------------------------------------------------------- 1 | package com.nil.reducer; 2 | 3 | import com.nil.kv.key.ComDimension; 4 | import com.nil.kv.value.CountDurationValue; 5 | import org.apache.hadoop.io.Text; 6 | import org.apache.hadoop.mapreduce.Reducer; 7 | 8 | import java.io.IOException; 9 | /** 10 | * @author lianyou 11 | * @version 1.0 12 | */ 13 | public class CountDurationReducer 14 | extends Reducer { 15 | CountDurationValue countDurationValue = new CountDurationValue(); 16 | 17 | @Override 18 | protected void reduce(ComDimension key, Iterable values, Context context) 19 | throws IOException, InterruptedException { 20 | int callSum = 0; 21 | int callDurationSum = 0; 22 | for (Text t : values) { 23 | callSum++; 24 | callDurationSum += Integer.valueOf(t.toString()); 25 | } 26 | countDurationValue.setCallSum(String.valueOf(callSum)); 27 | countDurationValue.setCallDurationSum(String.valueOf(callDurationSum)); 28 | 29 | context.write(key, countDurationValue); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /CTCC_analysis/src/main/java/com/nil/runner/CountDurationRunner.java: -------------------------------------------------------------------------------- 1 | package com.nil.runner; 2 | 3 | import com.nil.kv.key.ComDimension; 4 | import com.nil.kv.value.CountDurationValue; 5 | import com.nil.mapper.CountDurationMapper; 6 | import com.nil.outputformat.MysqlOutputFormat; 7 | import com.nil.reducer.CountDurationReducer; 8 | import org.apache.hadoop.conf.Configuration; 9 | import org.apache.hadoop.hbase.HBaseConfiguration; 10 | import org.apache.hadoop.hbase.TableName; 11 | import org.apache.hadoop.hbase.client.Admin; 12 | import org.apache.hadoop.hbase.client.Connection; 13 | import org.apache.hadoop.hbase.client.ConnectionFactory; 14 | import org.apache.hadoop.hbase.client.Scan; 15 | import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; 16 | import org.apache.hadoop.io.Text; 17 | import org.apache.hadoop.mapreduce.Job; 18 | import org.apache.hadoop.util.Tool; 19 | import org.apache.hadoop.util.ToolRunner; 20 | 21 | import java.io.IOException; 22 | /** 23 | * @author lianyou 24 | * @version 1.0 25 | */ 26 | public class CountDurationRunner implements Tool { 27 | private Configuration conf = null; 28 | 29 | @Override 30 | public void setConf(Configuration conf) { 31 | this.conf = HBaseConfiguration.create(conf); 32 | } 33 | 34 | @Override 35 | public Configuration getConf() { 36 | return this.conf; 37 | } 38 | 39 | @Override 40 | public int run(String[] args) throws Exception { 41 | // 得到conf 42 | Configuration conf = this.getConf(); 43 | // 实例化Job 44 | Job job = Job.getInstance(conf); 45 | job.setJarByClass(CountDurationRunner.class); 46 | // 组装Mapper InputForamt 47 | initHBaseInputConfig(job); 48 | // 组装Reducer Outputformat 49 | initReducerOutputConfig(job); 50 | return job.waitForCompletion(true) ? 0 : 1; 51 | } 52 | 53 | private void initHBaseInputConfig(Job job) { 54 | Connection connection = null; 55 | Admin admin = null; 56 | try { 57 | String tableName = "ctcc2:calllog"; 58 | connection = ConnectionFactory.createConnection(job.getConfiguration()); 59 | admin = connection.getAdmin(); 60 | if (!admin.tableExists(TableName.valueOf(tableName))) { 61 | throw new RuntimeException("无法找到目标表."); 62 | } 63 | Scan scan = new Scan(); 64 | // 可以优化 65 | TableMapReduceUtil.initTableMapperJob( 66 | tableName, scan, CountDurationMapper.class, ComDimension.class, Text.class, job, true); 67 | 68 | } catch (IOException e) { 69 | e.printStackTrace(); 70 | } finally { 71 | try { 72 | if (admin != null) { 73 | admin.close(); 74 | } 75 | if (connection != null && !connection.isClosed()) { 76 | connection.close(); 77 | } 78 | } catch (IOException e) { 79 | e.printStackTrace(); 80 | } 81 | } 82 | } 83 | 84 | private void initReducerOutputConfig(Job job) { 85 | job.setReducerClass(CountDurationReducer.class); 86 | job.setOutputKeyClass(ComDimension.class); 87 | job.setOutputValueClass(CountDurationValue.class); 88 | job.setOutputFormatClass(MysqlOutputFormat.class); 89 | } 90 | 91 | public static void main(String[] args) { 92 | try { 93 | int status = ToolRunner.run(new CountDurationRunner(), args); 94 | System.exit(status); 95 | } catch (Exception e) { 96 | e.printStackTrace(); 97 | } 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /CTCC_analysis/src/main/java/com/nil/utils/JdbcInstance.java: -------------------------------------------------------------------------------- 1 | package com.nil.utils; 2 | 3 | import java.sql.Connection; 4 | import java.sql.SQLException; 5 | /** 6 | * @author lianyou 7 | * @version 1.0 8 | */ 9 | public class JdbcInstance { 10 | private static Connection connection = null; 11 | 12 | private JdbcInstance() {} 13 | 14 | public static Connection getInstance() { 15 | try { 16 | if (connection == null || connection.isClosed() || !connection.isValid(3)) { 17 | connection = JdbcUtil.getConnection(); 18 | } 19 | } catch (SQLException e) { 20 | e.printStackTrace(); 21 | } 22 | return connection; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /CTCC_analysis/src/main/java/com/nil/utils/JdbcUtil.java: -------------------------------------------------------------------------------- 1 | package com.nil.utils; 2 | 3 | import java.sql.Connection; 4 | import java.sql.DriverManager; 5 | import java.sql.ResultSet; 6 | import java.sql.SQLException; 7 | import java.sql.Statement; 8 | /** 9 | * @author lianyou 10 | * @version 1.0 11 | */ 12 | public class JdbcUtil { 13 | /** 定义JDBC连接器实例化所需要的固定参数 */ 14 | private static final String MYSQL_DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; 15 | 16 | private static final String MYSQL_URL = 17 | "jdbc:mysql://localhost:3306/db_telecom?useUnicode=true&characterEncoding=UTF-8"; 18 | private static final String MYSQL_USERNAME = "root"; 19 | private static final String MYSQL_PASSWORD = "mac_2016"; 20 | 21 | /** 实例化JDBC连接器对象 */ 22 | public static Connection getConnection() { 23 | try { 24 | Class.forName(MYSQL_DRIVER_CLASS); 25 | return DriverManager.getConnection(MYSQL_URL, MYSQL_USERNAME, MYSQL_PASSWORD); 26 | } catch (ClassNotFoundException e) { 27 | e.printStackTrace(); 28 | } catch (SQLException e) { 29 | e.printStackTrace(); 30 | } 31 | return null; 32 | } 33 | 34 | /** 释放连接器资源 */ 35 | public static void close(Connection connection, Statement statement, ResultSet resultSet) { 36 | try { 37 | if (resultSet != null && !resultSet.isClosed()) { 38 | resultSet.close(); 39 | } 40 | 41 | if (statement != null && !statement.isClosed()) { 42 | statement.close(); 43 | } 44 | 45 | if (connection != null && !connection.isClosed()) { 46 | connection.close(); 47 | } 48 | } catch (SQLException e) { 49 | e.printStackTrace(); 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /CTCC_analysis/src/main/java/com/nil/utils/LRUCache.java: -------------------------------------------------------------------------------- 1 | package com.nil.utils; 2 | 3 | import java.util.LinkedHashMap; 4 | import java.util.Map; 5 | /** 6 | * @author lianyou 7 | * @version 1.0 8 | */ 9 | public class LRUCache extends LinkedHashMap { 10 | private static final long serialVersionUID = 1L; 11 | protected int maxElements; 12 | 13 | public LRUCache(int maxSize) { 14 | super(maxSize, 0.75F, true); 15 | this.maxElements = maxSize; 16 | } 17 | 18 | /** 19 | * (non-Javadoc) 20 | * 21 | * @see java.util.LinkedHashMap#removeEldestEntry(java.util.Map.Entry) 22 | */ 23 | @Override 24 | protected boolean removeEldestEntry(Map.Entry eldest) { 25 | return (size() > this.maxElements); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /CTCC_consumer/CTCC_consumer.iml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /CTCC_consumer/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | Nil_ctcc 7 | com.nil 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | CTCC_consumer 13 | 14 | CTCC_consumer 15 | 16 | http://www.example.com 17 | 18 | 19 | UTF-8 20 | 1.8 21 | 1.8 22 | 23 | 24 | 25 | 26 | junit 27 | junit 28 | 4.11 29 | test 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | maven-clean-plugin 38 | 3.0.0 39 | 40 | 41 | 42 | maven-resources-plugin 43 | 3.0.2 44 | 45 | 46 | maven-compiler-plugin 47 | 3.7.0 48 | 49 | 50 | maven-surefire-plugin 51 | 2.20.1 52 | 53 | 54 | maven-jar-plugin 55 | 3.0.2 56 | 57 | 58 | maven-install-plugin 59 | 2.5.2 60 | 61 | 62 | maven-deploy-plugin 63 | 2.8.2 64 | 65 | 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /CTCC_consumer/src/main/java/com/nil/hbase/CalleeWriteObserver.java: -------------------------------------------------------------------------------- 1 | package com.nil.hbase; 2 | 3 | import com.nil.utils.HBaseUtil; 4 | import com.nil.utils.PropertiesUtil; 5 | import org.apache.hadoop.hbase.TableName; 6 | import org.apache.hadoop.hbase.client.Durability; 7 | import org.apache.hadoop.hbase.client.Put; 8 | import org.apache.hadoop.hbase.client.Table; 9 | import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; 10 | import org.apache.hadoop.hbase.coprocessor.ObserverContext; 11 | import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; 12 | import org.apache.hadoop.hbase.regionserver.wal.WALEdit; 13 | import org.apache.hadoop.hbase.util.Bytes; 14 | 15 | import java.io.IOException; 16 | import java.text.ParseException; 17 | import java.text.SimpleDateFormat; 18 | /** 19 | * @author lianyou 20 | * @version 1.0 21 | */ 22 | public class CalleeWriteObserver extends BaseRegionObserver { 23 | SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss"); 24 | 25 | @Override 26 | public void postPut( 27 | ObserverContext e, Put put, WALEdit edit, Durability durability) 28 | throws IOException { 29 | // 1、获取你想要操作的目标表的名称 30 | String targetTableName = PropertiesUtil.getProperty("hbase.calllog.tablename"); 31 | // 2、当前操作的表(当前操作的表不一定是想要操作的表) 32 | String currentTableName = e.getEnvironment().getRegionInfo().getTable().getNameAsString(); 33 | 34 | if (!targetTableName.equals(currentTableName)) { 35 | return; 36 | } 37 | 38 | // regionCode_caller_buildTime_callee_flag_duration 39 | String oriRowKey = Bytes.toString(put.getRow()); 40 | 41 | String[] splitOriRowKey = oriRowKey.split("_"); 42 | 43 | String oldFlag = splitOriRowKey[4]; 44 | // 如果当前插入的是被叫数据，则直接返回 45 | if (oldFlag.equals("0")) { 46 | return; 47 | } 48 | 49 | int regions = Integer.valueOf(PropertiesUtil.getProperty("hbase.calllog.regions")); 50 | String caller = splitOriRowKey[1]; 51 | String callee = splitOriRowKey[3]; 52 | String buildTime = splitOriRowKey[2]; 53 | String duration = splitOriRowKey[5]; 54 | String flag = "0"; 55 | String regionCode = HBaseUtil.genRegionCode(callee, buildTime, regions); 56 | 57 | String calleeRowKey = 58 | HBaseUtil.genRowkey(regionCode, callee, buildTime, caller, flag, duration); 59 | 60 | // 生成时间错 61 | String buildTimeTs = ""; 62 | try { 63 | buildTimeTs = String.valueOf(sdf.parse(buildTime).getTime()); 64 | } catch (ParseException e1) { 65 | e1.printStackTrace(); 66 | } 67 | 68 | Put calleePut = new Put(Bytes.toBytes(calleeRowKey)); 69 | calleePut.addColumn(Bytes.toBytes("f2"), Bytes.toBytes("caller"), Bytes.toBytes(caller)); 70 | calleePut.addColumn(Bytes.toBytes("f2"), Bytes.toBytes("callee"), Bytes.toBytes(callee)); 71 | calleePut.addColumn(Bytes.toBytes("f2"), Bytes.toBytes("build_Time"), Bytes.toBytes(buildTime)); 72 | calleePut.addColumn( 73 | Bytes.toBytes("f2"), Bytes.toBytes("buildTimeTS"), Bytes.toBytes(buildTimeTs)); 74 | calleePut.addColumn(Bytes.toBytes("f2"), Bytes.toBytes("flag"), Bytes.toBytes(flag)); 75 | calleePut.addColumn(Bytes.toBytes("f2"), Bytes.toBytes("duration"), Bytes.toBytes(duration)); 76 | 77 | Table table = e.getEnvironment().getTable(TableName.valueOf(targetTableName)); 78 | table.put(calleePut); 79 | table.close(); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /CTCC_consumer/src/main/java/com/nil/hbase/HBaseDAO.java: -------------------------------------------------------------------------------- 1 | package com.nil.hbase; 2 | 3 | import com.nil.utils.ConnectionInstance; 4 | import com.nil.utils.HBaseUtil; 5 | import com.nil.utils.PropertiesUtil; 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.hbase.HBaseConfiguration; 8 | import org.apache.hadoop.hbase.TableName; 9 | import org.apache.hadoop.hbase.client.Connection; 10 | import org.apache.hadoop.hbase.client.HTable; 11 | import org.apache.hadoop.hbase.client.Put; 12 | import org.apache.hadoop.hbase.util.Bytes; 13 | 14 | import java.io.IOException; 15 | import java.text.ParseException; 16 | import java.text.SimpleDateFormat; 17 | import java.util.ArrayList; 18 | import java.util.List; 19 | /** 20 | * @author lianyou 21 | * @version 1.0 22 | */ 23 | public class HBaseDAO { 24 | private int regions; 25 | private String namespace; 26 | private String tableName; 27 | private static final Configuration conf; 28 | private HTable table; 29 | private Connection connection; 30 | private SimpleDateFormat sdf1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 31 | private SimpleDateFormat sdf2 = new SimpleDateFormat("yyyyMMddHHmmss"); 32 | 33 | private List cacheList = new ArrayList<>(); 34 | 35 | static { 36 | conf = HBaseConfiguration.create(); 37 | } 38 | 39 | public HBaseDAO() { 40 | try { 41 | regions = Integer.valueOf(PropertiesUtil.getProperty("hbase.calllog.regions")); 42 | namespace = PropertiesUtil.getProperty("hbase.calllog.namespace"); 43 | tableName = PropertiesUtil.getProperty("hbase.calllog.tablename"); 44 | if (!HBaseUtil.isExistTable(conf, tableName)) { 45 | HBaseUtil.initNameSpace(conf, namespace); 46 | HBaseUtil.createTable(conf, tableName, regions, "f1", "f2"); 47 | } 48 | 49 | } catch (Exception e) { 50 | e.printStackTrace(); 51 | } 52 | } 53 | 54 | /** 55 | * value格式 : 18576581848,18468618874,2018-07-02 07:30:49,0181 Rowkey格式 : 56 | * regionCode_caller_buildTime_callee_flag_duration 57 | * 58 | * @param value 59 | */ 60 | public void put(String value) { 61 | try { 62 | 63 | // 优化代码，做批处理 64 | if (cacheList.size() == 0) { 65 | connection = ConnectionInstance.getConnection(conf); 66 | table = (HTable) connection.getTable(TableName.valueOf(tableName)); 67 | table.setAutoFlushTo(false); 68 | table.setWriteBufferSize(2 * 1024 * 1024); 69 | } 70 | String[] splitOri = value.split(","); 71 | String caller = splitOri[0]; 72 | String callee = splitOri[1]; 73 | String buildTime = splitOri[2]; 74 | String duration = splitOri[3]; 75 | String regionCode = HBaseUtil.genRegionCode(caller, buildTime, regions); 76 | 77 | String buildTimeReplace = sdf2.format(sdf1.parse(buildTime)); 78 | String buildTimeTS = String.valueOf(sdf1.parse(buildTime).getTime()); 79 | 80 | // 生成rowkey 81 | String rowkey = 82 | HBaseUtil.genRowkey(regionCode, caller, buildTimeReplace, callee, "1", duration); 83 | // 向表中插入数据 84 | Put put = new Put(Bytes.toBytes(rowkey)); 85 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("caller"), Bytes.toBytes(caller)); 86 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("callee"), Bytes.toBytes(callee)); 87 | put.addColumn( 88 | Bytes.toBytes("f1"), Bytes.toBytes("buildTimeReplace"), Bytes.toBytes(buildTimeReplace)); 89 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("buildTimeTS"), Bytes.toBytes(buildTimeTS)); 90 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("flag"), Bytes.toBytes("1")); 91 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("duration"), Bytes.toBytes(duration)); 92 | 93 | cacheList.add(put); 94 | 95 | // 生成rowkey 96 | String rowkey2 = 97 | HBaseUtil.genRowkey(regionCode, callee, buildTimeReplace, caller, "0", duration); 98 | // 向表中插入数据 99 | Put put2 = new Put(Bytes.toBytes(rowkey2)); 100 | put2.addColumn(Bytes.toBytes("f2"), Bytes.toBytes("caller"), Bytes.toBytes(callee)); 101 | put2.addColumn(Bytes.toBytes("f2"), Bytes.toBytes("callee"), Bytes.toBytes(caller)); 102 | put2.addColumn( 103 | Bytes.toBytes("f2"), Bytes.toBytes("buildTimeReplace"), Bytes.toBytes(buildTimeReplace)); 104 | put2.addColumn(Bytes.toBytes("f2"), Bytes.toBytes("buildTimeTS"), Bytes.toBytes(buildTimeTS)); 105 | put2.addColumn(Bytes.toBytes("f2"), Bytes.toBytes("flag"), Bytes.toBytes("0")); 106 | put2.addColumn(Bytes.toBytes("f2"), Bytes.toBytes("duration"), Bytes.toBytes(duration)); 107 | 108 | cacheList.add(put); 109 | cacheList.add(put2); 110 | if (cacheList.size() >= 30) { 111 | table.put(cacheList); 112 | table.flushCommits(); 113 | table.close(); 114 | cacheList.clear(); 115 | } 116 | 117 | } catch (ParseException e) { 118 | e.printStackTrace(); 119 | } catch (IOException e) { 120 | e.printStackTrace(); 121 | } 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /CTCC_consumer/src/main/java/com/nil/kafka/HBaseConsumer.java: -------------------------------------------------------------------------------- 1 | package com.nil.kafka; 2 | 3 | import com.nil.hbase.HBaseDAO; 4 | import com.nil.utils.PropertiesUtil; 5 | import org.apache.kafka.clients.consumer.ConsumerRecord; 6 | import org.apache.kafka.clients.consumer.ConsumerRecords; 7 | import org.apache.kafka.clients.consumer.KafkaConsumer; 8 | 9 | import java.util.Arrays; 10 | /** 11 | * @author lianyou 12 | * @version 1.0 13 | */ 14 | public class HBaseConsumer { 15 | public static void main(String[] args) { 16 | KafkaConsumer kafkaConsumer = new KafkaConsumer<>(PropertiesUtil.properties); 17 | kafkaConsumer.subscribe(Arrays.asList(PropertiesUtil.getProperty("kafka.topics"))); 18 | 19 | HBaseDAO hd = new HBaseDAO(); 20 | while (true) { 21 | ConsumerRecords records = kafkaConsumer.poll(100); 22 | for (ConsumerRecord cr : records) { 23 | String value = cr.value(); 24 | // 18576581848,18468618874,2018-07-02 07:30:49,0181 25 | System.out.println(value); 26 | hd.put(value); 27 | } 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /CTCC_consumer/src/main/java/com/nil/utils/ConnectionInstance.java: -------------------------------------------------------------------------------- 1 | package com.nil.utils; 2 | 3 | import org.apache.hadoop.conf.Configuration; 4 | import org.apache.hadoop.hbase.client.Connection; 5 | import org.apache.hadoop.hbase.client.ConnectionFactory; 6 | 7 | import java.io.IOException; 8 | /** 9 | * @author lianyou 10 | * @version 1.0 11 | */ 12 | public class ConnectionInstance { 13 | private static Connection conn; 14 | 15 | public static synchronized Connection getConnection(Configuration conf) { 16 | try { 17 | if (conn == null || conn.isClosed()) { 18 | conn = ConnectionFactory.createConnection(conf); 19 | } 20 | } catch (IOException e) { 21 | e.printStackTrace(); 22 | } 23 | return conn; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /CTCC_consumer/src/main/java/com/nil/utils/HBaseUtil.java: -------------------------------------------------------------------------------- 1 | package com.nil.utils; 2 | 3 | import org.apache.hadoop.conf.Configuration; 4 | import org.apache.hadoop.hbase.HColumnDescriptor; 5 | import org.apache.hadoop.hbase.HTableDescriptor; 6 | import org.apache.hadoop.hbase.NamespaceDescriptor; 7 | import org.apache.hadoop.hbase.TableName; 8 | import org.apache.hadoop.hbase.client.Admin; 9 | import org.apache.hadoop.hbase.client.Connection; 10 | import org.apache.hadoop.hbase.client.ConnectionFactory; 11 | import org.apache.hadoop.hbase.client.HBaseAdmin; 12 | import org.apache.hadoop.hbase.util.Bytes; 13 | 14 | import java.io.IOException; 15 | import java.text.DecimalFormat; 16 | import java.util.Iterator; 17 | import java.util.TreeSet; 18 | /** 19 | * @author lianyou 20 | * @version 1.0 21 | */ 22 | public class HBaseUtil { 23 | /** 24 | * 判断表是否存在 25 | * 26 | * @param conf 27 | * @param tableName 28 | * @return 29 | */ 30 | public static boolean isExistTable(Configuration conf, String tableName) throws IOException { 31 | Connection connection = ConnectionFactory.createConnection(conf); 32 | HBaseAdmin admin = (HBaseAdmin) connection.getAdmin(); 33 | boolean result = admin.tableExists(TableName.valueOf(tableName)); 34 | System.out.println("此表不存在======================"); 35 | close(admin, connection); 36 | return result; 37 | } 38 | 39 | /** 40 | * 关闭流 41 | * 42 | * @param admin 43 | * @param connection 44 | */ 45 | private static void close(Admin admin, Connection connection) throws IOException { 46 | if (admin != null) { 47 | admin.close(); 48 | } 49 | if (connection != null) { 50 | connection.close(); 51 | } 52 | } 53 | 54 | /** 55 | * 创建命名空间 56 | * 57 | * @param conf 配置 58 | * @param namespace 命名空间 --- 库 59 | */ 60 | public static void initNameSpace(Configuration conf, String namespace) throws IOException { 61 | Connection connection = ConnectionFactory.createConnection(conf); 62 | Admin admin = connection.getAdmin(); 63 | // 命名空间描述器 64 | NamespaceDescriptor build = 65 | NamespaceDescriptor.create(namespace) 66 | .addConfiguration("CREATE_TIME", String.valueOf(System.currentTimeMillis())) 67 | .addConfiguration("AUTHOR", "Nil") 68 | .build(); 69 | admin.createNamespace(build); 70 | close(admin, connection); 71 | } 72 | 73 | /** 74 | * 创建表:协处理器 75 | * 76 | * @param conf 配置 77 | * @param tableName 表名 78 | * @param regions region个数 79 | * @param columnFamily 列簇 80 | */ 81 | public static void createTable( 82 | Configuration conf, String tableName, int regions, String... columnFamily) 83 | throws IOException { 84 | Connection connection = ConnectionFactory.createConnection(conf); 85 | Admin admin = connection.getAdmin(); 86 | 87 | // 判断表是否存在 88 | if (isExistTable(conf, tableName)) { 89 | return; 90 | } 91 | 92 | // 表描述器 93 | HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); 94 | for (String cf : columnFamily) { 95 | // 创建列、HColumnDescriptor列描述器 96 | htd.addFamily(new HColumnDescriptor(cf)); 97 | } 98 | // htd.addCoprocessor("com.nil.hbase.CalleeWriteObserver"); 99 | admin.createTable(htd, genSplitKeys(regions)); 100 | System.out.println("创建表成功================================"); 101 | close(admin, connection); 102 | } 103 | 104 | /** 105 | * 分区键 106 | * 107 | * @param regions 108 | * @return 109 | */ 110 | private static byte[][] genSplitKeys(int regions) { 111 | // 定义一个存放分区键的数组 112 | String[] keys = new String[regions]; 113 | // regions个数不会超过2位数，一个region能放10G 200万条约等于50 ~ 100MB 114 | DecimalFormat df = new DecimalFormat("00"); 115 | for (int i = 0; i < regions; i++) { 116 | keys[i] = df.format(i) + "|"; 117 | } 118 | 119 | byte[][] splitkeys = new byte[regions][]; 120 | // 生成byte[][]类型的分区键的时候,一定是保证分区键是有序的 121 | TreeSet treeSet = new TreeSet<>(Bytes.BYTES_COMPARATOR); 122 | for (int i = 0; i < regions; i++) { 123 | treeSet.add(Bytes.toBytes(keys[i])); 124 | } 125 | 126 | Iterator splitKeysIterator = treeSet.iterator(); 127 | int index = 0; 128 | while (splitKeysIterator.hasNext()) { 129 | byte[] b = splitKeysIterator.next(); 130 | splitkeys[index++] = b; 131 | } 132 | return splitkeys; 133 | } 134 | 135 | /** 136 | * 数据格式 137 | * 138 | * @param caller 18468618874 139 | * @param buildTime 2018-01-23 08:46:12 140 | * @param regions 自己后边写 141 | * @return 142 | */ 143 | public static String genRegionCode(String caller, String buildTime, int regions) { 144 | int len = caller.length(); 145 | // 取出电话号码后四位 1846861 8874 146 | String lastPhone = caller.substring(len - 4); 147 | // 取年月 148 | String ym = 149 | buildTime 150 | // 2018-01-23 => 20180123 151 | .replaceAll("-", "") 152 | .replaceAll(":", "") 153 | .replaceAll(" ", "") 154 | .substring(0, 6); 155 | // 做离散操作1 156 | Integer x = Integer.valueOf(lastPhone) ^ Integer.valueOf(ym); 157 | // 做离散操作2 158 | int y = x.hashCode(); 159 | // 生成分区号 160 | int regionCode = y % regions; 161 | // 格式化分区号 162 | DecimalFormat df = new DecimalFormat("00"); 163 | return df.format(regionCode); 164 | } 165 | 166 | /** 167 | * 组装RowKey regionCode_caller_buildTime_callee_flag_duration 168 | * 169 | * @param regionCode 随机的分区号 170 | * @param caller 主叫号码 171 | * @param buildTime 随机建立通话时间 172 | * @param callee 被叫号码 173 | * @param flag 标记: 1 \ 2 174 | * @param duration 通话持续时间 175 | * @return 返回RK 176 | */ 177 | public static String genRowkey( 178 | String regionCode, 179 | String caller, 180 | String buildTime, 181 | String callee, 182 | String flag, 183 | String duration) { 184 | StringBuilder sb = new StringBuilder(); 185 | sb.append(regionCode + "_") 186 | .append(caller + "_") 187 | .append(buildTime + "_") 188 | .append(callee + "_") 189 | .append(flag + "_") 190 | .append(duration); 191 | return sb.toString(); 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /CTCC_consumer/src/main/java/com/nil/utils/PropertiesUtil.java: -------------------------------------------------------------------------------- 1 | package com.nil.utils; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStream; 5 | import java.util.Properties; 6 | /** 7 | * @author lianyou 8 | * @version 1.0 9 | */ 10 | public class PropertiesUtil { 11 | public static Properties properties = null; 12 | 13 | static { 14 | InputStream inputStream = ClassLoader.getSystemResourceAsStream("hbase_consumer.properties"); 15 | properties = new Properties(); 16 | try { 17 | properties.load(inputStream); 18 | } catch (IOException e) { 19 | e.printStackTrace(); 20 | } 21 | } 22 | 23 | public static String getProperty(String key) { 24 | return properties.getProperty(key); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /CTCC_consumer/src/main/resources/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | fs.defaultFS 20 | hdfs://localhost:9000 21 | 22 | 23 | hadoop.tmp.dir 24 | /Users/lianyou/training/hadoop-2.7.3/tmp 25 | 26 | 27 | -------------------------------------------------------------------------------- /CTCC_consumer/src/main/resources/hbase-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 23 | 24 | 25 | hbase.rootdir 26 | hdfs://localhost:9000/hbase 27 | 28 | 29 | hbase.cluster.distributed 30 | true 31 | 32 | 33 | hbase.zookeeper.quorum 34 | localhost 35 | 36 | 37 | dfs.replication 38 | 1 39 | 40 | 41 | hbase.master,maxclockskew 42 | 180000 43 | 44 | 45 | hbase.master.info.port 46 | 60010 47 | 48 | 49 | 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /CTCC_consumer/src/main/resources/hbase_consumer.properties: -------------------------------------------------------------------------------- 1 | # \u8BBE\u7F6Ekafka\u7684brokerlist 2 | bootstrap.servers=localhost:9092 3 | # \u8BBE\u7F6E\u6D88\u8D39\u8005\u6240\u5C5E\u7684\u6D88\u8D39\u7EC4 4 | group.id=hbase_consumer_group 5 | # \u8BBE\u7F6E\u662F\u5426\u81EA\u52A8\u786E\u8BA4offset 6 | enable.auto.commit=true 7 | # \u81EA\u52A8\u786E\u8BA4offset\u7684\u65F6\u95F4\u95F4\u9694 8 | auto.commit.interval.ms=1000 9 | #\u8BBE\u7F6Ekey\uFF0Cvalue\u7684\u53CD\u5E8F\u5217\u5316\u7C7B\u7684\u5168\u540D 10 | key.deserializer=org.apache.kafka.common.serialization.StringDeserializer 11 | value.deserializer=org.apache.kafka.common.serialization.StringDeserializer 12 | # \u4EE5\u4E0B\u4E3A\u81EA\u5B9A\u4E49\u5C5E\u6027\u8BBE\u7F6E 13 | # \u8BBE\u7F6E\u672C\u6B21\u6D88\u8D39\u7684\u4E3B\u9898 14 | kafka.topics=calllog 15 | # \u8BBE\u7F6EHBase\u7684\u4E00\u4E9B\u53D8\u91CF 16 | # HBase \u7684regions,namespace,tablename \u8FD9\u4E09\u4E2A\u5C5E\u6027 17 | hbase.calllog.regions=6 18 | hbase.calllog.namespace=ctcc2 19 | hbase.calllog.tablename=ctcc2:calllog -------------------------------------------------------------------------------- /CTCC_consumer/src/main/resources/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | dfs.replication 20 | 1 21 | 22 | 23 | dfs.permissions 24 | false 25 | 26 | 27 | dfs.permissions.enabled 28 | false 29 | 30 | 31 | dfs.namenode.secondary.http-address 32 | localhost:50090 33 | 34 | 35 | hbase.table.sanity.checks 36 | false 37 | 38 | 39 | -------------------------------------------------------------------------------- /CTCC_consumer/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Define some default values that can be overridden by system properties 18 | hadoop.root.logger=INFO,console 19 | hadoop.log.dir=. 20 | hadoop.log.file=hadoop.log 21 | 22 | # Define the root logger to the system property "hadoop.root.logger". 23 | log4j.rootLogger=${hadoop.root.logger}, EventCounter 24 | 25 | # Logging Threshold 26 | log4j.threshold=ALL 27 | 28 | # Null Appender 29 | log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender 30 | 31 | # 32 | # Rolling File Appender - cap space usage at 5gb. 33 | # 34 | hadoop.log.maxfilesize=256MB 35 | hadoop.log.maxbackupindex=20 36 | log4j.appender.RFA=org.apache.log4j.RollingFileAppender 37 | log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file} 38 | 39 | log4j.appender.RFA.MaxFileSize=${hadoop.log.maxfilesize} 40 | log4j.appender.RFA.MaxBackupIndex=${hadoop.log.maxbackupindex} 41 | 42 | log4j.appender.RFA.layout=org.apache.log4j.PatternLayout 43 | 44 | # Pattern format: Date LogLevel LoggerName LogMessage 45 | log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 46 | # Debugging Pattern format 47 | #log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 48 | 49 | 50 | # 51 | # Daily Rolling File Appender 52 | # 53 | 54 | log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender 55 | log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file} 56 | 57 | # Rollover at midnight 58 | log4j.appender.DRFA.DatePattern=.yyyy-MM-dd 59 | 60 | log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout 61 | 62 | # Pattern format: Date LogLevel LoggerName LogMessage 63 | log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 64 | # Debugging Pattern format 65 | #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 66 | 67 | 68 | # 69 | # console 70 | # Add "console" to rootlogger above if you want to use this 71 | # 72 | 73 | log4j.appender.console=org.apache.log4j.ConsoleAppender 74 | log4j.appender.console.target=System.err 75 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 76 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n 77 | 78 | # 79 | # TaskLog Appender 80 | # 81 | 82 | #Default values 83 | hadoop.tasklog.taskid=null 84 | hadoop.tasklog.iscleanup=false 85 | hadoop.tasklog.noKeepSplits=4 86 | hadoop.tasklog.totalLogFileSize=100 87 | hadoop.tasklog.purgeLogSplits=true 88 | hadoop.tasklog.logsRetainHours=12 89 | 90 | log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender 91 | log4j.appender.TLA.taskId=${hadoop.tasklog.taskid} 92 | log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup} 93 | log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize} 94 | 95 | log4j.appender.TLA.layout=org.apache.log4j.PatternLayout 96 | log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 97 | 98 | # 99 | # HDFS block state change log from block manager 100 | # 101 | # Uncomment the following to suppress normal block state change 102 | # messages from BlockManager in NameNode. 103 | #log4j.logger.BlockStateChange=WARN 104 | 105 | # 106 | #Security appender 107 | # 108 | hadoop.security.logger=INFO,NullAppender 109 | hadoop.security.log.maxfilesize=256MB 110 | hadoop.security.log.maxbackupindex=20 111 | log4j.category.SecurityLogger=${hadoop.security.logger} 112 | hadoop.security.log.file=SecurityAuth-${user.name}.audit 113 | log4j.appender.RFAS=org.apache.log4j.RollingFileAppender 114 | log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} 115 | log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout 116 | log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 117 | log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize} 118 | log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex} 119 | 120 | # 121 | # Daily Rolling Security appender 122 | # 123 | log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender 124 | log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} 125 | log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout 126 | log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 127 | log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd 128 | 129 | # 130 | # hadoop configuration logging 131 | # 132 | 133 | # Uncomment the following line to turn off configuration deprecation warnings. 134 | # log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN 135 | 136 | # 137 | # hdfs audit logging 138 | # 139 | hdfs.audit.logger=INFO,NullAppender 140 | hdfs.audit.log.maxfilesize=256MB 141 | hdfs.audit.log.maxbackupindex=20 142 | log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger} 143 | log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false 144 | log4j.appender.RFAAUDIT=org.apache.log4j.RollingFileAppender 145 | log4j.appender.RFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log 146 | log4j.appender.RFAAUDIT.layout=org.apache.log4j.PatternLayout 147 | log4j.appender.RFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 148 | log4j.appender.RFAAUDIT.MaxFileSize=${hdfs.audit.log.maxfilesize} 149 | log4j.appender.RFAAUDIT.MaxBackupIndex=${hdfs.audit.log.maxbackupindex} 150 | 151 | # 152 | # mapred audit logging 153 | # 154 | mapred.audit.logger=INFO,NullAppender 155 | mapred.audit.log.maxfilesize=256MB 156 | mapred.audit.log.maxbackupindex=20 157 | log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger} 158 | log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false 159 | log4j.appender.MRAUDIT=org.apache.log4j.RollingFileAppender 160 | log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log 161 | log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout 162 | log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 163 | log4j.appender.MRAUDIT.MaxFileSize=${mapred.audit.log.maxfilesize} 164 | log4j.appender.MRAUDIT.MaxBackupIndex=${mapred.audit.log.maxbackupindex} 165 | 166 | # Custom Logging levels 167 | 168 | #log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG 169 | #log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG 170 | #log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=DEBUG 171 | 172 | # Jets3t library 173 | log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR 174 | 175 | # AWS SDK & S3A FileSystem 176 | log4j.logger.com.amazonaws=ERROR 177 | log4j.logger.com.amazonaws.http.AmazonHttpClient=ERROR 178 | log4j.logger.org.apache.hadoop.fs.s3a.S3AFileSystem=WARN 179 | 180 | # 181 | # Event Counter Appender 182 | # Sends counts of logging messages at different severity levels to Hadoop Metrics. 183 | # 184 | log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter 185 | 186 | # 187 | # Job Summary Appender 188 | # 189 | # Use following logger to send summary to separate file defined by 190 | # hadoop.mapreduce.jobsummary.log.file : 191 | # hadoop.mapreduce.jobsummary.logger=INFO,JSA 192 | # 193 | hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger} 194 | hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log 195 | hadoop.mapreduce.jobsummary.log.maxfilesize=256MB 196 | hadoop.mapreduce.jobsummary.log.maxbackupindex=20 197 | log4j.appender.JSA=org.apache.log4j.RollingFileAppender 198 | log4j.appender.JSA.File=${hadoop.log.dir}/${hadoop.mapreduce.jobsummary.log.file} 199 | log4j.appender.JSA.MaxFileSize=${hadoop.mapreduce.jobsummary.log.maxfilesize} 200 | log4j.appender.JSA.MaxBackupIndex=${hadoop.mapreduce.jobsummary.log.maxbackupindex} 201 | log4j.appender.JSA.layout=org.apache.log4j.PatternLayout 202 | log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n 203 | log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger} 204 | log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false 205 | 206 | # 207 | # Yarn ResourceManager Application Summary Log 208 | # 209 | # Set the ResourceManager summary log filename 210 | yarn.server.resourcemanager.appsummary.log.file=rm-appsummary.log 211 | # Set the ResourceManager summary log level and appender 212 | yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger} 213 | #yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY 214 | 215 | # To enable AppSummaryLogging for the RM, 216 | # set yarn.server.resourcemanager.appsummary.logger to 217 | # ,RMSUMMARY in hadoop-env.sh 218 | 219 | # Appender for ResourceManager Application Summary Log 220 | # Requires the following properties to be set 221 | # - hadoop.log.dir (Hadoop Log directory) 222 | # - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename) 223 | # - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender) 224 | 225 | log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger} 226 | log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false 227 | log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender 228 | log4j.appender.RMSUMMARY.File=${hadoop.log.dir}/${yarn.server.resourcemanager.appsummary.log.file} 229 | log4j.appender.RMSUMMARY.MaxFileSize=256MB 230 | log4j.appender.RMSUMMARY.MaxBackupIndex=20 231 | log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout 232 | log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 233 | 234 | # HS audit log configs 235 | #mapreduce.hs.audit.logger=INFO,HSAUDIT 236 | #log4j.logger.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=${mapreduce.hs.audit.logger} 237 | #log4j.additivity.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=false 238 | #log4j.appender.HSAUDIT=org.apache.log4j.DailyRollingFileAppender 239 | #log4j.appender.HSAUDIT.File=${hadoop.log.dir}/hs-audit.log 240 | #log4j.appender.HSAUDIT.layout=org.apache.log4j.PatternLayout 241 | #log4j.appender.HSAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 242 | #log4j.appender.HSAUDIT.DatePattern=.yyyy-MM-dd 243 | 244 | # Http Server Request Logs 245 | #log4j.logger.http.requests.namenode=INFO,namenoderequestlog 246 | #log4j.appender.namenoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender 247 | #log4j.appender.namenoderequestlog.Filename=${hadoop.log.dir}/jetty-namenode-yyyy_mm_dd.log 248 | #log4j.appender.namenoderequestlog.RetainDays=3 249 | 250 | #log4j.logger.http.requests.datanode=INFO,datanoderequestlog 251 | #log4j.appender.datanoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender 252 | #log4j.appender.datanoderequestlog.Filename=${hadoop.log.dir}/jetty-datanode-yyyy_mm_dd.log 253 | #log4j.appender.datanoderequestlog.RetainDays=3 254 | 255 | #log4j.logger.http.requests.resourcemanager=INFO,resourcemanagerrequestlog 256 | #log4j.appender.resourcemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender 257 | #log4j.appender.resourcemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-resourcemanager-yyyy_mm_dd.log 258 | #log4j.appender.resourcemanagerrequestlog.RetainDays=3 259 | 260 | #log4j.logger.http.requests.jobhistory=INFO,jobhistoryrequestlog 261 | #log4j.appender.jobhistoryrequestlog=org.apache.hadoop.http.HttpRequestLogAppender 262 | #log4j.appender.jobhistoryrequestlog.Filename=${hadoop.log.dir}/jetty-jobhistory-yyyy_mm_dd.log 263 | #log4j.appender.jobhistoryrequestlog.RetainDays=3 264 | 265 | #log4j.logger.http.requests.nodemanager=INFO,nodemanagerrequestlog 266 | #log4j.appender.nodemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender 267 | #log4j.appender.nodemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-nodemanager-yyyy_mm_dd.log 268 | #log4j.appender.nodemanagerrequestlog.RetainDays=3 -------------------------------------------------------------------------------- /CTCC_producer/CTCC_producer.iml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /CTCC_producer/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | Nil_ctcc 7 | com.nil 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | CTCC_producer 13 | 14 | CTCC_producer 15 | 16 | http://www.example.com 17 | 18 | 19 | UTF-8 20 | 1.8 21 | 1.8 22 | 23 | 24 | 25 | 26 | junit 27 | junit 28 | 4.12 29 | test 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | maven-clean-plugin 38 | 3.0.0 39 | 40 | 41 | 42 | maven-resources-plugin 43 | 3.0.2 44 | 45 | 46 | maven-compiler-plugin 47 | 3.7.0 48 | 49 | 50 | maven-surefire-plugin 51 | 2.20.1 52 | 53 | 54 | maven-jar-plugin 55 | 3.0.2 56 | 57 | 58 | maven-install-plugin 59 | 2.5.2 60 | 61 | 62 | maven-deploy-plugin 63 | 2.8.2 64 | 65 | 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /CTCC_producer/src/main/java/com/nil/ProductDriver.java: -------------------------------------------------------------------------------- 1 | package com.nil; 2 | 3 | /** 4 | * @author lianyou 5 | * @version 1.0 6 | * @date 2018/12/25 11:06 7 | */ 8 | public class ProductDriver { 9 | 10 | public static void main(String[] args) { 11 | // args = new String[] {"/Users/lianyou/training_data/xx.log"}; 12 | if (args == null || args.length <= 0) { 13 | System.out.println("==========请输出文件输出路径=========="); 14 | return; 15 | } 16 | 17 | ProductLog productLog = new ProductLog(); 18 | productLog.initPhone(); 19 | productLog.writeLog(args[0]); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /CTCC_producer/src/main/java/com/nil/ProductLog.java: -------------------------------------------------------------------------------- 1 | package com.nil; 2 | 3 | import java.io.FileNotFoundException; 4 | import java.io.FileOutputStream; 5 | import java.io.IOException; 6 | import java.io.OutputStreamWriter; 7 | import java.io.UnsupportedEncodingException; 8 | import java.text.DecimalFormat; 9 | import java.text.ParseException; 10 | import java.text.SimpleDateFormat; 11 | import java.util.ArrayList; 12 | import java.util.Date; 13 | import java.util.HashMap; 14 | import java.util.List; 15 | import java.util.Map; 16 | 17 | /** 18 | * 随机产生通话记录信息并保存到文件中 19 | * 20 | * @author lianyou 21 | * @version 1.0 22 | * @date 2018/12/11 08:45 23 | */ 24 | public class ProductLog { 25 | /** 开始时间 */ 26 | private String startTime = "2018-01-01"; 27 | /** 结束时间 */ 28 | private String endTime = "2018-12-31"; 29 | 30 | /** 存放生产的电话号码 "17078388295" */ 31 | private List mobileList = new ArrayList(); 32 | /** 存放生产的电话号码和姓名 : "17078388295", "李雁" */ 33 | private Map ownerMap = new HashMap();; 34 | /** 随机通话时间日期格式 */ 35 | SimpleDateFormat sdf_date = new SimpleDateFormat("yyyy-MM-dd"); 36 | 37 | SimpleDateFormat sdf_datetime = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 38 | /** 通话时间格式 */ 39 | DecimalFormat df = new DecimalFormat("0000"); 40 | /** 存放产生的通话记录 */ 41 | StringBuilder sb = new StringBuilder(); 42 | 43 | /** 44 | * 初始化信息 45 | * 46 | *

正常该部分信息应该来源于数据库 47 | */ 48 | public void initPhone() { 49 | mobileList.add("17078388295"); 50 | mobileList.add("13980337439"); 51 | mobileList.add("14575535933"); 52 | mobileList.add("19902496992"); 53 | mobileList.add("18549641558"); 54 | mobileList.add("17005930322"); 55 | mobileList.add("18468618874"); 56 | mobileList.add("18576581848"); 57 | mobileList.add("15978226424"); 58 | mobileList.add("15542823911"); 59 | mobileList.add("17526304161"); 60 | mobileList.add("15422018558"); 61 | mobileList.add("17269452013"); 62 | mobileList.add("17764278604"); 63 | mobileList.add("15711910344"); 64 | mobileList.add("15714728273"); 65 | mobileList.add("16061028454"); 66 | mobileList.add("16264433631"); 67 | mobileList.add("17601615878"); 68 | mobileList.add("15897468949"); 69 | 70 | ownerMap.put("17078388295", "李雁"); 71 | ownerMap.put("13980337439", "卫艺"); 72 | ownerMap.put("14575535933", "仰莉"); 73 | ownerMap.put("19902496992", "陶欣悦"); 74 | ownerMap.put("18549641558", "施梅梅"); 75 | ownerMap.put("17005930322", "金虹霖"); 76 | ownerMap.put("18468618874", "魏明艳"); 77 | ownerMap.put("18576581848", "华贞"); 78 | ownerMap.put("15978226424", "华啟倩"); 79 | ownerMap.put("15542823911", "仲采绿"); 80 | ownerMap.put("17526304161", "卫丹"); 81 | ownerMap.put("15422018558", "戚丽红"); 82 | ownerMap.put("17269452013", "何翠柔"); 83 | ownerMap.put("17764278604", "钱溶艳"); 84 | ownerMap.put("15711910344", "钱琳"); 85 | ownerMap.put("15714728273", "缪静欣"); 86 | ownerMap.put("16061028454", "焦秋菊"); 87 | ownerMap.put("16264433631", "吕访琴"); 88 | ownerMap.put("17601615878", "沈丹"); 89 | ownerMap.put("15897468949", "褚美丽"); 90 | } 91 | 92 | /** 93 | * 生产数据:16264433631,15714728273,2018-12-21 15:55:45,1595 94 | * 95 | *

对应名称:caller,callee.buildTime,Duration 96 | * 97 | *

翻译:主叫,被叫,通话建立时间,通话持续时间 98 | * 99 | * @return 100 | */ 101 | public String product() { 102 | String caller = null; 103 | String callee = null; 104 | String callerName = null; 105 | String calleeName = null; 106 | 107 | // 生成主叫的随机索引 108 | int callerIndex = (int) (Math.random() * mobileList.size()); 109 | // 通过随机索引获得主叫电话号码 110 | caller = mobileList.get(callerIndex); 111 | // 通过主叫号码,获得主叫姓名 112 | callerName = ownerMap.get(caller); 113 | while (true) { 114 | // 生成被叫的随机索引 115 | int calleeIndex = (int) (Math.random() * mobileList.size()); 116 | // 通过随机索引获得被叫电话号码 117 | callee = mobileList.get(calleeIndex); 118 | // 通过被叫号码,获得被叫姓名 119 | calleeName = ownerMap.get(callee); 120 | // 去重判断、防止自己给自己打电话 121 | if (!caller.equals(callee)) { 122 | break; 123 | } 124 | } 125 | // 随机产生通话建立时间 126 | String buildTime = randomBuildTime(startTime, endTime); 127 | // 随机产生通话持续时间 128 | DecimalFormat df = new DecimalFormat("0000"); 129 | String duration = df.format((int) (30 * 60 * Math.random())); 130 | StringBuilder sb = new StringBuilder(); 131 | sb.append(caller) 132 | .append(",") 133 | .append(callee) 134 | .append(",") 135 | .append(buildTime) 136 | .append(",") 137 | .append(duration) 138 | .append("\r\n"); 139 | return sb.toString(); 140 | } 141 | 142 | /** 143 | * 将日志写入文件 144 | * 145 | * @param filePath 日志文件存放目录 146 | */ 147 | public void writeLog(String filePath) { 148 | try { 149 | OutputStreamWriter osw = 150 | new OutputStreamWriter(new FileOutputStream(filePath, true), "UTF-8"); 151 | 152 | while (true) { 153 | Thread.sleep(500); 154 | String log = product(); 155 | System.out.println(log); 156 | osw.write(log); 157 | // 需要手动flush,确保每条数据都写入文件一次 158 | osw.flush(); 159 | } 160 | } catch (UnsupportedEncodingException e) { 161 | e.printStackTrace(); 162 | } catch (FileNotFoundException e) { 163 | e.printStackTrace(); 164 | } catch (InterruptedException e) { 165 | e.printStackTrace(); 166 | } catch (IOException e) { 167 | e.printStackTrace(); 168 | } 169 | } 170 | 171 | /** 172 | * 随机产生时间 173 | * 174 | *

startTimeTS + (endTimeTs - startTimeTs) * Math.random(); 175 | * 176 | * @param startTime 开始时间段 177 | * @param endTime 结束时间段 178 | * @return yyyy-MM-dd HH:mm:ss 179 | */ 180 | private String randomBuildTime(String startTime, String endTime) { 181 | 182 | try { 183 | SimpleDateFormat sdf1 = new SimpleDateFormat("yyyy-MM-dd"); 184 | Date startDate = sdf1.parse(startTime); 185 | Date endDate = sdf1.parse(endTime); 186 | 187 | if (endDate.getTime() <= startDate.getTime()) { 188 | return null; 189 | } 190 | long randomTS = 191 | startDate.getTime() + (long) ((endDate.getTime() - startDate.getTime()) * Math.random()); 192 | Date resultDate = new Date(randomTS); 193 | SimpleDateFormat sdf2 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 194 | String resultTimeString = sdf2.format(resultDate); 195 | return resultTimeString; 196 | } catch (ParseException e) { 197 | e.printStackTrace(); 198 | } 199 | return null; 200 | } 201 | } 202 | -------------------------------------------------------------------------------- /CTCC_web/CTCC_web.iml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /CTCC_web/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | Nil_ctcc 7 | com.nil 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | CTCC_web 13 | war 14 | 15 | CTCC_web Maven Webapp 16 | 17 | http://www.example.com 18 | 19 | 20 | UTF-8 21 | 1.7 22 | 1.7 23 | 24 | 25 | 26 | 27 | junit 28 | junit 29 | 4.11 30 | test 31 | 32 | 33 | 34 | 35 | CTCC_web 36 | 37 | 38 | 39 | maven-clean-plugin 40 | 3.1.0 41 | 42 | 43 | 44 | maven-resources-plugin 45 | 3.0.2 46 | 47 | 48 | maven-compiler-plugin 49 | 3.8.0 50 | 51 | 52 | maven-surefire-plugin 53 | 2.22.1 54 | 55 | 56 | maven-war-plugin 57 | 3.2.2 58 | 59 | 60 | maven-install-plugin 61 | 2.5.2 62 | 63 | 64 | maven-deploy-plugin 65 | 2.8.2 66 | 67 | 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /CTCC_web/src/main/java/com/nil/bean/CallLog.java: -------------------------------------------------------------------------------- 1 | package com.nil.bean; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Getter; 5 | import lombok.NoArgsConstructor; 6 | import lombok.Setter; 7 | 8 | /** 9 | * 用于存放返回给用户的数据 10 | * 11 | * @author lianyou 12 | * @version 1.0 13 | */ 14 | @AllArgsConstructor 15 | @NoArgsConstructor 16 | @Setter 17 | @Getter 18 | public class CallLog { 19 | private String id_date_contact; 20 | private String id_date_dimension; 21 | private String id_contact; 22 | private String call_sum; 23 | private String call_duration_sum; 24 | private String telephone; 25 | private String name; 26 | private String year; 27 | private String month; 28 | private String day; 29 | 30 | @Override 31 | public String toString() { 32 | return "CallLog{" 33 | + "call_sum='" 34 | + call_sum 35 | + '\'' 36 | + ", call_duration_sum='" 37 | + call_duration_sum 38 | + '\'' 39 | + ", telephone='" 40 | + telephone 41 | + '\'' 42 | + ", name='" 43 | + name 44 | + '\'' 45 | + ", year='" 46 | + year 47 | + '\'' 48 | + ", month='" 49 | + month 50 | + '\'' 51 | + ", day='" 52 | + day 53 | + '\'' 54 | + '}'; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /CTCC_web/src/main/java/com/nil/bean/QueryInfo.java: -------------------------------------------------------------------------------- 1 | package com.nil.bean; 2 | 3 | import lombok.Getter; 4 | import lombok.Setter; 5 | 6 | /** 7 | * 该类用于存放用户请求的数据 8 | * 9 | * @author lianyou 10 | * @version 1.0 11 | */ 12 | @Setter 13 | @Getter 14 | public class QueryInfo { 15 | private String telephone; 16 | private String year; 17 | private String month; 18 | private String day; 19 | 20 | public QueryInfo() { 21 | super(); 22 | } 23 | 24 | public QueryInfo(String telephone, String year, String month, String day) { 25 | super(); 26 | this.telephone = telephone; 27 | this.year = year; 28 | this.month = month; 29 | this.day = day; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /CTCC_web/src/main/java/com/nil/controller/CallLogHandler.java: -------------------------------------------------------------------------------- 1 | package com.nil.controller; 2 | 3 | import com.nil.bean.CallLog; 4 | import com.nil.bean.QueryInfo; 5 | import com.nil.dao.CallLogDAO; 6 | import org.springframework.context.ApplicationContext; 7 | import org.springframework.context.support.ClassPathXmlApplicationContext; 8 | import org.springframework.stereotype.Controller; 9 | import org.springframework.ui.Model; 10 | import org.springframework.web.bind.annotation.RequestMapping; 11 | 12 | import java.util.HashMap; 13 | import java.util.List; 14 | /** 15 | * @author lianyou 16 | * @version 1.0 17 | */ 18 | @Controller 19 | public class CallLogHandler { 20 | 21 | @RequestMapping("/queryCallLogList") 22 | public String queryCallLog(Model model, QueryInfo queryInfo) { 23 | ApplicationContext applicationContext = 24 | new ClassPathXmlApplicationContext("applicationContext.xml"); 25 | CallLogDAO callLogDAO = applicationContext.getBean(CallLogDAO.class); 26 | 27 | HashMap hashMap = new HashMap<>(); 28 | hashMap.put("telephone", queryInfo.getTelephone()); 29 | hashMap.put("year", queryInfo.getYear()); 30 | hashMap.put("month", queryInfo.getMonth()); 31 | hashMap.put("day", queryInfo.getDay()); 32 | 33 | List list = callLogDAO.getCallLogList(hashMap); 34 | 35 | StringBuilder dateSB = new StringBuilder(); 36 | StringBuilder callSumSB = new StringBuilder(); 37 | StringBuilder callDurationSumSB = new StringBuilder(); 38 | 39 | for (int i = 0; i < list.size(); i++) { 40 | CallLog callLog = list.get(i); 41 | // 1月, 2月, ....12月, 42 | dateSB.append(callLog.getMonth() + "月,"); 43 | callSumSB.append(callLog.getCall_sum() + ","); 44 | callDurationSumSB.append(callLog.getCall_duration_sum() + ","); 45 | } 46 | 47 | dateSB.deleteCharAt(dateSB.length() - 1); 48 | callSumSB.deleteCharAt(callSumSB.length() - 1); 49 | callDurationSumSB.deleteCharAt(callDurationSumSB.length() - 1); 50 | 51 | // 通过model返回数据 52 | model.addAttribute("telephone", list.get(0).getTelephone()); 53 | model.addAttribute("name", list.get(0).getName()); 54 | model.addAttribute("date", dateSB.toString()); 55 | model.addAttribute("count", callSumSB.toString()); 56 | model.addAttribute("duration", callDurationSumSB.toString()); 57 | 58 | return "jsp/CallLogListEchart"; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /CTCC_web/src/main/java/com/nil/dao/CallLogDAO.java: -------------------------------------------------------------------------------- 1 | package com.nil.dao; 2 | 3 | import com.nil.bean.CallLog; 4 | import org.springframework.beans.factory.annotation.Autowired; 5 | import org.springframework.jdbc.core.BeanPropertyRowMapper; 6 | import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; 7 | import org.springframework.stereotype.Repository; 8 | 9 | import java.util.HashMap; 10 | import java.util.List; 11 | /** 12 | * @author lianyou 13 | * @version 1.0 14 | */ 15 | @Repository 16 | public class CallLogDAO { 17 | @Autowired 18 | private NamedParameterJdbcTemplate namedParameterJdbcTemplate; 19 | 20 | public List getCallLogList(HashMap paramsMap) { 21 | // 按照年统计：统计某个用户，1年12个月的所有的数据（不精确到day） 22 | String sql = 23 | "SELECT `call_sum`, `call_duration_sum`, `telephone`, `name`, `year` , `month`, `day` FROM tb_dimension_date t4 INNER JOIN ( SELECT `id_date_dimension`, `call_sum`, `call_duration_sum`, `telephone`, `name` FROM tb_call t2 INNER JOIN ( SELECT `id`, `telephone`, `name` FROM tb_contacts WHERE telephone = :telephone ) t1 ON t2.id_contact = t1.id ) t3 ON t4.id = t3.id_date_dimension WHERE `year` = :year AND `month` != :month AND `day` = :day ORDER BY `year`, `month`;"; 24 | BeanPropertyRowMapper beanPropertyRowMapper = 25 | new BeanPropertyRowMapper<>(CallLog.class); 26 | List list = namedParameterJdbcTemplate.query(sql, paramsMap, beanPropertyRowMapper); 27 | return list; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /CTCC_web/src/main/resources/applicationContext.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /CTCC_web/src/main/resources/dbconfig.properties: -------------------------------------------------------------------------------- 1 | jdbc.user=root 2 | jdbc.password=mac_2016 3 | jdbc.jdbcUrl=jdbc:mysql://localhost:3306/db_telecom?useUnicode=true&characterEncoding=UTF-8 4 | jdbc.driverClass=com.mysql.cj.jdbc.Driver -------------------------------------------------------------------------------- /CTCC_web/src/main/webapp/WEB-INF/web.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | SpringMVC_CRUD 7 | 8 | 9 | dispatcherServlet 10 | org.springframework.web.servlet.DispatcherServlet 11 | 12 | contextConfigLocation 13 | classpath:applicationContext.xml 14 | 15 | 1 16 | 17 | 18 | dispatcherServlet 19 | / 20 | 21 | 22 | 23 | index.jsp 24 | 25 | 26 | 27 | CharacterEncodingFilter 28 | org.springframework.web.filter.CharacterEncodingFilter 29 | 30 | encoding 31 | utf-8 32 | 33 | 34 | forceEncoding 35 | true 36 | 37 | 38 | 39 | CharacterEncodingFilter 40 | /* 41 | 42 | 43 | 44 | default 45 | *.jpg 46 | 47 | 48 | default 49 | *.js 50 | 51 | 52 | default 53 | *.css 54 | 55 | -------------------------------------------------------------------------------- /CTCC_web/src/main/webapp/index.jsp: -------------------------------------------------------------------------------- 1 | <%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core" %> 2 | <%@ page language="java" contentType="text/html; charset=utf-8" 3 | pageEncoding="utf-8" %> 4 | <% 5 | String path = request.getContextPath(); 6 | %> 7 | 8 | 9 | 10 | 11 | Show Time 12 | 13 | 14 |

15 | 手机号码: 16 | 年: 17 | 月: 18 | 日: 19 | 20 |
21 | 22 | 23 | -------------------------------------------------------------------------------- /CTCC_web/src/main/webapp/jsp/CallLogListEchart.jsp: -------------------------------------------------------------------------------- 1 | <%-- 2 | Created by IntelliJ IDEA. 3 | User: Z 4 | Date: 2017/10/28 5 | Time: 14:36 6 | To change this template use File | Settings | File Templates. 7 | --%> 8 | <%@ page contentType="text/html;charset=UTF-8" language="java" isELIgnored="false" %> 9 | 10 | 11 | 12 | 显示通话记录 13 | 14 | <%----%> 15 | <%----%> 16 | <%----%> 17 | 18 | 19 | 26 |

通话月单查询：${requestScope.name}

27 | <%--

通话次数

--%> 28 | <%--

通话时长

--%> 29 |
30 |
31 | 239 | 240 | 241 | -------------------------------------------------------------------------------- /Nil_ctcc.iml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /ReadMe.md: -------------------------------------------------------------------------------- 1 | # 一、项目简介 2 | 大数据时代的到来，对数据的处理和分析有了很高的要求。本项目以通化数据去展示如何处理并分析大数据，并最终通过图表可视化展示。 3 | 4 | > 本项目主要分为以下几个部分 5 | 6 | 1. 生产日志【模拟通话】 7 | 2. 数据采集/消费 8 | 3. 数据分析 9 | 4. 数据展示 10 | 11 | # 二、项目实现 12 | 13 | 本项目由多个模块构成，这些模块之间可以独立运行。所有模块如下 14 | 15 | 1. CTCC_producer 实现数据生产 16 | 2. CTCC_consumer 实现数据采集（Flume）和消费 (Kafka),写入到HBase 17 | 3. CTCC_analysis 实现数据分析 18 | 4. CTCC_web 实现数据的可视化 19 | 20 | # 三、各模块测试流程 21 | 22 | ## 3.1、CTCC_producer 23 | 24 | ### 3.1.1、将源码打包为call_log.jar,并上传到服务器 25 | 26 | ``` 27 | $ cp call_log.jar ~/training_jobs/ctcc 28 | ``` 29 | ### 3.1.2、添加生成日志的脚本 30 | 31 | ``` 32 | $ vi ~/training_data/ctcc/call_log.sh 33 | 34 | #!/bin/bash 35 | java -cp ~/training_jobs/ctcc/call_log.jar com.nil.ProductDriver ~/training_data/ctcc/call_log.csv 36 | ``` 37 | ### 3.1.3、给脚本添加执行权限 38 | 39 | ``` 40 | $ chmod +x ~/training_data/ctcc/call_log.sh 41 | ``` 42 | ### 3.1.4、开始数据生产 43 | 44 | ``` 45 | $ bash ~/training_data/ctcc/call_log.sh 46 | ``` 47 | 48 | ## 3.2、CTCC_consumer 49 | 50 | ### 3.2.1、创建并查看主题 51 | ``` 52 | $ kafka-topics.sh --zookeeper localhost:2181 --delete --topic calllog 53 | $ kafka-topics.sh --zookeeper localhost:2181 --topic calllog --create --replication-factor 1 --partitions 3 54 | $ kafka-topics.sh --zookeeper localhost:2181 --list 55 | ``` 56 | ### 3.2.2、启动kafka消费者,等待flume输入 57 | 58 | > 低版本 59 | 60 | ``` 61 | $ kafka-console-consumer.sh --zookeeper localhost:2181 --topic calllog --from-beginning 62 | ``` 63 | > 高版本 64 | 65 | ``` 66 | $ kafka-console-consumer.sh --bootstrap-server localhost:2181 --topic calllog --from-beginning 67 | ``` 68 | 69 | ### 3.2.3、创建flume配置文件并启动相关任务 70 | 71 | ``` 72 | $ vi $FLUME_HOME/jobs/flume-task-ctcc.conf 73 | 74 | # define 75 | a1.sources = r1 76 | a1.sinks = k1 77 | a1.channels = c1 78 | 79 | # source +0 是从第0行开始 80 | a1.sources.r1.type = exec 81 | a1.sources.r1.command = tail -F -c +0 /Users/lianyou/training_data/ctcc/call_log.csv 82 | a1.sources.r1.shell = /bin/bash -c 83 | 84 | # sink 85 | a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink 86 | a1.sinks.k1.brokerList = localhost:9092 87 | a1.sinks.k1.topic =calllog 88 | a1.sinks.k1.batchSize = 20 89 | a1.sinks.k1.requiredAcks = 1 90 | 91 | # channel 92 | a1.channels.c1.type = memory 93 | a1.channels.c1.capacity = 1000 94 | a1.channels.c1.transactionCapacity = 100 95 | 96 | # bind 97 | a1.sources.r1.channels = c1 98 | a1.sinks.k1.channel = c1 99 | ``` 100 | 101 | ``` 102 | # flume-ng agent $FLUME_HOME/conf/ --name a1 --conf-file $FLUME_HOME/jobs/flume-task-ctcc.conf & 103 | ``` 104 | 105 | ## 3.3、 CTCC_analysis 实现数据分析 106 | 107 | ## 3.4、CTCC_web 实现数据的可视化 108 | 109 | ![最终效果](./index.png) -------------------------------------------------------------------------------- /doc/db_telecom.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Navicat MySQL Data Transfer 3 | 4 | Source Server : 192.168.216.20 5 | Source Server Version : 50173 6 | Source Host : 192.168.216.20:3306 7 | Source Database : db_telecom 8 | 9 | Target Server Type : MYSQL 10 | Target Server Version : 50173 11 | File Encoding : 65001 12 | 13 | Date: 2017-10-31 16:11:44 14 | */ 15 | 16 | SET FOREIGN_KEY_CHECKS=0; 17 | 18 | -- ---------------------------- 19 | -- Table structure for tb_call 20 | -- ---------------------------- 21 | DROP TABLE IF EXISTS `tb_call`; 22 | CREATE TABLE `tb_call` ( 23 | `id_date_contact` varchar(255) NOT NULL, 24 | `id_date_dimension` int(11) NOT NULL, 25 | `id_contact` int(11) NOT NULL, 26 | `call_sum` int(11) NOT NULL, 27 | `call_duration_sum` int(11) NOT NULL, 28 | PRIMARY KEY (`id_date_contact`) 29 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 30 | 31 | -- ---------------------------- 32 | -- Table structure for tb_contacts 33 | -- ---------------------------- 34 | DROP TABLE IF EXISTS `tb_contacts`; 35 | CREATE TABLE `tb_contacts` ( 36 | `id` int(11) NOT NULL AUTO_INCREMENT, 37 | `telephone` varchar(255) NOT NULL, 38 | `name` varchar(255) NOT NULL, 39 | PRIMARY KEY (`id`) 40 | ) ENGINE=InnoDB AUTO_INCREMENT=21 DEFAULT CHARSET=utf8; 41 | 42 | -- ---------------------------- 43 | -- Table structure for tb_dimension_date 44 | -- ---------------------------- 45 | DROP TABLE IF EXISTS `tb_dimension_date`; 46 | CREATE TABLE `tb_dimension_date` ( 47 | `id` int(11) NOT NULL AUTO_INCREMENT, 48 | `year` int(11) NOT NULL, 49 | `month` int(11) NOT NULL, 50 | `day` int(11) NOT NULL, 51 | PRIMARY KEY (`id`) 52 | ) ENGINE=InnoDB AUTO_INCREMENT=263 DEFAULT CHARSET=utf8; -------------------------------------------------------------------------------- /index.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nil-2008/ctcc/e19bc094a78e4c6345b299aee8d1aba45cfb16eb/index.png -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 4.0.0 6 | 7 | com.nil 8 | Nil_ctcc 9 | pom 10 | 1.0-SNAPSHOT 11 | 12 | CTCC_producer 13 | CTCC_consumer 14 | CTCC_analysis 15 | CTCC_web 16 | 17 | 18 | Nil_ctcc 19 | 20 | http://www.example.com 21 | 22 | 23 | UTF-8 24 | 1.8 25 | 1.8 26 | 2.7.3 27 | 4.12 28 | 1.18.4 29 | 1.3.1 30 | 2.0.1 31 | 8.0.13 32 | 1.8.10 33 | 2.5.0 34 | 35 | 36 | 37 | 38 | junit 39 | junit 40 | ${junit.version} 41 | 42 | 43 | mysql 44 | mysql-connector-java 45 | ${mysql.version} 46 | 47 | 48 | 49 | org.projectlombok 50 | lombok 51 | ${lombok.version} 52 | provided 53 | 54 | 55 | 56 | org.apache.hbase 57 | hbase-client 58 | ${hbase.version} 59 | 60 | 61 | org.apache.hbase 62 | hbase-server 63 | ${hbase.version} 64 | 65 | 66 | org.apache.hbase 67 | hbase-common 68 | ${hbase.version} 69 | 70 | 71 | 72 | org.apache.kafka 73 | kafka_2.11 74 | ${kafka.version} 75 | 76 | 77 | org.aspectj 78 | aspectjweaver 79 | ${aspectjweaver.version} 80 | 81 | 82 | com.google.protobuf 83 | protobuf-java 84 | ${protobuf-java.version} 85 | 86 | 87 | 88 | 89 | c3p0 90 | c3p0 91 | 0.9.1.2 92 | 93 | 94 | 95 | org.mybatis 96 | mybatis 97 | 3.4.6 98 | 99 | 100 | 101 | org.springframework 102 | spring-context-support 103 | 4.3.3.RELEASE 104 | 105 | 106 | org.springframework 107 | spring-jdbc 108 | 4.3.3.RELEASE 109 | 110 | 111 | org.springframework 112 | spring-webmvc 113 | 4.3.3.RELEASE 114 | 115 | 116 | org.mybatis 117 | mybatis-spring 118 | 1.3.0 119 | 120 | 121 | org.aspectj 122 | aspectjweaver 123 | 1.8.10 124 | 125 | 126 | javax.servlet 127 | servlet-api 128 | 2.5 129 | 130 | 131 | javax.servlet 132 | jstl 133 | 1.2 134 | 135 | 136 | org.springframework 137 | spring-beans 138 | 4.3.3.RELEASE 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | maven-clean-plugin 147 | 3.0.0 148 | 149 | 150 | 151 | maven-resources-plugin 152 | 3.0.2 153 | 154 | 155 | maven-compiler-plugin 156 | 3.7.0 157 | 158 | 159 | maven-surefire-plugin 160 | 2.20.1 161 | 162 | 163 | maven-jar-plugin 164 | 3.0.2 165 | 166 | 167 | maven-install-plugin 168 | 2.5.2 169 | 170 | 171 | maven-deploy-plugin 172 | 2.8.2 173 | 174 | 175 | 176 | 177 | 178 | --------------------------------------------------------------------------------