├── .gitattributes ├── .gitignore ├── README.md ├── assets ├── design.png ├── framework.png └── runing.png ├── mmc-lidea-common ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── mmc │ │ └── lidea │ │ └── common │ │ ├── bo │ │ ├── LideaAppBO.java │ │ ├── LideaLogBO.java │ │ ├── LideaLogErrorDetailBO.java │ │ ├── LideaMethodBO.java │ │ └── LideaServiceBO.java │ │ ├── context │ │ ├── Const.java │ │ └── KafkaConst.java │ │ └── entry │ │ ├── LideaAppEntry.java │ │ ├── LideaLogEntry.java │ │ ├── LideaMethodEntry.java │ │ └── LideaServiceEntry.java │ └── resources │ ├── hbase-clear.txt │ ├── hbase-create.txt │ └── hbase-drop.txt ├── mmc-lidea-stream ├── .gitignore ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── mmc │ │ │ └── lidea │ │ │ └── stream │ │ │ ├── Bootstrap.java │ │ │ ├── flink │ │ │ ├── LideaAppSinkFun.java │ │ │ ├── LideaBaseSinkFun.java │ │ │ ├── LideaDetailSinkFun.java │ │ │ ├── LideaMethodSinkFun.java │ │ │ ├── LideaMethodStatSinkFun.java │ │ │ ├── LideaServiceSinkFun.java │ │ │ ├── LogContentAcc.java │ │ │ ├── LogContentAgg.java │ │ │ ├── LogContentErrorFilter.java │ │ │ ├── LogContentFilter.java │ │ │ ├── LogContentSplitter.java │ │ │ ├── LogContentWinFun.java │ │ │ └── MessageWaterEmitter.java │ │ │ ├── job │ │ │ ├── LideaAppJob.java │ │ │ ├── LideaDetailJob.java │ │ │ ├── LideaLogJob.java │ │ │ ├── LideaMethodJob.java │ │ │ ├── LideaMethodStatJob.java │ │ │ └── LideaServiceJob.java │ │ │ ├── model │ │ │ ├── LogContent.java │ │ │ └── LogContentCount.java │ │ │ └── util │ │ │ ├── LogAppNameUtil.java │ │ │ ├── LogContentUtil.java │ │ │ ├── LogMethodNameUtil.java │ │ │ └── LogServiceNameUtil.java │ └── resources │ │ └── lidea.properties │ └── test │ └── java │ └── com │ └── mmc │ └── lidea │ └── stream │ └── producer │ ├── MessageProducer.java │ └── MessageProducerSingle.java ├── mmc-lidea-util ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── mmc │ │ └── lidea │ │ └── util │ │ ├── BytesUtils.java │ │ ├── Charsets.java │ │ ├── MD5Util.java │ │ ├── MemoryUsageExtrator.java │ │ ├── RandomUtil.java │ │ ├── StringUtil.java │ │ └── TimeUtil.java │ └── test │ └── java │ └── com │ └── mmc │ └── lidea │ └── util │ └── TimeUtilTest.java ├── mmc-lidea-web ├── .gitignore ├── .mvn │ └── wrapper │ │ ├── MavenWrapperDownloader.java │ │ ├── maven-wrapper.jar │ │ └── maven-wrapper.properties ├── mvnw ├── mvnw.cmd ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── mmc │ │ │ └── flink │ │ │ └── lidea │ │ │ ├── MmcLideaApplication.java │ │ │ ├── aop │ │ │ └── ExceptionHander.java │ │ │ ├── config │ │ │ └── HBaseConfiguration.java │ │ │ ├── controller │ │ │ ├── DetailController.java │ │ │ └── LideaController.java │ │ │ ├── dao │ │ │ ├── LideaAppDAO.java │ │ │ ├── LideaLogDAO.java │ │ │ ├── LideaLogErrorDAO.java │ │ │ ├── LideaMethodDAO.java │ │ │ └── LideaServiceDAO.java │ │ │ ├── dto │ │ │ ├── LideaAppResp.java │ │ │ ├── LideaLogErrorDetailResp.java │ │ │ ├── LideaLogReq.java │ │ │ ├── LideaLogResp.java │ │ │ ├── LideaMethodReq.java │ │ │ ├── LideaMethodResp.java │ │ │ ├── LideaServiceReq.java │ │ │ ├── LideaServiceResp.java │ │ │ └── ResultDTO.java │ │ │ ├── mapper │ │ │ ├── LideaAppResultsExtractor.java │ │ │ ├── LideaLogResultsExtractor.java │ │ │ ├── LideaMethodResultsExtractor.java │ │ │ └── LideaServiceResultsExtractor.java │ │ │ ├── service │ │ │ └── LideaLogService.java │ │ │ └── util │ │ │ └── RowKeyUtils.java │ └── resources │ │ ├── application-dev.yml │ │ ├── application.yml │ │ ├── logback-spring.xml │ │ ├── static │ │ └── v1 │ │ │ ├── css │ │ │ ├── basictable.css │ │ │ └── style.css │ │ │ ├── gallery │ │ │ ├── css │ │ │ │ └── style.css │ │ │ └── js │ │ │ │ ├── Draggable.min.js │ │ │ │ ├── TweenMax.min.js │ │ │ │ ├── index.js │ │ │ │ └── jquery.js │ │ │ ├── img │ │ │ └── 11.png │ │ │ └── js │ │ │ ├── Nora.js │ │ │ ├── app.js │ │ │ ├── detail.js │ │ │ ├── echarts.js │ │ │ ├── errorList.js │ │ │ ├── jquery-3.4.1.min.js │ │ │ ├── jquery.basictable.min.js │ │ │ ├── methodList.js │ │ │ └── serviceList.js │ │ └── templates │ │ └── pages │ │ └── v1 │ │ ├── app.html │ │ ├── detail.html │ │ ├── errorList.html │ │ ├── methodList.html │ │ └── serviceList.html │ └── test │ └── java │ └── com │ └── mmc │ └── flink │ └── lidea │ ├── MmcLideaApplicationTests.java │ ├── dao │ ├── DemoTest.java │ ├── LideaAppDAOTest.java │ ├── LideaLogDAOBatchTest.java │ ├── LideaLogDAOTest.java │ ├── LideaLogErrorDAOTest.java │ ├── LideaMethodDAOTest.java │ └── LideaServiceDAOTest.java │ ├── service │ └── LideaLogServiceTest.java │ └── test │ └── ByteTest.java └── pom.xml /.gitattributes: -------------------------------------------------------------------------------- 1 | *.js linguist-language=Java 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | target 4 | target/ 5 | .settings 6 | 7 | # Log file 8 | *.log 9 | 10 | # BlueJ files 11 | *.ctxt 12 | 13 | # Mobile Tools for Java (J2ME) 14 | .mtj.tmp/ 15 | 16 | # Package Files # 17 | *.jar 18 | *.war 19 | *.ear 20 | *.zip 21 | *.tar.gz 22 | *.rar 23 | 24 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 25 | hs_err_pid* 26 | 27 | #idea files 28 | *.iml 29 | .idea 30 | .idea/ 31 | 32 | #eclipse 33 | /.metadata/ 34 | .classpath 35 | .factorypath 36 | .project 37 | .settings 38 | *.bak 39 | 40 | # mac os 41 | .DS_Store 42 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Lidea 2 | 基于Flink和Hbase大型分布式系统实时监控平台 3 | 4 | # 效果图 5 | ![效果图](https://github.com/VIPJoey/Lidea/blob/master/assets/runing.png) 6 | 7 | # 系统架构 8 | ![系统架构](https://github.com/VIPJoey/Lidea/blob/master/assets/framework.png) 9 | 10 | # 设计原理 11 | ![设计原理](https://github.com/VIPJoey/Lidea/blob/master/assets/design.png) 12 | -------------------------------------------------------------------------------- /assets/design.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VIPJoey/Lidea/1fdd0c2bd59a77cbbba7efe0a1255006a9aca0a5/assets/design.png -------------------------------------------------------------------------------- /assets/framework.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VIPJoey/Lidea/1fdd0c2bd59a77cbbba7efe0a1255006a9aca0a5/assets/framework.png -------------------------------------------------------------------------------- /assets/runing.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VIPJoey/Lidea/1fdd0c2bd59a77cbbba7efe0a1255006a9aca0a5/assets/runing.png -------------------------------------------------------------------------------- /mmc-lidea-common/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | lidea 7 | com.mmc 8 | 1.0.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | mmc-lidea-common 13 | 14 | 15 | 16 | 17 | 18 | org.apache.hbase 19 | hbase-client 20 | 1.4.9 21 | 22 | 23 | org.projectlombok 24 | lombok 25 | 1.16.22 26 | 27 | 28 | 29 | com.mmc 30 | mmc-lidea-util 31 | 1.0.0-SNAPSHOT 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /mmc-lidea-common/src/main/java/com/mmc/lidea/common/bo/LideaAppBO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | 11 | package com.mmc.lidea.common.bo; 12 | 13 | import lombok.Data; 14 | 15 | import java.io.Serializable; 16 | 17 | /** 18 | * 应用名称. 19 | * @author Joey 20 | * @date 2019/8/6 18:36 21 | */ 22 | @Data 23 | public class LideaAppBO implements Serializable { 24 | 25 | private static final long serialVersionUID = -2296577378425040440L; 26 | 27 | public String appName; 28 | public String time; 29 | 30 | } 31 | -------------------------------------------------------------------------------- /mmc-lidea-common/src/main/java/com/mmc/lidea/common/bo/LideaLogBO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.common.bo; 11 | 12 | import lombok.Data; 13 | 14 | import java.io.Serializable; 15 | 16 | /** 17 | * @author Joey 18 | * @date 2019/7/26 17:57 19 | */ 20 | @Data 21 | public class LideaLogBO implements Serializable { 22 | 23 | private static final long serialVersionUID = 6673495081894666234L; 24 | 25 | /** 26 | * 记录时间. 27 | */ 28 | public String time; 29 | /** 30 | * 系统名称. 31 | */ 32 | public String appName; 33 | /** 34 | * 接口名称. 35 | */ 36 | public String serviceName; 37 | /** 38 | * 方法名称. 39 | */ 40 | public String methodName; 41 | /** 42 | * 访问量. 43 | */ 44 | public long count; 45 | /** 46 | * 平均响应时间(ms). 47 | */ 48 | public int avg; 49 | /** 50 | * 故障次数. 51 | */ 52 | public int exception; 53 | /** 54 | * 故障ID. 55 | */ 56 | public String traceIds; 57 | } 58 | -------------------------------------------------------------------------------- /mmc-lidea-common/src/main/java/com/mmc/lidea/common/bo/LideaLogErrorDetailBO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.common.bo; 11 | 12 | import lombok.Data; 13 | 14 | import java.io.Serializable; 15 | 16 | /** 17 | * @author Joey 18 | * @date 2019/7/26 17:57 19 | */ 20 | @Data 21 | public class LideaLogErrorDetailBO implements Serializable { 22 | 23 | private static final long serialVersionUID = 6673495081894666234L; 24 | 25 | public String time; 26 | public String traceId; 27 | public int type; 28 | public String localIp; 29 | public String remoteIp; 30 | public String appName; 31 | public String serviceName; 32 | public String methodName; 33 | public String args; 34 | public String response; 35 | public int cost; 36 | public String msg; 37 | public String customMsg; 38 | } 39 | -------------------------------------------------------------------------------- /mmc-lidea-common/src/main/java/com/mmc/lidea/common/bo/LideaMethodBO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.common.bo; 11 | 12 | import lombok.Data; 13 | 14 | import java.io.Serializable; 15 | 16 | /** 17 | * @author Joey 18 | * @date 2019/8/6 18:36 19 | */ 20 | @Data 21 | public class LideaMethodBO implements Serializable { 22 | 23 | private static final long serialVersionUID = -2296577378425040440L; 24 | 25 | public String appName; 26 | public String serviceName; 27 | public String methodName; 28 | public String time; 29 | public String count; 30 | 31 | } 32 | -------------------------------------------------------------------------------- /mmc-lidea-common/src/main/java/com/mmc/lidea/common/bo/LideaServiceBO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.common.bo; 11 | 12 | import lombok.Data; 13 | 14 | import java.io.Serializable; 15 | 16 | /** 17 | * 接口名称. 18 | * @author Joey 19 | * @date 2019/8/29 14:26 20 | */ 21 | @Data 22 | public class LideaServiceBO implements Serializable { 23 | 24 | private static final long serialVersionUID = -4111289278389251339L; 25 | 26 | public String appName; 27 | public String serviceName; 28 | 29 | } 30 | -------------------------------------------------------------------------------- /mmc-lidea-common/src/main/java/com/mmc/lidea/common/context/Const.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.common.context; 11 | 12 | import com.mmc.lidea.util.BytesUtils; 13 | 14 | /** 15 | * 常量. 16 | * 17 | * @author Joey 18 | * @date 2019/7/26 19:41 19 | */ 20 | public class Const { 21 | 22 | /** 23 | * appName + serviceName + method最大长度 md5 = 32. 24 | */ 25 | public static final int MAX_ROW_KEY_LEN = 32; 26 | /** 27 | * 固定8位时间长度. 28 | */ 29 | public static final int FIX_LONG_LENGTH = 8; 30 | /** 31 | * LideaLog表列族. 32 | */ 33 | public static final byte[] LIDEA_LOG_FEMILY = BytesUtils.toBytes("S"); 34 | /** 35 | * LideaLog表. 36 | */ 37 | public static final String LIDEA_LOG_TABLE = "LideaLog"; 38 | /** 39 | * LideaLogDetail表. 40 | */ 41 | public static final String LIDEA_LOG_DEATIL_TABLE = "LideaLogDetail"; 42 | /** 43 | * LideaApp表. 44 | */ 45 | public static final String LIDEA_APP_TABLE = "LideaApp"; 46 | /** 47 | * LideaMethod表. 48 | */ 49 | public static final String LIDEA_METHOD_TABLE = "LideaMethod"; 50 | /** 51 | * LideaService表. 52 | */ 53 | public static final String LIDEA_SERVICE_TABLE = "LideaService"; 54 | 55 | /** 56 | * success record. 57 | */ 58 | public static final int ERROR = 2; 59 | /** 60 | * error record. 61 | */ 62 | public static final int SUCCESS = 1; 63 | 64 | } 65 | -------------------------------------------------------------------------------- /mmc-lidea-common/src/main/java/com/mmc/lidea/common/context/KafkaConst.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | 11 | package com.mmc.lidea.common.context; 12 | 13 | /** 14 | * @author Joey 15 | * @date 2019/7/14 18:29 16 | */ 17 | public interface KafkaConst { 18 | 19 | String TOPIC = "flink-kafka-jane"; 20 | } 21 | -------------------------------------------------------------------------------- /mmc-lidea-common/src/main/java/com/mmc/lidea/common/entry/LideaAppEntry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.common.entry; 11 | 12 | import com.mmc.lidea.common.bo.LideaAppBO; 13 | import com.mmc.lidea.common.context.Const; 14 | import com.mmc.lidea.util.BytesUtils; 15 | import com.mmc.lidea.util.MD5Util; 16 | import org.apache.hadoop.hbase.client.Put; 17 | 18 | /** 19 | * 应用名称实体操作类. 20 | * @author Joey 21 | * @date 2019/8/29 10:48 22 | */ 23 | public class LideaAppEntry { 24 | 25 | public static Put of(LideaAppBO bo) { 26 | 27 | 28 | Put put = new Put(makeRowKey(bo)); 29 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("time"), BytesUtils.toBytes(bo.time)); 30 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("appName"), BytesUtils.toBytes(bo.appName)); 31 | 32 | return put; 33 | } 34 | 35 | 36 | private static byte[] makeRowKey(LideaAppBO bo) { 37 | return BytesUtils.toBytes(MD5Util.encrypt(bo.appName)); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /mmc-lidea-common/src/main/java/com/mmc/lidea/common/entry/LideaLogEntry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.common.entry; 11 | 12 | import com.mmc.lidea.common.bo.LideaLogBO; 13 | import com.mmc.lidea.common.context.Const; 14 | import com.mmc.lidea.util.BytesUtils; 15 | import com.mmc.lidea.util.MD5Util; 16 | import com.mmc.lidea.util.StringUtil; 17 | import com.mmc.lidea.util.TimeUtil; 18 | import org.apache.hadoop.hbase.client.Put; 19 | import org.apache.hadoop.hbase.util.Bytes; 20 | 21 | import java.util.Objects; 22 | 23 | /** 24 | * 日志内容实体类. 25 | * @author Joey 26 | * @date 2019/9/4 19:40 27 | */ 28 | public class LideaLogEntry { 29 | 30 | public static Put of(LideaLogBO bo) { 31 | 32 | Put put = new Put(makeRowKey(bo)); 33 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("time"), BytesUtils.toBytes(bo.time)); 34 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("appName"), BytesUtils.toBytes(bo.appName)); 35 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("serviceName"), BytesUtils.toBytes(bo.serviceName)); 36 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("methodName"), BytesUtils.toBytes(bo.methodName)); 37 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("count"), Bytes.toBytes(bo.count)); 38 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("avg"), Bytes.toBytes(bo.avg)); 39 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("exception"), Bytes.toBytes(bo.exception)); 40 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("traceIds"), Bytes.toBytes(bo.traceIds)); 41 | 42 | return put; 43 | } 44 | 45 | private static byte[] makeRowKey(LideaLogBO bo) { 46 | 47 | String base = StringUtil.format("{}{}{}", bo.getAppName(), bo.getServiceName(), bo.getMethodName()); 48 | long time = TimeUtil.stringToLong(bo.getTime()); 49 | 50 | return makeRowKey(MD5Util.encrypt(base), time); 51 | } 52 | 53 | /** 54 | * 行键. 55 | * @param md5 应用+服务+方法 的md5 56 | * @param timestamp 时间 57 | * 58 | * @return 59 | */ 60 | public static byte[] makeRowKey(String md5, long timestamp) { 61 | 62 | byte[] base = BytesUtils.toBytes(md5); // 32位 63 | byte[] rowKey = new byte[Const.MAX_ROW_KEY_LEN + Const.FIX_LONG_LENGTH]; 64 | 65 | long time = TimeUtil.reverseTimeMillis(timestamp); // 时间倒置 66 | 67 | BytesUtils.writeBytes(rowKey, 0, base); 68 | BytesUtils.writeLong(time, rowKey, Const.MAX_ROW_KEY_LEN); 69 | return rowKey; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /mmc-lidea-common/src/main/java/com/mmc/lidea/common/entry/LideaMethodEntry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.common.entry; 11 | 12 | import com.mmc.lidea.common.bo.LideaMethodBO; 13 | import com.mmc.lidea.common.context.Const; 14 | import com.mmc.lidea.util.BytesUtils; 15 | import com.mmc.lidea.util.MD5Util; 16 | import com.mmc.lidea.util.StringUtil; 17 | import org.apache.hadoop.hbase.client.Put; 18 | import org.apache.hadoop.hbase.client.Result; 19 | 20 | /** 21 | * 方法实体类. 22 | * @author Joey 23 | * @date 2019/8/29 15:01 24 | */ 25 | public class LideaMethodEntry { 26 | 27 | public static Put of(LideaMethodBO bo) { 28 | 29 | Put put = new Put(makeRowKey(bo)); 30 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("time"), BytesUtils.toBytes(bo.time)); 31 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("appName"), BytesUtils.toBytes(bo.appName)); 32 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("serviceName"), BytesUtils.toBytes(bo.serviceName)); 33 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("methodName"), BytesUtils.toBytes(bo.methodName)); 34 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("count"), BytesUtils.toBytes(bo.count)); 35 | 36 | return put; 37 | } 38 | 39 | public static byte[] makeRowKey(LideaMethodBO bo) { 40 | String base = StringUtil.format("{}{}", MD5Util.encrypt(bo.appName + bo.serviceName), bo.methodName); 41 | return BytesUtils.toBytes(base); 42 | } 43 | 44 | public static LideaMethodBO map(Result result) { 45 | LideaMethodBO bo = new LideaMethodBO(); 46 | bo.setTime(BytesUtils.toString(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("time")))); 47 | bo.setAppName(BytesUtils.toString(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("appName")))); 48 | bo.setServiceName(BytesUtils.toString(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("serviceName")))); 49 | bo.setMethodName(BytesUtils.toString(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("methodName")))); 50 | bo.setCount(BytesUtils.toString(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("count")))); 51 | return bo; 52 | 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /mmc-lidea-common/src/main/java/com/mmc/lidea/common/entry/LideaServiceEntry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.common.entry; 11 | 12 | import com.mmc.lidea.common.bo.LideaServiceBO; 13 | import com.mmc.lidea.common.context.Const; 14 | import com.mmc.lidea.util.BytesUtils; 15 | import com.mmc.lidea.util.MD5Util; 16 | import com.mmc.lidea.util.StringUtil; 17 | import org.apache.hadoop.hbase.client.Put; 18 | 19 | /** 20 | * 接口实体类. 21 | * 22 | * @author Joey 23 | * @date 2019/8/29 14:30 24 | */ 25 | public class LideaServiceEntry { 26 | 27 | public static Put of(LideaServiceBO bo) { 28 | 29 | Put put = new Put(makeRowKey(bo)); 30 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("serviceName"), BytesUtils.toBytes(bo.serviceName)); 31 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("appName"), BytesUtils.toBytes(bo.appName)); 32 | 33 | return put; 34 | } 35 | 36 | private static byte[] makeRowKey(LideaServiceBO bo) { 37 | 38 | String base = StringUtil.format("{}{}", bo.appName, MD5Util.encrypt(bo.serviceName)); 39 | 40 | return BytesUtils.toBytes(base); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /mmc-lidea-common/src/main/resources/hbase-clear.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | truncate 'LideaApp' 4 | 5 | truncate 'LideaService' 6 | 7 | truncate 'LideaMethod' 8 | 9 | truncate 'LideaLog' 10 | 11 | truncate 'LideaLogDetail' 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /mmc-lidea-common/src/main/resources/hbase-create.txt: -------------------------------------------------------------------------------- 1 | 2 | create 'LideaApp', { NAME => 'S', TTL => 864000} 3 | 4 | create 'LideaService', { NAME => 'S', TTL => 864000} 5 | 6 | create 'LideaMethod', { NAME => 'S', TTL => 864000} 7 | 8 | create 'LideaLog', { NAME => 'S', TTL => 259200} 9 | 10 | create 'LideaLogDetail', { NAME => 'S', TTL => 259200} -------------------------------------------------------------------------------- /mmc-lidea-common/src/main/resources/hbase-drop.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | disable 'LideaApp' 4 | 5 | disable 'LideaService' 6 | 7 | disable 'LideaMethod' 8 | 9 | disable 'LideaLog' 10 | 11 | disable 'LideaLogDetail' 12 | 13 | drop 'LideaApp' 14 | 15 | drop 'LideaService' 16 | 17 | drop 'LideaMethod' 18 | 19 | drop 'LideaLog' 20 | 21 | drop 'LideaLogDetail' 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /mmc-lidea-stream/.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | target 4 | target/ 5 | .settings 6 | 7 | # Log file 8 | *.log 9 | 10 | # BlueJ files 11 | *.ctxt 12 | 13 | # Mobile Tools for Java (J2ME) 14 | .mtj.tmp/ 15 | 16 | # Package Files # 17 | *.jar 18 | *.war 19 | *.ear 20 | *.zip 21 | *.tar.gz 22 | *.rar 23 | 24 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 25 | hs_err_pid* 26 | 27 | #idea files 28 | *.iml 29 | .idea 30 | .idea/ 31 | 32 | #eclipse 33 | /.metadata/ 34 | .classpath 35 | .factorypath 36 | .project 37 | .settings 38 | *.bak 39 | 40 | # mac os 41 | .DS_Store 42 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/Bootstrap.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream; 11 | 12 | /** 13 | * @author Joey 14 | * @date 2019/7/14 14:56 15 | */ 16 | public class Bootstrap { 17 | 18 | 19 | public static void main(String[] args) throws Exception { 20 | 21 | 22 | } 23 | 24 | 25 | 26 | } 27 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/flink/LideaAppSinkFun.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.flink; 11 | 12 | import com.mmc.lidea.common.bo.LideaAppBO; 13 | import com.mmc.lidea.common.entry.LideaAppEntry; 14 | import com.mmc.lidea.stream.model.LogContent; 15 | import com.mmc.lidea.stream.util.LogAppNameUtil; 16 | import org.apache.flink.api.java.utils.ParameterTool; 17 | import org.apache.flink.configuration.Configuration; 18 | import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; 19 | import org.apache.hadoop.hbase.HBaseConfiguration; 20 | import org.apache.hadoop.hbase.TableName; 21 | import org.apache.hadoop.hbase.client.*; 22 | 23 | import java.io.IOException; 24 | 25 | /** 26 | * @author Joey 27 | * @date 2019/8/2 17:18 28 | */ 29 | public class LideaAppSinkFun extends RichSinkFunction { 30 | 31 | private static final long serialVersionUID = 3868272239007101505L; 32 | 33 | private Connection conn = null; 34 | 35 | private BufferedMutator mutator; 36 | 37 | 38 | @Override 39 | public void open(Configuration parameters) throws Exception { 40 | super.open(parameters); 41 | 42 | ParameterTool para = (ParameterTool) 43 | getRuntimeContext().getExecutionConfig().getGlobalJobParameters(); 44 | String zkServer = para.getRequired("hbase.zookeeper.quorum"); 45 | String zkPort = para.getRequired("hbase.zookeeper.property.clientPort"); 46 | String tName = para.getRequired("lidea.log.app.name"); 47 | TableName tableName = TableName.valueOf(tName); 48 | 49 | org.apache.hadoop.conf.Configuration config = HBaseConfiguration.create(); 50 | config.set("hbase.zookeeper.quorum", zkServer); 51 | config.set("hbase.zookeeper.property.clientPort", zkPort); 52 | 53 | conn = ConnectionFactory.createConnection(config); 54 | 55 | // 连接表 56 | Table table = conn.getTable(tableName); 57 | 58 | // 设置缓存 59 | BufferedMutatorParams params = new BufferedMutatorParams(tableName); 60 | params.writeBufferSize(1024); 61 | mutator = conn.getBufferedMutator(params); 62 | 63 | } 64 | 65 | @Override 66 | public void close() throws Exception { 67 | mutator.flush(); 68 | conn.close(); 69 | } 70 | 71 | @Override 72 | public void invoke(LogContent bo, Context context) throws IOException { 73 | 74 | // 如果已经存在key,不用重复新增 75 | if (LogAppNameUtil.exists(bo.appName)) { 76 | return; 77 | } else { 78 | LogAppNameUtil.put(bo.appName); 79 | } 80 | 81 | LideaAppBO lideaAppBO = new LideaAppBO(); 82 | lideaAppBO.appName = bo.appName; 83 | lideaAppBO.time = bo.time; 84 | 85 | Put put = LideaAppEntry.of(lideaAppBO); 86 | 87 | mutator.mutate(put); 88 | 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/flink/LideaBaseSinkFun.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.flink; 11 | 12 | import com.mmc.lidea.common.context.Const; 13 | import com.mmc.lidea.stream.model.LogContentCount; 14 | import com.mmc.lidea.util.BytesUtils; 15 | import com.mmc.lidea.util.MD5Util; 16 | import com.mmc.lidea.util.StringUtil; 17 | import com.mmc.lidea.util.TimeUtil; 18 | import org.apache.flink.api.java.utils.ParameterTool; 19 | import org.apache.flink.configuration.Configuration; 20 | import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; 21 | import org.apache.hadoop.hbase.HBaseConfiguration; 22 | import org.apache.hadoop.hbase.TableName; 23 | import org.apache.hadoop.hbase.client.*; 24 | import org.apache.hadoop.hbase.util.Bytes; 25 | 26 | import java.io.IOException; 27 | import java.util.Objects; 28 | 29 | /** 30 | * @author Joey 31 | * @date 2019/8/2 17:18 32 | */ 33 | public class LideaBaseSinkFun extends RichSinkFunction { 34 | 35 | private static final long serialVersionUID = 3868272239007101505L; 36 | 37 | private Connection conn = null; 38 | private Table table = null; 39 | 40 | private static TableName tableName; 41 | 42 | private static final String click = "click"; 43 | private BufferedMutatorParams params; 44 | private BufferedMutator mutator; 45 | 46 | @Override 47 | public void open(Configuration parameters) throws Exception { 48 | super.open(parameters); 49 | 50 | ParameterTool para = (ParameterTool) 51 | getRuntimeContext().getExecutionConfig().getGlobalJobParameters(); 52 | String zkServer = para.getRequired("hbase.zookeeper.quorum"); 53 | String zkPort = para.getRequired("hbase.zookeeper.property.clientPort"); 54 | String tName = para.getRequired("lidea.log.base.name"); 55 | tableName = TableName.valueOf(tName); 56 | 57 | org.apache.hadoop.conf.Configuration config = HBaseConfiguration.create(); 58 | config.set("hbase.zookeeper.quorum", zkServer); 59 | config.set("hbase.zookeeper.property.clientPort", zkPort); 60 | 61 | conn = ConnectionFactory.createConnection(config); 62 | // Admin admin = conn.getAdmin(); 63 | // admin.listTableNames(); 64 | // if (!admin.tableExists(tableName)) { 65 | // HTableDescriptor tableDes = new HTableDescriptor(tableName); 66 | // 67 | // tableDes.addFamily(new HColumnDescriptor(click).setMaxVersions(3)); 68 | // 69 | // System.out.println("create table"); 70 | // admin.flush(tableName); 71 | // } 72 | // 连接表 73 | table = conn.getTable(tableName); 74 | 75 | // 设置缓存 76 | params = new BufferedMutatorParams(tableName); 77 | params.writeBufferSize(1024); 78 | mutator = conn.getBufferedMutator(params); 79 | 80 | } 81 | 82 | @Override 83 | public void close() throws Exception { 84 | mutator.flush(); 85 | conn.close(); 86 | } 87 | 88 | @Override 89 | public void invoke(LogContentCount bo, Context context) throws IOException { 90 | 91 | 92 | String base = StringUtil.format("{}{}{}", bo.appName, bo.serviceName, bo.methodName); 93 | long time = TimeUtil.reverseTimeMillis(TimeUtil.stringToLong(bo.getTime())); // 时间倒置 94 | 95 | byte[] md5 = Objects.requireNonNull(MD5Util.encrypt(base)).getBytes(); // 32位 96 | byte[] rowKey = new byte[Const.MAX_ROW_KEY_LEN + Const.FIX_LONG_LENGTH]; 97 | 98 | BytesUtils.writeBytes(rowKey, 0, md5); 99 | BytesUtils.writeLong(time, rowKey, Const.MAX_ROW_KEY_LEN); 100 | 101 | Put put = new Put(rowKey); 102 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("time"), BytesUtils.toBytes(bo.time)); 103 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("appName"), BytesUtils.toBytes(bo.appName)); 104 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("serviceName"), BytesUtils.toBytes(bo.serviceName)); 105 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("methodName"), BytesUtils.toBytes(bo.methodName)); 106 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("count"), Bytes.toBytes(bo.count)); 107 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("avg"), Bytes.toBytes(bo.avg)); 108 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("exception"), Bytes.toBytes(bo.exception)); 109 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("traceIds"), Bytes.toBytes(bo.traceIds)); 110 | 111 | mutator.mutate(put); 112 | 113 | 114 | } 115 | 116 | } 117 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/flink/LideaDetailSinkFun.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.flink; 11 | 12 | import com.mmc.lidea.common.context.Const; 13 | import com.mmc.lidea.stream.model.LogContent; 14 | import com.mmc.lidea.util.BytesUtils; 15 | import org.apache.flink.api.java.utils.ParameterTool; 16 | import org.apache.flink.configuration.Configuration; 17 | import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; 18 | import org.apache.hadoop.hbase.HBaseConfiguration; 19 | import org.apache.hadoop.hbase.TableName; 20 | import org.apache.hadoop.hbase.client.*; 21 | import org.apache.hadoop.hbase.util.Bytes; 22 | 23 | import java.io.IOException; 24 | import java.util.Objects; 25 | 26 | /** 27 | * @author Joey 28 | * @date 2019/8/4 17:16 29 | */ 30 | public class LideaDetailSinkFun extends RichSinkFunction { 31 | 32 | private static final long serialVersionUID = 8692984903123681090L; 33 | 34 | 35 | private Connection conn = null; 36 | private Table table = null; 37 | 38 | private static TableName tableName; 39 | 40 | private static final String click = "click"; 41 | private BufferedMutatorParams params; 42 | private BufferedMutator mutator; 43 | 44 | @Override 45 | public void open(Configuration parameters) throws Exception { 46 | super.open(parameters); 47 | 48 | ParameterTool para = (ParameterTool) 49 | getRuntimeContext().getExecutionConfig().getGlobalJobParameters(); 50 | String zkServer = para.getRequired("hbase.zookeeper.quorum"); 51 | String zkPort = para.getRequired("hbase.zookeeper.property.clientPort"); 52 | String tName = para.getRequired("lidea.log.detail.name"); 53 | tableName = TableName.valueOf(tName); 54 | 55 | org.apache.hadoop.conf.Configuration config = HBaseConfiguration.create(); 56 | config.set("hbase.zookeeper.quorum", zkServer); 57 | config.set("hbase.zookeeper.property.clientPort", zkPort); 58 | 59 | conn = ConnectionFactory.createConnection(config); 60 | // 连接表 61 | table = conn.getTable(tableName); 62 | 63 | // 设置缓存 64 | params = new BufferedMutatorParams(tableName); 65 | params.writeBufferSize(1024); 66 | mutator = conn.getBufferedMutator(params); 67 | 68 | } 69 | 70 | @Override 71 | public void close() throws Exception { 72 | mutator.flush(); 73 | conn.close(); 74 | } 75 | 76 | @Override 77 | public void invoke(LogContent bo, Context context) throws IOException { 78 | 79 | 80 | byte[] rowKey = Objects.requireNonNull(bo.traceId.getBytes()); // 直接用traceId作为rowKey 81 | 82 | 83 | Put put = new Put(rowKey); 84 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("time"), BytesUtils.toBytes(bo.time)); 85 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("appName"), BytesUtils.toBytes(bo.appName)); 86 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("serviceName"), BytesUtils.toBytes(bo.serviceName)); 87 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("methodName"), BytesUtils.toBytes(bo.methodName)); 88 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("localIp"), BytesUtils.toBytes(bo.localIp)); 89 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("remoteIp"), BytesUtils.toBytes(bo.remoteIp)); 90 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("args"), BytesUtils.toBytes(bo.args)); 91 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("response"), BytesUtils.toBytes(bo.response)); 92 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("cost"), Bytes.toBytes(bo.cost)); 93 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("msg"), BytesUtils.toBytes(bo.msg)); 94 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("customMsg"), BytesUtils.toBytes(bo.customMsg)); 95 | 96 | mutator.mutate(put); 97 | 98 | 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/flink/LideaMethodSinkFun.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.flink; 11 | 12 | import com.mmc.lidea.common.bo.LideaMethodBO; 13 | import com.mmc.lidea.common.entry.LideaMethodEntry; 14 | import com.mmc.lidea.stream.model.LogContent; 15 | import com.mmc.lidea.stream.util.LogMethodNameUtil; 16 | import com.mmc.lidea.util.MD5Util; 17 | import com.mmc.lidea.util.StringUtil; 18 | import org.apache.flink.api.java.utils.ParameterTool; 19 | import org.apache.flink.configuration.Configuration; 20 | import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; 21 | import org.apache.hadoop.hbase.HBaseConfiguration; 22 | import org.apache.hadoop.hbase.TableName; 23 | import org.apache.hadoop.hbase.client.*; 24 | 25 | import java.io.IOException; 26 | 27 | /** 28 | * @author Joey 29 | * @date 2019/8/2 17:18 30 | */ 31 | public class LideaMethodSinkFun extends RichSinkFunction { 32 | 33 | private static final long serialVersionUID = 3868272239007101505L; 34 | 35 | private Connection conn = null; 36 | 37 | private BufferedMutator mutator; 38 | 39 | 40 | @Override 41 | public void open(Configuration parameters) throws Exception { 42 | super.open(parameters); 43 | 44 | ParameterTool para = (ParameterTool) 45 | getRuntimeContext().getExecutionConfig().getGlobalJobParameters(); 46 | String zkServer = para.getRequired("hbase.zookeeper.quorum"); 47 | String zkPort = para.getRequired("hbase.zookeeper.property.clientPort"); 48 | String tName = para.getRequired("lidea.log.method.name"); 49 | TableName tableName = TableName.valueOf(tName); 50 | 51 | org.apache.hadoop.conf.Configuration config = HBaseConfiguration.create(); 52 | config.set("hbase.zookeeper.quorum", zkServer); 53 | config.set("hbase.zookeeper.property.clientPort", zkPort); 54 | 55 | conn = ConnectionFactory.createConnection(config); 56 | 57 | // 连接表 58 | Table table = conn.getTable(tableName); 59 | 60 | // 设置缓存 61 | BufferedMutatorParams params = new BufferedMutatorParams(tableName); 62 | params.writeBufferSize(1024); 63 | mutator = conn.getBufferedMutator(params); 64 | 65 | } 66 | 67 | @Override 68 | public void close() throws Exception { 69 | mutator.flush(); 70 | conn.close(); 71 | } 72 | 73 | @Override 74 | public void invoke(LogContent bo, Context context) throws IOException { 75 | 76 | // 过滤一下 77 | String key = MD5Util.encrypt(StringUtil.format("{}{}{}", bo.appName, bo.serviceName, bo.methodName)); 78 | if (LogMethodNameUtil.exists(key)) { 79 | return; 80 | } 81 | LogMethodNameUtil.put(key); 82 | 83 | LideaMethodBO lideaMethodBO = new LideaMethodBO(); 84 | lideaMethodBO.time = bo.time; 85 | lideaMethodBO.appName = bo.appName; 86 | lideaMethodBO.serviceName = bo.serviceName; 87 | lideaMethodBO.methodName = bo.methodName; 88 | lideaMethodBO.count = "0"; 89 | 90 | Put put = LideaMethodEntry.of(lideaMethodBO); 91 | 92 | mutator.mutate(put); 93 | 94 | } 95 | 96 | } 97 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/flink/LideaMethodStatSinkFun.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.flink; 11 | 12 | import com.mmc.lidea.common.bo.LideaMethodBO; 13 | import com.mmc.lidea.common.context.Const; 14 | import com.mmc.lidea.common.entry.LideaMethodEntry; 15 | import com.mmc.lidea.stream.model.LogContentCount; 16 | import com.mmc.lidea.util.BytesUtils; 17 | import com.mmc.lidea.util.StringUtil; 18 | import org.apache.flink.api.java.utils.ParameterTool; 19 | import org.apache.flink.configuration.Configuration; 20 | import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; 21 | import org.apache.hadoop.hbase.HBaseConfiguration; 22 | import org.apache.hadoop.hbase.TableName; 23 | import org.apache.hadoop.hbase.client.*; 24 | 25 | import java.io.IOException; 26 | import java.util.concurrent.ConcurrentHashMap; 27 | 28 | /** 29 | * @author Joey 30 | * @date 2019/8/2 17:18 31 | */ 32 | public class LideaMethodStatSinkFun extends RichSinkFunction { 33 | 34 | private static final long serialVersionUID = 3868272239007101505L; 35 | 36 | private Connection conn = null; 37 | private Table table = null; 38 | 39 | private static TableName tableName; 40 | 41 | private BufferedMutatorParams params; 42 | private BufferedMutator mutator; 43 | 44 | private ConcurrentHashMap countCache = new ConcurrentHashMap<>(); 45 | 46 | @Override 47 | public void open(Configuration parameters) throws Exception { 48 | super.open(parameters); 49 | 50 | ParameterTool para = (ParameterTool) 51 | getRuntimeContext().getExecutionConfig().getGlobalJobParameters(); 52 | String zkServer = para.getRequired("hbase.zookeeper.quorum"); 53 | String zkPort = para.getRequired("hbase.zookeeper.property.clientPort"); 54 | String tName = para.getRequired("lidea.log.method.name"); 55 | tableName = TableName.valueOf(tName); 56 | 57 | org.apache.hadoop.conf.Configuration config = HBaseConfiguration.create(); 58 | config.set("hbase.zookeeper.quorum", zkServer); 59 | config.set("hbase.zookeeper.property.clientPort", zkPort); 60 | 61 | conn = ConnectionFactory.createConnection(config); 62 | 63 | // 连接表 64 | // table = conn.getTable(tableName); 65 | 66 | // 设置缓存 67 | params = new BufferedMutatorParams(tableName); 68 | params.writeBufferSize(1024); 69 | mutator = conn.getBufferedMutator(params); 70 | 71 | } 72 | 73 | @Override 74 | public void close() throws Exception { 75 | mutator.flush(); 76 | conn.close(); 77 | } 78 | 79 | @Override 80 | public void invoke(LogContentCount bo, Context context) throws IOException { 81 | 82 | LideaMethodBO dto = new LideaMethodBO(); 83 | dto.appName = bo.appName; 84 | dto.serviceName = bo.serviceName; 85 | dto.methodName = bo.methodName; 86 | 87 | byte[] rowKey = LideaMethodEntry.makeRowKey(dto); 88 | String base = StringUtil.format("{}{}{}", bo.appName, bo.serviceName, bo.methodName); 89 | Put put = new Put(rowKey); 90 | 91 | long count = bo.count; 92 | 93 | // 如果缓存里没有全局统计值,到hbase里找 94 | if (!countCache.containsKey(base)) { 95 | Get get = new Get(rowKey); 96 | table = conn.getTable(tableName); 97 | Result result = table.get(get); 98 | if (!result.isEmpty()) { 99 | // 放入缓存 100 | LideaMethodBO src = LideaMethodEntry.map(result); 101 | count += Long.valueOf(src.count); 102 | } 103 | } else { 104 | 105 | // 累加 106 | count += countCache.get(base); 107 | } 108 | 109 | // 重新设置缓存 110 | countCache.put(base, count); 111 | 112 | put.addColumn(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("count"), BytesUtils.toBytes(String.valueOf(count))); 113 | 114 | mutator.mutate(put); 115 | 116 | 117 | } 118 | 119 | } 120 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/flink/LideaServiceSinkFun.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.flink; 11 | 12 | import com.mmc.lidea.common.bo.LideaServiceBO; 13 | import com.mmc.lidea.common.entry.LideaServiceEntry; 14 | import com.mmc.lidea.stream.model.LogContent; 15 | import com.mmc.lidea.stream.util.LogServiceNameUtil; 16 | import com.mmc.lidea.util.MD5Util; 17 | import com.mmc.lidea.util.StringUtil; 18 | import org.apache.flink.api.java.utils.ParameterTool; 19 | import org.apache.flink.configuration.Configuration; 20 | import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; 21 | import org.apache.hadoop.hbase.HBaseConfiguration; 22 | import org.apache.hadoop.hbase.TableName; 23 | import org.apache.hadoop.hbase.client.*; 24 | 25 | import java.io.IOException; 26 | 27 | /** 28 | * @author Joey 29 | * @date 2019/8/2 17:18 30 | */ 31 | public class LideaServiceSinkFun extends RichSinkFunction { 32 | 33 | private static final long serialVersionUID = 3868272239007101505L; 34 | 35 | private Connection conn = null; 36 | 37 | private BufferedMutator mutator; 38 | 39 | 40 | @Override 41 | public void open(Configuration parameters) throws Exception { 42 | super.open(parameters); 43 | 44 | ParameterTool para = (ParameterTool) 45 | getRuntimeContext().getExecutionConfig().getGlobalJobParameters(); 46 | String zkServer = para.getRequired("hbase.zookeeper.quorum"); 47 | String zkPort = para.getRequired("hbase.zookeeper.property.clientPort"); 48 | String tName = para.getRequired("lidea.log.service.name"); 49 | TableName tableName = TableName.valueOf(tName); 50 | 51 | org.apache.hadoop.conf.Configuration config = HBaseConfiguration.create(); 52 | config.set("hbase.zookeeper.quorum", zkServer); 53 | config.set("hbase.zookeeper.property.clientPort", zkPort); 54 | 55 | conn = ConnectionFactory.createConnection(config); 56 | 57 | // 连接表 58 | Table table = conn.getTable(tableName); 59 | 60 | // 设置缓存 61 | BufferedMutatorParams params = new BufferedMutatorParams(tableName); 62 | params.writeBufferSize(1024); 63 | mutator = conn.getBufferedMutator(params); 64 | 65 | } 66 | 67 | @Override 68 | public void close() throws Exception { 69 | mutator.flush(); 70 | conn.close(); 71 | } 72 | 73 | @Override 74 | public void invoke(LogContent bo, Context context) throws IOException { 75 | 76 | String key = MD5Util.encrypt(StringUtil.format("{}{}", 77 | bo.appName, bo.serviceName)); 78 | 79 | // 如果已经存在key,不用重复新增 80 | if (LogServiceNameUtil.exists(key)) { 81 | return; 82 | } else { 83 | LogServiceNameUtil.put(key); 84 | } 85 | 86 | LideaServiceBO lideaServiceBO = new LideaServiceBO(); 87 | lideaServiceBO.appName = bo.appName; 88 | lideaServiceBO.serviceName = bo.serviceName; 89 | 90 | Put put = LideaServiceEntry.of(lideaServiceBO); 91 | 92 | mutator.mutate(put); 93 | 94 | } 95 | 96 | } 97 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/flink/LogContentAcc.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.flink; 11 | 12 | import java.util.ArrayList; 13 | import java.util.List; 14 | 15 | /** 16 | * @author Joey 17 | * @date 2019/8/2 14:00 18 | */ 19 | public class LogContentAcc { 20 | 21 | public int access; 22 | public int exception; 23 | public int cost; 24 | public List traceIds = new ArrayList<>(); 25 | } 26 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/flink/LogContentAgg.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.flink; 11 | 12 | import com.mmc.lidea.common.context.Const; 13 | import com.mmc.lidea.stream.model.LogContent; 14 | import org.apache.flink.api.common.functions.AggregateFunction; 15 | 16 | /** 17 | * @author Joey 18 | * @date 2019/8/1 19:54 19 | */ 20 | public class LogContentAgg implements AggregateFunction { 21 | 22 | private static final long serialVersionUID = -8449536961395080960L; 23 | 24 | 25 | /** 26 | * Creates a new accumulator, starting a new aggregate. 27 | * 28 | *

The new accumulator is typically meaningless unless a value is added 29 | * via {@link #add(Object, Object)}. 30 | * 31 | *

The accumulator is the state of a running aggregation. When a program has multiple 32 | * aggregates in progress (such as per key and window), the state (per key and window) 33 | * is the size of the accumulator. 34 | * 35 | * @return A new accumulator, corresponding to an empty aggregate. 36 | */ 37 | @Override 38 | public LogContentAcc createAccumulator() { 39 | return new LogContentAcc(); 40 | } 41 | 42 | /** 43 | * Adds the given input value to the given accumulator, returning the 44 | * new accumulator value. 45 | * 46 | *

For efficiency, the input accumulator may be modified and returned. 47 | * 48 | * @param value The value to add 49 | * @param accumulator The accumulator to add the value to 50 | */ 51 | @Override 52 | public LogContentAcc add(LogContent value, LogContentAcc accumulator) { 53 | 54 | if (null != value) { 55 | accumulator.access += 1; 56 | accumulator.cost += value.cost; 57 | if (value.type == Const.ERROR) { 58 | accumulator.exception += 1; 59 | accumulator.traceIds.add(value.traceId); 60 | } 61 | } 62 | 63 | return accumulator; 64 | } 65 | 66 | /** 67 | * Gets the result of the aggregation from the accumulator. 68 | * 69 | * @param accumulator The accumulator of the aggregation 70 | * @return The final aggregation result. 71 | */ 72 | @Override 73 | public LogContentAcc getResult(LogContentAcc accumulator) { 74 | 75 | return accumulator; 76 | } 77 | 78 | /** 79 | * Merges two accumulators, returning an accumulator with the merged state. 80 | * 81 | *

This function may reuse any of the given accumulators as the target for the merge 82 | * and return that. The assumption is that the given accumulators will not be used any 83 | * more after having been passed to this function. 84 | * 85 | * @param a An accumulator to merge 86 | * @param b Another accumulator to merge 87 | * @return The accumulator with the merged state 88 | */ 89 | @Override 90 | public LogContentAcc merge(LogContentAcc a, LogContentAcc b) { 91 | return null; 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/flink/LogContentErrorFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.flink; 11 | 12 | import com.mmc.lidea.common.context.Const; 13 | import com.mmc.lidea.stream.model.LogContent; 14 | import com.mmc.lidea.stream.util.LogContentUtil; 15 | 16 | /** 17 | * @author Joey 18 | * @date 2019/8/4 16:45 19 | */ 20 | public class LogContentErrorFilter implements org.apache.flink.api.common.functions.FilterFunction { 21 | 22 | private static final long serialVersionUID = -8905991913035855962L; 23 | 24 | /** 25 | * The filter function that evaluates the predicate. 26 | * 27 | *

IMPORTANT: The system assumes that the function does not 28 | * modify the elements on which the predicate is applied. Violating this assumption 29 | * can lead to incorrect results. 30 | * 31 | * @param element The value to be filtered. 32 | * @return True for values that should be retained, false for values to be filtered out. 33 | * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation 34 | * to fail and may trigger recovery. 35 | */ 36 | @Override 37 | public boolean filter(String element) throws Exception { 38 | 39 | LogContent content = LogContentUtil.valueOf(element); 40 | if (null == content) 41 | return false; 42 | 43 | // 只能故障数据通过 44 | // 必须要有traceId 45 | return (content.type == Const.ERROR) && (content.traceId != null); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/flink/LogContentFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.flink; 11 | 12 | /** 13 | * @author Joey 14 | * @date 2019/8/1 19:44 15 | */ 16 | public class LogContentFilter implements org.apache.flink.api.common.functions.FilterFunction { 17 | 18 | private static final long serialVersionUID = -1475916466164561184L; 19 | 20 | /** 21 | * The filter function that evaluates the predicate. 22 | * 23 | *

IMPORTANT: The system assumes that the function does not 24 | * modify the elements on which the predicate is applied. Violating this assumption 25 | * can lead to incorrect results. 26 | * 27 | * @param value The value to be filtered. 28 | * @return True for values that should be retained, false for values to be filtered out. 29 | * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation 30 | * to fail and may trigger recovery. 31 | */ 32 | @Override 33 | public boolean filter(String value) throws Exception { 34 | return value.split("\\|").length > 12; // filt the not suitable record, default is 13. 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/flink/LogContentSplitter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.flink; 11 | 12 | import com.mmc.lidea.stream.model.LogContent; 13 | import com.mmc.lidea.stream.util.LogContentUtil; 14 | import org.apache.flink.api.common.functions.MapFunction; 15 | 16 | /** 17 | * @author Joey 18 | * @date 2019/8/1 19:31 19 | */ 20 | public class LogContentSplitter implements MapFunction { 21 | 22 | private static final long serialVersionUID = -5055197964573569120L; 23 | 24 | 25 | /** 26 | * The mapping method. Takes an element from the input data set and transforms 27 | * it into exactly one element. 28 | * 29 | * @param element The input value. 30 | * @return The transformed value 31 | * @throws Exception This method may throw exceptions. Throwing an exception will cause the operation 32 | * to fail and may trigger recovery. 33 | */ 34 | @Override 35 | public LogContent map(String element) throws Exception { 36 | 37 | return LogContentUtil.valueOf(element); 38 | } 39 | 40 | 41 | } 42 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/flink/LogContentWinFun.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.flink; 11 | 12 | import com.mmc.lidea.stream.model.LogContentCount; 13 | import com.mmc.lidea.util.TimeUtil; 14 | import org.apache.flink.api.java.tuple.Tuple; 15 | import org.apache.flink.api.java.tuple.Tuple3; 16 | import org.apache.flink.streaming.api.functions.windowing.WindowFunction; 17 | import org.apache.flink.streaming.api.windowing.windows.TimeWindow; 18 | import org.apache.flink.util.Collector; 19 | 20 | /** 21 | * @author Joey 22 | * @date 2019/8/1 19:54 23 | */ 24 | public class LogContentWinFun implements WindowFunction { 25 | 26 | private static final long serialVersionUID = 2539721015512513020L; 27 | 28 | 29 | /** 30 | * Evaluates the window and outputs none or several elements. 31 | * 32 | * @param orgin The key for which this window is evaluated. 33 | * @param window The window that is being evaluated. 34 | * @param input The elements in the window being evaluated. 35 | * @param out A collector for emitting elements. 36 | * @throws Exception The function may throw exceptions to fail the program and trigger recovery. 37 | */ 38 | @Override 39 | public void apply(Tuple orgin, TimeWindow window, Iterable input, Collector out) throws Exception { 40 | 41 | Tuple3 tuple = (Tuple3) orgin; 42 | LogContentAcc acc = input.iterator().next(); 43 | 44 | long begin = window.getEnd(); 45 | String time = TimeUtil.timestampToString(begin); 46 | 47 | LogContentCount result = new LogContentCount(); 48 | result.time = time; 49 | result.appName = tuple.f0; // 系统 50 | result.serviceName = tuple.f1; // 接口 51 | result.methodName = tuple.f2; // 方法 52 | result.count = acc.access; // 调用次数 53 | result.avg = acc.cost / acc.access; // 平均响应时间 54 | result.exception = acc.exception; 55 | result.traceIds = String.join(",", acc.traceIds); 56 | out.collect(result); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/flink/MessageWaterEmitter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.flink; 11 | 12 | import com.mmc.lidea.util.TimeUtil; 13 | import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks; 14 | import org.apache.flink.streaming.api.watermark.Watermark; 15 | 16 | import javax.annotation.Nullable; 17 | 18 | /** 19 | * @author Joey 20 | * @date 2019/7/14 15:00 21 | */ 22 | public class MessageWaterEmitter implements AssignerWithPunctuatedWatermarks { 23 | 24 | private static final long serialVersionUID = -6218113758475504882L; 25 | 26 | /** 27 | * Asks this implementation if it wants to emit a watermark. This method is called right after 28 | * the {@link #extractTimestamp(Object, long)} method. 29 | * 30 | *

The returned watermark will be emitted only if it is non-null and its timestamp 31 | * is larger than that of the previously emitted watermark (to preserve the contract of 32 | * ascending watermarks). If a null value is returned, or the timestamp of the returned 33 | * watermark is smaller than that of the last emitted one, then no new watermark will 34 | * be generated. 35 | * 36 | *

For an example how to use this method, see the documentation of 37 | * {@link AssignerWithPunctuatedWatermarks this class}. 38 | * 39 | * @param lastElement 40 | * @param extractedTimestamp 41 | * @return {@code Null}, if no watermark should be emitted, or the next watermark to emit. 42 | */ 43 | @Nullable 44 | @Override 45 | public Watermark checkAndGetNextWatermark(String lastElement, long extractedTimestamp) { 46 | 47 | if (null != lastElement && lastElement.contains("|")) { 48 | 49 | String[] parts = lastElement.split("\\|"); 50 | // long timestamp = Long.parseLong(parts[0]); 51 | long timestamp = TimeUtil.stringToLong(parts[0], TimeUtil.yyyyMMddHHmmssSSS); 52 | return new Watermark(timestamp); 53 | } 54 | return new Watermark(extractedTimestamp); 55 | } 56 | 57 | /** 58 | * Assigns a timestamp to an element, in milliseconds since the Epoch. 59 | * 60 | *

The method is passed the previously assigned timestamp of the element. 61 | * That previous timestamp may have been assigned from a previous assigner, 62 | * by ingestion time. If the element did not carry a timestamp before, this value is 63 | * {@code Long.MIN_VALUE}. 64 | * 65 | * @param element The element that the timestamp will be assigned to. 66 | * @param previousElementTimestamp The previous internal timestamp of the element, 67 | * or a negative value, if no timestamp has been assigned yet. 68 | * @return The new timestamp. 69 | */ 70 | @Override 71 | public long extractTimestamp(String element, long previousElementTimestamp) { 72 | if (element != null && element.contains("|")) { 73 | String[] parts = element.split("\\|"); 74 | // long timestamp = Long.parseLong(parts[0]); 75 | long timestamp = TimeUtil.stringToLong(parts[0], TimeUtil.yyyyMMddHHmmssSSS); 76 | 77 | return timestamp; 78 | } 79 | return previousElementTimestamp; 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/job/LideaAppJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.job; 11 | 12 | import com.mmc.lidea.common.context.KafkaConst; 13 | import com.mmc.lidea.stream.Bootstrap; 14 | import com.mmc.lidea.stream.flink.LideaAppSinkFun; 15 | import com.mmc.lidea.stream.flink.LogContentFilter; 16 | import com.mmc.lidea.stream.flink.LogContentSplitter; 17 | import com.mmc.lidea.stream.flink.MessageWaterEmitter; 18 | import com.mmc.lidea.stream.model.LogContent; 19 | import com.mmc.lidea.stream.util.LogAppNameUtil; 20 | import org.apache.flink.api.common.serialization.SimpleStringSchema; 21 | import org.apache.flink.api.java.utils.ParameterTool; 22 | import org.apache.flink.streaming.api.TimeCharacteristic; 23 | import org.apache.flink.streaming.api.datastream.DataStream; 24 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 25 | import org.apache.flink.streaming.api.windowing.time.Time; 26 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010; 27 | 28 | import java.io.InputStream; 29 | import java.util.Properties; 30 | 31 | /** 32 | * @author Joey 33 | * @date 2019/8/4 16:39 34 | */ 35 | public class LideaAppJob { 36 | 37 | 38 | public static void main(String[] args) throws Exception { 39 | 40 | 41 | final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 42 | env.enableCheckpointing(5000); // 非常关键,一定要设置启动检查点!! 43 | env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); 44 | env.setParallelism(1); // 这个不需要太多资源 45 | 46 | String confName = "lidea.properties"; 47 | InputStream in = Bootstrap.class.getClassLoader().getResourceAsStream(confName); 48 | ParameterTool parameterTool = ParameterTool.fromPropertiesFile(in); 49 | env.getConfig().setGlobalJobParameters(parameterTool); 50 | 51 | Properties props = new Properties(); 52 | props.setProperty("bootstrap.servers", parameterTool.get("kafka.bootstrap.servers", "localhost:9092")); 53 | props.setProperty("group.id", "lidea-app-group"); 54 | 55 | FlinkKafkaConsumer010 consumer = 56 | new FlinkKafkaConsumer010<>(KafkaConst.TOPIC, new SimpleStringSchema(), props); 57 | consumer.assignTimestampsAndWatermarks(new MessageWaterEmitter()); // 水位 58 | 59 | // 分离出日志格式 60 | DataStream mapStream = env.addSource(consumer) 61 | .filter(new LogContentFilter()) 62 | .map(new LogContentSplitter()); 63 | 64 | 65 | // 写入APP数据 66 | addBaseJob(mapStream); 67 | 68 | env.execute("Record the app name."); 69 | 70 | } 71 | 72 | private static void addBaseJob(DataStream mapStream) { 73 | 74 | mapStream.filter(l -> !LogAppNameUtil.exists(l.appName)).keyBy("traceId") 75 | .timeWindow(Time.seconds(10)) 76 | ; 77 | 78 | mapStream.addSink(new LideaAppSinkFun()); 79 | } 80 | 81 | } 82 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/job/LideaDetailJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.job; 11 | 12 | import com.mmc.lidea.common.context.KafkaConst; 13 | import com.mmc.lidea.stream.Bootstrap; 14 | import com.mmc.lidea.stream.flink.LideaDetailSinkFun; 15 | import com.mmc.lidea.stream.flink.LogContentErrorFilter; 16 | import com.mmc.lidea.stream.flink.LogContentSplitter; 17 | import com.mmc.lidea.stream.flink.MessageWaterEmitter; 18 | import com.mmc.lidea.stream.model.LogContent; 19 | import org.apache.flink.api.common.serialization.SimpleStringSchema; 20 | import org.apache.flink.api.java.utils.ParameterTool; 21 | import org.apache.flink.streaming.api.TimeCharacteristic; 22 | import org.apache.flink.streaming.api.datastream.DataStream; 23 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 24 | import org.apache.flink.streaming.api.windowing.time.Time; 25 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010; 26 | 27 | import java.io.InputStream; 28 | import java.util.Properties; 29 | 30 | /** 31 | * @author Joey 32 | * @date 2019/8/4 16:39 33 | */ 34 | public class LideaDetailJob { 35 | 36 | 37 | public static void main(String[] args) throws Exception { 38 | 39 | 40 | final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 41 | env.enableCheckpointing(5000); // 非常关键,一定要设置启动检查点!! 42 | env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); 43 | 44 | String confName = "lidea.properties"; 45 | InputStream in = Bootstrap.class.getClassLoader().getResourceAsStream(confName); 46 | ParameterTool parameterTool = ParameterTool.fromPropertiesFile(in); 47 | env.getConfig().setGlobalJobParameters(parameterTool); 48 | 49 | Properties props = new Properties(); 50 | props.setProperty("bootstrap.servers", parameterTool.get("kafka.bootstrap.servers", "localhost:9092")); 51 | props.setProperty("group.id", "lidea-detail-group"); 52 | 53 | FlinkKafkaConsumer010 consumer = 54 | new FlinkKafkaConsumer010<>(KafkaConst.TOPIC, new SimpleStringSchema(), props); 55 | consumer.assignTimestampsAndWatermarks(new MessageWaterEmitter()); 56 | 57 | // 分离出日志格式 58 | DataStream mapStream = env.addSource(consumer) 59 | .filter(new LogContentErrorFilter()) 60 | .map(new LogContentSplitter()); 61 | 62 | 63 | // 写入故障数据 64 | addBaseJob(mapStream); 65 | // keyStream.print().setParallelism(1); // 打印调试 66 | 67 | env.execute("Record the exception detail."); 68 | 69 | } 70 | 71 | private static void addBaseJob(DataStream mapStream) { 72 | 73 | mapStream.keyBy("traceId") 74 | .timeWindow(Time.seconds(10)) 75 | ; 76 | 77 | // mapStream.print().setParallelism(1); // 打印调试 78 | mapStream.addSink(new LideaDetailSinkFun()); 79 | } 80 | 81 | } 82 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/job/LideaLogJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.job; 11 | 12 | import com.mmc.lidea.common.context.KafkaConst; 13 | import com.mmc.lidea.stream.flink.*; 14 | import com.mmc.lidea.stream.model.LogContent; 15 | import com.mmc.lidea.stream.model.LogContentCount; 16 | import org.apache.flink.api.common.serialization.SimpleStringSchema; 17 | import org.apache.flink.api.java.utils.ParameterTool; 18 | import org.apache.flink.streaming.api.TimeCharacteristic; 19 | import org.apache.flink.streaming.api.datastream.DataStream; 20 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 21 | import org.apache.flink.streaming.api.windowing.time.Time; 22 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010; 23 | 24 | import java.io.InputStream; 25 | import java.util.Properties; 26 | 27 | /** 28 | * @author Joey 29 | * @date 2019/7/14 14:56 30 | */ 31 | public class LideaLogJob { 32 | 33 | 34 | public static void main(String[] args) throws Exception { 35 | 36 | final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 37 | env.enableCheckpointing(5000); // 非常关键,一定要设置启动检查点!! 38 | env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); 39 | 40 | String confName = "lidea.properties"; 41 | InputStream in = LideaLogJob.class.getClassLoader().getResourceAsStream(confName); 42 | ParameterTool parameterTool = ParameterTool.fromPropertiesFile(in); 43 | env.getConfig().setGlobalJobParameters(parameterTool); 44 | 45 | Properties props = new Properties(); 46 | props.setProperty("bootstrap.servers", parameterTool.get("kafka.bootstrap.servers", "localhost:9092")); 47 | props.setProperty("group.id", "flink-base-group"); 48 | 49 | FlinkKafkaConsumer010 consumer = 50 | new FlinkKafkaConsumer010<>(KafkaConst.TOPIC, new SimpleStringSchema(), props); 51 | consumer.assignTimestampsAndWatermarks(new MessageWaterEmitter()); 52 | 53 | // 分离出日志格式 54 | DataStream mapStream = env.addSource(consumer) 55 | .filter(new LogContentFilter()) 56 | .map(new LogContentSplitter()); 57 | 58 | // 统计调用次数、平均响应时间、故障次数 59 | addBaseJob(mapStream); 60 | 61 | env.execute("Calc access, avg time, exception count."); 62 | 63 | } 64 | 65 | private static void addBaseJob(DataStream mapStream) { 66 | // 统计调用次数 67 | DataStream keyStream = mapStream 68 | .keyBy("appName", "serviceName", "methodName") 69 | .timeWindow(Time.seconds(10)) 70 | .aggregate(new LogContentAgg(), new LogContentWinFun()); 71 | 72 | keyStream.addSink(new LideaBaseSinkFun()); 73 | } 74 | 75 | 76 | } 77 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/job/LideaMethodJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.job; 11 | 12 | import com.mmc.lidea.common.context.KafkaConst; 13 | import com.mmc.lidea.stream.Bootstrap; 14 | import com.mmc.lidea.stream.flink.LideaMethodSinkFun; 15 | import com.mmc.lidea.stream.flink.LogContentFilter; 16 | import com.mmc.lidea.stream.flink.LogContentSplitter; 17 | import com.mmc.lidea.stream.flink.MessageWaterEmitter; 18 | import com.mmc.lidea.stream.model.LogContent; 19 | import com.mmc.lidea.stream.util.LogMethodNameUtil; 20 | import com.mmc.lidea.util.MD5Util; 21 | import com.mmc.lidea.util.StringUtil; 22 | import org.apache.flink.api.common.serialization.SimpleStringSchema; 23 | import org.apache.flink.api.java.utils.ParameterTool; 24 | import org.apache.flink.streaming.api.TimeCharacteristic; 25 | import org.apache.flink.streaming.api.datastream.DataStream; 26 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 27 | import org.apache.flink.streaming.api.windowing.time.Time; 28 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010; 29 | 30 | import java.io.InputStream; 31 | import java.util.Properties; 32 | 33 | /** 34 | * @author Joey 35 | * @date 2019/8/4 16:39 36 | */ 37 | public class LideaMethodJob { 38 | 39 | 40 | public static void main(String[] args) throws Exception { 41 | 42 | 43 | final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 44 | env.enableCheckpointing(5000); // 非常关键,一定要设置启动检查点!! 45 | env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); 46 | env.setParallelism(1); // 这个不需要太多资源 47 | 48 | String confName = "lidea.properties"; 49 | InputStream in = Bootstrap.class.getClassLoader().getResourceAsStream(confName); 50 | ParameterTool parameterTool = ParameterTool.fromPropertiesFile(in); 51 | env.getConfig().setGlobalJobParameters(parameterTool); 52 | 53 | Properties props = new Properties(); 54 | props.setProperty("bootstrap.servers", parameterTool.get("kafka.bootstrap.servers", "localhost:9092")); 55 | props.setProperty("group.id", "lidea-method-group"); 56 | 57 | FlinkKafkaConsumer010 consumer = 58 | new FlinkKafkaConsumer010<>(KafkaConst.TOPIC, new SimpleStringSchema(), props); 59 | consumer.assignTimestampsAndWatermarks(new MessageWaterEmitter()); // 水位 60 | 61 | // 分离出日志格式 62 | DataStream mapStream = env.addSource(consumer) 63 | .filter(new LogContentFilter()) 64 | .map(new LogContentSplitter()); 65 | 66 | 67 | // 写入APP数据 68 | addBaseJob(mapStream); 69 | 70 | env.execute("Record the method name."); 71 | 72 | } 73 | 74 | private static void addBaseJob(DataStream mapStream) { 75 | 76 | mapStream.filter(value -> { 77 | 78 | String key = MD5Util.encrypt(StringUtil.format("{}{}{}", 79 | value.appName, value.serviceName, value.methodName)); 80 | 81 | return !LogMethodNameUtil.exists(key); 82 | 83 | }).keyBy("traceId").timeWindow(Time.seconds(10)); 84 | 85 | mapStream.addSink(new LideaMethodSinkFun()); 86 | } 87 | 88 | } 89 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/job/LideaMethodStatJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.job; 11 | 12 | import com.mmc.lidea.common.context.KafkaConst; 13 | import com.mmc.lidea.stream.flink.*; 14 | import com.mmc.lidea.stream.model.LogContent; 15 | import com.mmc.lidea.stream.model.LogContentCount; 16 | import org.apache.flink.api.common.serialization.SimpleStringSchema; 17 | import org.apache.flink.api.java.utils.ParameterTool; 18 | import org.apache.flink.streaming.api.TimeCharacteristic; 19 | import org.apache.flink.streaming.api.datastream.DataStream; 20 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 21 | import org.apache.flink.streaming.api.windowing.time.Time; 22 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010; 23 | 24 | import java.io.InputStream; 25 | import java.util.Properties; 26 | 27 | /** 28 | * @author Joey 29 | * @date 2019/7/14 14:56 30 | */ 31 | public class LideaMethodStatJob { 32 | 33 | 34 | public static void main(String[] args) throws Exception { 35 | 36 | final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 37 | env.enableCheckpointing(5000); // 非常关键,一定要设置启动检查点!! 38 | env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); 39 | 40 | String confName = "lidea.properties"; 41 | InputStream in = LideaMethodStatJob.class.getClassLoader().getResourceAsStream(confName); 42 | ParameterTool parameterTool = ParameterTool.fromPropertiesFile(in); 43 | env.getConfig().setGlobalJobParameters(parameterTool); 44 | 45 | Properties props = new Properties(); 46 | props.setProperty("bootstrap.servers", parameterTool.get("kafka.bootstrap.servers", "localhost:9092")); 47 | props.setProperty("group.id", "flink-method-stat-group"); 48 | 49 | FlinkKafkaConsumer010 consumer = 50 | new FlinkKafkaConsumer010<>(KafkaConst.TOPIC, new SimpleStringSchema(), props); 51 | consumer.assignTimestampsAndWatermarks(new MessageWaterEmitter()); 52 | 53 | // 分离出日志格式 54 | DataStream mapStream = env.addSource(consumer) 55 | .filter(new LogContentFilter()) 56 | .map(new LogContentSplitter()); 57 | 58 | // 统计调用次数 59 | addBaseJob(mapStream); 60 | 61 | env.execute("Calc access count of methods simply."); 62 | 63 | } 64 | 65 | private static void addBaseJob(DataStream mapStream) { 66 | // 统计调用次数 67 | DataStream keyStream = mapStream 68 | .keyBy("appName", "serviceName", "methodName") 69 | .timeWindow(Time.seconds(10)) 70 | .aggregate(new LogContentAgg(), new LogContentWinFun()); 71 | 72 | keyStream.addSink(new LideaMethodStatSinkFun()); 73 | } 74 | 75 | 76 | } 77 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/job/LideaServiceJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.job; 11 | 12 | import com.mmc.lidea.common.context.KafkaConst; 13 | import com.mmc.lidea.stream.Bootstrap; 14 | import com.mmc.lidea.stream.flink.LideaServiceSinkFun; 15 | import com.mmc.lidea.stream.flink.LogContentFilter; 16 | import com.mmc.lidea.stream.flink.LogContentSplitter; 17 | import com.mmc.lidea.stream.flink.MessageWaterEmitter; 18 | import com.mmc.lidea.stream.model.LogContent; 19 | import com.mmc.lidea.stream.util.LogServiceNameUtil; 20 | import com.mmc.lidea.util.MD5Util; 21 | import com.mmc.lidea.util.StringUtil; 22 | import org.apache.flink.api.common.serialization.SimpleStringSchema; 23 | import org.apache.flink.api.java.utils.ParameterTool; 24 | import org.apache.flink.streaming.api.TimeCharacteristic; 25 | import org.apache.flink.streaming.api.datastream.DataStream; 26 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 27 | import org.apache.flink.streaming.api.windowing.time.Time; 28 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010; 29 | 30 | import java.io.InputStream; 31 | import java.util.Properties; 32 | 33 | /** 34 | * @author Joey 35 | * @date 2019/8/4 16:39 36 | */ 37 | public class LideaServiceJob { 38 | 39 | 40 | public static void main(String[] args) throws Exception { 41 | 42 | 43 | final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 44 | env.enableCheckpointing(5000); // 非常关键,一定要设置启动检查点!! 45 | env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); 46 | env.setParallelism(1); // 这个不需要太多资源 47 | 48 | String confName = "lidea.properties"; 49 | InputStream in = Bootstrap.class.getClassLoader().getResourceAsStream(confName); 50 | ParameterTool parameterTool = ParameterTool.fromPropertiesFile(in); 51 | env.getConfig().setGlobalJobParameters(parameterTool); 52 | 53 | Properties props = new Properties(); 54 | props.setProperty("bootstrap.servers", parameterTool.get("kafka.bootstrap.servers", "localhost:9092")); 55 | props.setProperty("group.id", "lidea-service-group"); 56 | 57 | FlinkKafkaConsumer010 consumer = 58 | new FlinkKafkaConsumer010<>(KafkaConst.TOPIC, new SimpleStringSchema(), props); 59 | consumer.assignTimestampsAndWatermarks(new MessageWaterEmitter()); // 水位 60 | 61 | // 分离出日志格式 62 | DataStream mapStream = env.addSource(consumer) 63 | .filter(new LogContentFilter()) 64 | .map(new LogContentSplitter()); 65 | 66 | // 写入APP数据 67 | addBaseJob(mapStream); 68 | 69 | env.execute("Record the interface name."); 70 | 71 | } 72 | 73 | private static void addBaseJob(DataStream mapStream) { 74 | 75 | mapStream.filter(value -> { 76 | String key = MD5Util.encrypt(StringUtil.format("{}{}", 77 | value.appName, value.serviceName)); 78 | 79 | return !LogServiceNameUtil.exists(key); 80 | 81 | }).keyBy("traceId").timeWindow(Time.seconds(10)); 82 | 83 | mapStream.addSink(new LideaServiceSinkFun()); 84 | } 85 | 86 | } 87 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/model/LogContent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.model; 11 | 12 | /** 13 | * @author Joey 14 | * @date 2019/8/1 19:20 15 | */ 16 | public class LogContent { 17 | 18 | public String time; 19 | public String traceId; 20 | public int type; 21 | public String localIp; 22 | public String remoteIp; 23 | public String appName; 24 | public String serviceName; 25 | public String methodName; 26 | public String args; 27 | public String response; 28 | public int cost; 29 | public String msg; 30 | public String customMsg; 31 | 32 | } 33 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/model/LogContentCount.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.model; 11 | 12 | import com.mmc.lidea.util.StringUtil; 13 | import lombok.Data; 14 | 15 | /** 16 | * @author Joey 17 | * @date 2019/8/1 19:59 18 | */ 19 | @Data 20 | public class LogContentCount { 21 | 22 | /** 23 | * 时间. 24 | */ 25 | public String time; 26 | /** 27 | * 应用名称. 28 | */ 29 | public String appName; 30 | /** 31 | * 接口名称. 32 | */ 33 | public String serviceName; 34 | /** 35 | * 方法名称. 36 | */ 37 | public String methodName; 38 | 39 | /** 40 | * 调用次数. 41 | */ 42 | public long count; 43 | /** 44 | * 平均响应时间. 45 | */ 46 | public int avg; 47 | /** 48 | * 故障次数. 49 | */ 50 | public int exception; 51 | /** 52 | * 故障ID. 53 | */ 54 | public String traceIds; 55 | 56 | @Override 57 | public String toString() { 58 | return StringUtil.format("{}, {}, {}, {}, {}, {}, {}, {}", 59 | time, appName, serviceName, methodName, count, avg, exception, traceIds); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/util/LogAppNameUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.util; 11 | 12 | import java.util.HashMap; 13 | import java.util.Map; 14 | 15 | /** 16 | * @author Joey 17 | * @date 2019/8/6 19:47 18 | */ 19 | public class LogAppNameUtil { 20 | 21 | private static Map names = new HashMap<>(); 22 | 23 | public static boolean exists(String name) { 24 | 25 | return names.containsKey(name); 26 | } 27 | 28 | public static void put(String pwd) { 29 | names.put(pwd, "1"); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/util/LogContentUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.util; 11 | 12 | import com.mmc.lidea.stream.model.LogContent; 13 | 14 | /** 15 | * @author Joey 16 | * @date 2019/8/4 16:49 17 | */ 18 | public class LogContentUtil { 19 | 20 | public static LogContent valueOf(String element) { 21 | if (element != null && element.contains("|")) { 22 | 23 | try { 24 | 25 | 26 | String[] parts = element.split("\\|"); 27 | 28 | if (parts.length < 13) { 29 | System.out.println(Thread.currentThread().getName() + ", bad message, length < 13"); 30 | return null; 31 | } 32 | 33 | int i = 0; 34 | LogContent content = new LogContent(); 35 | content.time = parts[i++]; 36 | content.traceId = parts[i++]; 37 | content.type = Integer.valueOf(parts[i++]); 38 | content.localIp = parts[i++]; 39 | content.remoteIp = parts[i++]; 40 | content.appName = parts[i++]; 41 | content.serviceName = parts[i++]; 42 | content.methodName = parts[i++]; 43 | content.args = parts[i++]; 44 | content.response = parts[i++]; 45 | content.cost = Integer.valueOf(parts[i++]); 46 | content.msg = parts[i++]; 47 | content.customMsg = parts[i++]; 48 | return content; 49 | 50 | } catch (Exception e) { 51 | e.printStackTrace(); 52 | return null; 53 | } 54 | } 55 | return null; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/util/LogMethodNameUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.util; 11 | 12 | import java.util.HashMap; 13 | import java.util.Map; 14 | 15 | /** 16 | * @author Joey 17 | * @date 2019/8/6 22:07 18 | */ 19 | public class LogMethodNameUtil { 20 | 21 | private static Map methods = new HashMap<>(); 22 | 23 | public static boolean exists(String pwd) { 24 | 25 | return methods.containsKey(pwd); 26 | } 27 | 28 | public static void put(String pwd) { 29 | methods.put(pwd, "1"); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/java/com/mmc/lidea/stream/util/LogServiceNameUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.util; 11 | 12 | import java.util.HashMap; 13 | import java.util.Map; 14 | 15 | /** 16 | * @author Joey 17 | * @date 2019/8/29 14:15 18 | */ 19 | public class LogServiceNameUtil { 20 | 21 | private static Map names = new HashMap<>(); 22 | 23 | public static boolean exists(String name) { 24 | 25 | return names.containsKey(name); 26 | } 27 | 28 | public static void put(String name) { 29 | names.put(name, "1"); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/main/resources/lidea.properties: -------------------------------------------------------------------------------- 1 | hbase.zookeeper.quorum=10.204.49.54 2 | hbase.zookeeper.property.clientPort=2181 3 | 4 | lidea.log.app.name=LideaApp 5 | lidea.log.service.name=LideaService 6 | lidea.log.method.name=LideaMethod 7 | lidea.log.base.name=LideaLog 8 | lidea.log.detail.name=LideaLogDetail 9 | 10 | kafka.bootstrap.servers=10.204.58.157:9092 -------------------------------------------------------------------------------- /mmc-lidea-stream/src/test/java/com/mmc/lidea/stream/producer/MessageProducer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.producer; 11 | 12 | import com.mmc.lidea.common.context.KafkaConst; 13 | import com.mmc.lidea.util.StringUtil; 14 | import com.mmc.lidea.util.TimeUtil; 15 | import org.apache.kafka.clients.producer.*; 16 | 17 | import java.util.Properties; 18 | import java.util.UUID; 19 | 20 | import static com.mmc.lidea.util.RandomUtil.getRandomNumberInRange; 21 | 22 | /** 23 | * @author Joey 24 | * @date 2019/7/14 18:22 25 | */ 26 | public class MessageProducer { 27 | 28 | public static void main(String[] args) throws InterruptedException { 29 | 30 | 31 | Properties props = new Properties(); 32 | props.put("bootstrap.servers", "localhost:9092"); 33 | // props.put("bootstrap.servers", "10.204.58.157:9092"); 34 | props.put("acks", "all"); 35 | props.put("retries", 0); 36 | props.put("batch.size", 16384); 37 | props.put("linger.ms", 1); 38 | props.put("buffer.memory", 33554432); 39 | props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 40 | props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 41 | 42 | Producer producer = new KafkaProducer<>(props); 43 | int totalMessageCount = 1000; 44 | for (int i = 0; i < totalMessageCount; i++) { 45 | 46 | long time = System.currentTimeMillis(); 47 | System.out.println(time); 48 | // String value = makeMessage(time); 49 | String value = makeMessage(TimeUtil.timestampToString(time, TimeUtil.yyyyMMddHHmmssSSS)); 50 | 51 | System.out.println(value); 52 | 53 | producer.send(new ProducerRecord<>(KafkaConst.TOPIC, value), new Callback() { 54 | @Override 55 | public void onCompletion(RecordMetadata metadata, Exception exception) { 56 | if (exception != null) { 57 | System.out.println("Failed to send message with exception " + exception); 58 | } 59 | } 60 | }); 61 | Thread.sleep(getRandomNumberInRange(1, 10) * 1000); 62 | } 63 | producer.close(); 64 | 65 | } 66 | 67 | private static String[] methods = new String[] { 68 | "getCabinetInfo", 69 | "addSendExpressCompany", 70 | "updateChargeEmp", 71 | "getCabinetVisualInfoWithDetailByCode", 72 | "getVisualData" 73 | }; 74 | 75 | private static String[] interfaces = new String[]{ 76 | "com.demo.api.CabinetServiceFacade", 77 | "com.demo.api.CabinetServiceAdvanceFacade", 78 | "com.demo.api.CabinetNetCodeFacade", 79 | "com.demo.api.CabinetServiceAllSlowFacade" 80 | }; 81 | private static String[] servers = new String[]{ 82 | "demo", 83 | "cabinet-conf-server", 84 | "cabinet-heart-server", 85 | "cabinet-control-server" 86 | }; 87 | 88 | private static String makeMessage(Object time) { 89 | 90 | // 日志格式 91 | // time|traceId|type|localIp|remoteIP|appName|serviceName|methodName|args|response|cost|msg|customMsg 92 | String tpl = StringUtil.format("{}|{}|{}|{}|{}|{}|{}|{}|{}|{}|{}|{}|{}", 93 | time, // 时间 94 | UUID.randomUUID().toString().replace("-", ""), 95 | getRandomNumberInRange(1, 3), // 1:成功 2:异常 96 | "10.204.240.74", 97 | "10.204.240.74", 98 | getServerName(), 99 | getInterFaceName(), 100 | getMethodName(), 101 | "{\"id\": 888, \"name\": \"zhangsan\"} ", 102 | "{\"sucess\": true}", 103 | getRandomNumberInRange(1, 8000), 104 | "no msg", 105 | "no data" 106 | ); 107 | 108 | return tpl; 109 | } 110 | 111 | private static String getMethodName() { 112 | 113 | // return "getCabinetInfo"; 114 | return methods[getRandomNumberInRange(0, 4)]; 115 | 116 | } 117 | 118 | private static String getServerName() { 119 | // return servers[getRandomNumberInRange(0, 0)]; 120 | return servers[getRandomNumberInRange(0, 3)]; 121 | } 122 | 123 | private static String getInterFaceName() { 124 | // return interfaces[getRandomNumberInRange(0, 0)]; 125 | return interfaces[getRandomNumberInRange(0, 3)]; 126 | } 127 | 128 | 129 | } 130 | -------------------------------------------------------------------------------- /mmc-lidea-stream/src/test/java/com/mmc/lidea/stream/producer/MessageProducerSingle.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.stream.producer; 11 | 12 | import com.mmc.lidea.common.context.KafkaConst; 13 | import com.mmc.lidea.util.StringUtil; 14 | import com.mmc.lidea.util.TimeUtil; 15 | import org.apache.kafka.clients.producer.*; 16 | 17 | import java.util.Properties; 18 | import java.util.UUID; 19 | 20 | import static com.mmc.lidea.util.RandomUtil.getRandomNumberInRange; 21 | 22 | /** 23 | * @author Joey 24 | * @date 2019/7/14 18:22 25 | */ 26 | public class MessageProducerSingle { 27 | 28 | public static void main(String[] args) throws InterruptedException { 29 | 30 | 31 | Properties props = new Properties(); 32 | props.put("bootstrap.servers", "localhost:9092"); 33 | // props.put("bootstrap.servers", "10.204.58.157:9092"); 34 | props.put("acks", "all"); 35 | props.put("retries", 0); 36 | props.put("batch.size", 16384); 37 | props.put("linger.ms", 1); 38 | props.put("buffer.memory", 33554432); 39 | props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 40 | props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 41 | 42 | Producer producer = new KafkaProducer<>(props); 43 | int totalMessageCount = 1000; 44 | for (int i = 0; i < totalMessageCount; i++) { 45 | 46 | long time = System.currentTimeMillis(); 47 | System.out.println(time); 48 | // String value = makeMessage(time); 49 | String value = makeMessage(TimeUtil.timestampToString(time, TimeUtil.yyyyMMddHHmmssSSS)); 50 | 51 | System.out.println(value); 52 | 53 | producer.send(new ProducerRecord<>(KafkaConst.TOPIC, value), new Callback() { 54 | @Override 55 | public void onCompletion(RecordMetadata metadata, Exception exception) { 56 | if (exception != null) { 57 | System.out.println("Failed to send message with exception " + exception); 58 | } 59 | } 60 | }); 61 | Thread.sleep(getRandomNumberInRange(1, 10) * 1000); 62 | } 63 | producer.close(); 64 | 65 | } 66 | 67 | private static String[] methods = new String[] { 68 | "getCabinetInfo", 69 | "addSendExpressCompany", 70 | "updateChargeEmp", 71 | "getCabinetVisualInfoWithDetailByCode", 72 | "getVisualData" 73 | }; 74 | 75 | private static String[] interfaces = new String[]{ 76 | "com.demo.api.CabinetServiceFacade", 77 | "com.demo.api.CabinetServiceAdvanceFacade", 78 | "com.demo.api.CabinetNetCodeFacade", 79 | "com.demo.api.CabinetServiceAllSlowFacade" 80 | }; 81 | private static String[] servers = new String[]{ 82 | "demo", 83 | "cabinet-conf-server", 84 | "cabinet-heart-server", 85 | "cabinet-control-server" 86 | }; 87 | 88 | private static String makeMessage(Object time) { 89 | 90 | // 日志格式 91 | // time|traceId|type|localIp|remoteIP|appName|serviceName|methodName|args|response|cost|msg|customMsg 92 | String tpl = StringUtil.format("{}|{}|{}|{}|{}|{}|{}|{}|{}|{}|{}|{}|{}", 93 | time, // 时间 94 | UUID.randomUUID().toString().replace("-", ""), 95 | getRandomNumberInRange(1, 3), // 1:成功 2:异常 96 | "10.204.240.74", 97 | "10.204.240.74", 98 | getServerName(), 99 | getInterFaceName(), 100 | getMethodName(), 101 | "{\"id\": 888, \"name\": \"zhangsan\"} ", 102 | "{\"sucess\": true}", 103 | getRandomNumberInRange(1, 8000), 104 | "no msg", 105 | "no data" 106 | ); 107 | 108 | return tpl; 109 | } 110 | 111 | private static String getMethodName() { 112 | 113 | return "getCabinetInfo"; 114 | // return methods[getRandomNumberInRange(0, 4)]; 115 | 116 | } 117 | 118 | private static String getServerName() { 119 | return "demo"; 120 | // return servers[getRandomNumberInRange(0, 3)]; 121 | } 122 | 123 | private static String getInterFaceName() { 124 | return "com.demo.api.CabinetServiceFacade"; 125 | // return interfaces[getRandomNumberInRange(0, 3)]; 126 | } 127 | 128 | 129 | } 130 | -------------------------------------------------------------------------------- /mmc-lidea-util/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | lidea 7 | com.mmc 8 | 1.0.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | mmc-lidea-util 13 | 14 | 15 | 16 | 17 | junit 18 | junit 19 | 4.12 20 | test 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /mmc-lidea-util/src/main/java/com/mmc/lidea/util/Charsets.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 NAVER Corp. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mmc.lidea.util; 18 | 19 | import java.nio.charset.Charset; 20 | 21 | /** 22 | * @author Woonduk Kang(emeroad) 23 | */ 24 | public final class Charsets { 25 | private Charsets() { 26 | } 27 | 28 | public static final Charset US_ASCII = Charset.forName("US-ASCII"); 29 | public static final String US_ASCII_NAME = US_ASCII.name(); 30 | 31 | public static final Charset UTF_8 = Charset.forName("UTF-8"); 32 | public static final String UTF_8_NAME = UTF_8.name(); 33 | 34 | 35 | } 36 | -------------------------------------------------------------------------------- /mmc-lidea-util/src/main/java/com/mmc/lidea/util/MD5Util.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.util; 11 | 12 | import java.security.MessageDigest; 13 | 14 | /** 15 | * MD5加密工具类. 16 | * @author Joey 17 | * @date 2018/6/24 16:40 18 | */ 19 | public class MD5Util { 20 | 21 | public static String encrypt(String pwd) { 22 | 23 | //用于加密的字符 24 | char md5String[] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 25 | 'A', 'B', 'C', 'D', 'E', 'F' }; 26 | try { 27 | //使用平台的默认字符集将此 String 编码为 byte序列,并将结果存储到一个新的 byte数组中 28 | byte[] btInput = pwd.getBytes(); 29 | 30 | //信息摘要是安全的单向哈希函数,它接收任意大小的数据,并输出固定长度的哈希值。 31 | MessageDigest mdInst = MessageDigest.getInstance("MD5"); 32 | 33 | //MessageDigest对象通过使用 update方法处理数据, 使用指定的byte数组更新摘要 34 | mdInst.update(btInput); 35 | 36 | // 摘要更新之后,通过调用digest()执行哈希计算,获得密文 37 | byte[] md = mdInst.digest(); 38 | 39 | // 把密文转换成十六进制的字符串形式 40 | int j = md.length; 41 | char str[] = new char[j * 2]; 42 | int k = 0; 43 | for (byte byte0 : md) { // i = 0 44 | str[k++] = md5String[byte0 >>> 4 & 0xf]; // 5 45 | str[k++] = md5String[byte0 & 0xf]; // F 46 | } 47 | 48 | //返回经过加密后的字符串 49 | return new String(str); 50 | 51 | } catch (Exception e) { 52 | e.printStackTrace(); 53 | return null; 54 | } 55 | } 56 | } -------------------------------------------------------------------------------- /mmc-lidea-util/src/main/java/com/mmc/lidea/util/MemoryUsageExtrator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.util; 11 | 12 | 13 | import com.sun.management.OperatingSystemMXBean; 14 | 15 | import java.lang.management.ManagementFactory; 16 | 17 | /** 18 | * @author Joey 19 | * @date 2019/7/14 18:24 20 | */ 21 | public class MemoryUsageExtrator { 22 | 23 | private static OperatingSystemMXBean mxBean = 24 | (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); 25 | 26 | /** 27 | * Get current free memory size in bytes 28 | * @return free RAM size 29 | */ 30 | public static long currentFreeMemorySizeInBytes() { 31 | return mxBean.getFreePhysicalMemorySize(); 32 | } 33 | 34 | 35 | } 36 | -------------------------------------------------------------------------------- /mmc-lidea-util/src/main/java/com/mmc/lidea/util/RandomUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.util; 11 | 12 | import java.util.Random; 13 | 14 | /** 15 | * @author Joey 16 | * @date 2019/7/28 17:01 17 | */ 18 | public class RandomUtil { 19 | 20 | /** 21 | * 任意范围随机数. 22 | * 23 | * @param min 起始值 24 | * @param max 结束值 25 | * @return 随机值 26 | */ 27 | public static int getRandomNumberInRange(int min, int max) { 28 | 29 | Random r = new Random(); 30 | return r.ints(min, (max + 1)).findFirst().getAsInt(); 31 | 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /mmc-lidea-util/src/main/java/com/mmc/lidea/util/StringUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.util; 11 | 12 | /** 13 | * 字符串工具类. 14 | * @author Joey 15 | * 2016年10月17日 下午4:08:55 16 | */ 17 | public class StringUtil { 18 | 19 | /** 20 | * 简单格式化{}样式的字符串.
21 | * String str = "aaa{} bbb{} ccc{}";
22 | * System.out.println(StringUtil.format(str, "1", "2", "3")); 23 | * @param src 源字符串 24 | * @param param 跟源字符串{}匹配的个数字符串 25 | * @return 26 | */ 27 | public static String format(String src, Object... param) { 28 | int i = 0; 29 | int index = 0; 30 | StringBuffer sb = new StringBuffer(src); 31 | while (-1 != (index = sb.indexOf("{}"))) { 32 | sb.replace(index, index + 2, String.valueOf(param[i++])); 33 | } 34 | return sb.toString(); 35 | } 36 | 37 | public static String formatNullForEmpty(String src, Object... param) { 38 | int i = 0; 39 | int index = 0; 40 | StringBuffer sb = new StringBuffer(src); 41 | while (-1 != (index = sb.indexOf("{}"))) { 42 | Object obj = param[i++]; 43 | sb.replace(index, index + 2, String.valueOf(null == obj ? "" : obj)); 44 | } 45 | return sb.toString(); 46 | } 47 | 48 | public static boolean equals(String str1,String str2) { 49 | return (null == str1 && null == str2) || (null != str1 && str1.equals(str2)); 50 | } 51 | 52 | public static void main(String[] args) { 53 | String str = "aaa{} bbb{} ccc{}"; 54 | System.out.println(StringUtil.format(str, "1", "2", "3")); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /mmc-lidea-util/src/main/java/com/mmc/lidea/util/TimeUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.lidea.util; 11 | 12 | import java.time.Instant; 13 | import java.time.LocalDateTime; 14 | import java.time.ZoneId; 15 | import java.time.ZoneOffset; 16 | import java.time.format.DateTimeFormatter; 17 | 18 | /** 19 | * @author Joey 20 | * @date 2019/7/20 20:19 21 | */ 22 | public class TimeUtil { 23 | 24 | public static final String yyyyMMddHHmmss = "yyyy-MM-dd HH:mm:ss"; 25 | public static final String yyyyMMddHHmmssSSS = "yyyy-MM-dd HH:mm:ss.SSS"; 26 | 27 | /** 28 | * 时间戳转字符串. 29 | * 30 | * @param time 31 | * @return 32 | */ 33 | public static String timestampToString(Long time) { 34 | DateTimeFormatter ftf = DateTimeFormatter.ofPattern(yyyyMMddHHmmss); 35 | return ftf.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(time), ZoneId.systemDefault())); 36 | } 37 | 38 | /** 39 | * 时间戳转字符串. 40 | * 41 | * @param time 42 | * @param pattern 43 | * @return 44 | */ 45 | public static String timestampToString(Long time, String pattern) { 46 | DateTimeFormatter ftf = DateTimeFormatter.ofPattern(pattern); 47 | return ftf.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(time), ZoneId.systemDefault())); 48 | } 49 | 50 | /** 51 | * 字符串转时间戳. 52 | * 53 | * @param ltime 54 | * @return 55 | */ 56 | public static long stringToLong(String ltime) { 57 | DateTimeFormatter ftf = DateTimeFormatter.ofPattern(yyyyMMddHHmmss); 58 | LocalDateTime parse = LocalDateTime.parse(ltime, ftf); 59 | return LocalDateTime.from(parse).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); 60 | } 61 | 62 | /** 63 | * 字符串转时间戳. 64 | * 65 | * @param ltime 66 | * @param pattern 67 | * @return 68 | */ 69 | public static long stringToLong(String ltime, String pattern) { 70 | DateTimeFormatter ftf = DateTimeFormatter.ofPattern(pattern); 71 | LocalDateTime parse = LocalDateTime.parse(ltime, ftf); 72 | return LocalDateTime.from(parse).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); 73 | } 74 | 75 | /** 76 | * 翻转时间. 77 | * 78 | * @param currentTimeMillis 79 | * @return 80 | */ 81 | public static long reverseTimeMillis(long currentTimeMillis) { 82 | return Long.MAX_VALUE - currentTimeMillis; 83 | } 84 | 85 | /** 86 | * 时间转字符串. 87 | * 88 | * @param now 89 | * @return 90 | */ 91 | public static String dateToString(LocalDateTime now) { 92 | DateTimeFormatter ftf = DateTimeFormatter.ofPattern(yyyyMMddHHmmss); 93 | return ftf.format(now); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /mmc-lidea-util/src/test/java/com/mmc/lidea/util/TimeUtilTest.java: -------------------------------------------------------------------------------- 1 | package com.mmc.lidea.util; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | 6 | public class TimeUtilTest { 7 | 8 | @Test 9 | public void test() { 10 | 11 | 12 | long time = System.currentTimeMillis(); 13 | 14 | String stime = TimeUtil.timestampToString(time, TimeUtil.yyyyMMddHHmmssSSS); 15 | long ntime = TimeUtil.stringToLong(stime, TimeUtil.yyyyMMddHHmmssSSS); 16 | 17 | System.out.println(StringUtil.format("{} {} {}", time, stime, ntime)); 18 | Assert.assertEquals(time, ntime); 19 | 20 | } 21 | 22 | @Test 23 | public void testEqual() { 24 | 25 | 26 | long timestamp = 1565057370000L; 27 | 28 | String timestr = "2019-08-06 10:09:30"; 29 | 30 | long ntime = TimeUtil.stringToLong(timestr); 31 | Assert.assertEquals(timestamp, ntime); 32 | 33 | String ntimestr = TimeUtil.timestampToString(timestamp); 34 | Assert.assertEquals(timestr, ntimestr); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /mmc-lidea-web/.gitignore: -------------------------------------------------------------------------------- 1 | HELP.md 2 | target/ 3 | !.mvn/wrapper/maven-wrapper.jar 4 | !**/src/main/** 5 | !**/src/test/** 6 | 7 | ### STS ### 8 | .apt_generated 9 | .classpath 10 | .factorypath 11 | .project 12 | .settings 13 | .springBeans 14 | .sts4-cache 15 | 16 | ### IntelliJ IDEA ### 17 | .idea 18 | *.iws 19 | *.iml 20 | *.ipr 21 | 22 | ### NetBeans ### 23 | /nbproject/private/ 24 | /nbbuild/ 25 | /dist/ 26 | /nbdist/ 27 | /.nb-gradle/ 28 | build/ 29 | 30 | ### VS Code ### 31 | .vscode/ 32 | -------------------------------------------------------------------------------- /mmc-lidea-web/.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VIPJoey/Lidea/1fdd0c2bd59a77cbbba7efe0a1255006a9aca0a5/mmc-lidea-web/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /mmc-lidea-web/.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.0/apache-maven-3.6.0-bin.zip 2 | -------------------------------------------------------------------------------- /mmc-lidea-web/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | org.springframework.boot 7 | spring-boot-starter-parent 8 | 2.1.6.RELEASE 9 | 10 | 11 | com.mmc.flink 12 | mmc-lidea-web 13 | 0.0.1-SNAPSHOT 14 | mmc-lidea-web 15 | Demo project for Spring Boot 16 | 17 | 18 | 1.8 19 | 20 | 21 | 22 | 23 | 24 | org.springframework.boot 25 | spring-boot-starter-web 26 | 27 | 28 | 29 | org.springframework.boot 30 | spring-boot-starter-thymeleaf 31 | 32 | 33 | 34 | org.springframework.boot 35 | spring-boot-starter-test 36 | test 37 | 38 | 39 | 40 | org.springframework.data 41 | spring-data-hadoop-hbase 42 | 2.5.0.RELEASE 43 | 44 | 45 | 46 | org.apache.hbase 47 | hbase-client 48 | 1.4.9 49 | 50 | 51 | 52 | org.springframework.data 53 | spring-data-hadoop 54 | 2.5.0.RELEASE 55 | 56 | 57 | org.projectlombok 58 | lombok 59 | 1.16.22 60 | 61 | 62 | com.alibaba 63 | fastjson 64 | 1.2.47 65 | 66 | 67 | 68 | com.mmc 69 | mmc-lidea-common 70 | 1.0.0-SNAPSHOT 71 | 72 | 73 | com.mmc 74 | mmc-lidea-util 75 | 1.0.0-SNAPSHOT 76 | 77 | 78 | org.aspectj 79 | aspectjweaver 80 | 1.8.10 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | org.springframework.boot 98 | spring-boot-maven-plugin 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | dev 107 | 108 | true 109 | 110 | 111 | dev 112 | 113 | 114 | 115 | prd 116 | 117 | prd 118 | 119 | 120 | 121 | 122 | 123 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/MmcLideaApplication.java: -------------------------------------------------------------------------------- 1 | package com.mmc.flink.lidea; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class MmcLideaApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(MmcLideaApplication.class, args); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/aop/ExceptionHander.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.aop; 11 | 12 | import com.mmc.flink.lidea.dto.ResultDTO; 13 | import lombok.extern.slf4j.Slf4j; 14 | import org.aspectj.lang.ProceedingJoinPoint; 15 | import org.aspectj.lang.annotation.Around; 16 | import org.aspectj.lang.annotation.Aspect; 17 | import org.aspectj.lang.annotation.Pointcut; 18 | import org.springframework.stereotype.Component; 19 | 20 | /** 21 | * 统一异常处理. 22 | * 23 | * @author Joey 24 | * @date 2019/3/12 15:50 25 | */ 26 | @Component 27 | @Aspect 28 | @Slf4j 29 | public class ExceptionHander { 30 | 31 | public ExceptionHander() { 32 | log.info("ExceptionHander.ExceptionHander"); 33 | } 34 | 35 | @Pointcut("execution(* com.mmc.flink.lidea.controller.DetailController.*(..))") 36 | public void hander() { 37 | 38 | } 39 | 40 | @Around("hander()") 41 | public Object around(ProceedingJoinPoint pjp) throws Throwable { 42 | Object result; 43 | try { 44 | result = pjp.proceed(); 45 | return result; 46 | 47 | } catch (Throwable t) { 48 | //如果没有统一日志拦截,打印堆栈信息 49 | log.error("an uncaught exception occur, e:", t); 50 | return ResultDTO.handleException(t.getMessage(), null, t); 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/config/HBaseConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.config; 11 | 12 | import org.springframework.beans.factory.annotation.Value; 13 | import org.springframework.context.annotation.Bean; 14 | import org.springframework.context.annotation.Configuration; 15 | import org.springframework.data.hadoop.hbase.HbaseTemplate; 16 | 17 | /** 18 | * @author Joey 19 | * @date 2019/7/26 17:21 20 | */ 21 | @Configuration 22 | public class HBaseConfiguration { 23 | 24 | @Value("${hbase.zookeeper.quorum}") 25 | private String zookeeperQuorum; 26 | 27 | @Value("${hbase.zookeeper.property.clientPort}") 28 | private String clientPort; 29 | 30 | @Value("${zookeeper.znode.parent}") 31 | private String znodeParent; 32 | 33 | @Bean 34 | public HbaseTemplate hbaseTemplate() { 35 | org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration(); 36 | conf.set("hbase.zookeeper.quorum", zookeeperQuorum); 37 | conf.set("hbase.zookeeper.property.clientPort", clientPort); 38 | conf.set("zookeeper.znode.parent", znodeParent); 39 | return new HbaseTemplate(conf); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/controller/DetailController.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.controller; 11 | 12 | import com.mmc.flink.lidea.dto.*; 13 | import com.mmc.flink.lidea.service.LideaLogService; 14 | import lombok.extern.slf4j.Slf4j; 15 | import org.springframework.web.bind.annotation.RequestMapping; 16 | import org.springframework.web.bind.annotation.RestController; 17 | 18 | import javax.annotation.Resource; 19 | 20 | /** 21 | * @author Joey 22 | * @date 2019/7/28 16:11 23 | */ 24 | @RestController 25 | @RequestMapping("/lidea/detail/") 26 | @Slf4j 27 | public class DetailController { 28 | 29 | @Resource 30 | private LideaLogService lidiaLogService; 31 | 32 | @RequestMapping("/apps") 33 | public ResultDTO getApps() { 34 | 35 | LideaAppResp data = lidiaLogService.listApps(); 36 | 37 | return ResultDTO.handleSuccess("SUCESS", data); 38 | 39 | } 40 | 41 | @RequestMapping("/services") 42 | public ResultDTO getServices(LideaServiceReq req) { 43 | 44 | LideaServiceResp data = lidiaLogService.listServices(req); 45 | 46 | return ResultDTO.handleSuccess("SUCESS", data); 47 | 48 | } 49 | 50 | @RequestMapping("/methods") 51 | public ResultDTO methods(LideaMethodReq req) { 52 | 53 | LideaMethodResp data = lidiaLogService.listMethods(req); 54 | 55 | return ResultDTO.handleSuccess("SUCESS", data); 56 | 57 | } 58 | 59 | @RequestMapping("/access") 60 | public ResultDTO getAccess(LideaLogReq req) { 61 | 62 | if (null == req) { 63 | throw new RuntimeException("req can't be null."); 64 | } 65 | 66 | 67 | LideaLogResp data = lidiaLogService.listAccess(req); 68 | 69 | return ResultDTO.handleSuccess("SUCESS", data); 70 | } 71 | 72 | @RequestMapping("/error") 73 | public ResultDTO getError(LideaLogReq req) { 74 | 75 | if (null == req) { 76 | return ResultDTO.handleError("req can't be null.", null); 77 | } 78 | 79 | LideaLogErrorDetailResp data = lidiaLogService.listError(req); 80 | 81 | return ResultDTO.handleSuccess("SUCESS", data); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/controller/LideaController.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.controller; 11 | 12 | import com.alibaba.fastjson.JSON; 13 | import com.mmc.flink.lidea.dto.LideaLogReq; 14 | import com.mmc.flink.lidea.dto.LideaMethodReq; 15 | import com.mmc.flink.lidea.dto.LideaServiceReq; 16 | import org.springframework.stereotype.Controller; 17 | import org.springframework.ui.Model; 18 | import org.springframework.web.bind.annotation.RequestMapping; 19 | import org.springframework.web.servlet.mvc.support.RedirectAttributes; 20 | 21 | /** 22 | * @author Joey 23 | * @date 2019/7/28 15:01 24 | */ 25 | @Controller 26 | @RequestMapping("/lidea") 27 | public class LideaController { 28 | 29 | @RequestMapping({"", "ServerTest", "ServerTest.*", "index", "index.*"}) 30 | public String hello() { 31 | return app(); 32 | } 33 | 34 | @RequestMapping("/app") 35 | public String app() { 36 | 37 | return "/pages/v1/app.html"; 38 | } 39 | 40 | @RequestMapping("/serviceList") 41 | public String serviceList(LideaServiceReq req, Model model) { 42 | 43 | model.addAttribute("appName", req.getAppName()); 44 | 45 | return "/pages/v1/serviceList.html"; 46 | } 47 | 48 | @RequestMapping("/methodList") 49 | public String methodList(LideaMethodReq req, Model model) { 50 | 51 | model.addAttribute("appName", req.getAppName()); 52 | model.addAttribute("serviceName", req.getServiceName()); 53 | 54 | return "/pages/v1/methodList.html"; 55 | } 56 | 57 | @RequestMapping("/detail") 58 | public String detail(LideaLogReq req, RedirectAttributes redirectAttributes) { 59 | 60 | redirectAttributes.addAttribute("appName", req.getAppName()); 61 | redirectAttributes.addAttribute("serviceName", req.getServiceName()); 62 | redirectAttributes.addAttribute("methodName", req.getMethodName()); 63 | 64 | return "redirect:/lidea/redetail"; 65 | } 66 | 67 | @RequestMapping("/redetail") 68 | public String reDetail(LideaLogReq req, Model model) { 69 | 70 | model.addAttribute("appName", req.getAppName()); 71 | model.addAttribute("serviceName", req.getServiceName()); 72 | model.addAttribute("methodName", req.getMethodName()); 73 | 74 | return "/pages/v1/detail.html"; 75 | } 76 | 77 | 78 | @RequestMapping("/errorList") 79 | public String errorList(LideaLogReq req, Model model) { 80 | 81 | model.addAttribute("from", req.getFrom()); 82 | model.addAttribute("to", req.getTo()); 83 | model.addAttribute("size", req.getSize()); 84 | model.addAttribute("req", JSON.toJSONString(req)); 85 | 86 | return "pages/v1/errorList.html"; 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/dao/LideaAppDAO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.dao; 11 | 12 | import com.mmc.lidea.common.bo.LideaAppBO; 13 | import com.mmc.lidea.common.context.Const; 14 | import com.mmc.lidea.common.entry.LideaAppEntry; 15 | import com.mmc.flink.lidea.mapper.LideaAppResultsExtractor; 16 | import org.apache.hadoop.hbase.client.Put; 17 | import org.apache.hadoop.hbase.client.Scan; 18 | import org.springframework.data.hadoop.hbase.HbaseTemplate; 19 | import org.springframework.stereotype.Service; 20 | 21 | import javax.annotation.Resource; 22 | import java.util.List; 23 | 24 | /** 25 | * @author Joey 26 | * @date 2019/8/6 18:29 27 | */ 28 | @Service("lideaAppDAO") 29 | public class LideaAppDAO { 30 | 31 | @Resource 32 | private HbaseTemplate hbaseTemplate; 33 | 34 | public LideaAppBO put(LideaAppBO bo) { 35 | 36 | Put put = LideaAppEntry.of(bo); 37 | 38 | return hbaseTemplate.execute(Const.LIDEA_APP_TABLE, (htable) -> { 39 | 40 | htable.put(put); 41 | return bo; 42 | 43 | }); 44 | 45 | } 46 | 47 | public List scan() { 48 | 49 | Scan scan = new Scan(); 50 | scan.setCaching(30); 51 | scan.addFamily(Const.LIDEA_LOG_FEMILY); 52 | 53 | return hbaseTemplate.find(Const.LIDEA_APP_TABLE, scan, new LideaAppResultsExtractor()); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/dao/LideaLogDAO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.dao; 11 | 12 | import com.alibaba.fastjson.JSON; 13 | import com.mmc.flink.lidea.dto.LideaLogReq; 14 | import com.mmc.flink.lidea.dto.LideaLogResp; 15 | import com.mmc.flink.lidea.mapper.LideaLogResultsExtractor; 16 | import com.mmc.lidea.common.bo.LideaLogBO; 17 | import com.mmc.lidea.common.context.Const; 18 | import com.mmc.lidea.common.entry.LideaLogEntry; 19 | import com.mmc.lidea.util.MD5Util; 20 | import com.mmc.lidea.util.StringUtil; 21 | import lombok.extern.slf4j.Slf4j; 22 | import org.apache.hadoop.hbase.client.Get; 23 | import org.apache.hadoop.hbase.client.Put; 24 | import org.apache.hadoop.hbase.client.Result; 25 | import org.apache.hadoop.hbase.client.Scan; 26 | import org.springframework.data.hadoop.hbase.HbaseTemplate; 27 | import org.springframework.data.hadoop.hbase.ResultsExtractor; 28 | import org.springframework.stereotype.Service; 29 | 30 | import javax.annotation.Resource; 31 | import java.util.Collections; 32 | import java.util.List; 33 | 34 | /** 35 | * @author Joey 36 | * @date 2019/7/26 17:44 37 | */ 38 | @Slf4j 39 | @Service("lideaLogDAO") 40 | public class LideaLogDAO { 41 | 42 | @Resource 43 | private HbaseTemplate hbaseTemplate; 44 | 45 | 46 | public void put(LideaLogBO bo) { 47 | 48 | Put put = LideaLogEntry.of(bo); 49 | 50 | hbaseTemplate.execute(Const.LIDEA_LOG_TABLE, (htable) -> { 51 | 52 | htable.put(put); 53 | return null; 54 | }); 55 | 56 | } 57 | 58 | public LideaLogResp scan(LideaLogReq req) { 59 | 60 | Scan scan = createScan(req); 61 | 62 | ResultsExtractor> extractor = new LideaLogResultsExtractor(); 63 | List list = hbaseTemplate.find(Const.LIDEA_LOG_TABLE, scan, extractor); 64 | 65 | LideaLogResp resp = new LideaLogResp(); 66 | resp.setData(list); 67 | resp.setAppName(req.getAppName()); 68 | resp.setServiceName(req.getServiceName()); 69 | resp.setMethodName(req.getMethodName()); 70 | 71 | return resp; 72 | } 73 | 74 | private Scan createScan(LideaLogReq bo) { 75 | 76 | Scan scan = new Scan(); 77 | scan.setCaching(60 * 24); 78 | 79 | String base = StringUtil.format("{}{}{}", bo.getAppName(), bo.getServiceName(), bo.getMethodName()); 80 | 81 | byte[] traceIndexStartKey = LideaLogEntry.makeRowKey(MD5Util.encrypt(base), bo.getFrom()); 82 | 83 | byte[] traceIndexEndKey = LideaLogEntry.makeRowKey(MD5Util.encrypt(base), bo.getTo()); 84 | 85 | // start key is replaced by end key because key has been reversed 86 | 87 | scan.withStartRow(traceIndexEndKey); 88 | scan.withStopRow(traceIndexStartKey); 89 | 90 | // scan.withStartRow(traceIndexStartKey); 91 | // scan.withStopRow(traceIndexEndKey); 92 | 93 | scan.addFamily(Const.LIDEA_LOG_FEMILY); 94 | scan.setId("LideaLogScan"); 95 | 96 | return scan; 97 | 98 | } 99 | 100 | public LideaLogResp get(LideaLogReq req) { 101 | 102 | log.info("req : {}", JSON.toJSONString(req)); 103 | 104 | LideaLogResp resp = new LideaLogResp(); 105 | resp.setAppName(req.getAppName()); 106 | resp.setServiceName(req.getServiceName()); 107 | resp.setMethodName(req.getMethodName()); 108 | 109 | List data = hbaseTemplate.execute(Const.LIDEA_LOG_TABLE, (table) -> { 110 | 111 | String base = StringUtil.format("{}{}{}", req.getAppName(), req.getServiceName(), req.getMethodName()); 112 | 113 | byte[] rowKey = LideaLogEntry.makeRowKey(MD5Util.encrypt(base), req.getFrom()); 114 | Get get = new Get(rowKey); 115 | 116 | Result result = table.get(get); 117 | 118 | LideaLogBO bo = LideaLogResultsExtractor.map(result); 119 | 120 | return Collections.singletonList(bo); 121 | }); 122 | 123 | resp.setData(data); 124 | 125 | return resp; 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/dao/LideaLogErrorDAO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.dao; 11 | 12 | import com.mmc.lidea.common.bo.LideaLogErrorDetailBO; 13 | import com.mmc.lidea.common.context.Const; 14 | import com.mmc.flink.lidea.mapper.LideaLogResultsExtractor; 15 | import com.mmc.lidea.util.BytesUtils; 16 | import org.apache.hadoop.hbase.client.Get; 17 | import org.apache.hadoop.hbase.client.Result; 18 | import org.springframework.data.hadoop.hbase.HbaseTemplate; 19 | import org.springframework.stereotype.Service; 20 | 21 | import javax.annotation.Resource; 22 | import java.util.ArrayList; 23 | import java.util.Collections; 24 | import java.util.List; 25 | 26 | /** 27 | * @author Joey 28 | * @date 2019/7/26 17:44 29 | */ 30 | @Service("lideaLogErrorDAO") 31 | public class LideaLogErrorDAO { 32 | 33 | @Resource 34 | private HbaseTemplate hbaseTemplate; 35 | 36 | 37 | public List list(List traceIds) { 38 | 39 | 40 | return hbaseTemplate.execute(Const.LIDEA_LOG_DEATIL_TABLE, table -> { 41 | 42 | List bos = new ArrayList<>(); 43 | 44 | if (traceIds.size() > 1) { 45 | 46 | List list = new ArrayList<>(); 47 | 48 | for (String traceId : traceIds) { 49 | 50 | Get get = new Get(BytesUtils.toBytes(traceId)); 51 | 52 | list.add(get); 53 | } 54 | 55 | Result[] rs = table.get(list); 56 | 57 | for (Result r : rs) { 58 | LideaLogErrorDetailBO bo = LideaLogResultsExtractor.mapErrorDetail(r); 59 | bos.add(bo); 60 | } 61 | return bos; 62 | 63 | } else { 64 | 65 | Get get = new Get(BytesUtils.toBytes(traceIds.get(0))); 66 | Result r = table.get(get); 67 | if (r.isEmpty()) { 68 | return null; 69 | } 70 | LideaLogErrorDetailBO bo = LideaLogResultsExtractor.mapErrorDetail(r); 71 | 72 | return Collections.singletonList(bo); 73 | } 74 | }); 75 | 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/dao/LideaMethodDAO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.dao; 11 | 12 | import com.alibaba.fastjson.JSON; 13 | import com.mmc.lidea.common.bo.LideaMethodBO; 14 | import com.mmc.lidea.common.context.Const; 15 | import com.mmc.lidea.common.entry.LideaMethodEntry; 16 | import com.mmc.flink.lidea.dto.LideaMethodReq; 17 | import com.mmc.flink.lidea.dto.LideaMethodResp; 18 | import com.mmc.flink.lidea.mapper.LideaMethodResultsExtractor; 19 | import com.mmc.lidea.util.BytesUtils; 20 | import com.mmc.lidea.util.MD5Util; 21 | import lombok.extern.slf4j.Slf4j; 22 | import org.apache.hadoop.hbase.client.Get; 23 | import org.apache.hadoop.hbase.client.Put; 24 | import org.apache.hadoop.hbase.client.Result; 25 | import org.apache.hadoop.hbase.client.Scan; 26 | import org.apache.hadoop.hbase.filter.PrefixFilter; 27 | import org.springframework.data.hadoop.hbase.HbaseTemplate; 28 | import org.springframework.stereotype.Service; 29 | 30 | import javax.annotation.Resource; 31 | import java.util.Collections; 32 | import java.util.List; 33 | 34 | /** 35 | * @author Joey 36 | * @date 2019/8/6 18:29 37 | */ 38 | @Slf4j 39 | @Service("lideaMethodDAO") 40 | public class LideaMethodDAO { 41 | 42 | @Resource 43 | private HbaseTemplate hbaseTemplate; 44 | 45 | public LideaMethodBO put(LideaMethodBO bo) { 46 | 47 | Put put = LideaMethodEntry.of(bo); 48 | 49 | return hbaseTemplate.execute(Const.LIDEA_METHOD_TABLE, (htable) -> { 50 | 51 | htable.put(put); 52 | return bo; 53 | 54 | }); 55 | 56 | } 57 | 58 | public List scan(LideaMethodReq req) { 59 | 60 | Scan scan = new Scan(); 61 | scan.setCaching(50); 62 | scan.addFamily(Const.LIDEA_LOG_FEMILY); 63 | scan.setFilter(new PrefixFilter(BytesUtils.toBytes(MD5Util.encrypt(req.getAppName() + req.getServiceName())))); 64 | 65 | return hbaseTemplate.find(Const.LIDEA_METHOD_TABLE, scan, new LideaMethodResultsExtractor()); 66 | } 67 | 68 | public LideaMethodResp get(LideaMethodReq req) { 69 | 70 | log.info("req : {}", JSON.toJSONString(req)); 71 | 72 | LideaMethodResp resp = new LideaMethodResp(); 73 | resp.setAppName(req.getAppName()); 74 | resp.setServiceName(req.getServiceName()); 75 | resp.setMethodName(req.getMethodName()); 76 | 77 | List data = hbaseTemplate.execute(Const.LIDEA_METHOD_TABLE, (table) -> { 78 | 79 | LideaMethodBO bo = new LideaMethodBO(); 80 | bo.setAppName(req.getAppName()); 81 | bo.setServiceName(req.getServiceName()); 82 | bo.setMethodName(req.getMethodName()); 83 | byte[] rowKey = LideaMethodEntry.makeRowKey(bo); 84 | 85 | Get get = new Get(rowKey); 86 | 87 | Result result = table.get(get); 88 | 89 | LideaMethodBO lideaMethodBO = LideaMethodEntry.map(result); 90 | 91 | return Collections.singletonList(lideaMethodBO); 92 | }); 93 | 94 | resp.setData(data); 95 | 96 | return resp; 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/dao/LideaServiceDAO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.dao; 11 | 12 | import com.mmc.lidea.common.bo.LideaServiceBO; 13 | import com.mmc.lidea.common.context.Const; 14 | import com.mmc.lidea.common.entry.LideaServiceEntry; 15 | import com.mmc.flink.lidea.dto.LideaServiceReq; 16 | import com.mmc.flink.lidea.mapper.LideaServiceResultsExtractor; 17 | import org.apache.hadoop.hbase.client.Put; 18 | import org.apache.hadoop.hbase.client.Scan; 19 | import org.apache.hadoop.hbase.filter.PrefixFilter; 20 | import org.springframework.data.hadoop.hbase.HbaseTemplate; 21 | import org.springframework.stereotype.Service; 22 | 23 | import javax.annotation.Resource; 24 | import java.util.List; 25 | 26 | /** 27 | * @author Joey 28 | * @date 2019/8/29 14:31 29 | */ 30 | @Service("lideaServiceDAO") 31 | public class LideaServiceDAO { 32 | 33 | @Resource 34 | private HbaseTemplate hbaseTemplate; 35 | 36 | public LideaServiceBO put(LideaServiceBO bo) { 37 | 38 | Put put = LideaServiceEntry.of(bo); 39 | 40 | return hbaseTemplate.execute(Const.LIDEA_SERVICE_TABLE, (htable) -> { 41 | 42 | htable.put(put); 43 | return bo; 44 | 45 | }); 46 | } 47 | 48 | public List scan(LideaServiceReq req) { 49 | 50 | Scan scan = new Scan(); 51 | scan.setCaching(50); 52 | scan.addFamily(Const.LIDEA_LOG_FEMILY); 53 | scan.setFilter(new PrefixFilter(req.getAppName().getBytes())); 54 | 55 | return hbaseTemplate.find(Const.LIDEA_SERVICE_TABLE, scan, new LideaServiceResultsExtractor()); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/dto/LideaAppResp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.dto; 11 | 12 | import com.mmc.lidea.common.bo.LideaAppBO; 13 | import lombok.Data; 14 | 15 | import java.util.List; 16 | 17 | /** 18 | * @author Joey 19 | * @date 2019/7/26 20:47 20 | */ 21 | @Data 22 | public class LideaAppResp { 23 | 24 | 25 | /** 26 | * 数据. 27 | */ 28 | private List data; 29 | 30 | } 31 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/dto/LideaLogErrorDetailResp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.dto; 11 | 12 | import com.mmc.lidea.common.bo.LideaLogErrorDetailBO; 13 | import lombok.Data; 14 | 15 | import java.util.List; 16 | 17 | /** 18 | * @author Joey 19 | * @date 2019/7/26 20:47 20 | */ 21 | @Data 22 | public class LideaLogErrorDetailResp { 23 | 24 | /** 25 | * 系统名称. 26 | */ 27 | private String appName; 28 | /** 29 | * 接口名称. 30 | */ 31 | private String serviceName; 32 | /** 33 | * 方法名称. 34 | */ 35 | private String methodName; 36 | /** 37 | * 数据. 38 | */ 39 | private List data; 40 | 41 | } 42 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/dto/LideaLogReq.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.dto; 11 | 12 | import lombok.Data; 13 | 14 | /** 15 | * @author Joey 16 | * @date 2019/7/26 20:46 17 | */ 18 | @Data 19 | public class LideaLogReq { 20 | 21 | private long from; 22 | private long to; 23 | private int size; 24 | 25 | 26 | /** 27 | * 系统名称. 28 | */ 29 | private String appName; 30 | /** 31 | * 接口名称. 32 | */ 33 | private String serviceName; 34 | /** 35 | * 方法名称. 36 | */ 37 | private String methodName; 38 | 39 | 40 | } 41 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/dto/LideaLogResp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.dto; 11 | 12 | import com.mmc.lidea.common.bo.LideaLogBO; 13 | import lombok.Data; 14 | 15 | import java.util.List; 16 | 17 | /** 18 | * @author Joey 19 | * @date 2019/7/26 20:47 20 | */ 21 | @Data 22 | public class LideaLogResp { 23 | 24 | /** 25 | * 系统名称. 26 | */ 27 | private String appName; 28 | /** 29 | * 接口名称. 30 | */ 31 | private String serviceName; 32 | /** 33 | * 方法名称. 34 | */ 35 | private String methodName; 36 | /** 37 | * 数据. 38 | */ 39 | private List data; 40 | 41 | } 42 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/dto/LideaMethodReq.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.dto; 11 | 12 | import lombok.Data; 13 | 14 | /** 15 | * @author Joey 16 | * @date 2019/8/6 20:41 17 | */ 18 | @Data 19 | public class LideaMethodReq { 20 | 21 | /** 22 | * 系统名称. 23 | */ 24 | private String appName; 25 | /** 26 | * 接口名称. 27 | */ 28 | private String serviceName; 29 | /** 30 | * 方法名称. 31 | */ 32 | private String methodName; 33 | 34 | } 35 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/dto/LideaMethodResp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.dto; 11 | 12 | import com.mmc.lidea.common.bo.LideaMethodBO; 13 | import lombok.Data; 14 | 15 | import java.util.List; 16 | 17 | /** 18 | * @author Joey 19 | * @date 2019/8/6 20:41 20 | */ 21 | @Data 22 | public class LideaMethodResp { 23 | 24 | /** 25 | * 系统名称. 26 | */ 27 | private String appName; 28 | /** 29 | * 接口名称. 30 | */ 31 | private String serviceName; 32 | /** 33 | * 方法名称. 34 | */ 35 | private String methodName; 36 | 37 | /** 38 | * 数据. 39 | */ 40 | private List data; 41 | 42 | } 43 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/dto/LideaServiceReq.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.dto; 11 | 12 | import lombok.Data; 13 | 14 | /** 15 | * @author Joey 16 | * @date 2019/8/29 14:49 17 | */ 18 | @Data 19 | public class LideaServiceReq { 20 | 21 | /** 22 | * 系统名称. 23 | */ 24 | private String appName; 25 | } 26 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/dto/LideaServiceResp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.dto; 11 | 12 | import com.mmc.lidea.common.bo.LideaServiceBO; 13 | import lombok.Data; 14 | 15 | import java.util.List; 16 | 17 | /** 18 | * @author Joey 19 | * @date 2019/8/29 15:27 20 | */ 21 | @Data 22 | public class LideaServiceResp { 23 | 24 | /** 25 | * 系统名称. 26 | */ 27 | private String appName; 28 | 29 | /** 30 | * 数据. 31 | */ 32 | private List data; 33 | } 34 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/dto/ResultDTO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.dto; 11 | 12 | import lombok.Data; 13 | import org.apache.commons.lang.StringUtils; 14 | 15 | /** 16 | * common result. 17 | * 18 | * @author Joey 19 | * @date 2018/6/17 9:30 20 | */ 21 | @Data 22 | public class ResultDTO { 23 | 24 | private static final long serialVersionUID = 1789151585L; 25 | private static final int DEFAULT_EXCEPTION_CODE = -1; // 默认异常码 26 | private static final int DEFAULT_SUCCESS_CODE = 1; // 默认成功吗 27 | private static final int DEFAULT_ERROR_CODE = 0; // 默认错误吗 28 | 29 | private int code; // 响应码 30 | private boolean success; // 执行结果标识 31 | private String msg; // 消息 32 | private String remark; // 备注 33 | private T data; // 附带数据 34 | private Throwable exception; // 异常 35 | 36 | public static ResultDTO handleSuccess(String msg, T data) { 37 | 38 | ResultDTO ret = new ResultDTO<>(); 39 | ret.setCode(DEFAULT_SUCCESS_CODE); 40 | ret.setSuccess(true); 41 | ret.setMsg(msg); 42 | ret.setRemark("success"); 43 | ret.setData(data); 44 | ret.setException(null); 45 | return ret; 46 | 47 | } 48 | public static ResultDTO handleError(String msg, T data) { 49 | ResultDTO ret = new ResultDTO<>(); 50 | ret.setCode(DEFAULT_ERROR_CODE); 51 | ret.setMsg(msg); 52 | ret.setSuccess(false); 53 | ret.setRemark("occur an error"); 54 | ret.setData(data); 55 | ret.setException(null); 56 | return ret; 57 | } 58 | public static ResultDTO handleException(String msg, T data, Throwable e) { 59 | ResultDTO ret = new ResultDTO<>(); 60 | ret.setCode(DEFAULT_EXCEPTION_CODE); 61 | ret.setSuccess(false); 62 | ret.setMsg(null == msg ? e.getMessage() : msg); 63 | ret.setRemark("occur an exception"); 64 | ret.setData(data); 65 | ret.setException(e); 66 | return ret; 67 | 68 | } 69 | 70 | public static ResultDTO createExceptionResult(Throwable e, Class clazz) { 71 | 72 | return createExceptionResult("", e, clazz); 73 | } 74 | 75 | public static ResultDTO createExceptionResult(String msg, Throwable e, Class clazz) { 76 | 77 | ResultDTO ret = new ResultDTO<>(); 78 | ret.setCode(DEFAULT_EXCEPTION_CODE); 79 | ret.setSuccess(false); 80 | ret.setMsg(StringUtils.isEmpty(msg) ? e.getMessage() : msg); 81 | ret.setRemark("occur an exception"); 82 | ret.setData(null); 83 | ret.setException(e); 84 | return ret; 85 | } 86 | 87 | public static ResultDTO createErrorResult(String msg, Class clazz) { 88 | ResultDTO ret = new ResultDTO<>(); 89 | ret.setCode(DEFAULT_ERROR_CODE); 90 | ret.setMsg(msg); 91 | ret.setSuccess(false); 92 | ret.setRemark("occur an error"); 93 | ret.setData(null); 94 | ret.setException(null); 95 | return ret; 96 | } 97 | 98 | public static ResultDTO createSuccessResult(String msg, Class clazz) { 99 | return createSuccessResult(msg, null, clazz); 100 | } 101 | 102 | public static ResultDTO createSuccessResult(String msg, T data, Class clazz) { 103 | ResultDTO ret = new ResultDTO<>(); 104 | ret.setCode(DEFAULT_SUCCESS_CODE); 105 | ret.setSuccess(true); 106 | ret.setMsg(msg); 107 | ret.setRemark("success"); 108 | ret.setData(data); 109 | ret.setException(null); 110 | return ret; 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/mapper/LideaAppResultsExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.mapper; 11 | 12 | import com.mmc.lidea.common.bo.LideaAppBO; 13 | import com.mmc.lidea.common.context.Const; 14 | import com.mmc.lidea.util.BytesUtils; 15 | import org.apache.hadoop.hbase.client.Result; 16 | import org.apache.hadoop.hbase.client.ResultScanner; 17 | import org.springframework.data.hadoop.hbase.HbaseTemplate; 18 | import org.springframework.data.hadoop.hbase.ResultsExtractor; 19 | 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | 23 | /** 24 | * @author Joey 25 | * @date 2019/7/26 21:11 26 | */ 27 | public class LideaAppResultsExtractor implements ResultsExtractor> { 28 | 29 | 30 | /** 31 | * Implementations must implement this method to process the entire {@link ResultScanner}. 32 | * 33 | * @param results {@link ResultScanner} to extract data from. Implementations should not close this; it will be closed 34 | * automatically by the calling {@link HbaseTemplate} 35 | * @return an arbitrary result object, or null if none (the extractor will typically be stateful in the latter case). 36 | * @throws Exception if an Hbase exception is encountered 37 | */ 38 | @Override 39 | public List extractData(ResultScanner results) throws Exception { 40 | 41 | 42 | List rs = new ArrayList<>(); 43 | for (Result result : results) { 44 | LideaAppBO bo = map(result); 45 | rs.add(bo); 46 | } 47 | 48 | return rs; 49 | } 50 | 51 | private LideaAppBO map(Result result) { 52 | 53 | LideaAppBO bo = new LideaAppBO(); 54 | bo.setTime(BytesUtils.toString(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("time")))); 55 | bo.setAppName(BytesUtils.toString(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("appName")))); 56 | return bo; 57 | 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/mapper/LideaLogResultsExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.mapper; 11 | 12 | import com.mmc.lidea.common.bo.LideaLogBO; 13 | import com.mmc.lidea.common.bo.LideaLogErrorDetailBO; 14 | import com.mmc.lidea.common.context.Const; 15 | import com.mmc.lidea.util.BytesUtils; 16 | import org.apache.hadoop.hbase.client.Result; 17 | import org.apache.hadoop.hbase.client.ResultScanner; 18 | import org.apache.hadoop.hbase.util.Bytes; 19 | import org.springframework.data.hadoop.hbase.HbaseTemplate; 20 | import org.springframework.data.hadoop.hbase.ResultsExtractor; 21 | 22 | import java.util.ArrayList; 23 | import java.util.List; 24 | 25 | /** 26 | * @author Joey 27 | * @date 2019/7/26 21:11 28 | */ 29 | public class LideaLogResultsExtractor implements ResultsExtractor> { 30 | 31 | /** 32 | * Implementations must implement this method to process the entire {@link ResultScanner}. 33 | * 34 | * @param results {@link ResultScanner} to extract data from. Implementations should not close this; it will be closed 35 | * automatically by the calling {@link HbaseTemplate} 36 | * @return an arbitrary result object, or null if none (the extractor will typically be stateful in the latter case). 37 | * @throws Exception if an Hbase exception is encountered 38 | */ 39 | @Override 40 | public List extractData(ResultScanner results) throws Exception { 41 | 42 | List rs = new ArrayList<>(); 43 | for (Result result : results) { 44 | LideaLogBO bo = map(result); 45 | rs.add(bo); 46 | } 47 | 48 | return rs; 49 | } 50 | 51 | public static LideaLogBO map(Result result) { 52 | LideaLogBO bo = new LideaLogBO(); 53 | bo.setTime(BytesUtils.toString(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("time")))); 54 | bo.setAppName(BytesUtils.toString(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("appName")))); 55 | bo.setServiceName(BytesUtils.toString(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("serviceName")))); 56 | bo.setMethodName(BytesUtils.toString(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("methodName")))); 57 | bo.setCount(Bytes.toLong(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("count")))); 58 | bo.setAvg(Bytes.toInt(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("avg")))); 59 | bo.setException(Bytes.toInt(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("exception")))); 60 | bo.setTraceIds(Bytes.toString(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("traceIds")))); 61 | return bo; 62 | } 63 | 64 | public static LideaLogErrorDetailBO mapErrorDetail(Result result) { 65 | LideaLogErrorDetailBO bo = new LideaLogErrorDetailBO(); 66 | bo.setTraceId(BytesUtils.toString(result.getRow())); 67 | bo.setTime(BytesUtils.toString(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("time")))); 68 | bo.setLocalIp(BytesUtils.toString(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("localIp")))); 69 | bo.setRemoteIp(BytesUtils.toString(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("remoteIp")))); 70 | bo.setAppName(BytesUtils.toString(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("appName")))); 71 | bo.setServiceName(BytesUtils.toString(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("serviceName")))); 72 | bo.setMethodName(BytesUtils.toString(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("methodName")))); 73 | bo.setArgs(Bytes.toString(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("args")))); 74 | bo.setResponse(BytesUtils.toString(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("response")))); 75 | bo.setCost(Bytes.toInt(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("cost")))); 76 | bo.setMsg(Bytes.toString(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("msg")))); 77 | bo.setCustomMsg(Bytes.toString(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("customMsg")))); 78 | return bo; 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/mapper/LideaMethodResultsExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.mapper; 11 | 12 | import com.mmc.lidea.common.bo.LideaMethodBO; 13 | import com.mmc.lidea.common.entry.LideaMethodEntry; 14 | import org.apache.hadoop.hbase.client.Result; 15 | import org.apache.hadoop.hbase.client.ResultScanner; 16 | import org.springframework.data.hadoop.hbase.HbaseTemplate; 17 | import org.springframework.data.hadoop.hbase.ResultsExtractor; 18 | 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | 22 | /** 23 | * @author Joey 24 | * @date 2019/7/26 21:11 25 | */ 26 | public class LideaMethodResultsExtractor implements ResultsExtractor> { 27 | 28 | 29 | /** 30 | * Implementations must implement this method to process the entire {@link ResultScanner}. 31 | * 32 | * @param results {@link ResultScanner} to extract data from. Implementations should not close this; it will be closed 33 | * automatically by the calling {@link HbaseTemplate} 34 | * @return an arbitrary result object, or null if none (the extractor will typically be stateful in the latter case). 35 | * @throws Exception if an Hbase exception is encountered 36 | */ 37 | @Override 38 | public List extractData(ResultScanner results) throws Exception { 39 | 40 | 41 | List rs = new ArrayList<>(); 42 | for (Result result : results) { 43 | LideaMethodBO bo = map(result); 44 | rs.add(bo); 45 | } 46 | 47 | return rs; 48 | } 49 | 50 | private LideaMethodBO map(Result result) { 51 | 52 | return LideaMethodEntry.map(result); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/mapper/LideaServiceResultsExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.mapper; 11 | 12 | import com.mmc.lidea.common.bo.LideaServiceBO; 13 | import com.mmc.lidea.common.context.Const; 14 | import com.mmc.lidea.util.BytesUtils; 15 | import org.apache.hadoop.hbase.client.Result; 16 | import org.apache.hadoop.hbase.client.ResultScanner; 17 | import org.springframework.data.hadoop.hbase.HbaseTemplate; 18 | import org.springframework.data.hadoop.hbase.ResultsExtractor; 19 | 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | 23 | /** 24 | * @author Joey 25 | * @date 2019/7/26 21:11 26 | */ 27 | public class LideaServiceResultsExtractor implements ResultsExtractor> { 28 | 29 | 30 | /** 31 | * Implementations must implement this method to process the entire {@link ResultScanner}. 32 | * 33 | * @param results {@link ResultScanner} to extract data from. Implementations should not close this; it will be closed 34 | * automatically by the calling {@link HbaseTemplate} 35 | * @return an arbitrary result object, or null if none (the extractor will typically be stateful in the latter case). 36 | * @throws Exception if an Hbase exception is encountered 37 | */ 38 | @Override 39 | public List extractData(ResultScanner results) throws Exception { 40 | 41 | 42 | List rs = new ArrayList<>(); 43 | for (Result result : results) { 44 | LideaServiceBO bo = map(result); 45 | rs.add(bo); 46 | } 47 | 48 | return rs; 49 | } 50 | 51 | private LideaServiceBO map(Result result) { 52 | 53 | LideaServiceBO bo = new LideaServiceBO(); 54 | bo.setAppName(BytesUtils.toString(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("appName")))); 55 | bo.setServiceName(BytesUtils.toString(result.getValue(Const.LIDEA_LOG_FEMILY, BytesUtils.toBytes("serviceName")))); 56 | return bo; 57 | 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/service/LideaLogService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.service; 11 | 12 | import com.mmc.lidea.common.bo.LideaAppBO; 13 | import com.mmc.lidea.common.bo.LideaLogErrorDetailBO; 14 | import com.mmc.lidea.common.bo.LideaMethodBO; 15 | import com.mmc.lidea.common.bo.LideaServiceBO; 16 | import com.mmc.flink.lidea.dao.*; 17 | import com.mmc.flink.lidea.dto.*; 18 | import org.springframework.stereotype.Service; 19 | 20 | import javax.annotation.Resource; 21 | import java.util.Arrays; 22 | import java.util.List; 23 | import java.util.stream.Collectors; 24 | 25 | /** 26 | * @author Joey 27 | * @date 2019/8/5 14:30 28 | */ 29 | @Service("lideaLogService") 30 | public class LideaLogService { 31 | 32 | @Resource 33 | private LideaLogDAO lideaLogDAO; 34 | 35 | @Resource 36 | private LideaLogErrorDAO lideaLogErrorDAO; 37 | 38 | @Resource 39 | private LideaAppDAO lideaAppDAO; 40 | 41 | @Resource 42 | private LideaMethodDAO lideaMethodDAO; 43 | 44 | @Resource 45 | private LideaServiceDAO lideaServiceDAO; 46 | 47 | public LideaLogErrorDetailResp listError(LideaLogReq req) { 48 | 49 | // 获取traceId列表 50 | LideaLogResp traceResp = null; 51 | if (req.getFrom() == req.getTo()) { 52 | 53 | traceResp = lideaLogDAO.get(req); 54 | 55 | } else { 56 | 57 | traceResp = lideaLogDAO.scan(req); 58 | } 59 | 60 | // 获取trace详情 61 | List traceIds = traceResp.getData().stream().flatMap(m -> Arrays.stream(m.getTraceIds().split(","))).collect(Collectors.toList()); 62 | 63 | List datas = lideaLogErrorDAO.list(traceIds); 64 | 65 | LideaLogErrorDetailResp resp = new LideaLogErrorDetailResp(); 66 | resp.setAppName(req.getAppName()); 67 | resp.setServiceName(req.getServiceName()); 68 | resp.setMethodName(req.getMethodName()); 69 | 70 | resp.setData(datas); 71 | 72 | return resp; 73 | } 74 | 75 | public LideaAppResp listApps() { 76 | 77 | LideaAppResp resp = new LideaAppResp(); 78 | List data = lideaAppDAO.scan(); 79 | 80 | resp.setData(data); 81 | return resp; 82 | } 83 | 84 | public LideaServiceResp listServices(LideaServiceReq req) { 85 | 86 | LideaServiceResp resp = new LideaServiceResp(); 87 | 88 | List data = lideaServiceDAO.scan(req); 89 | 90 | resp.setAppName(req.getAppName()); 91 | resp.setData(data); 92 | 93 | return resp; 94 | } 95 | 96 | public LideaMethodResp listMethods(LideaMethodReq req) { 97 | 98 | LideaMethodResp resp = new LideaMethodResp(); 99 | 100 | List data = lideaMethodDAO.scan(req); 101 | 102 | resp.setAppName(req.getAppName()); 103 | resp.setServiceName(req.getServiceName()); 104 | resp.setData(data); 105 | 106 | return resp; 107 | } 108 | 109 | public LideaLogResp listAccess(LideaLogReq req) { 110 | 111 | return lideaLogDAO.scan(req); 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/java/com/mmc/flink/lidea/util/RowKeyUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 NAVER Corp. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.mmc.flink.lidea.util; 18 | 19 | import com.mmc.lidea.util.BytesUtils; 20 | 21 | /** 22 | * @author emeroad 23 | */ 24 | public final class RowKeyUtils { 25 | private RowKeyUtils() { 26 | } 27 | 28 | public static byte[] concatFixedByteAndLong(byte[] fixedBytes, int maxFixedLength, long l) { 29 | if (fixedBytes == null) { 30 | throw new NullPointerException("fixedBytes must not null"); 31 | } 32 | if (fixedBytes.length > maxFixedLength) { 33 | throw new IndexOutOfBoundsException("fixedBytes.length too big. length:" + fixedBytes.length); 34 | } 35 | byte[] rowKey = new byte[maxFixedLength + 8]; 36 | BytesUtils.writeBytes(rowKey, 0, fixedBytes); 37 | BytesUtils.writeLong(l, rowKey, maxFixedLength); 38 | return rowKey; 39 | } 40 | 41 | 42 | 43 | 44 | 45 | } 46 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/resources/application-dev.yml: -------------------------------------------------------------------------------- 1 | server: 2 | port: 10001 3 | spring: 4 | thymeleaf: 5 | # 开发环境禁用页面缓存 6 | cache: false 7 | encoding: utf-8 8 | mode: HTML5 9 | hbase: 10 | zookeeper: 11 | quorum: 10.204.49.54 12 | property: 13 | clientPort: 2181 14 | 15 | zookeeper: 16 | znode: 17 | parent: /hbase 18 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | spring: 2 | profiles: 3 | active: @spring.profiles.active@ 4 | 5 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/resources/static/v1/css/basictable.css: -------------------------------------------------------------------------------- 1 | /* 2 | * jQuery Basic Table 3 | * Author: Jerry Low 4 | */ 5 | 6 | table.bt thead, 7 | table.bt tbody th { 8 | display: none; 9 | } 10 | 11 | table.bt tfoot th, 12 | table.bt tfoot td, 13 | table.bt tbody td { 14 | border: none; 15 | display: block; 16 | vertical-align: top; 17 | } 18 | 19 | table.bt tfoot th:before, 20 | table.bt tfoot td:before, 21 | table.bt tbody td:before { 22 | content: attr(data-th) ": "; 23 | font-weight: bold; 24 | width: 6.5em; 25 | display: inline-block; 26 | } 27 | 28 | table.bt tfoot th.bt-hide, 29 | table.bt tfoot td.bt-hide, 30 | table.bt tbody td.bt-hide { 31 | display: none; 32 | } 33 | 34 | table.bt tfoot th .bt-content, 35 | table.bt tfoot td .bt-content, 36 | table.bt tbody td .bt-content { 37 | vertical-align: top; 38 | } 39 | 40 | .bt-wrapper.active { 41 | max-height: 310px; 42 | overflow: auto; 43 | -webkit-overflow-scrolling: touch; 44 | } 45 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/resources/static/v1/gallery/css/style.css: -------------------------------------------------------------------------------- 1 | body{ 2 | height:0; /*not sure but codepen has some kind of height*/ 3 | } 4 | .container .img-holder { 5 | width:250px; 6 | height:180px; 7 | position:absolute; 8 | background-color:#fff; 9 | box-shadow:0px 1px 6px 3px #ccc; 10 | visiblity:hidden; 11 | opacity:0; 12 | top:50px; 13 | left:395px; 14 | cursor:pointer; 15 | z-index:99; 16 | } 17 | 18 | .container .img-holder img {margin-top:5px;} 19 | 20 | .container{ 21 | position:relative; 22 | height:300px; 23 | width:1040px; 24 | left:20px; 25 | top:100px; 26 | border:2px solid #ccc; 27 | text-indent:4px; 28 | font-weight:bold; 29 | } 30 | 31 | .controller{ 32 | top:120px; 33 | left:20px; 34 | width:300px; 35 | margin:0 auto; 36 | position:relative; 37 | } 38 | 39 | .scrolller-container{ 40 | width:800px; 41 | height:15px; 42 | border:1px solid #ccc; 43 | border-radius:10px; 44 | position:relative; 45 | top:150px; 46 | left:130px; 47 | cursor:pointer; 48 | } 49 | .scrolller-container .scroller{ 50 | width:100px; 51 | height:15px; 52 | background-color:#ddd; 53 | border-radius:10px; 54 | position:absolute; 55 | left:0px; 56 | } 57 | 58 | /* For reflection */ 59 | .container .img-ref{ 60 | position:relative; 61 | left:0; 62 | z-index:-1; 63 | } 64 | .container .img-ref img { 65 | -webkit-transform: scaleY(-1); 66 | -moz-transform: scaleY(-1); 67 | -ms-transform: scaleY(-1); 68 | -o-transform: scaleY(-1); 69 | transform: scaleY(-1); 70 | } 71 | .container .img-ref:after { 72 | content: ""; 73 | width: 100%; 74 | height: 300px; 75 | position: absolute; 76 | bottom: 0; 77 | left: 0; 78 | background-color: #000000; 79 | background: -webkit-gradient(linear, left top, left bottom, color-stop(38%, rgba(255, 255, 255, 0)), color-stop(40% rgba(255, 255, 255, 0.67)), color-stop(65% #fff)); 80 | background: -webkit-linear-gradient(top, rgba(255, 255, 255, 0) 38%, rgba(255, 255, 255, 0.67) 40%, #fff 65%); 81 | background: -moz-linear-gradient(top, rgba(255, 255, 255, 0) 38%, rgba(255, 255, 255, 0.67) 40%, #fff 65%); 82 | background: -ms-linear-gradient(top, rgba(255, 255, 255, 0) 38%, rgba(255, 255, 255, 0.67) 40%, #fff 65%); 83 | background: -o-linear-gradient(top, rgba(255, 255, 255, 0) 38%, rgba(255, 255, 255, 0.67) 40%, #fff 65%); 84 | background: linear-gradient(top, rgba(255, 255, 255, 0) 38%, rgba(0, 0, 0, 0.67) 40%, #fff 65%); 85 | } -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/resources/static/v1/img/11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VIPJoey/Lidea/1fdd0c2bd59a77cbbba7efe0a1255006a9aca0a5/mmc-lidea-web/src/main/resources/static/v1/img/11.png -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/resources/static/v1/js/Nora.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Nora工具包.
3 | * 请放在jq后加载. 4 | */ 5 | 6 | var Nora = window.Nora || { 7 | /** 8 | * 返回整响应结果. 9 | * @param url 10 | * @param prams 11 | * @param successFun 12 | * @param failFun 13 | */ 14 | "Ajax": function(url, prams, successFun, failFun) { 15 | $.ajax({ 16 | type : "POST", 17 | url : url, 18 | data: prams, 19 | success : function(result){ 20 | if (typeof(successFun) == "function") { 21 | successFun(result); 22 | } 23 | }, 24 | error: function(xhr, msg, e) { 25 | if (typeof(failFun) == "function") { 26 | failFun(xhr, msg, e); 27 | } else { 28 | alert("Nora.request函数调用出错:" + msg + " " + e); 29 | } 30 | } 31 | }); 32 | }, 33 | /** 34 | * 只返回数据部分. 35 | * @param url 36 | * @param prams 37 | * @param successFun 38 | * @param failFun 39 | */ 40 | "Request": function(url, prams, successFun, failFun) { 41 | $.ajax({ 42 | type : "POST", 43 | url : url, 44 | data: prams, 45 | success : function(result){ 46 | if (typeof(successFun) == "function") { 47 | successFun(result.data); 48 | } 49 | }, 50 | error: function(xhr, msg, e) { 51 | if (typeof(failFun) == "function") { 52 | failFun(xhr, msg, e); 53 | } else { 54 | alert("Nora.request函数调用出错:" + msg + " " + e); 55 | } 56 | } 57 | }); 58 | }, 59 | "Util": { 60 | "StringUtil": { 61 | "format": function(source, args) { 62 | var result = source; 63 | if (arguments.length > 1) { 64 | for (var i = 1; i < arguments.length; i++) { 65 | if (arguments[i] != undefined) { 66 | result = result.replace("{}", arguments[i]); 67 | } 68 | } 69 | } 70 | return result; 71 | } 72 | }, 73 | "DateUtil": { 74 | "format": function(source, pattern) { 75 | var o = { 76 | "M+": source.getMonth() + 1, //月份 77 | "d+": source.getDate(), //日 78 | "h+": source.getHours(), //小时 79 | "m+": source.getMinutes(), //分 80 | "s+": source.getSeconds(), //秒 81 | "q+": Math.floor((source.getMonth() + 3) / 3), //季度 82 | "S": source.getMilliseconds() //毫秒 83 | }; 84 | if (/(y+)/.test(pattern)) pattern = pattern.replace(RegExp.$1, (source.getFullYear() + "").substr(4 - RegExp.$1.length)); 85 | for (var k in o) 86 | if (new RegExp("(" + k + ")").test(pattern)) 87 | pattern = pattern.replace(RegExp.$1, (RegExp.$1.length == 1) ? (o[k]) : (("00" + o[k]).substr(("" + o[k]).length))); 88 | return pattern; 89 | } 90 | }, 91 | "TemplateUtil": { 92 | "formate": function(dta, tmpl) { 93 | var format = { 94 | name: function(x) { 95 | return x 96 | } 97 | }; 98 | return tmpl.replace(/{(\w+)}/g, function(m1, m2) { 99 | if (!m2) 100 | return ""; 101 | return (format && format[m2]) ? format[m2](dta[m2]) : dta[m2]; 102 | }); 103 | } 104 | } 105 | } 106 | }; 107 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/resources/static/v1/js/app.js: -------------------------------------------------------------------------------- 1 | 2 | function bindDataToGallery(resp) { 3 | 4 | for (let i = 0; i < resp.data.length; i++) { 5 | 6 | let d = resp.data[i]; 7 | 8 | let tpl = $("#divTpl").html(); 9 | let html = Nora.Util.StringUtil.format(tpl, d.appName, d.appName); 10 | 11 | $("#divContainer").append(html); 12 | } 13 | // 防止控件bug抖动,加样本填充 14 | for (let i = resp.data.length; i < 2; i++) { 15 | 16 | let tpl = $("#divTpl").html(); 17 | let html = Nora.Util.StringUtil.format(tpl, "demo" + i, "demo" + i); 18 | 19 | $("#divContainer").append(html); 20 | } 21 | 22 | let mid = resp.data.length / 2; 23 | 24 | initContainer(); 25 | setJump(); 26 | setDrag(); 27 | bindEvent(); 28 | 29 | showImgAt(mid); 30 | 31 | } 32 | 33 | function openServicePage(e) { 34 | 35 | let appName = $(e).attr("data-name"); 36 | $("#txtAppName").val(appName); 37 | 38 | window.location.href = "/lidea/serviceList?appName=" + appName; 39 | //$("#frmMethodList").submit(); 40 | } 41 | 42 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/resources/static/v1/js/errorList.js: -------------------------------------------------------------------------------- 1 | 2 | 3 | function bindDataToGrid(resp) { 4 | 5 | let tpl = $("#tplRow").html(); 6 | 7 | for (let i = 0; i < resp.data.length; i++) { 8 | let d = resp.data[i]; 9 | let html = Nora.Util.StringUtil.format(tpl, 10 | d.time, 11 | d.serviceName, 12 | d.methodName, 13 | wrap(d.traceId), 14 | d.localIp, 15 | d.remoteIp, 16 | d.cost, 17 | d.args, 18 | d.response, 19 | d.msg, 20 | d.customMsg 21 | 22 | ); 23 | $("#tbList tbody").append("" + html + ""); 24 | } 25 | 26 | $("#tbList").basictable(); 27 | } 28 | 29 | function wrap(src) { 30 | 31 | return (null == src) ? "" : src; 32 | } -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/resources/static/v1/js/jquery.basictable.min.js: -------------------------------------------------------------------------------- 1 | /* 2 | * jQuery Basic Table 3 | * Author: Jerry Low 4 | */(function(e){e.fn.basictable=function(t){var n=function(t,n){var i=[];n.tableWrap&&t.wrap('

');var s="";t.find("thead tr th").length?s="thead th":t.find("tbody tr th").length?s="tbody tr th":t.find("th").length?s="tr:first th":s="tr:first td",e.each(t.find(s),function(){var t=e(this),n=parseInt(t.attr("colspan"),10)||1,r=t.closest("tr").index();i[r]||(i[r]=[]);for(var s=0;s')}else t.addClass("bt-hide")})},i=function(t){e.each(t.find("td"),function(){var t=e(this),n=t.children(".bt-content").html();t.html(n)})},s=function(t,n){n.forceResponsive?e(window).width()<=n.breakpoint?o(t,n):u(t,n):t.removeClass("bt").outerWidth()>t.parent().width()?o(t,n):u(t,n)},o=function(e,t){e.addClass("bt"),t.tableWrap&&e.parent(".bt-wrapper").addClass("active")},u=function(e,t){e.removeClass("bt"),t.tableWrap&&e.parent(".bt-wrapper").removeClass("active")},a=function(e,t){e.find("td").removeAttr("data-th"),t.tableWrap&&e.unwrap(),t.contentWrap&&i(e),e.removeData("basictable")},f=function(e){e.data("basictable")&&s(e,e.data("basictable"))};this.each(function(){var r=e(this);if(r.length===0||r.data("basictable"))return r.data("basictable")&&(t=="destroy"?a(r,r.data("basictable")):t==="start"?o(r,r.data("basictable")):t==="stop"?u(r,r.data("basictable")):s(r,r.data("basictable"))),!1;var i=e.extend({},e.fn.basictable.defaults,t),l={breakpoint:i.breakpoint,contentWrap:i.contentWrap,forceResponsive:i.forceResponsive,noResize:i.noResize,tableWrap:i.tableWrap,showEmptyCells:i.showEmptyCells};r.data("basictable",l),n(r,r.data("basictable")),l.noResize||(s(r,r.data("basictable")),e(window).bind("resize.basictable",function(){f(r)}))})},e.fn.basictable.defaults={breakpoint:568,contentWrap:!0,forceResponsive:!0,noResize:!1,tableWrap:!1,showEmptyCells:!1}})(jQuery); -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/resources/static/v1/js/methodList.js: -------------------------------------------------------------------------------- 1 | function bindDataToMethods(resp) { 2 | 3 | let tpl = $("#tplRow").html(); 4 | 5 | for (let i = 0; i < resp.data.length; i++) { 6 | let d = resp.data[i]; 7 | let html = Nora.Util.StringUtil.format(tpl, 8 | d.time, 9 | d.serviceName, 10 | Nora.Util.StringUtil.format("{}", d.appName, d.serviceName, d.methodName, d.methodName), 11 | d.count 12 | ); 13 | $("#divContainer").append(html); 14 | 15 | } 16 | 17 | $("#tbList").basictable(); 18 | 19 | } 20 | 21 | function showDetail(e) { 22 | 23 | let appName = $(e).attr("data-appName"); 24 | let serviceName = $(e).attr("data-service"); 25 | let methodName = $(e).attr("data-method"); 26 | 27 | $("#txtAppName").val(appName); 28 | $("#txtServiceName").val(serviceName); 29 | $("#txtMethodName").val(methodName); 30 | $("#frmDetail").submit(); 31 | 32 | } 33 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/resources/static/v1/js/serviceList.js: -------------------------------------------------------------------------------- 1 | 2 | function bindDataToServices(resp) { 3 | 4 | let tpl = $("#tplRow").html(); 5 | 6 | for (let i = 0; i < resp.data.length; i++) { 7 | let d = resp.data[i]; 8 | let html = Nora.Util.StringUtil.format(tpl, 9 | d.appName, 10 | d.serviceName, 11 | Nora.Util.StringUtil.format("查看", d.appName, d.serviceName) 12 | ); 13 | $("#divContainer").append(html); 14 | 15 | } 16 | 17 | $("#tbList").basictable(); 18 | } 19 | 20 | function openMethodPage(sender) { 21 | 22 | let appName = $(sender).attr("data-appName"); 23 | let serviceName = $(sender).attr("data-serviceName"); 24 | 25 | window.location.href = "/lidea/methodList?appName=" + appName + "&serviceName=" + serviceName; 26 | //$("#frmMethodList").submit(); 27 | } -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/resources/templates/pages/v1/app.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Lidea 6 | 7 | 8 | 9 | 10 | 11 |
12 | 13 | 14 |
15 | 16 |
17 | 18 | 19 |
20 | 21 | 22 |
23 | Go to: 26 |
27 | 28 |
29 |
30 |
31 | 32 | 33 |
34 | 35 |
36 | 37 | 38 | 48 | 49 |
50 |
51 | 52 |
53 |
54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 88 | 89 | 90 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/resources/templates/pages/v1/detail.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Lidea 6 | 7 | 8 | 9 | 10 | 11 |
12 | 13 |
14 | 15 |
16 | 17 |
18 |
19 | 20 | 21 | 22 | 23 | 24 | 25 |
26 |
27 | 28 | 29 | 30 | 31 | 32 | 33 | 76 | 77 | 78 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/resources/templates/pages/v1/errorList.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Lidea 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 |
TimeServiceNameMethodTraceIdLocalRemoteCostArgRespMsgCustom
43 |
44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 |
{}{}{}{}{}{}{}{}{}{}{}
59 |
60 | 61 |
62 | 63 | 64 | 65 | 66 | 109 | 110 | 111 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/resources/templates/pages/v1/methodList.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Lidea 6 | 7 | 8 | 9 | 10 | 13 | 14 | 15 | 16 | 17 |
18 | 19 |
20 | 21 | 22 | 23 | 24 | 25 |
26 | 27 |
28 | 49 | 50 |
51 |
52 | 53 | 54 | 55 |
56 |
57 | 58 | 59 | 60 | 61 | 62 | 93 | 94 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/main/resources/templates/pages/v1/serviceList.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Lidea 6 | 7 | 8 | 9 | 10 | 13 | 14 | 15 | 16 | 17 |
18 | 19 |
20 | 21 | 22 | 23 | 24 | 25 |
26 | 27 |
28 | 47 | 48 |
49 |
50 | 51 | 52 | 53 |
54 |
55 | 56 | 57 | 58 | 59 | 60 | 89 | 90 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/test/java/com/mmc/flink/lidea/MmcLideaApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.mmc.flink.lidea; 2 | 3 | import org.junit.Test; 4 | import org.junit.runner.RunWith; 5 | import org.springframework.boot.test.context.SpringBootTest; 6 | import org.springframework.test.context.junit4.SpringRunner; 7 | 8 | @RunWith(SpringRunner.class) 9 | @SpringBootTest 10 | public class MmcLideaApplicationTests { 11 | 12 | @Test 13 | public void contextLoads() { 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/test/java/com/mmc/flink/lidea/dao/DemoTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.dao; 11 | 12 | import com.alibaba.fastjson.JSON; 13 | import com.mmc.lidea.common.bo.LideaAppBO; 14 | import com.mmc.lidea.common.bo.LideaLogBO; 15 | import com.mmc.lidea.common.bo.LideaMethodBO; 16 | import com.mmc.lidea.common.bo.LideaServiceBO; 17 | import com.mmc.lidea.util.RandomUtil; 18 | import com.mmc.lidea.util.TimeUtil; 19 | import org.junit.FixMethodOrder; 20 | import org.junit.Test; 21 | import org.junit.runner.RunWith; 22 | import org.junit.runners.MethodSorters; 23 | import org.springframework.boot.test.context.SpringBootTest; 24 | import org.springframework.test.context.junit4.SpringRunner; 25 | 26 | import javax.annotation.Resource; 27 | import java.time.LocalDateTime; 28 | import java.util.UUID; 29 | 30 | /** 31 | * @author Joey 32 | * @date 2019/8/30 15:41 33 | */ 34 | @RunWith(SpringRunner.class) 35 | @SpringBootTest 36 | @FixMethodOrder(MethodSorters.NAME_ASCENDING) 37 | public class DemoTest { 38 | 39 | private final String APP_NAME = "demo"; 40 | private final String SERVICE_NAME = "com.demo.api.CabinetServiceFacade"; 41 | private final String METHOD_NAME = "getCabinetInfo"; 42 | 43 | @Resource 44 | private LideaAppDAO lideaAppDAO; 45 | @Resource 46 | private LideaServiceDAO lideaServiceDAO; 47 | @Resource 48 | private LideaMethodDAO lideaMethodDAO; 49 | @Resource 50 | private LideaLogDAO lideaLogDAO; 51 | 52 | @Test 53 | public void testAll() { 54 | 55 | testPutApp(APP_NAME); 56 | testPutApp("just_test"); 57 | testPutService(); 58 | testPutMethod(); 59 | testPutLog(); 60 | } 61 | 62 | private void testPutApp(String appName) { 63 | 64 | LideaAppBO bo = new LideaAppBO(); 65 | bo.appName = appName; 66 | bo.time = TimeUtil.dateToString(LocalDateTime.now()); 67 | 68 | bo = lideaAppDAO.put(bo); 69 | 70 | System.out.println("=========================================================================================="); 71 | System.out.println(JSON.toJSONString(bo)); 72 | 73 | } 74 | 75 | private void testPutService() { 76 | 77 | LideaServiceBO bo = new LideaServiceBO(); 78 | bo.appName = APP_NAME; 79 | bo.serviceName = SERVICE_NAME; 80 | 81 | bo = lideaServiceDAO.put(bo); 82 | 83 | System.out.println("=========================================================================================="); 84 | System.out.println(JSON.toJSONString(bo)); 85 | 86 | } 87 | 88 | private void testPutMethod() { 89 | 90 | LideaMethodBO bo = new LideaMethodBO(); 91 | bo.setTime(TimeUtil.dateToString(LocalDateTime.now())); 92 | bo.setAppName(APP_NAME); 93 | bo.setServiceName(SERVICE_NAME); 94 | bo.setMethodName(METHOD_NAME); 95 | bo.setCount("100"); 96 | 97 | bo = lideaMethodDAO.put(bo); 98 | 99 | System.out.println("=========================================================================================="); 100 | System.out.println(JSON.toJSONString(bo)); 101 | } 102 | 103 | private void testPutLog() { 104 | 105 | // 测试插入一条记录 106 | LideaLogBO bo = new LideaLogBO(); 107 | bo.setTime(TimeUtil.dateToString(LocalDateTime.now())); 108 | bo.setAppName(APP_NAME); 109 | bo.setServiceName(SERVICE_NAME); 110 | bo.setMethodName(METHOD_NAME); 111 | bo.setCount(RandomUtil.getRandomNumberInRange(1, 100)); 112 | bo.setAvg(RandomUtil.getRandomNumberInRange(100, 999)); 113 | bo.setException(RandomUtil.getRandomNumberInRange(1, 100)); 114 | bo.setTraceIds(UUID.randomUUID().toString().replace("-", "")); 115 | 116 | lideaLogDAO.put(bo); 117 | 118 | System.out.println("=========================================================================================="); 119 | System.out.println(JSON.toJSONString(bo)); 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/test/java/com/mmc/flink/lidea/dao/LideaAppDAOTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.dao; 11 | 12 | import com.alibaba.fastjson.JSON; 13 | import com.mmc.lidea.common.bo.LideaAppBO; 14 | import com.mmc.lidea.util.TimeUtil; 15 | import org.junit.FixMethodOrder; 16 | import org.junit.Test; 17 | import org.junit.runner.RunWith; 18 | import org.junit.runners.MethodSorters; 19 | import org.springframework.boot.test.context.SpringBootTest; 20 | import org.springframework.test.context.junit4.SpringRunner; 21 | 22 | import javax.annotation.Resource; 23 | import java.time.LocalDateTime; 24 | import java.util.List; 25 | 26 | /** 27 | * @author Joey 28 | * @date 2019/8/6 18:30 29 | */ 30 | @RunWith(SpringRunner.class) 31 | @SpringBootTest 32 | @FixMethodOrder(MethodSorters.NAME_ASCENDING) 33 | public class LideaAppDAOTest { 34 | 35 | 36 | @Resource 37 | private LideaAppDAO lideaAppDAO; 38 | 39 | @Test 40 | public void testPut() { 41 | 42 | LideaAppBO bo = new LideaAppBO(); 43 | bo.appName = "demo"; 44 | bo.time = TimeUtil.dateToString(LocalDateTime.now()); 45 | 46 | bo = lideaAppDAO.put(bo); 47 | 48 | System.out.println("=========================================================================================="); 49 | System.out.println(JSON.toJSONString(bo)); 50 | 51 | } 52 | 53 | @Test 54 | public void testScan() { 55 | 56 | 57 | List list = lideaAppDAO.scan(); 58 | 59 | System.out.println("=========================================================================================="); 60 | System.out.println(JSON.toJSONString(list)); 61 | } 62 | 63 | } -------------------------------------------------------------------------------- /mmc-lidea-web/src/test/java/com/mmc/flink/lidea/dao/LideaLogDAOBatchTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.dao; 11 | 12 | import com.alibaba.fastjson.JSON; 13 | import com.mmc.flink.lidea.dto.LideaLogReq; 14 | import com.mmc.flink.lidea.dto.LideaLogResp; 15 | import com.mmc.lidea.common.bo.LideaLogBO; 16 | import com.mmc.lidea.util.RandomUtil; 17 | import com.mmc.lidea.util.StringUtil; 18 | import com.mmc.lidea.util.TimeUtil; 19 | import org.junit.FixMethodOrder; 20 | import org.junit.Test; 21 | import org.junit.runner.RunWith; 22 | import org.junit.runners.MethodSorters; 23 | import org.springframework.boot.test.context.SpringBootTest; 24 | import org.springframework.test.context.junit4.SpringRunner; 25 | 26 | import javax.annotation.Resource; 27 | import java.time.LocalDateTime; 28 | import java.util.UUID; 29 | 30 | /** 31 | * @author Joey 32 | * @date 2019/7/26 17:39 33 | */ 34 | @RunWith(SpringRunner.class) 35 | @SpringBootTest 36 | @FixMethodOrder(MethodSorters.NAME_ASCENDING) 37 | public class LideaLogDAOBatchTest { 38 | 39 | @Resource 40 | private LideaLogDAO lideaLogDAO; 41 | 42 | @Test 43 | public void testPut1() throws InterruptedException { 44 | 45 | // 测试插入5条记录 46 | for (int i = 0; i < 5; i++) { 47 | 48 | LideaLogBO bo = new LideaLogBO(); 49 | String time = TimeUtil.dateToString(LocalDateTime.now().minusDays(RandomUtil.getRandomNumberInRange(0, 2))); 50 | System.out.println("time: " + time); 51 | bo.setTime(time); 52 | bo.setAppName("demo"); 53 | bo.setServiceName("com.demo.api.CabinetServiceFacade"); 54 | bo.setMethodName("getCabinetInfo"); 55 | bo.setCount(RandomUtil.getRandomNumberInRange(1, 100)); 56 | bo.setAvg(RandomUtil.getRandomNumberInRange(100, 999)); 57 | bo.setException(RandomUtil.getRandomNumberInRange(1, 100)); 58 | bo.setTraceIds(UUID.randomUUID().toString().replace("-", "")); 59 | 60 | lideaLogDAO.put(bo); 61 | 62 | System.out.println("=========================================================================================="); 63 | System.out.println(JSON.toJSONString(bo)); 64 | 65 | Thread.sleep(1000 * RandomUtil.getRandomNumberInRange(0, 10)); 66 | } 67 | } 68 | 69 | @Test 70 | public void testScan1() { 71 | 72 | LideaLogReq req = new LideaLogReq(); 73 | 74 | req.setAppName("demo"); 75 | req.setServiceName("com.demo.api.CabinetServiceFacade"); 76 | req.setMethodName("getCabinetInfo"); 77 | 78 | req.setFrom(TimeUtil.stringToLong("2019-09-20 00:00:00")); 79 | req.setTo(TimeUtil.stringToLong("2019-09-24 23:00:00")); 80 | 81 | LideaLogResp resp = lideaLogDAO.scan(req); 82 | System.out.println("=========================================================================================="); 83 | System.out.println("size: " + resp.getData().size()); 84 | resp.getData().forEach(d -> { 85 | System.out.println(StringUtil.format("{}|{}|{}|{}", d.time, d.appName, d.serviceName, d.methodName)); 86 | }); 87 | } 88 | 89 | @Test 90 | public void testGet1() { 91 | 92 | LideaLogReq req = new LideaLogReq(); 93 | 94 | req.setAppName("demo"); 95 | req.setServiceName("com.demo.api.CabinetServiceFacade"); 96 | req.setMethodName("getCabinetInfo"); 97 | 98 | System.out.println(TimeUtil.timestampToString(1565266170000L, TimeUtil.yyyyMMddHHmmssSSS)); 99 | 100 | req.setFrom(1565266170000L); 101 | req.setTo(1565266170000L); 102 | 103 | LideaLogResp resp = lideaLogDAO.get(req); 104 | System.out.println("=========================================================================================="); 105 | System.out.println(JSON.toJSONString(resp)); 106 | 107 | } 108 | 109 | } 110 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/test/java/com/mmc/flink/lidea/dao/LideaLogDAOTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.dao; 11 | 12 | import com.alibaba.fastjson.JSON; 13 | import com.mmc.lidea.common.bo.LideaLogBO; 14 | import com.mmc.flink.lidea.dto.LideaLogReq; 15 | import com.mmc.flink.lidea.dto.LideaLogResp; 16 | import com.mmc.lidea.util.RandomUtil; 17 | import com.mmc.lidea.util.TimeUtil; 18 | import org.junit.FixMethodOrder; 19 | import org.junit.Test; 20 | import org.junit.runner.RunWith; 21 | import org.junit.runners.MethodSorters; 22 | import org.springframework.boot.test.context.SpringBootTest; 23 | import org.springframework.test.context.junit4.SpringRunner; 24 | 25 | import javax.annotation.Resource; 26 | import java.time.LocalDateTime; 27 | import java.util.UUID; 28 | 29 | /** 30 | * @author Joey 31 | * @date 2019/7/26 17:39 32 | */ 33 | @RunWith(SpringRunner.class) 34 | @SpringBootTest 35 | @FixMethodOrder(MethodSorters.NAME_ASCENDING) 36 | public class LideaLogDAOTest { 37 | 38 | @Resource 39 | private LideaLogDAO lideaLogDAO; 40 | 41 | @Test 42 | public void testPut() { 43 | 44 | // 测试插入一条记录 45 | LideaLogBO bo = new LideaLogBO(); 46 | bo.setTime(TimeUtil.dateToString(LocalDateTime.now())); 47 | bo.setAppName("demo"); 48 | bo.setServiceName("com.demo.api.CabinetServiceFacade"); 49 | bo.setMethodName("getCabinetInfo"); 50 | bo.setCount(RandomUtil.getRandomNumberInRange(1, 100)); 51 | bo.setAvg(RandomUtil.getRandomNumberInRange(100, 999)); 52 | bo.setException(RandomUtil.getRandomNumberInRange(1, 100)); 53 | bo.setTraceIds(UUID.randomUUID().toString().replace("-", "")); 54 | 55 | lideaLogDAO.put(bo); 56 | 57 | System.out.println("=========================================================================================="); 58 | System.out.println(JSON.toJSONString(bo)); 59 | } 60 | 61 | @Test 62 | public void testScan() { 63 | 64 | LideaLogReq req = new LideaLogReq(); 65 | 66 | req.setAppName("demo"); 67 | req.setServiceName("com.demo.api.CabinetServiceFacade"); 68 | req.setMethodName("getCabinetInfo"); 69 | 70 | req.setFrom(TimeUtil.stringToLong("2019-07-20 00:00:00")); 71 | req.setTo(TimeUtil.stringToLong("2019-08-29 23:00:00")); 72 | 73 | LideaLogResp resp = lideaLogDAO.scan(req); 74 | System.out.println("=========================================================================================="); 75 | System.out.println(JSON.toJSONString(resp)); 76 | } 77 | 78 | @Test 79 | public void testGet() { 80 | 81 | LideaLogReq req = new LideaLogReq(); 82 | 83 | req.setAppName("demo"); 84 | req.setServiceName("com.demo.api.CabinetServiceFacade"); 85 | req.setMethodName("getCabinetInfo"); 86 | 87 | req.setFrom(TimeUtil.stringToLong("2019-08-05 14:26:39")); 88 | req.setTo(TimeUtil.stringToLong("2019-08-05 14:26:39")); 89 | 90 | LideaLogResp resp = lideaLogDAO.get(req); 91 | System.out.println("=========================================================================================="); 92 | System.out.println(JSON.toJSONString(resp)); 93 | 94 | } 95 | 96 | @Test 97 | public void testGet1() { 98 | 99 | LideaLogReq req = new LideaLogReq(); 100 | 101 | req.setAppName("demo"); 102 | req.setServiceName("com.demo.api.CabinetServiceFacade"); 103 | req.setMethodName("getCabinetInfo"); 104 | 105 | System.out.println(TimeUtil.timestampToString(1565266170000L, TimeUtil.yyyyMMddHHmmssSSS)); 106 | 107 | req.setFrom(1565266170000L); 108 | req.setTo(1565266170000L); 109 | 110 | LideaLogResp resp = lideaLogDAO.get(req); 111 | System.out.println("=========================================================================================="); 112 | System.out.println(JSON.toJSONString(resp)); 113 | 114 | } 115 | 116 | } 117 | -------------------------------------------------------------------------------- /mmc-lidea-web/src/test/java/com/mmc/flink/lidea/dao/LideaLogErrorDAOTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.dao; 11 | 12 | import com.alibaba.fastjson.JSON; 13 | import com.mmc.lidea.common.bo.LideaLogErrorDetailBO; 14 | import org.junit.FixMethodOrder; 15 | import org.junit.Test; 16 | import org.junit.runner.RunWith; 17 | import org.junit.runners.MethodSorters; 18 | import org.springframework.boot.test.context.SpringBootTest; 19 | import org.springframework.test.context.junit4.SpringRunner; 20 | 21 | import javax.annotation.Resource; 22 | 23 | import java.util.ArrayList; 24 | import java.util.List; 25 | 26 | /** 27 | * @author Joey 28 | * @date 2019/8/5 16:31 29 | */ 30 | @RunWith(SpringRunner.class) 31 | @SpringBootTest 32 | @FixMethodOrder(MethodSorters.NAME_ASCENDING) 33 | public class LideaLogErrorDAOTest { 34 | 35 | @Resource 36 | private 37 | LideaLogErrorDAO lideaLogErrorDAO; 38 | 39 | @Test 40 | public void list() { 41 | 42 | List traceIds = new ArrayList<>(); 43 | traceIds.add("a3f2f2660d6c4a869626c878be92961d"); 44 | 45 | List ret = lideaLogErrorDAO.list(traceIds); 46 | 47 | 48 | System.out.println("=========================================================================================="); 49 | System.out.println(JSON.toJSONString(ret)); 50 | 51 | } 52 | } -------------------------------------------------------------------------------- /mmc-lidea-web/src/test/java/com/mmc/flink/lidea/dao/LideaMethodDAOTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.dao; 11 | 12 | import com.alibaba.fastjson.JSON; 13 | import com.mmc.lidea.common.bo.LideaMethodBO; 14 | import com.mmc.flink.lidea.dto.LideaMethodReq; 15 | import com.mmc.flink.lidea.dto.LideaMethodResp; 16 | import com.mmc.lidea.util.TimeUtil; 17 | import org.junit.FixMethodOrder; 18 | import org.junit.Test; 19 | import org.junit.runner.RunWith; 20 | import org.junit.runners.MethodSorters; 21 | import org.springframework.boot.test.context.SpringBootTest; 22 | import org.springframework.test.context.junit4.SpringRunner; 23 | 24 | import javax.annotation.Resource; 25 | 26 | import java.time.LocalDateTime; 27 | import java.util.List; 28 | 29 | /** 30 | * @author Joey 31 | * @date 2019/8/6 21:47 32 | */ 33 | @RunWith(SpringRunner.class) 34 | @SpringBootTest 35 | @FixMethodOrder(MethodSorters.NAME_ASCENDING) 36 | public class LideaMethodDAOTest { 37 | 38 | @Resource 39 | private LideaMethodDAO lideaMethodDAO; 40 | 41 | @Test 42 | public void put() { 43 | 44 | LideaMethodBO bo = new LideaMethodBO(); 45 | bo.setTime(TimeUtil.dateToString(LocalDateTime.now())); 46 | bo.setAppName("demo"); 47 | bo.setServiceName("com.demo.api.CabinetServiceFacade"); 48 | bo.setMethodName("getCabinetInfo"); 49 | bo.setCount("100"); 50 | 51 | bo = lideaMethodDAO.put(bo); 52 | 53 | System.out.println("=========================================================================================="); 54 | System.out.println(JSON.toJSONString(bo)); 55 | } 56 | 57 | @Test 58 | public void scan() { 59 | 60 | LideaMethodReq req = new LideaMethodReq(); 61 | req.setAppName("demo"); 62 | req.setServiceName("com.demo.api.CabinetServiceFacade"); 63 | 64 | List list = lideaMethodDAO.scan(req); 65 | 66 | System.out.println("=========================================================================================="); 67 | System.out.println(JSON.toJSONString(list)); 68 | 69 | } 70 | 71 | @Test 72 | public void testGet() { 73 | 74 | LideaMethodReq req = new LideaMethodReq(); 75 | 76 | req.setAppName("demo"); 77 | req.setServiceName("com.demo.api.CabinetServiceFacade"); 78 | req.setMethodName("getCabinetInfo"); 79 | 80 | 81 | LideaMethodResp resp = lideaMethodDAO.get(req); 82 | System.out.println("=========================================================================================="); 83 | System.out.println(JSON.toJSONString(resp)); 84 | 85 | } 86 | } -------------------------------------------------------------------------------- /mmc-lidea-web/src/test/java/com/mmc/flink/lidea/dao/LideaServiceDAOTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010-2020 Founder Ltd. All Rights Reserved. 3 | * 4 | * This software is the confidential and proprietary information of 5 | * Founder. You shall not disclose such Confidential Information 6 | * and shall use it only in accordance with the terms of the agreements 7 | * you entered into with Founder. 8 | * 9 | */ 10 | package com.mmc.flink.lidea.dao; 11 | 12 | import com.alibaba.fastjson.JSON; 13 | import com.mmc.lidea.common.bo.LideaServiceBO; 14 | import com.mmc.flink.lidea.dto.LideaServiceReq; 15 | import org.junit.FixMethodOrder; 16 | import org.junit.Test; 17 | import org.junit.runner.RunWith; 18 | import org.junit.runners.MethodSorters; 19 | import org.springframework.boot.test.context.SpringBootTest; 20 | import org.springframework.test.context.junit4.SpringRunner; 21 | 22 | import javax.annotation.Resource; 23 | 24 | import java.util.List; 25 | 26 | /** 27 | * @author Joey 28 | * @date 2019/8/29 14:32 29 | */ 30 | @RunWith(SpringRunner.class) 31 | @SpringBootTest 32 | @FixMethodOrder(MethodSorters.NAME_ASCENDING) 33 | public class LideaServiceDAOTest { 34 | 35 | @Resource 36 | private LideaServiceDAO lideaServiceDAO; 37 | 38 | @Test 39 | public void testPut() { 40 | 41 | LideaServiceBO bo = new LideaServiceBO(); 42 | bo.appName = "demo"; 43 | bo.serviceName = "com.demo.api.CabinetServiceFacade"; 44 | 45 | bo = lideaServiceDAO.put(bo); 46 | 47 | System.out.println("=========================================================================================="); 48 | System.out.println(JSON.toJSONString(bo)); 49 | 50 | } 51 | 52 | @Test 53 | public void scan() { 54 | 55 | LideaServiceReq req = new LideaServiceReq(); 56 | req.setAppName("demo"); 57 | 58 | List list = lideaServiceDAO.scan(req); 59 | 60 | System.out.println("=========================================================================================="); 61 | System.out.println(JSON.toJSONString(list)); 62 | 63 | } 64 | 65 | 66 | 67 | } -------------------------------------------------------------------------------- /mmc-lidea-web/src/test/java/com/mmc/flink/lidea/service/LideaLogServiceTest.java: -------------------------------------------------------------------------------- 1 | package com.mmc.flink.lidea.service; 2 | 3 | import com.alibaba.fastjson.JSON; 4 | import com.mmc.flink.lidea.dto.LideaLogErrorDetailResp; 5 | import com.mmc.flink.lidea.dto.LideaLogReq; 6 | import com.mmc.flink.lidea.dto.LideaMethodReq; 7 | import com.mmc.flink.lidea.dto.LideaMethodResp; 8 | import com.mmc.lidea.util.TimeUtil; 9 | import org.junit.FixMethodOrder; 10 | import org.junit.Test; 11 | import org.junit.runner.RunWith; 12 | import org.junit.runners.MethodSorters; 13 | import org.springframework.boot.test.context.SpringBootTest; 14 | import org.springframework.test.context.junit4.SpringRunner; 15 | 16 | import javax.annotation.Resource; 17 | 18 | @RunWith(SpringRunner.class) 19 | @SpringBootTest 20 | @FixMethodOrder(MethodSorters.NAME_ASCENDING) 21 | public class LideaLogServiceTest { 22 | 23 | @Resource 24 | private LideaLogService lideaLogService; 25 | 26 | @Test 27 | public void testList() { 28 | 29 | LideaLogReq req = new LideaLogReq(); 30 | req.setAppName("demo"); 31 | req.setServiceName("com.demo.api.CabinetServiceFacade"); 32 | req.setMethodName("getCabinetInfo"); 33 | req.setFrom(TimeUtil.stringToLong("2019-08-06 10:09:30")); 34 | req.setTo(TimeUtil.stringToLong("2019-08-06 10:09:30")); 35 | req.setSize(100); 36 | 37 | LideaLogErrorDetailResp resp = lideaLogService.listError(req); 38 | 39 | System.out.println("=========================================================================================="); 40 | System.out.println(JSON.toJSONString(resp)); 41 | } 42 | @Test 43 | public void testList1() { 44 | 45 | LideaLogReq req = new LideaLogReq(); 46 | req.setAppName("demo"); 47 | req.setServiceName("com.demo.api.CabinetServiceFacade"); 48 | req.setMethodName("getCabinetInfo"); 49 | req.setFrom(1565057370000L); 50 | req.setTo(1565057370000L); 51 | req.setSize(100); 52 | 53 | LideaLogErrorDetailResp resp = lideaLogService.listError(req); 54 | 55 | System.out.println("=========================================================================================="); 56 | System.out.println(JSON.toJSONString(resp)); 57 | } 58 | 59 | @Test 60 | public void listMethods() { 61 | 62 | LideaMethodReq req = new LideaMethodReq(); 63 | req.setAppName("demo"); 64 | req.setServiceName("com.demo.api.CabinetServiceFacade"); 65 | 66 | LideaMethodResp resp = lideaLogService.listMethods(req); 67 | 68 | System.out.println("=========================================================================================="); 69 | System.out.println(JSON.toJSONString(resp)); 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 4.0.0 7 | 8 | com.mmc 9 | lidea 10 | 1.0.0-SNAPSHOT 11 | 12 | 13 | pom 14 | 15 | 16 | 17 | 18 | 19 | mmc-lidea-stream 20 | mmc-lidea-web 21 | mmc-lidea-util 22 | mmc-lidea-common 23 | 24 | 25 | 26 | 27 | 28 | 29 | --------------------------------------------------------------------------------