├── .gitignore ├── SparkStreaming_1.5.2-Kafka_0.8.1.1 ├── .gitignore ├── README.md ├── src │ └── main │ │ ├── resources │ │ └── conf.properties │ │ ├── java │ │ └── org │ │ │ └── hna │ │ │ ├── dao │ │ │ └── MysqlDao.java │ │ │ ├── utils │ │ │ └── Config.java │ │ │ └── domain │ │ │ └── RequestLog.java │ │ └── scala │ │ └── org │ │ └── hna │ │ └── stream │ │ ├── KafkaTest.scala │ │ └── KafkaStreaming.scala └── pom.xml ├── README.md └── SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1 ├── README.md ├── src ├── main │ ├── java │ │ └── com │ │ │ └── gh │ │ │ ├── bean │ │ │ ├── alert │ │ │ │ ├── AlertData.java │ │ │ │ ├── KeyValue.java │ │ │ │ └── AlertDataInfo.java │ │ │ ├── logfile │ │ │ │ ├── LogFileRedis.java │ │ │ │ ├── LogFileContainer.java │ │ │ │ ├── LogInfoContainer.java │ │ │ │ ├── LogInfoMySql.java │ │ │ │ ├── LogInfoRedis.java │ │ │ │ ├── LogFileNginx.java │ │ │ │ ├── LogFileMySql.java │ │ │ │ ├── DataContainer.java │ │ │ │ ├── DataMySql.java │ │ │ │ ├── DataRedis.java │ │ │ │ ├── DataNginx.java │ │ │ │ └── LogInfoNginx.java │ │ │ ├── appcapability │ │ │ │ ├── AppMySql.java │ │ │ │ ├── AppRedis.java │ │ │ │ ├── AppMySql2.java │ │ │ │ ├── AppNginx.java │ │ │ │ ├── AppRedis2.java │ │ │ │ ├── AppMySqlData2.java │ │ │ │ ├── AppRedisData2.java │ │ │ │ ├── AppNginxData.java │ │ │ │ ├── AppMySqlData.java │ │ │ │ ├── AppRedisData.java │ │ │ │ ├── AppNginxStats.java │ │ │ │ ├── AppMysqlStats.java │ │ │ │ └── AppRedisStats.java │ │ │ ├── containercapability │ │ │ │ ├── Container2.java │ │ │ │ ├── Container.java │ │ │ │ ├── ContainerData2.java │ │ │ │ ├── ContainerData.java │ │ │ │ ├── ContainerFileSystem.java │ │ │ │ └── ContainerStats.java │ │ │ └── GZ.java │ │ │ └── utils │ │ │ ├── AlertInfoUtil.java │ │ │ ├── JsonUtil.java │ │ │ ├── ConfigUtil.java │ │ │ ├── DateUtil.java │ │ │ └── HttpUtil.java │ ├── resources │ │ ├── log4j.properties │ │ ├── conf_local.properties │ │ ├── conf.properties │ │ └── conf_aws.properties │ └── scala │ │ └── com │ │ └── gh │ │ └── yun │ │ ├── SaveToESLog2.scala │ │ ├── SaveToESCap3.scala │ │ ├── SaveToESCap2.scala │ │ ├── SaveToESLog3.scala │ │ ├── parse │ │ ├── ParseNginx.scala │ │ ├── ParseMysql.scala │ │ ├── ParseRedis.scala │ │ └── ParseContainer.scala │ │ ├── AppNginxCap.scala │ │ ├── SaveToESLog.scala │ │ ├── AppMysqlCap.scala │ │ ├── AppRedisCap.scala │ │ ├── SaveToESCap.scala │ │ └── ContainerCap.scala └── test │ └── java │ └── com │ └── gh │ └── EsSpark.scala └── pom.xml /.gitignore: -------------------------------------------------------------------------------- 1 | /.git 2 | /target 3 | /.project 4 | /.classpath 5 | /.idea 6 | /*.eml 7 | /*.iml 8 | -------------------------------------------------------------------------------- /SparkStreaming_1.5.2-Kafka_0.8.1.1/.gitignore: -------------------------------------------------------------------------------- 1 | /.git 2 | /target 3 | /.project 4 | /.classpath 5 | /.idea 6 | /*.eml 7 | /*.iml 8 | -------------------------------------------------------------------------------- /SparkStreaming_1.5.2-Kafka_0.8.1.1/README.md: -------------------------------------------------------------------------------- 1 | # SparkStreaming-Kafka 2 | Streaming 消费kafka数据 3 | 4 | 环境： 5 | spark: 1.5.2 6 | kafka: 0.8.1.1 -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | #SparkStreaming-1.5.2-Kafka_2.10 2 | Streaming 消费kafka数据 3 | 4 | #SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.0.1 5 | Streaming 消费kafka数据并写入ES 6 | 7 | 8 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/README.md: -------------------------------------------------------------------------------- 1 | # SparkStreaming-Kafka-ES 2 | Streaming 消费kafka数据并写入ES 3 | 4 | 环境： 5 | spark: 2.0.2 6 | kafka: 0.10.0.1 7 | ES: 5.1.1 8 | 9 | 10 | # 程序业务说明 11 | 12 | 容器云日志告警查询处理程序 -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/alert/AlertData.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.alert; 2 | 3 | import java.io.Serializable; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | 7 | /** 8 | * Created by GH-GAN on 2016/11/25. 9 | */ 10 | public class AlertData implements Serializable { 11 | AlertDataInfo [] alert_data; 12 | 13 | public AlertDataInfo[] getAlert_data() { 14 | return alert_data; 15 | } 16 | 17 | public void setAlert_data(AlertDataInfo[] alert_data) { 18 | this.alert_data = alert_data; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, stdout 2 | 3 | #Console 4 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 5 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 6 | log4j.appender.stdout.layout.ConversionPattern=[%-4p]%d{yyyy-MM-dd HH:mm:ss} - %m%n 7 | 8 | #LogFile 9 | log4j.appender.file=org.apache.log4j.FileAppender 10 | log4j.appender.file.File=../logs/jklog.log 11 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 12 | log4j.appender.file.layout.ConversionPattern=%d{ABSOLUTE} %5p %c{1}:%L - %m%n 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/logfile/LogFileRedis.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.logfile; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by GH-GAN on 2016/11/24. 7 | */ 8 | public class LogFileRedis implements Serializable { 9 | String type; 10 | DataRedis data; 11 | 12 | public String getType() { 13 | return type; 14 | } 15 | 16 | public void setType(String type) { 17 | this.type = type; 18 | } 19 | 20 | public DataRedis getData() { 21 | return data; 22 | } 23 | 24 | public void setData(DataRedis data) { 25 | this.data = data; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/appcapability/AppMySql.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.appcapability; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by GH-GAN on 2016/11/25. 7 | */ 8 | public class AppMySql implements Serializable { 9 | String type; 10 | AppMySqlData data; 11 | 12 | public String getType() { 13 | return type; 14 | } 15 | 16 | public void setType(String type) { 17 | this.type = type; 18 | } 19 | 20 | public AppMySqlData getData() { 21 | return data; 22 | } 23 | 24 | public void setData(AppMySqlData data) { 25 | this.data = data; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/appcapability/AppRedis.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.appcapability; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by GH-GAN on 2016/11/25. 7 | */ 8 | public class AppRedis implements Serializable { 9 | String type; 10 | AppRedisData data; 11 | 12 | public String getType() { 13 | return type; 14 | } 15 | 16 | public void setType(String type) { 17 | this.type = type; 18 | } 19 | 20 | public AppRedisData getData() { 21 | return data; 22 | } 23 | 24 | public void setData(AppRedisData data) { 25 | this.data = data; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/appcapability/AppMySql2.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.appcapability; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by GH-GAN on 2016/11/25. 7 | */ 8 | public class AppMySql2 implements Serializable { 9 | String type; 10 | AppMySqlData2 data; 11 | 12 | public String getType() { 13 | return type; 14 | } 15 | 16 | public void setType(String type) { 17 | this.type = type; 18 | } 19 | 20 | public AppMySqlData2 getData() { 21 | return data; 22 | } 23 | 24 | public void setData(AppMySqlData2 data) { 25 | this.data = data; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/appcapability/AppNginx.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.appcapability; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by GH-GAN on 2016/11/25. 7 | */ 8 | public class AppNginx implements Serializable { 9 | String type = ""; 10 | AppNginxData data; 11 | 12 | public String getType() { 13 | return type; 14 | } 15 | 16 | public void setType(String type) { 17 | this.type = type; 18 | } 19 | 20 | public AppNginxData getData() { 21 | return data; 22 | } 23 | 24 | public void setData(AppNginxData data) { 25 | this.data = data; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/appcapability/AppRedis2.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.appcapability; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by GH-GAN on 2016/11/25. 7 | */ 8 | public class AppRedis2 implements Serializable { 9 | String type; 10 | AppRedisData2 data; 11 | 12 | public String getType() { 13 | return type; 14 | } 15 | 16 | public void setType(String type) { 17 | this.type = type; 18 | } 19 | 20 | public AppRedisData2 getData() { 21 | return data; 22 | } 23 | 24 | public void setData(AppRedisData2 data) { 25 | this.data = data; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/containercapability/Container2.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.containercapability; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by GH-GAN on 2016/11/25. 7 | */ 8 | public class Container2 implements Serializable { 9 | String type; 10 | ContainerData2 data; 11 | 12 | public String getType() { 13 | return type; 14 | } 15 | 16 | public void setType(String type) { 17 | this.type = type; 18 | } 19 | 20 | public ContainerData2 getData() { 21 | return data; 22 | } 23 | 24 | public void setData(ContainerData2 data) { 25 | this.data = data; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/logfile/LogFileContainer.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.logfile; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by GH-GAN on 2016/11/24. 7 | */ 8 | public class LogFileContainer implements Serializable { 9 | String type; 10 | DataContainer data; 11 | 12 | public String getType() { 13 | return type; 14 | } 15 | 16 | public void setType(String type) { 17 | this.type = type; 18 | } 19 | 20 | public DataContainer getData() { 21 | return data; 22 | } 23 | 24 | public void setData(DataContainer data) { 25 | this.data = data; 26 | } 27 | 28 | 29 | } 30 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/containercapability/Container.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.containercapability; 2 | 3 | import java.io.Serializable; 4 | import java.util.ArrayList; 5 | 6 | /** 7 | * Created by GH-GAN on 2016/11/25. 8 | */ 9 | public class Container implements Serializable { 10 | String type; 11 | ContainerData [] data; 12 | 13 | public String getType() { 14 | return type; 15 | } 16 | 17 | public void setType(String type) { 18 | this.type = type; 19 | } 20 | 21 | public ContainerData[] getData() { 22 | return data; 23 | } 24 | 25 | public void setData(ContainerData[] data) { 26 | this.data = data; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/resources/conf_local.properties: -------------------------------------------------------------------------------- 1 | 2 | sparkmaster=local[2] 3 | # sparkstream (s) 4 | capStreamtime=60 5 | logStreamSaveTime=10 6 | capStreamSaveTime=6 7 | mysqlCapcheckpoint=D:\\tmp\\checkpoint 8 | redisCapcheckpoint=D:\\tmp\\checkpoint 9 | nginxCapcheckpoint=D:\\tmp\\checkpoint 10 | containerCapcheckpoint=D:\\tmp\\checkpoint4 11 | saveCapcheckpoint=D:\\tmp\\checkpoint3 12 | saveLogcheckpoint=D:\\tmp\\checkpoint 13 | brokers=192.168.100.180:8074,192.168.100.181:8074,192.168.100.182:8074 14 | esnodes=192.168.100.154:8070,192.168.100.162:8070,192.168.100.163:8070,192.168.100.164:8070,192.168.100.166:8070,192.168.100.167:8070,192.168.100.168:8070,192.168.100.169:8070 15 | gjGetUrl=http://223.202.32.56:8077/alert/v1/rule 16 | alertUrl=http://223.202.32.56:8077/alert/v1/info/receive -------------------------------------------------------------------------------- /SparkStreaming_1.5.2-Kafka_0.8.1.1/src/main/resources/conf.properties: -------------------------------------------------------------------------------- 1 | 2 | #msyql 3 | standalone_jdbcUrl=jdbc:mysql://172.16.0.22:3306/testdb1?characterEncoding=utf8&autoReconnect=true&failOverReadOnly=false 4 | standalone_userName=test1 5 | standalone_userPasswd=hna1234! 6 | standalone_master=spark://data05:7077,data06:7077 7 | standalone_checkpoint=/mnt/data0/storm/spark-checkpoint 8 | standalone_zklist=data25:2181,data26:2181,data27:2181,data28:2181,data29:2181 9 | standalone_kafkatopic=test 10 | 11 | #cdh 12 | cdh_jdbcUrl=jdbc:mysql://192.168.128.11:3306/testdb1?characterEncoding=utf8&autoReconnect=true&failOverReadOnly=false 13 | cdh_userName=test1 14 | cdh_userPasswd=hna@11234! 15 | cdh_master=yarn-cluster 16 | cdh_checkpoint=/mnt/data0/checkpoint 17 | cdh_zklist=cdh-node01:2181,cdh-node07:2181,cdh-node08:2181 18 | cdh_kafkatopic=nginx-log -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/alert/KeyValue.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.alert; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by GH-GAN on 2016/12/1. 7 | */ 8 | public class KeyValue implements Serializable { 9 | String p_key; 10 | String p_value; 11 | 12 | public KeyValue() { 13 | } 14 | 15 | public KeyValue(String p_key, String p_value) { 16 | this.p_key = p_key; 17 | this.p_value = p_value; 18 | } 19 | 20 | public String getP_key() { 21 | return p_key; 22 | } 23 | 24 | public void setP_key(String p_key) { 25 | this.p_key = p_key; 26 | } 27 | 28 | public String getP_value() { 29 | return p_value; 30 | } 31 | 32 | public void setP_value(String p_value) { 33 | this.p_value = p_value; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/logfile/LogInfoContainer.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.logfile; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by GH-GAN on 2016/11/24. 7 | */ 8 | public class LogInfoContainer implements Serializable { 9 | String log_time; 10 | String source; 11 | String message; 12 | 13 | public String getMessage() { 14 | return message; 15 | } 16 | 17 | public void setMessage(String message) { 18 | this.message = message; 19 | } 20 | 21 | public String getSource() { 22 | return source; 23 | } 24 | 25 | public void setSource(String source) { 26 | this.source = source; 27 | } 28 | 29 | public String getLog_time() { 30 | return log_time; 31 | } 32 | 33 | public void setLog_time(String log_time) { 34 | this.log_time = log_time; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/logfile/LogInfoMySql.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.logfile; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by GH-GAN on 2016/11/24. 7 | */ 8 | public class LogInfoMySql implements Serializable { 9 | String log_time; 10 | String warn_type; 11 | String message; 12 | 13 | public String getLog_time() { 14 | return log_time; 15 | } 16 | 17 | public void setLog_time(String log_time) { 18 | this.log_time = log_time; 19 | } 20 | 21 | public String getWarn_type() { 22 | return warn_type; 23 | } 24 | 25 | public void setWarn_type(String warn_type) { 26 | this.warn_type = warn_type; 27 | } 28 | 29 | public String getMessage() { 30 | return message; 31 | } 32 | 33 | public void setMessage(String message) { 34 | this.message = message; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/logfile/LogInfoRedis.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.logfile; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by GH-GAN on 2016/11/24. 7 | */ 8 | public class LogInfoRedis implements Serializable { 9 | String log_time; 10 | String warn_type; 11 | String message; 12 | 13 | public String getLog_time() { 14 | return log_time; 15 | } 16 | 17 | public void setLog_time(String log_time) { 18 | this.log_time = log_time; 19 | } 20 | 21 | public String getWarn_type() { 22 | return warn_type; 23 | } 24 | 25 | public void setWarn_type(String warn_type) { 26 | this.warn_type = warn_type; 27 | } 28 | 29 | public String getMessage() { 30 | return message; 31 | } 32 | 33 | public void setMessage(String message) { 34 | this.message = message; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/resources/conf.properties: -------------------------------------------------------------------------------- 1 | 2 | sparkmaster=spark://ggh-yrzkf2-1:8077,ggh-yrzkf2-2:8077 3 | # sparkstream (s) 4 | capStreamtime=60 5 | logStreamSaveTime=15 6 | capStreamSaveTime=23 7 | mysqlCapcheckpoint=/root/checkpoint/mysqlCapcheckpoint 8 | nginxCapcheckpoint=/root/checkpoint/nginxCapcheckpoint 9 | redisCapcheckpoint=/root/checkpoint/redisCapcheckpoint 10 | containerCapcheckpoint=/root/checkpoint/containerCapcheckpoint 11 | saveCapcheckpoint=/root/checkpoint/saveCapcheckpoint 12 | saveLogcheckpoint=/root/checkpoint/saveLogcheckpoint 13 | brokers=192.168.100.180:8074,192.168.100.181:8074,192.168.100.182:8074 14 | esnodes=192.168.100.154:8070,192.168.100.162:8070,192.168.100.163:8070,192.168.100.164:8070,192.168.100.166:8070,192.168.100.167:8070,192.168.100.168:8070,192.168.100.169:8070 15 | gjGetUrl=http://223.202.32.56:8077/alert/v1/rule 16 | alertUrl=http://223.202.32.56:8077/alert/v1/info/receive -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/logfile/LogFileNginx.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.logfile; 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnore; 4 | 5 | import java.io.Serializable; 6 | 7 | /** 8 | * Created by GH-GAN on 2016/11/24. 9 | */ 10 | public class LogFileNginx implements Serializable { 11 | String type; 12 | @JsonIgnore 13 | String date; 14 | DataNginx data; 15 | 16 | public String getDate() { 17 | return date; 18 | } 19 | 20 | public void setDate(String date) { 21 | this.date = date; 22 | } 23 | 24 | public String getType() { 25 | return type; 26 | } 27 | 28 | public void setType(String type) { 29 | this.type = type; 30 | } 31 | 32 | public DataNginx getData() { 33 | return data; 34 | } 35 | 36 | public void setData(DataNginx data) { 37 | this.data = data; 38 | } 39 | 40 | 41 | } 42 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/GZ.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean; 2 | 3 | /** 4 | * Created by GH-GAN on 2016/12/7. 5 | */ 6 | public class GZ { 7 | String key; 8 | String condition; 9 | Double value; 10 | 11 | public GZ(String key, String condition, Double value) { 12 | this.key = key; 13 | this.condition = condition; 14 | this.value = value; 15 | } 16 | 17 | public String getKey() { 18 | return key; 19 | } 20 | 21 | public void setKey(String key) { 22 | this.key = key; 23 | } 24 | 25 | public String getCondition() { 26 | return condition; 27 | } 28 | 29 | public void setCondition(String condition) { 30 | this.condition = condition; 31 | } 32 | 33 | public Double getValue() { 34 | return value; 35 | } 36 | 37 | public void setValue(Double value) { 38 | this.value = value; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/resources/conf_aws.properties: -------------------------------------------------------------------------------- 1 | 2 | #sparkmaster=spark://ip-10-0-128-134:8077,ip-10-0-128-158:8077 3 | sparkmaster=spark://ip-10-0-128-158:8077 4 | # sparkstream (s) 5 | capStreamtime=60 6 | logStreamSaveTime=15 7 | capStreamSaveTime=23 8 | mysqlCapcheckpoint=/root/checkpoint/mysqlCapcheckpoint 9 | nginxCapcheckpoint=/root/checkpoint/nginxCapcheckpoint 10 | redisCapcheckpoint=/root/checkpoint/redisCapcheckpoint 11 | containerCapcheckpoint=/root/checkpoint/containerCapcheckpoint 12 | saveCapcheckpoint=/root/checkpoint/saveCapcheckpoint 13 | saveLogcheckpoint=/root/checkpoint/saveLogcheckpoint 14 | #brokers=10.0.128.132:9092,10.0.128.133:9092 15 | brokers=10.0.128.132:9092 16 | esnodes=10.0.128.137:9200,10.0.128.136:9200,10.0.128.135:9200 17 | gjGetUrl=http://internal-hna-caas-product-lb-internel-476060353.cn-north-1.elb.amazonaws.com.cn:8077/alert/v1/rule 18 | alertUrl=http://internal-hna-caas-product-lb-internel-476060353.cn-north-1.elb.amazonaws.com.cn:8077/alert/v1/info/receive -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/test/java/com/gh/EsSpark.scala: -------------------------------------------------------------------------------- 1 | package com.gh 2 | 3 | /** 4 | * Created by GH-GAN on 2016/6/3. 5 | */ 6 | 7 | import org.apache.spark.{SparkConf, SparkContext} 8 | //import org.elasticsearch.spark._ 9 | object EsSpark { 10 | def main(args: Array[String]) { 11 | val conf = new SparkConf().setMaster("local").setAppName("es-test-11"); 12 | conf.set("es.index.auto.create", "true") 13 | conf.set("es.nodes", "192.168.100.154:8070,192.168.100.162:8070,192.168.100.163:8070,192.168.100.164:8070,192.168.100.166:8070,192.168.100.167:8070,192.168.100.168:8070,192.168.100.169:8070") 14 | 15 | val sc = new SparkContext(conf) 16 | 17 | 18 | // 字段 19 | // df.columns.foreach(col => println(col)) 20 | 21 | 22 | // 数据插入es 23 | val numbers = Map("one" -> 1, "two" -> 2, "three" -> 3) 24 | val airports = Map("arrival" -> "Otopeni", "SFO" -> "San Fran") 25 | // sc.makeRDD(Seq(numbers, airports)).saveToEs("es-test-13/docs") 26 | 27 | // sc.stop() 28 | 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/logfile/LogFileMySql.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.logfile; 2 | 3 | import com.fasterxml.jackson.annotation.JsonFilter; 4 | import com.fasterxml.jackson.annotation.JsonIgnore; 5 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 6 | 7 | import java.io.Serializable; 8 | 9 | /** 10 | * Created by GH-GAN on 2016/11/24. 11 | */ 12 | public class LogFileMySql implements Serializable { 13 | String type; 14 | @JsonIgnore 15 | String date = ""; 16 | DataMySql data; 17 | public String getDate() { 18 | return date; 19 | } 20 | public void setDate(String date) { 21 | this.date = date; 22 | } 23 | 24 | public String getType() { 25 | return type; 26 | } 27 | 28 | public void setType(String type) { 29 | this.type = type; 30 | } 31 | 32 | public DataMySql getData() { 33 | return data; 34 | } 35 | 36 | public void setData(DataMySql data) { 37 | this.data = data; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /SparkStreaming_1.5.2-Kafka_0.8.1.1/src/main/java/org/hna/dao/MysqlDao.java: -------------------------------------------------------------------------------- 1 | package org.hna.dao; 2 | 3 | import java.sql.Connection; 4 | import java.sql.DriverManager; 5 | import java.sql.PreparedStatement; 6 | 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import org.hna.utils.Config; 11 | 12 | public class MysqlDao { 13 | private static Logger log = LoggerFactory.getLogger(MysqlDao.class); 14 | private static Connection con = null; 15 | // private static ObjectMapper mapper = new ObjectMapper(); 16 | public MysqlDao(){ 17 | init(); 18 | } 19 | public void init(){ 20 | try { 21 | Class.forName("com.mysql.jdbc.Driver"); 22 | con = DriverManager.getConnection(Config.url, Config.userName, Config.userPasswd); 23 | } catch (Exception e) { 24 | log.error("mysql connection error!!!"); 25 | e.printStackTrace(); 26 | } 27 | } 28 | public boolean save(String sql) { 29 | PreparedStatement prepareStatement = null; 30 | try{ 31 | prepareStatement = con.prepareStatement(sql); 32 | prepareStatement.executeUpdate(); 33 | }catch (Exception e){ 34 | e.printStackTrace(); 35 | }finally { 36 | try{ 37 | prepareStatement.close(); 38 | }catch (Exception e){ 39 | e.printStackTrace(); 40 | } 41 | } 42 | return true; 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /SparkStreaming_1.5.2-Kafka_0.8.1.1/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | groupId 8 | spark_kafka 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 2.10 13 | 1.5.2 14 | 15 | 16 | 17 | 18 | org.apache.spark 19 | spark-streaming_${scala.binary.version} 20 | ${spark.version} 21 | 22 | 23 | mysql 24 | mysql-connector-java 25 | 5.1.38 26 | 27 | 28 | org.apache.spark 29 | spark-streaming-kafka_${scala.binary.version} 30 | ${spark.version} 31 | 32 | 33 | org.codehaus.jackson 34 | jackson-mapper-asl 35 | 1.9.13 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | ggh 8 | yun-log 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 2.11 13 | 2.0.2 14 | 5.1.1 15 | 16 | 17 | 18 | 23 | 24 | 25 | org.codehaus.jackson 26 | jackson-mapper-asl 27 | 1.9.13 28 | 29 | 30 | 31 | org.apache.spark 32 | spark-streaming-kafka-0-10_2.11 33 | 2.0.2 34 | 35 | 36 | 37 | junit 38 | junit 39 | 4.12 40 | test 41 | 42 | 43 | -------------------------------------------------------------------------------- /SparkStreaming_1.5.2-Kafka_0.8.1.1/src/main/java/org/hna/utils/Config.java: -------------------------------------------------------------------------------- 1 | package org.hna.utils; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.io.InputStream; 7 | import java.util.Properties; 8 | 9 | /** 10 | * Created by HP on 2016/2/16. 11 | */ 12 | public class Config { 13 | private static Logger log = LoggerFactory.getLogger(Config.class); 14 | public static String url; 15 | public static String userName; 16 | public static String userPasswd; 17 | public static String master; 18 | public static String checkpoint; 19 | public static String zklist; 20 | public static String kafkatopic; 21 | 22 | public static String cdh = "cdh_"; 23 | public static String pre = "standalone_"; 24 | public static String env = pre; // 切换环境 25 | 26 | static{ 27 | InputStream in = Config.class.getResourceAsStream("/conf.properties"); 28 | Properties prop = new Properties(); 29 | try{ 30 | prop.load(in); 31 | 32 | url = prop.getProperty(env+"jdbcUrl"); 33 | userName = prop.getProperty(env+"userName"); 34 | userPasswd = prop.getProperty(env+"userPasswd"); 35 | 36 | master = prop.getProperty(env+"master"); 37 | checkpoint = prop.getProperty(env+"checkpoint"); 38 | zklist = prop.getProperty(env+"zklist"); 39 | kafkatopic = prop.getProperty(env+"kafkatopic"); 40 | 41 | }catch(Exception e){ 42 | log.error("conf file load error !!!"); 43 | } 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/containercapability/ContainerData2.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.containercapability; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by GH-GAN on 2016/11/25. 7 | */ 8 | public class ContainerData2 implements Serializable { 9 | String timestamp; 10 | String container_uuid; 11 | String environment_id; 12 | String container_name; 13 | String namespace; 14 | ContainerStats stats; 15 | 16 | public String getContainer_name() { 17 | return (null == container_name) ? "" :container_name; 18 | } 19 | 20 | public void setContainer_name(String container_name) { 21 | this.container_name = container_name; 22 | } 23 | 24 | public String getNamespace() { 25 | return namespace; 26 | } 27 | 28 | public void setNamespace(String namespace) { 29 | this.namespace = namespace; 30 | } 31 | 32 | public String getTimestamp() { 33 | return timestamp; 34 | } 35 | 36 | public void setTimestamp(String timestamp) { 37 | this.timestamp = timestamp; 38 | } 39 | 40 | public String getContainer_uuid() { 41 | return container_uuid; 42 | } 43 | 44 | public void setContainer_uuid(String container_uuid) { 45 | this.container_uuid = container_uuid; 46 | } 47 | 48 | public String getEnvironment_id() { 49 | return (null == environment_id) ? "" : environment_id; 50 | } 51 | 52 | public void setEnvironment_id(String environment_id) { 53 | this.environment_id = environment_id; 54 | } 55 | 56 | public ContainerStats getStats() { 57 | return stats; 58 | } 59 | 60 | public void setStats(ContainerStats stats) { 61 | this.stats = stats; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/logfile/DataContainer.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.logfile; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by GH-GAN on 2016/11/24. 7 | */ 8 | public class DataContainer implements Serializable { 9 | String container_uuid; 10 | String environment_id; 11 | String container_name; 12 | String namespace; 13 | String timestamp; 14 | LogInfoContainer log_info; 15 | 16 | public String getContainer_name() { 17 | return container_name; 18 | } 19 | 20 | public void setContainer_name(String container_name) { 21 | this.container_name = container_name; 22 | } 23 | 24 | public String getNamespace() { 25 | return namespace; 26 | } 27 | 28 | public void setNamespace(String namespace) { 29 | this.namespace = namespace; 30 | } 31 | 32 | public String getContainer_uuid() { 33 | return (null == container_uuid) ? "" : container_uuid; 34 | } 35 | 36 | public void setContainer_uuid(String container_uuid) { 37 | this.container_uuid = container_uuid; 38 | } 39 | 40 | public String getEnvironment_id() { 41 | return (null == environment_id) ? "" : environment_id; 42 | } 43 | 44 | public void setEnvironment_id(String environment_id) { 45 | this.environment_id = environment_id; 46 | } 47 | 48 | public String getTimestamp() { 49 | return timestamp; 50 | } 51 | 52 | public void setTimestamp(String timestamp) { 53 | this.timestamp = timestamp; 54 | } 55 | 56 | public LogInfoContainer getLog_info() { 57 | return log_info; 58 | } 59 | 60 | public void setLog_info(LogInfoContainer log_info) { 61 | this.log_info = log_info; 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/appcapability/AppMySqlData2.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.appcapability; 2 | 3 | import java.io.Serializable; 4 | import java.util.ArrayList; 5 | 6 | /** 7 | * Created by GH-GAN on 2016/11/25. 8 | */ 9 | public class AppMySqlData2 implements Serializable { 10 | String timestamp; 11 | String container_uuid; 12 | String environment_id; 13 | String container_name; 14 | String namespace; 15 | AppMysqlStats stats; 16 | 17 | public String getContainer_name() { 18 | return container_name; 19 | } 20 | 21 | public void setContainer_name(String container_name) { 22 | this.container_name = container_name; 23 | } 24 | 25 | public String getNamespace() { 26 | return namespace; 27 | } 28 | 29 | public void setNamespace(String namespace) { 30 | this.namespace = namespace; 31 | } 32 | 33 | public String getTimestamp() { 34 | return timestamp; 35 | } 36 | 37 | public void setTimestamp(String timestamp) { 38 | this.timestamp = timestamp; 39 | } 40 | 41 | public String getContainer_uuid() { 42 | return (null == container_uuid) ? "" : container_uuid; 43 | } 44 | 45 | public void setContainer_uuid(String container_uuid) { 46 | this.container_uuid = container_uuid; 47 | } 48 | 49 | public String getEnvironment_id() { 50 | return (null == environment_id) ? "" : environment_id; 51 | } 52 | 53 | public void setEnvironment_id(String environment_id) { 54 | this.environment_id = environment_id; 55 | } 56 | 57 | public AppMysqlStats getStats() { 58 | return stats; 59 | } 60 | 61 | public void setStats(AppMysqlStats stats) { 62 | this.stats = stats; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/appcapability/AppRedisData2.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.appcapability; 2 | 3 | import java.io.Serializable; 4 | import java.util.ArrayList; 5 | 6 | /** 7 | * Created by GH-GAN on 2016/11/25. 8 | */ 9 | public class AppRedisData2 implements Serializable { 10 | String timestamp; 11 | String container_uuid; 12 | String environment_id; 13 | String container_name; 14 | String namespace; 15 | AppRedisStats stats; 16 | 17 | public String getTimestamp() { 18 | return timestamp; 19 | } 20 | 21 | public void setTimestamp(String timestamp) { 22 | this.timestamp = timestamp; 23 | } 24 | 25 | public String getContainer_name() { 26 | return container_name; 27 | } 28 | 29 | public void setContainer_name(String container_name) { 30 | this.container_name = container_name; 31 | } 32 | 33 | public String getNamespace() { 34 | return namespace; 35 | } 36 | 37 | public void setNamespace(String namespace) { 38 | this.namespace = namespace; 39 | } 40 | 41 | public String getContainer_uuid() { 42 | return (null == container_uuid) ? "" : container_uuid; 43 | } 44 | 45 | public void setContainer_uuid(String container_uuid) { 46 | this.container_uuid = container_uuid; 47 | } 48 | 49 | public String getEnvironment_id() { 50 | return (null == environment_id) ? "" : environment_id; 51 | } 52 | 53 | public void setEnvironment_id(String environment_id) { 54 | this.environment_id = environment_id; 55 | } 56 | 57 | public AppRedisStats getStats() { 58 | return stats; 59 | } 60 | 61 | public void setStats(AppRedisStats stats) { 62 | this.stats = stats; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/appcapability/AppNginxData.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.appcapability; 2 | 3 | import java.io.Serializable; 4 | import java.util.ArrayList; 5 | 6 | /** 7 | * Created by GH-GAN on 2016/11/25. 8 | */ 9 | public class AppNginxData implements Serializable { 10 | String container_uuid = ""; 11 | String environment_id = ""; 12 | String container_name; 13 | String namespace; 14 | String timestamp; 15 | AppNginxStats stats; 16 | 17 | public String getContainer_name() { 18 | return container_name; 19 | } 20 | 21 | public void setContainer_name(String container_name) { 22 | this.container_name = container_name; 23 | } 24 | 25 | public String getNamespace() { 26 | return namespace; 27 | } 28 | 29 | public void setNamespace(String namespace) { 30 | this.namespace = namespace; 31 | } 32 | 33 | public String getContainer_uuid() { 34 | return (null == container_uuid) ? "" : container_uuid; 35 | } 36 | 37 | public void setContainer_uuid(String container_uuid) { 38 | this.container_uuid = container_uuid; 39 | } 40 | 41 | public String getEnvironment_id() { 42 | return (null == environment_id) ? "" : environment_id; 43 | } 44 | 45 | public void setEnvironment_id(String environment_id) { 46 | this.environment_id = environment_id; 47 | } 48 | 49 | public String getTimestamp() { 50 | return timestamp; 51 | } 52 | 53 | public void setTimestamp(String timestamp) { 54 | this.timestamp = timestamp; 55 | } 56 | 57 | public AppNginxStats getStats() { 58 | return stats; 59 | } 60 | 61 | public void setStats(AppNginxStats stats) { 62 | this.stats = stats; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/appcapability/AppMySqlData.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.appcapability; 2 | 3 | import java.io.Serializable; 4 | import java.util.ArrayList; 5 | 6 | /** 7 | * Created by GH-GAN on 2016/11/25. 8 | */ 9 | public class AppMySqlData implements Serializable { 10 | String timestamp; 11 | String container_uuid; 12 | String environment_id; 13 | String container_name; 14 | String namespace; 15 | ArrayList stats; 16 | 17 | public String getContainer_name() { 18 | return container_name; 19 | } 20 | 21 | public void setContainer_name(String container_name) { 22 | this.container_name = container_name; 23 | } 24 | 25 | public String getNamespace() { 26 | return namespace; 27 | } 28 | 29 | public void setNamespace(String namespace) { 30 | this.namespace = namespace; 31 | } 32 | 33 | public String getTimestamp() { 34 | return timestamp; 35 | } 36 | 37 | public void setTimestamp(String timestamp) { 38 | this.timestamp = timestamp; 39 | } 40 | 41 | public String getContainer_uuid() { 42 | return (null == container_uuid) ? "" : container_uuid; 43 | } 44 | 45 | public void setContainer_uuid(String container_uuid) { 46 | this.container_uuid = container_uuid; 47 | } 48 | 49 | public String getEnvironment_id() { 50 | return (null == environment_id) ? "" : environment_id; 51 | } 52 | 53 | public void setEnvironment_id(String environment_id) { 54 | this.environment_id = environment_id; 55 | } 56 | 57 | public ArrayList getStats() { 58 | return stats; 59 | } 60 | 61 | public void setStats(ArrayList stats) { 62 | this.stats = stats; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/appcapability/AppRedisData.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.appcapability; 2 | 3 | import java.io.Serializable; 4 | import java.util.ArrayList; 5 | 6 | /** 7 | * Created by GH-GAN on 2016/11/25. 8 | */ 9 | public class AppRedisData implements Serializable { 10 | String timestamp; 11 | String container_uuid; 12 | String environment_id; 13 | String container_name; 14 | String namespace; 15 | ArrayList stats; 16 | 17 | public String getTimestamp() { 18 | return timestamp; 19 | } 20 | 21 | public void setTimestamp(String timestamp) { 22 | this.timestamp = timestamp; 23 | } 24 | 25 | public String getContainer_name() { 26 | return container_name; 27 | } 28 | 29 | public void setContainer_name(String container_name) { 30 | this.container_name = container_name; 31 | } 32 | 33 | public String getNamespace() { 34 | return namespace; 35 | } 36 | 37 | public void setNamespace(String namespace) { 38 | this.namespace = namespace; 39 | } 40 | 41 | public String getContainer_uuid() { 42 | return (null == container_uuid) ? "" : container_uuid; 43 | } 44 | 45 | public void setContainer_uuid(String container_uuid) { 46 | this.container_uuid = container_uuid; 47 | } 48 | 49 | public String getEnvironment_id() { 50 | return (null == environment_id) ? "" : environment_id; 51 | } 52 | 53 | public void setEnvironment_id(String environment_id) { 54 | this.environment_id = environment_id; 55 | } 56 | 57 | public ArrayList getStats() { 58 | return stats; 59 | } 60 | 61 | public void setStats(ArrayList stats) { 62 | this.stats = stats; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/containercapability/ContainerData.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.containercapability; 2 | 3 | import com.gh.bean.appcapability.AppRedisData; 4 | 5 | import java.io.Serializable; 6 | import java.util.ArrayList; 7 | 8 | /** 9 | * Created by GH-GAN on 2016/11/25. 10 | */ 11 | public class ContainerData implements Serializable { 12 | String timestamp; 13 | String container_uuid; 14 | String environment_id; 15 | String container_name; 16 | String namespace; 17 | ContainerStats [] stats; 18 | 19 | public String getContainer_name() { 20 | return (null == container_name) ? "" :container_name; 21 | } 22 | 23 | public void setContainer_name(String container_name) { 24 | this.container_name = container_name; 25 | } 26 | 27 | public String getNamespace() { 28 | return namespace; 29 | } 30 | 31 | public void setNamespace(String namespace) { 32 | this.namespace = namespace; 33 | } 34 | 35 | public String getTimestamp() { 36 | return timestamp; 37 | } 38 | 39 | public void setTimestamp(String timestamp) { 40 | this.timestamp = timestamp; 41 | } 42 | 43 | public String getContainer_uuid() { 44 | return container_uuid; 45 | } 46 | 47 | public void setContainer_uuid(String container_uuid) { 48 | this.container_uuid = container_uuid; 49 | } 50 | 51 | public String getEnvironment_id() { 52 | return environment_id; 53 | } 54 | 55 | public void setEnvironment_id(String environment_id) { 56 | this.environment_id = environment_id; 57 | } 58 | 59 | public ContainerStats[] getStats() { 60 | return stats; 61 | } 62 | 63 | public void setStats(ContainerStats[] stats) { 64 | this.stats = stats; 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/utils/AlertInfoUtil.java: -------------------------------------------------------------------------------- 1 | package com.gh.utils; 2 | 3 | import com.gh.bean.alert.AlertData; 4 | import com.gh.bean.alert.AlertDataInfo; 5 | import com.gh.bean.alert.KeyValue; 6 | 7 | import java.util.ArrayList; 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | /** 12 | * Created by GH-GAN on 2016/11/29. 13 | */ 14 | public class AlertInfoUtil { 15 | public final static String SUCCESS = "200"; 16 | public final static String FAILED = "500"; 17 | public final static String ALERT_TYPE_M = "M"; 18 | public final static String ALERT_TYPE_L = "L"; 19 | public final static String ALERT_DIM_C = "C"; 20 | public final static String ALERT_DIM_A = "A"; 21 | 22 | public static AlertDataInfo toWarnBean(String status, String alert_type, String alert_dim, String key, String start_time, String end_time, ArrayList list,String msg){ 23 | String[] keys = key.split("#"); 24 | String environment_id = keys[0]; 25 | String container_uuid = keys[1]; 26 | String _type = keys[2]; 27 | 28 | AlertDataInfo warn = new AlertDataInfo(); 29 | warn.setStatus(status); 30 | warn.setAlert_type(alert_type); 31 | warn.setAlert_dim(alert_dim); 32 | warn.setApp_type(_type); 33 | warn.setMsg(msg); 34 | warn.setEnvironment_id(environment_id); 35 | warn.setContainer_uuid(container_uuid); 36 | if (keys.length > 3) warn.setContainer_name(keys[3]); 37 | if (keys.length > 4) warn.setNamespace(keys[4]); 38 | warn.setStart_time(start_time); 39 | warn.setEnd_time(end_time); 40 | warn.setData(list); 41 | 42 | return warn; 43 | } 44 | 45 | public static String toAlertJson(AlertDataInfo [] alert_data){ 46 | AlertData ad = new AlertData(); 47 | ad.setAlert_data(alert_data); 48 | return JsonUtil.formatJson(ad); 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/logfile/DataMySql.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.logfile; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by GH-GAN on 2016/11/24. 7 | */ 8 | public class DataMySql implements Serializable { 9 | String container_uuid; 10 | String environment_id; 11 | String container_name; 12 | String namespace; 13 | String app_file; 14 | String timestamp; 15 | LogInfoMySql log_info; 16 | 17 | public String getContainer_name() { 18 | return container_name; 19 | } 20 | 21 | public void setContainer_name(String container_name) { 22 | this.container_name = container_name; 23 | } 24 | 25 | public String getNamespace() { 26 | return namespace; 27 | } 28 | 29 | public void setNamespace(String namespace) { 30 | this.namespace = namespace; 31 | } 32 | 33 | public String getContainer_uuid() { 34 | return (null == container_uuid) ? "" : container_uuid; 35 | } 36 | 37 | public void setContainer_uuid(String container_uuid) { 38 | this.container_uuid = container_uuid; 39 | } 40 | 41 | public String getEnvironment_id() { 42 | return (null == environment_id) ? "" : environment_id; 43 | } 44 | 45 | public void setEnvironment_id(String environment_id) { 46 | this.environment_id = environment_id; 47 | } 48 | 49 | public String getTimestamp() { 50 | return timestamp; 51 | } 52 | 53 | public void setTimestamp(String timestamp) { 54 | this.timestamp = timestamp; 55 | } 56 | 57 | public LogInfoMySql getLog_info() { 58 | return log_info; 59 | } 60 | 61 | public void setLog_info(LogInfoMySql log_info) { 62 | this.log_info = log_info; 63 | } 64 | 65 | public String getApp_file() { 66 | return app_file; 67 | } 68 | 69 | public void setApp_file(String app_file) { 70 | this.app_file = app_file; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/logfile/DataRedis.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.logfile; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by GH-GAN on 2016/11/24. 7 | */ 8 | public class DataRedis implements Serializable { 9 | String container_uuid; 10 | String environment_id; 11 | String container_name; 12 | String namespace; 13 | String app_file; 14 | String timestamp; 15 | LogInfoRedis log_info; 16 | 17 | public String getContainer_name() { 18 | return container_name; 19 | } 20 | 21 | public void setContainer_name(String container_name) { 22 | this.container_name = container_name; 23 | } 24 | 25 | public String getNamespace() { 26 | return namespace; 27 | } 28 | 29 | public void setNamespace(String namespace) { 30 | this.namespace = namespace; 31 | } 32 | 33 | public String getApp_file() { 34 | return app_file; 35 | } 36 | 37 | public void setApp_file(String app_file) { 38 | this.app_file = app_file; 39 | } 40 | 41 | public String getContainer_uuid() { 42 | return (null == container_uuid) ? "" : container_uuid; 43 | } 44 | 45 | public void setContainer_uuid(String container_uuid) { 46 | this.container_uuid = container_uuid; 47 | } 48 | 49 | public String getEnvironment_id() { 50 | return (null == environment_id) ? "" : environment_id; 51 | } 52 | 53 | public void setEnvironment_id(String environment_id) { 54 | this.environment_id = environment_id; 55 | } 56 | 57 | public String getTimestamp() { 58 | return timestamp; 59 | } 60 | 61 | public void setTimestamp(String timestamp) { 62 | this.timestamp = timestamp; 63 | } 64 | 65 | public LogInfoRedis getLog_info() { 66 | return log_info; 67 | } 68 | 69 | public void setLog_info(LogInfoRedis log_info) { 70 | this.log_info = log_info; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/logfile/DataNginx.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.logfile; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by GH-GAN on 2016/11/24. 7 | */ 8 | public class DataNginx implements Serializable { 9 | String container_uuid; 10 | String environment_id; 11 | String container_name; 12 | String namespace; 13 | String app_file; 14 | String timestamp; 15 | LogInfoNginx log_info; 16 | 17 | public String getContainer_name() { 18 | return container_name; 19 | } 20 | 21 | public void setContainer_name(String container_name) { 22 | this.container_name = container_name; 23 | } 24 | 25 | public String getNamespace() { 26 | return namespace; 27 | } 28 | 29 | public void setNamespace(String namespace) { 30 | this.namespace = namespace; 31 | } 32 | 33 | public String getApp_file() { 34 | return app_file; 35 | } 36 | 37 | public void setApp_file(String app_file) { 38 | this.app_file = app_file; 39 | } 40 | 41 | public String getContainer_uuid() { 42 | return (null == container_uuid) ? "" : container_uuid; 43 | } 44 | 45 | public void setContainer_uuid(String container_uuid) { 46 | this.container_uuid = container_uuid; 47 | } 48 | 49 | public String getEnvironment_id() { 50 | return (null == environment_id) ? "" : environment_id; 51 | } 52 | 53 | public void setEnvironment_id(String environment_id) { 54 | this.environment_id = environment_id; 55 | } 56 | 57 | public String getTimestamp() { 58 | return timestamp; 59 | } 60 | 61 | public void setTimestamp(String timestamp) { 62 | this.timestamp = timestamp; 63 | } 64 | 65 | public LogInfoNginx getLog_info() { 66 | return log_info; 67 | } 68 | 69 | public void setLog_info(LogInfoNginx log_info) { 70 | this.log_info = log_info; 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/utils/JsonUtil.java: -------------------------------------------------------------------------------- 1 | package com.gh.utils; 2 | 3 | import java.io.IOException; 4 | import java.io.Serializable; 5 | 6 | import org.codehaus.jackson.JsonNode; 7 | import org.codehaus.jackson.annotate.JsonAutoDetect; 8 | import org.codehaus.jackson.annotate.JsonMethod; 9 | import org.codehaus.jackson.map.ObjectMapper; 10 | import org.codehaus.jackson.map.SerializationConfig; 11 | 12 | public class JsonUtil implements Serializable { 13 | public static ObjectMapper mapper = new ObjectMapper(); 14 | static { 15 | mapper.setVisibility(JsonMethod.FIELD, JsonAutoDetect.Visibility.ANY); 16 | //mapper.disable(SerializationConfig.Feature.FAIL_ON_EMPTY_BEANS); 17 | } 18 | 19 | // 将 list,array,map,bean等转换为 json格式 20 | public static String formatJson(Object obj){ 21 | try { 22 | return mapper.writeValueAsString(obj); 23 | } catch (Exception e) { 24 | e.printStackTrace(); 25 | } 26 | return null; 27 | } 28 | 29 | //取出json中某filed 30 | public static String get(String json,String filed){ 31 | JsonNode readTree = null; 32 | try { 33 | readTree = mapper.readTree(json); 34 | } catch (Exception e) { 35 | e.printStackTrace(); 36 | } 37 | return readTree.isNull() ? null : readTree.get(filed).toString(); 38 | } 39 | 40 | //获取json tree 41 | public static JsonNode getJsonNode(String json){ 42 | //Person pp2 = mm.readValue(json, Person.class); 将json直接转换为bean 43 | JsonNode readTree = null; // 将json读为tree树 44 | try { 45 | if ("".equals(json) || null == json) return null; 46 | readTree = mapper.readTree(json); 47 | } catch (Exception e) { 48 | readTree = null; 49 | e.printStackTrace(); 50 | } 51 | return readTree; 52 | } 53 | 54 | //将json直接转换为bean 55 | public static T getBean(String json,Class cl){ 56 | T bean = null; 57 | try { 58 | bean = mapper.readValue(json, cl); 59 | } catch (Exception e) { 60 | bean = null; 61 | e.printStackTrace(); 62 | } 63 | return bean; 64 | } 65 | 66 | 67 | } 68 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/appcapability/AppNginxStats.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.appcapability; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by GH-GAN on 2016/11/29. 7 | */ 8 | public class AppNginxStats implements Serializable { 9 | String timestamp; 10 | Double active_connections; 11 | Double accepts; 12 | Double handled; 13 | Double requests; 14 | Double reading; 15 | Double witing; 16 | Double waiting; 17 | 18 | public String getTimestamp() { 19 | return timestamp; 20 | } 21 | 22 | public void setTimestamp(String timestamp) { 23 | this.timestamp = timestamp; 24 | } 25 | 26 | public Double getActive_connections() { 27 | return active_connections; 28 | } 29 | 30 | public void setActive_connections(Double active_connections) { 31 | this.active_connections = active_connections; 32 | } 33 | 34 | public Double getAccepts() { 35 | return accepts; 36 | } 37 | 38 | public void setAccepts(Double accepts) { 39 | this.accepts = accepts; 40 | } 41 | 42 | public Double getHandled() { 43 | return handled; 44 | } 45 | 46 | public void setHandled(Double handled) { 47 | this.handled = handled; 48 | } 49 | 50 | public Double getRequests() { 51 | return requests; 52 | } 53 | 54 | public void setRequests(Double requests) { 55 | this.requests = requests; 56 | } 57 | 58 | public Double getReading() { 59 | return reading; 60 | } 61 | 62 | public void setReading(Double reading) { 63 | this.reading = reading; 64 | } 65 | 66 | public Double getWiting() { 67 | return witing; 68 | } 69 | 70 | public void setWiting(Double witing) { 71 | this.witing = witing; 72 | } 73 | 74 | public Double getWaiting() { 75 | return waiting; 76 | } 77 | 78 | public void setWaiting(Double waiting) { 79 | this.waiting = waiting; 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/containercapability/ContainerFileSystem.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.containercapability; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by GH-GAN on 2016/12/6. 7 | */ 8 | public class ContainerFileSystem implements Serializable{ 9 | String container_filesystem_name; 10 | String container_filesystem_type; 11 | Double container_filesystem_capacity; 12 | Double container_filesystem_usage; 13 | 14 | public ContainerFileSystem(){} 15 | 16 | public ContainerFileSystem(String container_filesystem_name, String container_filesystem_type, Double container_filesystem_capacity, Double container_filesystem_usage) { 17 | this.container_filesystem_name = container_filesystem_name; 18 | this.container_filesystem_type = container_filesystem_type; 19 | this.container_filesystem_capacity = container_filesystem_capacity; 20 | this.container_filesystem_usage = container_filesystem_usage; 21 | } 22 | 23 | public String getContainer_filesystem_name() { 24 | return container_filesystem_name; 25 | } 26 | 27 | public void setContainer_filesystem_name(String container_filesystem_name) { 28 | this.container_filesystem_name = container_filesystem_name; 29 | } 30 | 31 | public String getContainer_filesystem_type() { 32 | return container_filesystem_type; 33 | } 34 | 35 | public void setContainer_filesystem_type(String container_filesystem_type) { 36 | this.container_filesystem_type = container_filesystem_type; 37 | } 38 | 39 | public Double getContainer_filesystem_capacity() { 40 | return container_filesystem_capacity; 41 | } 42 | 43 | public void setContainer_filesystem_capacity(Double container_filesystem_capacity) { 44 | this.container_filesystem_capacity = container_filesystem_capacity; 45 | } 46 | 47 | public Double getContainer_filesystem_usage() { 48 | return container_filesystem_usage; 49 | } 50 | 51 | public void setContainer_filesystem_usage(Double container_filesystem_usage) { 52 | this.container_filesystem_usage = container_filesystem_usage; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/logfile/LogInfoNginx.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.logfile; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by GH-GAN on 2016/11/24. 7 | */ 8 | public class LogInfoNginx implements Serializable { 9 | String log_time; 10 | String remote; 11 | String host; 12 | String user; 13 | String method; 14 | String path; 15 | int code; 16 | int size; 17 | String referer; 18 | String agent; 19 | 20 | public String getLog_time() { 21 | return log_time; 22 | } 23 | 24 | public void setLog_time(String log_time) { 25 | this.log_time = log_time; 26 | } 27 | 28 | public String getRemote() { 29 | return remote; 30 | } 31 | 32 | public void setRemote(String remote) { 33 | this.remote = remote; 34 | } 35 | 36 | public String getHost() { 37 | return host; 38 | } 39 | 40 | public void setHost(String host) { 41 | this.host = host; 42 | } 43 | 44 | public String getUser() { 45 | return user; 46 | } 47 | 48 | public void setUser(String user) { 49 | this.user = user; 50 | } 51 | 52 | public String getMethod() { 53 | return method; 54 | } 55 | 56 | public void setMethod(String method) { 57 | this.method = method; 58 | } 59 | 60 | public String getPath() { 61 | return path; 62 | } 63 | 64 | public void setPath(String path) { 65 | this.path = path; 66 | } 67 | 68 | public int getCode() { 69 | return code; 70 | } 71 | 72 | public void setCode(int code) { 73 | this.code = code; 74 | } 75 | 76 | public int getSize() { 77 | return size; 78 | } 79 | 80 | public void setSize(int size) { 81 | this.size = size; 82 | } 83 | 84 | public String getReferer() { 85 | return referer; 86 | } 87 | 88 | public void setReferer(String referer) { 89 | this.referer = referer; 90 | } 91 | 92 | public String getAgent() { 93 | return agent; 94 | } 95 | 96 | public void setAgent(String agent) { 97 | this.agent = agent; 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/utils/ConfigUtil.java: -------------------------------------------------------------------------------- 1 | package com.gh.utils; 2 | 3 | import java.io.InputStream; 4 | import java.util.Properties; 5 | 6 | /** 7 | * Created by GH-GAN on 2016/12/8. 8 | */ 9 | public class ConfigUtil { 10 | public static String sparkmaster; 11 | public static int capStreamtime; 12 | public static int logStreamSaveTime; 13 | public static int capStreamSaveTime; 14 | public static String mysqlCapcheckpoint; 15 | public static String nginxCapcheckpoint; 16 | public static String redisCapcheckpoint; 17 | public static String containerCapcheckpoint; 18 | public static String saveCapcheckpoint; 19 | public static String saveLogcheckpoint; 20 | public static String brokers; 21 | public static String esnodes; 22 | public static String alertUrl; 23 | public static String gjGetUrl; 24 | 25 | static { 26 | // InputStream in = ConfigUtil.class.getResourceAsStream("/conf_aws.properties"); 27 | InputStream in = ConfigUtil.class.getResourceAsStream("/conf.properties"); 28 | // InputStream in = ConfigUtil.class.getResourceAsStream("/conf_local.properties"); 29 | Properties prop = new Properties(); 30 | try { 31 | prop.load(in); 32 | sparkmaster = prop.getProperty("sparkmaster"); 33 | capStreamtime = Integer.parseInt(prop.getProperty("capStreamtime")); 34 | logStreamSaveTime = Integer.parseInt(prop.getProperty("logStreamSaveTime")); 35 | capStreamSaveTime = Integer.parseInt(prop.getProperty("capStreamSaveTime")); 36 | mysqlCapcheckpoint = prop.getProperty("mysqlCapcheckpoint"); 37 | nginxCapcheckpoint = prop.getProperty("nginxCapcheckpoint"); 38 | redisCapcheckpoint = prop.getProperty("redisCapcheckpoint"); 39 | containerCapcheckpoint = prop.getProperty("containerCapcheckpoint"); 40 | saveCapcheckpoint = prop.getProperty("saveCapcheckpoint"); 41 | saveLogcheckpoint = prop.getProperty("saveLogcheckpoint"); 42 | brokers = prop.getProperty("brokers"); 43 | esnodes = prop.getProperty("esnodes"); 44 | alertUrl = prop.getProperty("alertUrl"); 45 | gjGetUrl = prop.getProperty("gjGetUrl"); 46 | } catch (Exception e) { 47 | System.err.println("conf file load error !!!"); 48 | } 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/alert/AlertDataInfo.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.alert; 2 | 3 | 4 | import java.io.Serializable; 5 | import java.util.ArrayList; 6 | import java.util.Map; 7 | 8 | /** 9 | * Created by GH-GAN on 2016/11/25. 10 | */ 11 | public class AlertDataInfo implements Serializable { 12 | String status = "200"; 13 | String alert_type = ""; 14 | String alert_dim = ""; 15 | String app_type = ""; 16 | String msg = ""; 17 | String environment_id = "123"; 18 | String container_uuid = ""; 19 | String container_name = ""; 20 | String namespace = ""; 21 | String start_time = ""; 22 | String end_time = ""; 23 | ArrayList data; 24 | 25 | public String getContainer_name() { 26 | return container_name; 27 | } 28 | 29 | public void setContainer_name(String container_name) { 30 | this.container_name = container_name; 31 | } 32 | 33 | public String getStatus() { 34 | return status; 35 | } 36 | 37 | public void setStatus(String status) { 38 | this.status = status; 39 | } 40 | 41 | public String getAlert_type() { 42 | return alert_type; 43 | } 44 | 45 | public void setAlert_type(String alert_type) { 46 | this.alert_type = alert_type; 47 | } 48 | 49 | public String getAlert_dim() { 50 | return alert_dim; 51 | } 52 | 53 | public void setAlert_dim(String alert_dim) { 54 | this.alert_dim = alert_dim; 55 | } 56 | 57 | public String getApp_type() { 58 | return app_type; 59 | } 60 | 61 | public void setApp_type(String app_type) { 62 | this.app_type = app_type; 63 | } 64 | 65 | public String getMsg() { 66 | return msg; 67 | } 68 | 69 | public void setMsg(String msg) { 70 | this.msg = msg; 71 | } 72 | 73 | public String getEnvironment_id() { 74 | return environment_id; 75 | } 76 | 77 | public void setEnvironment_id(String environment_id) { 78 | this.environment_id = environment_id; 79 | } 80 | 81 | public String getContainer_uuid() { 82 | return container_uuid; 83 | } 84 | 85 | public void setContainer_uuid(String container_uuid) { 86 | this.container_uuid = container_uuid; 87 | } 88 | 89 | public String getStart_time() { 90 | return start_time; 91 | } 92 | 93 | public void setStart_time(String start_time) { 94 | this.start_time = start_time; 95 | } 96 | 97 | public String getEnd_time() { 98 | return end_time; 99 | } 100 | 101 | public void setEnd_time(String end_time) { 102 | this.end_time = end_time; 103 | } 104 | 105 | public ArrayList getData() { 106 | return data; 107 | } 108 | 109 | public String getNamespace() { 110 | return namespace; 111 | } 112 | 113 | public void setNamespace(String namespace) { 114 | this.namespace = namespace; 115 | } 116 | 117 | public void setData(ArrayList data) { 118 | this.data = data; 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /SparkStreaming_1.5.2-Kafka_0.8.1.1/src/main/scala/org/hna/stream/KafkaTest.scala: -------------------------------------------------------------------------------- 1 | package org.hna.stream 2 | import org.hna.dao.MysqlDao 3 | import org.hna.utils.Config 4 | import org.apache.spark._ 5 | import org.apache.spark.streaming._ 6 | import org.apache.spark.streaming.dstream.DStream 7 | import org.apache.spark.streaming.kafka.KafkaUtils 8 | /** 9 | * Created by HP on 2016/1/7. 10 | */ 11 | object KafkaTest { 12 | // val mysql = new MysqlDao() 13 | def main(args: Array[String]) { 14 | val sparkconf = new SparkConf().setAppName("kafka").setMaster(Config.master) 15 | val sc = new SparkContext(sparkconf) 16 | val ssc = new StreamingContext(sc,Seconds(5)) 17 | ssc.checkpoint(Config.checkpoint) 18 | 19 | val zkList = Config.zklist 20 | val groupId = "test-consumer-group" 21 | val topicMap = Config.kafkatopic.split(",").map(x => (x,2)).toMap 22 | val lines = KafkaUtils.createStream(ssc,zkList,groupId,topicMap).map(_._2) 23 | // parseAndSave(lines) 24 | lines.print() 25 | ssc.start() 26 | ssc.awaitTermination() 27 | } 28 | 29 | def parseAndSave(lines : DStream[String]) = { 30 | lines.foreachRDD(rdd => { 31 | rdd.foreach(f = record => { 32 | var line = record.toString 33 | println("line======>"+line) 34 | /* try { 35 | //解码两次 36 | line = URLDecoder.decode(line, "utf-8") 37 | line = URLDecoder.decode(line, "utf-8") 38 | 39 | val json = JSON.parseFull(line) 40 | json match { 41 | case Some(map: Map[String, Any]) => 42 | val sql = getSql(map) 43 | mysql.save(sql) 44 | case None => println("json parse error !!! => None") 45 | } 46 | } catch { 47 | case e: Exception => e.printStackTrace() 48 | println("json parse error !!! ") 49 | }*/ 50 | }) 51 | }) 52 | } 53 | def getSql(map:Map[String,Any]) : String ={ 54 | val sql = new StringBuffer() 55 | sql.append("INSERT INTO request_log (website,time_local,remote_addr,remote_port,request,request_body,response_time,request_time,status,body_bytes_sent,http_referer,http_user_agent,headers,jsession,http_cookie,resp_body)") 56 | sql.append("VALUES") 57 | sql.append("(") 58 | sql.append("\""+map.get("website").get.toString+"\"") 59 | sql.append(",\""+map.get("time_local").get.toString+"\"") 60 | sql.append(",\""+map.get("remote_addr").get.toString+"\"") 61 | sql.append(",\""+map.get("remote_port").get.toString+"\"") 62 | sql.append(",\""+map.get("request").get.toString+"\"") 63 | sql.append(",\""+map.get("request_body").get.toString+"\"") 64 | sql.append(",\""+map.get("response_time").get.toString+"\"") 65 | sql.append(",\""+map.get("request_time").get.toString+"\"") 66 | sql.append(",\""+map.get("status").get.toString+"\"") 67 | sql.append(",\""+map.get("body_bytes_sent").get.toString+"\"") 68 | sql.append(",\""+map.get("http_referer").get.toString+"\"") 69 | sql.append(",\""+map.get("http_user_agent").get.toString+"\"") 70 | sql.append(",\""+map.get("headers").get.toString+"\"") 71 | sql.append(",\""+map.get("jsession").get.toString +"\"") 72 | sql.append(",\""+ map.get("http_cookie").get.toString +"\"") 73 | sql.append(",\""+map.get("resp_body").get.toString+"\"") 74 | sql.append(")") 75 | 76 | sql.toString 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /SparkStreaming_1.5.2-Kafka_0.8.1.1/src/main/scala/org/hna/stream/KafkaStreaming.scala: -------------------------------------------------------------------------------- 1 | package org.hna.stream 2 | 3 | import java.net.URLDecoder 4 | import org.hna.dao.MysqlDao 5 | import org.hna.utils.Config 6 | import org.apache.spark._ 7 | import org.apache.spark.streaming._ 8 | import org.apache.spark.streaming.dstream.DStream 9 | import org.apache.spark.streaming.kafka.KafkaUtils 10 | 11 | import scala.util.parsing.json._ 12 | 13 | object KafkaStreaming { 14 | val mysql = new MysqlDao() 15 | def main(args: Array[String]) { 16 | val sparkconf = new SparkConf().setAppName("kafka").setMaster(Config.master) 17 | val sc = new SparkContext(sparkconf) 18 | val ssc = new StreamingContext(sc,Seconds(5)) 19 | ssc.checkpoint(Config.checkpoint) 20 | 21 | val zkList = Config.zklist 22 | val groupId = "test-consumer-group" 23 | val topicMap = Config.kafkatopic.split(",").map(x => (x,2)).toMap 24 | val lines = KafkaUtils.createStream(ssc,zkList,groupId,topicMap).map(_._2) 25 | parseAndSave(lines) 26 | //lines.print() 27 | ssc.start() 28 | ssc.awaitTermination() 29 | } 30 | 31 | def parseAndSave(lines : DStream[String]) = { 32 | lines.foreachRDD(rdd => { 33 | rdd.foreach(f = record => { 34 | var line = record.toString 35 | //println("line=>"+line) 36 | try { 37 | //解码两次 38 | line = URLDecoder.decode(line, "utf-8") 39 | line = URLDecoder.decode(line, "utf-8") 40 | 41 | val json = JSON.parseFull(line) 42 | json match { 43 | case Some(map: Map[String, Any]) => 44 | val sql = getSql(map) 45 | mysql.save(sql) 46 | case None => println("json parse error !!! => None") 47 | } 48 | } catch { 49 | case e: Exception => e.printStackTrace() 50 | println("json parse error !!! ") 51 | } 52 | }) 53 | }) 54 | } 55 | def getSql(map:Map[String,Any]) : String ={ 56 | val sql = new StringBuffer() 57 | sql.append("INSERT INTO request_log (website,time_local,remote_addr,remote_port,request,request_body,response_time,request_time,status,body_bytes_sent,http_referer,http_user_agent,headers,jsession,http_cookie,resp_body)") 58 | sql.append("VALUES") 59 | sql.append("(") 60 | sql.append("\""+map.get("website").get.toString+"\"") 61 | sql.append(",\""+map.get("time_local").get.toString+"\"") 62 | sql.append(",\""+map.get("remote_addr").get.toString+"\"") 63 | sql.append(",\""+map.get("remote_port").get.toString+"\"") 64 | sql.append(",\""+map.get("request").get.toString+"\"") 65 | sql.append(",\""+map.get("request_body").get.toString+"\"") 66 | sql.append(",\""+map.get("response_time").get.toString+"\"") 67 | sql.append(",\""+map.get("request_time").get.toString+"\"") 68 | sql.append(",\""+map.get("status").get.toString+"\"") 69 | sql.append(",\""+map.get("body_bytes_sent").get.toString+"\"") 70 | sql.append(",\""+map.get("http_referer").get.toString+"\"") 71 | sql.append(",\""+map.get("http_user_agent").get.toString+"\"") 72 | sql.append(",\""+map.get("headers").get.toString+"\"") 73 | sql.append(",\""+map.get("jsession").get.toString +"\"") 74 | sql.append(",\""+ map.get("http_cookie").get.toString +"\"") 75 | sql.append(",\""+map.get("resp_body").get.toString+"\"") 76 | sql.append(")") 77 | 78 | sql.toString 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/utils/DateUtil.java: -------------------------------------------------------------------------------- 1 | package com.gh.utils; 2 | 3 | import org.joda.time.DateTimeZone; 4 | 5 | import java.text.ParseException; 6 | import java.text.SimpleDateFormat; 7 | import java.time.ZonedDateTime; 8 | import java.util.Calendar; 9 | import java.util.Date; 10 | import java.util.TimeZone; 11 | 12 | /** 13 | * Created by GH-GAN on 2016/11/24. 14 | */ 15 | public class DateUtil { 16 | 17 | /*public static SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 18 | public static SimpleDateFormat df2 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); 19 | 20 | public static SimpleDateFormat df_utc_1 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); 21 | public static SimpleDateFormat df_utc_2 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); 22 | public static SimpleDateFormat df_utc_3 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); 23 | public static SimpleDateFormat df_utc_base3 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSFFFFFF'Z'"); 24 | 25 | public static TimeZone utc_0 = TimeZone.getTimeZone("UTC"); 26 | 27 | static { 28 | df_utc_1.setTimeZone(utc_0); 29 | df_utc_2.setTimeZone(utc_0); 30 | df_utc_3.setTimeZone(utc_0); 31 | df_utc_base3.setTimeZone(utc_0); 32 | } 33 | 34 | public static String formatToUTC_0(String timestap){ 35 | try { 36 | Date parse = df.parse(timestap); 37 | return df_utc_2.format(parse); 38 | } catch (ParseException e) { 39 | e.printStackTrace(); 40 | } 41 | return null; 42 | } 43 | 44 | public static String formatToGeneral(String timestap){ 45 | try { 46 | Date parse = df_utc_2.parse(timestap); 47 | return df.format(parse); 48 | } catch (ParseException e) { 49 | e.printStackTrace(); 50 | } 51 | return null; 52 | } 53 | 54 | public static String baseUTCToGeneral(String timestap){ 55 | try { 56 | Date parse = df_utc_1.parse(timestap); 57 | return df.format(parse); 58 | } catch (ParseException e) { 59 | e.printStackTrace(); 60 | } 61 | return null; 62 | } 63 | */ 64 | public static String getYYYYMMdd(String timestamp){ 65 | ZonedDateTime zdt = ZonedDateTime.parse(timestamp); 66 | ZonedDateTime localnow = ZonedDateTime.now(); 67 | 68 | int local_offset_sec = localnow.getOffset().getTotalSeconds(); 69 | int zdt_offset_sec = zdt.getOffset().getTotalSeconds(); 70 | 71 | StringBuffer sb = new StringBuffer(); 72 | if (local_offset_sec != zdt_offset_sec){ 73 | int sec = local_offset_sec - zdt_offset_sec; 74 | ZonedDateTime _new_zdt = zdt.withZoneSameLocal(localnow.getZone()); 75 | ZonedDateTime new_zdt = _new_zdt.plusSeconds(sec); 76 | sb.append(new_zdt.getYear()); 77 | if (new_zdt.getMonthValue() < 10){ 78 | sb.append("-0" + new_zdt.getMonthValue()); 79 | }else { 80 | sb.append("-" + new_zdt.getMonthValue()); 81 | } 82 | if (new_zdt.getDayOfMonth() < 10){ 83 | sb.append("-0" + new_zdt.getDayOfMonth()); 84 | }else { 85 | sb.append("-" + new_zdt.getDayOfMonth()); 86 | } 87 | }else { 88 | sb.append(zdt.getYear()); 89 | if (zdt.getMonthValue() < 10){ 90 | sb.append("-0" + zdt.getMonthValue()); 91 | }else { 92 | sb.append("-" + zdt.getMonthValue()); 93 | } 94 | if (zdt.getDayOfMonth() < 10){ 95 | sb.append("-0" + zdt.getDayOfMonth()); 96 | }else { 97 | sb.append("-" + zdt.getDayOfMonth()); 98 | } 99 | } 100 | return sb.toString(); 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/scala/com/gh/yun/SaveToESLog2.scala: -------------------------------------------------------------------------------- 1 | package com.gh.yun 2 | 3 | /** 4 | * Created by GH-GAN on 2016/6/3. 5 | * 保存入es时，将app和容器只分为两个队列 6 | */ 7 | 8 | import com.gh.utils.{ConfigUtil, JsonUtil} 9 | import org.apache.kafka.common.serialization.StringDeserializer 10 | import org.apache.log4j.{Level, Logger} 11 | import org.apache.spark.SparkConf 12 | import org.apache.spark.streaming.dstream.DStream 13 | import org.apache.spark.streaming.kafka010.ConsumerStrategies._ 14 | import org.apache.spark.streaming.kafka010.KafkaUtils 15 | import org.apache.spark.streaming.kafka010.LocationStrategies._ 16 | import org.apache.spark.streaming.{Seconds, StreamingContext} 17 | import org.codehaus.jackson.JsonNode 18 | import org.elasticsearch.spark.streaming._ 19 | 20 | object SaveToESLog2 { 21 | def main(args: Array[String]) { 22 | Logger.getRootLogger.setLevel(Level.WARN) 23 | val conf = new SparkConf().setMaster(ConfigUtil.sparkmaster).setAppName("saveToesLog"); 24 | conf.set("es.index.auto.create", "true") 25 | conf.set("es.nodes", ConfigUtil.esnodes) 26 | 27 | val ssc = new StreamingContext(conf,Seconds(ConfigUtil.logStreamSaveTime)) 28 | ssc.checkpoint(ConfigUtil.saveLogcheckpoint) 29 | 30 | var brokers = ConfigUtil.brokers 31 | val _topics = "log-file".split(",").toSet 32 | val group = "saveToesLog" 33 | 34 | val kafkaParams = Map[String, Object]( 35 | "bootstrap.servers" -> brokers, 36 | "key.deserializer" -> classOf[StringDeserializer], 37 | "value.deserializer" -> classOf[StringDeserializer], 38 | "group.id" -> group, 39 | "auto.offset.reset" -> "latest", 40 | "heartbeat.interval.ms" -> "6000", 41 | "session.timeout.ms" -> "20000", 42 | "max.partition.fetch.bytes" -> "10485760", //1048576000 43 | "max.poll.records" -> "10000", // message.max.bytes 44 | "enable.auto.commit" -> (false: java.lang.Boolean) 45 | ) 46 | 47 | val stream = KafkaUtils.createDirectStream[String, String]( 48 | ssc, 49 | PreferConsistent, 50 | Subscribe[String, String](_topics, kafkaParams) 51 | ) 52 | 53 | val datas = stream.map(line => { 54 | var node : JsonNode = JsonUtil.getJsonNode(line.value()) 55 | (node,line.value()) 56 | }).filter(x => x._1 != null && x._1.get("type") != null) 57 | //environment_id/container_uuid not null/"" 58 | .filter(x => { 59 | var environment_id = "" 60 | var container_uuid = "" 61 | try{ 62 | val data = x._1.get("data") 63 | environment_id = data.get("environment_id").asText() 64 | container_uuid = data.get("container_uuid").asText() 65 | }catch { case ex : Exception => ex.printStackTrace() } 66 | 67 | !"".equals(environment_id) && !"".equals(container_uuid) 68 | }) 69 | .map(node => (node._1.get("type").asText(),node._2)) 70 | 71 | // log 72 | ParseLogContainerSaveToEs(datas) // log_container 73 | ParseLogFileSaveToEs(datas) // log_app 74 | 75 | /*datas.foreachRDD(x => { 76 | x.foreach(y => println(y)) 77 | })*/ 78 | 79 | ssc.start() 80 | ssc.awaitTermination() 81 | 82 | } 83 | 84 | def ParseLogContainerSaveToEs(datas : DStream[(String,String)]): Unit ={ 85 | val container = datas.filter(_._1.equals("container")) 86 | container.map(_._2).saveJsonToEs("yun-{data.environment_id}-{data.container_uuid}-containerlogfile-container-{data.log_info.log_time:YYYY-MM-dd}/container-log",Map("es.batch.size.bytes"->"1048576","es.batch.size.entries"->"1000","es.batch.write.refresh"->"false")) 87 | } 88 | 89 | def ParseLogFileSaveToEs(datas : DStream[(String,String)]): Unit ={ 90 | val logfile = datas.filter(x => x._1.equals("nginx") || x._1.equals("mysql") || x._1.equals("redis")) 91 | logfile.map(_._2).saveJsonToEs("yun-{data.environment_id}-{data.container_uuid}-applogfile-{type}-{data.log_info.log_time:YYYY-MM-dd}/{type}-log",Map("es.batch.size.bytes"->"1048576","es.batch.size.entries"->"1000","es.batch.write.refresh"->"false")) 92 | } 93 | 94 | } 95 | -------------------------------------------------------------------------------- /SparkStreaming_1.5.2-Kafka_0.8.1.1/src/main/java/org/hna/domain/RequestLog.java: -------------------------------------------------------------------------------- 1 | package org.hna.domain; 2 | 3 | public class RequestLog { 4 | 5 | /** 6 | * { 7 | * "website":"JDBJPRD000", 8 | * "time_local":"02/Dec/2015:23:59:02 +0800", 9 | * "remote_addr":"120.52.73.1", 10 | * "remote_port":"41494", 11 | * "request":"/403.html", 12 | * "request_body":"dstCity=PEK&dstCity_m=&orgCity_m=&flightDate=2016-01-07&orgCity=LHW&tripType=ONEWAY&returnDate=", 13 | * "response_time":"2015-12-02 23:59:02", 14 | * "request_time":"", 15 | * "status":"403", 16 | * "body_bytes_sent":"2754", 17 | * "http_referer":"http://www.jdair.net/index.jsp", 18 | * "http_user_agent":"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.154 Safari/537.36 LBBROWSER", 19 | * "headers":"", 20 | * "jsession":"-", 21 | * "http_cookie":"-", 22 | * "resp_body":"" 23 | * } 24 | */ 25 | private String website; 26 | private String time_local; 27 | private String remote_addr; 28 | private String remote_port; 29 | private String request; 30 | private String request_body; 31 | private String response_time; 32 | private String request_time; 33 | private String status; 34 | private String body_bytes_sent; 35 | private String http_referer; 36 | private String http_user_agent; 37 | private String headers; 38 | private String jsession; 39 | private String http_cookie; 40 | private String resp_body; 41 | 42 | public String getResp_body() { 43 | return resp_body; 44 | } 45 | public void setResp_body(String resp_body) { 46 | this.resp_body = resp_body; 47 | } 48 | public String getWebsite() { 49 | return website; 50 | } 51 | public void setWebsite(String website) { 52 | this.website = website; 53 | } 54 | public String getTime_local() { 55 | return time_local; 56 | } 57 | public void setTime_local(String time_local) { 58 | this.time_local = time_local; 59 | } 60 | public String getRemote_addr() { 61 | return remote_addr; 62 | } 63 | public void setRemote_addr(String remote_addr) { 64 | this.remote_addr = remote_addr; 65 | } 66 | public String getRemote_port() { 67 | return remote_port; 68 | } 69 | public void setRemote_port(String remote_port) { 70 | this.remote_port = remote_port; 71 | } 72 | public String getRequest() { 73 | return request; 74 | } 75 | public void setRequest(String request) { 76 | this.request = request; 77 | } 78 | public String getRequest_body() { 79 | return request_body; 80 | } 81 | public void setRequest_body(String request_body) { 82 | this.request_body = request_body; 83 | } 84 | public String getResponse_time() { 85 | return response_time; 86 | } 87 | public void setResponse_time(String response_time) { 88 | this.response_time = response_time; 89 | } 90 | public String getRequest_time() { 91 | return request_time; 92 | } 93 | public void setRequest_time(String request_time) { 94 | this.request_time = request_time; 95 | } 96 | public String getStatus() { 97 | return status; 98 | } 99 | public void setStatus(String status) { 100 | this.status = status; 101 | } 102 | public String getBody_bytes_sent() { 103 | return body_bytes_sent; 104 | } 105 | public void setBody_bytes_sent(String body_bytes_sent) { 106 | this.body_bytes_sent = body_bytes_sent; 107 | } 108 | public String getHttp_referer() { 109 | return http_referer; 110 | } 111 | public void setHttp_referer(String http_referer) { 112 | this.http_referer = http_referer; 113 | } 114 | public String getHttp_user_agent() { 115 | return http_user_agent; 116 | } 117 | public void setHttp_user_agent(String http_user_agent) { 118 | this.http_user_agent = http_user_agent; 119 | } 120 | public String getHeaders() { 121 | return headers; 122 | } 123 | public void setHeaders(String headers) { 124 | this.headers = headers; 125 | } 126 | public String getJsession() { 127 | return jsession; 128 | } 129 | public void setJsession(String jsession) { 130 | this.jsession = jsession; 131 | } 132 | public String getHttp_cookie() { 133 | return http_cookie; 134 | } 135 | public void setHttp_cookie(String http_cookie) { 136 | this.http_cookie = http_cookie; 137 | } 138 | 139 | 140 | } 141 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/scala/com/gh/yun/SaveToESCap3.scala: -------------------------------------------------------------------------------- 1 | package com.gh.yun 2 | 3 | /** 4 | * Created by GH-GAN on 2016/6/3. 5 | * 保存入es时，将app和容器只分为两个队列 6 | * 整理index，indexName： yun-环境id-日志类型-时间/容器id 7 | */ 8 | 9 | import com.gh.utils.{ConfigUtil, JsonUtil} 10 | import com.gh.yun.parse.{ParseContainer, ParseMysql, ParseNginx, ParseRedis} 11 | import org.apache.kafka.common.serialization.StringDeserializer 12 | import org.apache.log4j.{Level, Logger} 13 | import org.apache.spark.SparkConf 14 | import org.apache.spark.streaming.dstream.DStream 15 | import org.apache.spark.streaming.kafka010.ConsumerStrategies._ 16 | import org.apache.spark.streaming.kafka010.KafkaUtils 17 | import org.apache.spark.streaming.kafka010.LocationStrategies._ 18 | import org.apache.spark.streaming.{Seconds, StreamingContext} 19 | import org.codehaus.jackson.JsonNode 20 | import org.elasticsearch.spark.streaming._ 21 | 22 | import scala.collection.mutable.ArrayBuffer 23 | 24 | object SaveToESCap3 { 25 | def main(args: Array[String]) { 26 | Logger.getRootLogger.setLevel(Level.WARN) 27 | val conf = new SparkConf().setMaster(ConfigUtil.sparkmaster).setAppName("saveToesCap"); 28 | conf.set("es.index.auto.create", "true") 29 | conf.set("es.nodes", ConfigUtil.esnodes) 30 | conf.set("spark.streaming.kafka.consumer.poll.ms", "1024") 31 | 32 | val ssc = new StreamingContext(conf,Seconds(ConfigUtil.capStreamSaveTime)) 33 | ssc.checkpoint(ConfigUtil.saveCapcheckpoint) 34 | 35 | var brokers = ConfigUtil.brokers 36 | val _topics = "capability-mysql,capability-redis,capability-container,capability-nginx".split(",").toSet 37 | // val _topics = "capability-container".split(",").toSet 38 | val group = "saveToesCap" 39 | 40 | val kafkaParams = Map[String, Object]( 41 | "bootstrap.servers" -> brokers, 42 | "key.deserializer" -> classOf[StringDeserializer], 43 | "value.deserializer" -> classOf[StringDeserializer], 44 | "group.id" -> group, 45 | "auto.offset.reset" -> "latest", 46 | "heartbeat.interval.ms" -> "6000", 47 | "session.timeout.ms" -> "20000", 48 | "max.partition.fetch.bytes" -> "10485760", 49 | "max.poll.records" -> "10000", // message.max.bytes 50 | "enable.auto.commit" -> (false: java.lang.Boolean) 51 | ) 52 | 53 | val stream = KafkaUtils.createDirectStream[String, String]( 54 | ssc, 55 | PreferConsistent, 56 | Subscribe[String, String](_topics, kafkaParams) 57 | ) 58 | 59 | val datas = stream.map(line => { 60 | var node : JsonNode = JsonUtil.getJsonNode(line.value()) 61 | node 62 | }).filter(x => x != null && x.get("type") != null) 63 | .map(node => (node.get("type").asText(),node)) 64 | 65 | // cap 66 | ParseContainerSaveToEs(datas) 67 | ParseAppSaveToEs(datas) 68 | 69 | // datas.foreachRDD(x => { 70 | // x.foreach(y => println(y._1 + "============================================" + y._2)) 71 | // }) 72 | 73 | ssc.start() 74 | ssc.awaitTermination() 75 | 76 | } 77 | 78 | def ParseAppSaveToEs(datas : DStream[(String,JsonNode)]): Unit ={ 79 | val app = datas.filter(x => x._1.equals("nginx") || x._1.equals("mysql") || x._1.equals("redis")) 80 | app.flatMap(lines => { 81 | val arr = ArrayBuffer[String]() 82 | if (lines._1.equals("nginx")){ 83 | arr.++=(new ParseNginx().statsToArray(lines._1,lines._2)) 84 | } 85 | if (lines._1.equals("mysql")){ 86 | arr.++=(new ParseMysql().statsToArray(lines._1,lines._2)) 87 | } 88 | if (lines._1.equals("redis")){ 89 | arr.++=(new ParseRedis().statsToArray(lines._1,lines._2)) 90 | } 91 | arr 92 | }).saveJsonToEs("yun-{data.environment_id}-app-{type}-{data.stats.timestamp:YYYY-MM-dd}/{data.container_uuid}",Map("es.batch.size.bytes"->"1048576","es.batch.size.entries"->"500","es.batch.write.refresh"->"false")) 93 | } 94 | 95 | def ParseContainerSaveToEs(datas : DStream[(String,JsonNode)]): Unit ={ 96 | val container = datas.filter(x => x._1.equals("container")) 97 | val ss = container.flatMap(lines => { 98 | new ParseContainer().statsToArray(lines._1,lines._2) 99 | }).saveJsonToEs("yun-{data.environment_id}-container-container-{data.stats.timestamp:YYYY-MM-dd}/{data.container_uuid}",Map("es.batch.size.bytes"->"1048576","es.batch.size.entries"->"500","es.batch.write.refresh"->"false")) 100 | 101 | } 102 | 103 | } 104 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/scala/com/gh/yun/SaveToESCap2.scala: -------------------------------------------------------------------------------- 1 | package com.gh.yun 2 | 3 | /** 4 | * Created by GH-GAN on 2016/6/3. 5 | * 保存入es时，将app和容器只分为两个队列 6 | */ 7 | 8 | import com.gh.bean.appcapability._ 9 | import com.gh.bean.containercapability._ 10 | import com.gh.utils.{ConfigUtil, JsonUtil} 11 | import com.gh.yun.parse.{ParseContainer, ParseMysql, ParseNginx, ParseRedis} 12 | import org.apache.kafka.common.serialization.StringDeserializer 13 | import org.apache.log4j.{Level, Logger} 14 | import org.apache.spark.SparkConf 15 | import org.apache.spark.streaming.dstream.DStream 16 | import org.apache.spark.streaming.kafka010.ConsumerStrategies._ 17 | import org.apache.spark.streaming.kafka010.KafkaUtils 18 | import org.apache.spark.streaming.kafka010.LocationStrategies._ 19 | import org.apache.spark.streaming.{Seconds, StreamingContext} 20 | import org.codehaus.jackson.JsonNode 21 | import org.elasticsearch.spark.streaming._ 22 | 23 | import scala.collection.mutable.ArrayBuffer 24 | 25 | object SaveToESCap2 { 26 | def main(args: Array[String]) { 27 | Logger.getRootLogger.setLevel(Level.WARN) 28 | val conf = new SparkConf().setMaster(ConfigUtil.sparkmaster).setAppName("saveToesCap"); 29 | conf.set("es.index.auto.create", "true") 30 | conf.set("es.nodes", ConfigUtil.esnodes) 31 | conf.set("spark.streaming.kafka.consumer.poll.ms", "1024") 32 | 33 | val ssc = new StreamingContext(conf,Seconds(ConfigUtil.capStreamSaveTime)) 34 | ssc.checkpoint(ConfigUtil.saveCapcheckpoint) 35 | 36 | var brokers = ConfigUtil.brokers 37 | val _topics = "capability-mysql,capability-redis,capability-container,capability-nginx".split(",").toSet 38 | // val _topics = "capability-container".split(",").toSet 39 | val group = "saveToesCap" 40 | 41 | val kafkaParams = Map[String, Object]( 42 | "bootstrap.servers" -> brokers, 43 | "key.deserializer" -> classOf[StringDeserializer], 44 | "value.deserializer" -> classOf[StringDeserializer], 45 | "group.id" -> group, 46 | "auto.offset.reset" -> "latest", 47 | "heartbeat.interval.ms" -> "6000", 48 | "session.timeout.ms" -> "20000", 49 | "max.partition.fetch.bytes" -> "10485760", 50 | "max.poll.records" -> "10000", // message.max.bytes 51 | "enable.auto.commit" -> (false: java.lang.Boolean) 52 | ) 53 | 54 | val stream = KafkaUtils.createDirectStream[String, String]( 55 | ssc, 56 | PreferConsistent, 57 | Subscribe[String, String](_topics, kafkaParams) 58 | ) 59 | 60 | val datas = stream.map(line => { 61 | var node : JsonNode = JsonUtil.getJsonNode(line.value()) 62 | node 63 | }).filter(x => x != null && x.get("type") != null) 64 | .map(node => (node.get("type").asText(),node)) 65 | 66 | // cap 67 | ParseContainerSaveToEs(datas) 68 | ParseAppSaveToEs(datas) 69 | 70 | // datas.foreachRDD(x => { 71 | // x.foreach(y => println(y._1 + "============================================" + y._2)) 72 | // }) 73 | 74 | ssc.start() 75 | ssc.awaitTermination() 76 | 77 | } 78 | 79 | def ParseAppSaveToEs(datas : DStream[(String,JsonNode)]): Unit ={ 80 | val app = datas.filter(x => x._1.equals("nginx") || x._1.equals("mysql") || x._1.equals("redis")) 81 | app.flatMap(lines => { 82 | val arr = ArrayBuffer[String]() 83 | if (lines._1.equals("nginx")){ 84 | arr.++=(new ParseNginx().statsToArray(lines._1,lines._2)) 85 | } 86 | if (lines._1.equals("mysql")){ 87 | arr.++=(new ParseMysql().statsToArray(lines._1,lines._2)) 88 | } 89 | if (lines._1.equals("redis")){ 90 | arr.++=(new ParseRedis().statsToArray(lines._1,lines._2)) 91 | } 92 | arr 93 | }).saveJsonToEs("yun-{data.environment_id}-{data.container_uuid}-app-{type}-{data.stats.timestamp:YYYY-MM-dd}/{type}-cap",Map("es.batch.size.bytes"->"1048576","es.batch.size.entries"->"500","es.batch.write.refresh"->"false")) 94 | } 95 | 96 | def ParseContainerSaveToEs(datas : DStream[(String,JsonNode)]): Unit ={ 97 | val container = datas.filter(_._1.equals("container")) 98 | val ss = container.flatMap(lines => { 99 | new ParseContainer().statsToArray(lines._1,lines._2) 100 | }).saveJsonToEs("yun-{data.environment_id}-{data.container_uuid}-container-container-{data.stats.timestamp:YYYY-MM-dd}/container-cap",Map("es.batch.size.bytes"->"1048576","es.batch.size.entries"->"500","es.batch.write.refresh"->"false")) 101 | 102 | } 103 | 104 | } 105 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/scala/com/gh/yun/SaveToESLog3.scala: -------------------------------------------------------------------------------- 1 | package com.gh.yun 2 | 3 | /** 4 | * Created by GH-GAN on 2016/6/3. 5 | * 保存入es时，将app和容器只分为两个队列 6 | * 整理index，根据目前单个index数据很少，为减少index数量，将容器id，作为index的type 7 | * indexName： yun-环境id-日志类型-时间/容器id 8 | */ 9 | 10 | import com.gh.utils.{ConfigUtil, JsonUtil} 11 | import org.apache.kafka.common.serialization.StringDeserializer 12 | import org.apache.log4j.{Level, Logger} 13 | import org.apache.spark.SparkConf 14 | import org.apache.spark.streaming.dstream.DStream 15 | import org.apache.spark.streaming.kafka010.ConsumerStrategies._ 16 | import org.apache.spark.streaming.kafka010.KafkaUtils 17 | import org.apache.spark.streaming.kafka010.LocationStrategies._ 18 | import org.apache.spark.streaming.{Seconds, StreamingContext} 19 | import org.codehaus.jackson.JsonNode 20 | import org.elasticsearch.spark.streaming._ 21 | 22 | object SaveToESLog3 { 23 | def main(args: Array[String]) { 24 | Logger.getRootLogger.setLevel(Level.WARN) 25 | val conf = new SparkConf().setMaster(ConfigUtil.sparkmaster).setAppName("saveToesLog"); 26 | conf.set("es.index.auto.create", "true") 27 | conf.set("es.nodes", ConfigUtil.esnodes) 28 | 29 | val ssc = new StreamingContext(conf,Seconds(ConfigUtil.logStreamSaveTime)) 30 | ssc.checkpoint(ConfigUtil.saveLogcheckpoint) 31 | 32 | var brokers = ConfigUtil.brokers 33 | // val _topics = "log-file".split(",").toSet 34 | val _topics = "log-file,custom-log".split(",").toSet 35 | val group = "saveToesLog" 36 | 37 | val kafkaParams = Map[String, Object]( 38 | "bootstrap.servers" -> brokers, 39 | "key.deserializer" -> classOf[StringDeserializer], 40 | "value.deserializer" -> classOf[StringDeserializer], 41 | "group.id" -> group, 42 | "auto.offset.reset" -> "latest", 43 | "heartbeat.interval.ms" -> "6000", 44 | "session.timeout.ms" -> "20000", 45 | "max.partition.fetch.bytes" -> "10485760", //1048576000 46 | "max.poll.records" -> "10000", // message.max.bytes 47 | "enable.auto.commit" -> (false: java.lang.Boolean) 48 | ) 49 | 50 | val stream = KafkaUtils.createDirectStream[String, String]( 51 | ssc, 52 | PreferConsistent, 53 | Subscribe[String, String](_topics, kafkaParams) 54 | ) 55 | 56 | val datas = stream.map(line => { 57 | var node : JsonNode = JsonUtil.getJsonNode(line.value()) 58 | (node,line.value()) 59 | }).filter(x => x._1 != null && x._1.get("type") != null) 60 | //environment_id/container_uuid not null/"" 61 | .filter(x => { 62 | var environment_id = "" 63 | var container_uuid = "" 64 | try{ 65 | val data = x._1.get("data") 66 | environment_id = data.get("environment_id").asText() 67 | container_uuid = data.get("container_uuid").asText() 68 | }catch { case ex : Exception => ex.printStackTrace() } 69 | 70 | !"".equals(environment_id) && !"".equals(container_uuid) 71 | }) 72 | .map(line => (line._1.get("type").asText(),line._2)) 73 | 74 | // log 75 | ParseLogContainerSaveToEs(datas) // log_container 76 | ParseLogFileSaveToEs(datas) // log_file 77 | ParseCustomLogSaveToEs(datas) // custom-log 78 | 79 | /*datas.foreachRDD(x => { 80 | x.foreach(y => println(y)) 81 | })*/ 82 | 83 | ssc.start() 84 | ssc.awaitTermination() 85 | 86 | } 87 | 88 | def ParseLogContainerSaveToEs(datas : DStream[(String,String)]): Unit ={ 89 | val containerLogFile = datas.filter(x => x._1.equals("container")) 90 | containerLogFile.map(_._2).saveJsonToEs("yun-{data.environment_id}-containerlogfile-container-{data.log_info.log_time:YYYY-MM-dd}/{data.container_uuid}",Map("es.batch.size.bytes"->"1048576","es.batch.size.entries"->"1000","es.batch.write.refresh"->"false")) 91 | } 92 | 93 | def ParseLogFileSaveToEs(datas : DStream[(String,String)]): Unit ={ 94 | val appLogFile = datas.filter(x => x._1.equals("nginx") || x._1.equals("mysql") || x._1.equals("redis")) 95 | appLogFile.map(_._2).saveJsonToEs("yun-{data.environment_id}-applogfile-{type}-{data.log_info.log_time:YYYY-MM-dd}/{data.container_uuid}",Map("es.batch.size.bytes"->"1048576","es.batch.size.entries"->"1000","es.batch.write.refresh"->"false")) 96 | } 97 | 98 | def ParseCustomLogSaveToEs(datas : DStream[(String,String)]): Unit ={ 99 | val customlog = datas.filter(x => x._1.equals("custom_log")) 100 | customlog.map(_._2).saveJsonToEs("yun-customlog-{data.log_info.log_time:YYYY-MM-dd}/{data.environment_id}-{data.container_uuid}",Map("es.batch.size.bytes"->"1048576","es.batch.size.entries"->"1000","es.batch.write.refresh"->"false")) 101 | } 102 | 103 | } 104 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/scala/com/gh/yun/parse/ParseNginx.scala: -------------------------------------------------------------------------------- 1 | package com.gh.yun.parse 2 | 3 | import com.gh.bean.appcapability.{AppNginx, AppNginxData, AppNginxStats} 4 | import com.gh.utils.{DateUtil, JsonUtil} 5 | import org.codehaus.jackson.JsonNode 6 | 7 | import scala.collection.mutable.ArrayBuffer 8 | 9 | /** 10 | * Created by GH-GAN on 2016/12/20. 11 | */ 12 | class ParseNginx { 13 | def statsToArray(_type: String,node : JsonNode): ArrayBuffer[String] ={ 14 | val arr = ArrayBuffer[String]() 15 | try{ 16 | var data = node.get("data") 17 | val environment_id = data.get("environment_id").asText() 18 | val container_uuid = data.get("container_uuid").asText() 19 | val container_name = data.get("container_name").asText() 20 | val namespace = data.get("namespace").asText() 21 | val data_timestamp = data.get("timestamp").asText() 22 | 23 | if (!"".equals(environment_id) && !"".equals(container_uuid)){ 24 | var _stats = data.get("stats") 25 | for (i <- 0 to (_stats.size() - 1) ){ 26 | val stats = _stats.get(i) 27 | val timestamp = stats.get("timestamp").asText() 28 | val active_connections = stats.get("active_connections").asDouble() 29 | val accepts = stats.get("accepts").asDouble() 30 | val handled = stats.get("handled").asDouble() 31 | val requests = stats.get("requests").asDouble() 32 | val reading = stats.get("reading").asDouble() 33 | val writing = stats.get("writing").asDouble() 34 | val waiting = stats.get("waiting").asDouble() 35 | 36 | val _ng_stats = new AppNginxStats() 37 | _ng_stats.setTimestamp(timestamp) 38 | _ng_stats.setActive_connections(active_connections) 39 | _ng_stats.setAccepts(accepts) 40 | _ng_stats.setHandled(handled) 41 | _ng_stats.setRequests(requests) 42 | _ng_stats.setReading(reading) 43 | _ng_stats.setWiting(writing) 44 | _ng_stats.setWaiting(waiting) 45 | 46 | val _ng_data = new AppNginxData() 47 | _ng_data.setEnvironment_id(environment_id) 48 | _ng_data.setContainer_uuid(container_uuid) 49 | _ng_data.setContainer_name(container_name) 50 | _ng_data.setNamespace(namespace) 51 | _ng_data.setTimestamp(data_timestamp) 52 | _ng_data.setStats(_ng_stats) 53 | 54 | val _ng = new AppNginx() 55 | _ng.setType(_type) 56 | _ng.setData(_ng_data) 57 | 58 | arr.+=(JsonUtil.formatJson(_ng)) 59 | } 60 | } 61 | }catch { 62 | case ex : Exception => ex.printStackTrace() 63 | } 64 | arr 65 | } 66 | 67 | def statsToArray2(_type: String,node : JsonNode): ArrayBuffer[(String,String)] ={ 68 | val arr = ArrayBuffer[(String,String)]() 69 | try{ 70 | var data = node.get("data") 71 | val environment_id = data.get("environment_id").asText().trim 72 | val container_uuid = data.get("container_uuid").asText().trim 73 | val container_name = data.get("container_name").asText() 74 | val namespace = data.get("namespace").asText() 75 | val data_timestamp = data.get("timestamp").asText() 76 | 77 | if (!"".equals(environment_id) && !"".equals(container_uuid)){ 78 | var _stats = data.get("stats") 79 | for (i <- 0 until _stats.size()){ 80 | val stats = _stats.get(i) 81 | val timestamp = stats.get("timestamp").asText() 82 | val active_connections = stats.get("active_connections").asDouble() 83 | val accepts = stats.get("accepts").asDouble() 84 | val handled = stats.get("handled").asDouble() 85 | val requests = stats.get("requests").asDouble() 86 | val reading = stats.get("reading").asDouble() 87 | val writing = stats.get("writing").asDouble() 88 | val waiting = stats.get("waiting").asDouble() 89 | 90 | val _ng_stats = new AppNginxStats() 91 | _ng_stats.setTimestamp(timestamp) 92 | _ng_stats.setActive_connections(active_connections) 93 | _ng_stats.setAccepts(accepts) 94 | _ng_stats.setHandled(handled) 95 | _ng_stats.setRequests(requests) 96 | _ng_stats.setReading(reading) 97 | _ng_stats.setWiting(writing) 98 | _ng_stats.setWaiting(waiting) 99 | 100 | val _ng_data = new AppNginxData() 101 | _ng_data.setEnvironment_id(environment_id) 102 | _ng_data.setContainer_uuid(container_uuid) 103 | _ng_data.setContainer_name(container_name) 104 | _ng_data.setNamespace(namespace) 105 | _ng_data.setTimestamp(data_timestamp) 106 | _ng_data.setStats(_ng_stats) 107 | 108 | val _ng = new AppNginx() 109 | _ng.setType(_type) 110 | _ng.setData(_ng_data) 111 | 112 | // index yun-{data.environment_id}-{data.container_uuid}-app-nginx-{data.stats.timestamp:YYYY-MM-dd} 113 | val index = "yun-" + environment_id + "-" + container_uuid + "-app-nginx-" + DateUtil.getYYYYMMdd(timestamp) 114 | arr.+=((index,JsonUtil.formatJson(_ng))) 115 | } 116 | } 117 | }catch { 118 | case ex : Exception => ex.printStackTrace() 119 | } 120 | arr 121 | } 122 | 123 | } 124 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/utils/HttpUtil.java: -------------------------------------------------------------------------------- 1 | package com.gh.utils; 2 | 3 | import com.gh.bean.GZ; 4 | import com.gh.bean.alert.AlertDataInfo; 5 | import org.apache.log4j.Logger; 6 | import org.codehaus.jackson.JsonNode; 7 | 8 | import java.io.*; 9 | import java.net.HttpURLConnection; 10 | import java.net.URL; 11 | import java.net.URLConnection; 12 | import java.util.HashMap; 13 | import java.util.Map; 14 | import java.util.concurrent.Executors; 15 | import java.util.concurrent.ScheduledExecutorService; 16 | import java.util.concurrent.TimeUnit; 17 | 18 | /** 19 | * Created by GH-GAN on 2016/11/28. 20 | */ 21 | public class HttpUtil { 22 | URLConnection conn = null; 23 | private static Logger logger = Logger.getLogger(HttpUtil.class); 24 | 25 | public static Map gz_map = new HashMap(); 26 | 27 | static { 28 | gz_map.put("app_mysql_connection",new GZ("connection","GT",0.89)); 29 | gz_map.put("app_redis_memory",new GZ("memory","GT",0.8)); 30 | gz_map.put("app_redis_hits",new GZ("hits","GT",0.6)); 31 | gz_map.put("app_nginx_accept",new GZ("accept","LTE",0.8)); 32 | 33 | gz_map.put("container_cpu",new GZ("cpu","GT",0.9)); 34 | gz_map.put("container_memory",new GZ("memory","GT",0.7)); 35 | gz_map.put("container_network_tx",new GZ("network_tx","GT",2.0)); 36 | gz_map.put("container_network_rx",new GZ("network_rx","GT",2.0)); 37 | gz_map.put("container_disk",new GZ("disk","GT",0.8)); 38 | 39 | ScheduledExecutorService ses = Executors.newScheduledThreadPool(1); 40 | ses.scheduleAtFixedRate(new Runnable() { 41 | public void run() { 42 | initGJ(); 43 | } 44 | },1,60, TimeUnit.SECONDS); 45 | 46 | } 47 | public static void initGJ(){ 48 | String gz = HttpUtil.Get(ConfigUtil.gjGetUrl); 49 | JsonNode jsonNode = JsonUtil.getJsonNode(gz); 50 | JsonNode app = jsonNode.get("app"); 51 | for (int i=0;i < app.size() - 1 ; i++){ 52 | JsonNode jsonNode1 = app.get(i); 53 | String app_type = jsonNode1.get("app_type").asText(); 54 | JsonNode app_param = jsonNode1.get("app_param"); 55 | for (int j=0;j ") 27 | System.exit(1) 28 | }*/ 29 | Logger.getRootLogger.setLevel(Level.WARN) 30 | 31 | val sparkConf = new SparkConf().setAppName("capability-nginx").setMaster(ConfigUtil.sparkmaster) 32 | val ssc = new StreamingContext(sparkConf, Seconds(ConfigUtil.capStreamtime)) 33 | ssc.checkpoint(ConfigUtil.nginxCapcheckpoint) 34 | 35 | var brokers = ConfigUtil.brokers 36 | val _topics = "capability-nginx".split(",").toSet 37 | val group = "capability-nginx-js" 38 | 39 | val kafkaParams = Map[String, Object]( 40 | "bootstrap.servers" -> brokers, 41 | "key.deserializer" -> classOf[StringDeserializer], 42 | "value.deserializer" -> classOf[StringDeserializer], 43 | "group.id" -> group, 44 | "auto.offset.reset" -> "latest", //当前偏移不在服务器上时,按最新开始 45 | "heartbeat.interval.ms" -> "6000", 46 | "session.timeout.ms" -> "20000", 47 | "max.partition.fetch.bytes" -> "1048576000", 48 | "max.poll.records" -> "5000000", // message.max.bytes 49 | "enable.auto.commit" -> (false: java.lang.Boolean) 50 | ) 51 | 52 | val stream = KafkaUtils.createDirectStream[String, String]( 53 | ssc, 54 | PreferConsistent, 55 | Subscribe[String, String](_topics, kafkaParams) 56 | ) 57 | 58 | val datas = parseLog(stream) 59 | val mysql_group = datas.groupByKey() 60 | compute(mysql_group) 61 | 62 | ssc.start() 63 | ssc.awaitTermination() 64 | } 65 | 66 | //一条包含多条记录 67 | def parseLog(stream : InputDStream[ConsumerRecord[String, String]]) : DStream[(String,(String,Double,Double))] ={ 68 | val datas = stream.map(line => { 69 | var node : JsonNode = JsonUtil.getJsonNode(line.value()) 70 | node 71 | }).filter(x => (x != null && x.get("type") != null)) 72 | .filter(x => { 73 | val _type = x.get("type").asText().trim 74 | var environment_id = "" 75 | var container_uuid = "" 76 | try{ 77 | val data = x.get("data") 78 | environment_id = data.get("environment_id").asText().trim 79 | container_uuid = data.get("container_uuid").asText().trim 80 | }catch { case ex : Exception => ex.printStackTrace() } 81 | 82 | !"".equals(_type) && !"".equals(environment_id) && !"".equals(container_uuid) && null != environment_id && null != container_uuid 83 | }) 84 | .flatMap(node => { 85 | val _type = node.get("type").asText() 86 | val environment_id = node.get("data").get("environment_id").asText() 87 | val container_uuid = node.get("data").get("container_uuid").asText() 88 | 89 | var container_name = "" 90 | var namespace = "" 91 | try { 92 | container_name = node.get("data").get("container_name").asText() 93 | namespace = node.get("data").get("namespace").asText() 94 | }catch { 95 | case ex : Exception => { 96 | println("--------> container_name/namespace is null") 97 | ex.printStackTrace() 98 | } 99 | } 100 | 101 | val _stats = node.get("data").get("stats") 102 | val arr = ArrayBuffer[(String,(String,Double,Double))]() 103 | for (i <- 0 to (_stats.size() - 1) ){ 104 | val stats = node.get("data").get("stats").get(i) 105 | val timestamp = stats.get("timestamp").asText() 106 | val accepts = stats.get("accepts").asDouble() 107 | val requests = stats.get("requests").asDouble() 108 | arr.+=((environment_id+"#"+container_uuid+"#"+_type + "#" + container_name + "#" + namespace, (timestamp,accepts,requests))) 109 | } 110 | arr 111 | }) 112 | datas 113 | } 114 | 115 | def compute(mysql_group : DStream[(String,Iterable[(String,Double,Double)])]): Unit ={ 116 | val warn = mysql_group.map(x => { 117 | val count = x._2.size 118 | 119 | val start_log = x._2.head 120 | val end_log = x._2.last 121 | val start_time = start_log._1 122 | val end_time = end_log._1 123 | 124 | // accepts 125 | val accepts_sum = x._2.map(_._2).reduce(_+_) 126 | val accepts_avg = accepts_sum / count 127 | 128 | // requests 129 | val requests_sum = x._2.map(_._3).reduce(_+_) 130 | val requests_avg = requests_sum / count 131 | 132 | val accept = accepts_avg / requests_avg 133 | 134 | (x._1,start_time,end_time,accept) 135 | }).map(x => aAlert(x)) // 阈值 136 | .filter(_._4.size() > 0) // 是否有告警信息 137 | 138 | warn.foreachRDD( record => { 139 | val alerts = record.map(line => { 140 | AlertInfoUtil.toWarnBean(AlertInfoUtil.SUCCESS,AlertInfoUtil.ALERT_TYPE_M,AlertInfoUtil.ALERT_DIM_A,line._1, line._2,line._3,line._4,line._5) 141 | }) 142 | if (!alerts.isEmpty()){ 143 | val collect = alerts.collect() 144 | if (collect.size > 0) new HttpUtil().alerts(collect) //告警 145 | } 146 | }) 147 | 148 | } 149 | 150 | def aAlert(line : (String,String,String,Double)): (String,String,String,util.ArrayList[KeyValue],String) ={ 151 | val list = new util.ArrayList[KeyValue]() 152 | 153 | val gz = HttpUtil.gz_map.get("app_nginx_accept") 154 | gz.getCondition match { 155 | case "GTE" => if(line._4 >= gz.getValue) list.add(new KeyValue("accept",line._4.toString)) 156 | case "GT" => if(line._4 > gz.getValue) list.add(new KeyValue("accept",line._4.toString)) 157 | case "LTE" => if(line._4 <= gz.getValue) list.add(new KeyValue("accept",line._4.toString)) 158 | case "LT" => if(line._4 < gz.getValue) list.add(new KeyValue("accept",line._4.toString)) 159 | case "EQ" => if(line._4 == gz.getValue) list.add(new KeyValue("accept",line._4.toString)) 160 | } 161 | 162 | (line._1,line._2,line._3,list,"") 163 | } 164 | 165 | } 166 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/scala/com/gh/yun/SaveToESLog.scala: -------------------------------------------------------------------------------- 1 | package com.gh.yun 2 | 3 | /** 4 | * Created by GH-GAN on 2016/6/3. 5 | */ 6 | 7 | import java.util 8 | 9 | import com.gh.bean.appcapability._ 10 | import com.gh.bean.containercapability._ 11 | import com.gh.bean.logfile.{LogFileContainer, LogFileMySql, LogFileNginx, LogFileRedis} 12 | import com.gh.utils.{ConfigUtil, DateUtil, JsonUtil} 13 | import org.apache.kafka.common.serialization.StringDeserializer 14 | import org.apache.log4j.{Level, Logger} 15 | import org.apache.spark.SparkConf 16 | import org.apache.spark.streaming.dstream.DStream 17 | import org.apache.spark.streaming.kafka010.ConsumerStrategies._ 18 | import org.apache.spark.streaming.kafka010.KafkaUtils 19 | import org.apache.spark.streaming.kafka010.LocationStrategies._ 20 | import org.apache.spark.streaming.{Seconds, StreamingContext} 21 | import org.codehaus.jackson.JsonNode 22 | import org.elasticsearch.spark.streaming._ 23 | import org.elasticsearch.spark._ 24 | import scala.collection.mutable.ArrayBuffer 25 | 26 | object SaveToESLog { 27 | def main(args: Array[String]) { 28 | Logger.getRootLogger.setLevel(Level.WARN) 29 | val conf = new SparkConf().setMaster(ConfigUtil.sparkmaster).setAppName("saveToesLog"); 30 | conf.set("es.index.auto.create", "true") 31 | conf.set("es.nodes", ConfigUtil.esnodes) 32 | 33 | val ssc = new StreamingContext(conf,Seconds(ConfigUtil.logStreamSaveTime)) 34 | ssc.checkpoint(ConfigUtil.saveLogcheckpoint) 35 | 36 | var brokers = ConfigUtil.brokers 37 | val _topics = "log-file".split(",").toSet 38 | val group = "saveToesLog" 39 | 40 | val kafkaParams = Map[String, Object]( 41 | "bootstrap.servers" -> brokers, 42 | "key.deserializer" -> classOf[StringDeserializer], 43 | "value.deserializer" -> classOf[StringDeserializer], 44 | "group.id" -> group, 45 | "auto.offset.reset" -> "latest", 46 | "heartbeat.interval.ms" -> "6000", 47 | "session.timeout.ms" -> "20000", 48 | "max.partition.fetch.bytes" -> "10485760", //1048576000 49 | "max.poll.records" -> "10000", // message.max.bytes 50 | "enable.auto.commit" -> (false: java.lang.Boolean) 51 | ) 52 | 53 | val stream = KafkaUtils.createDirectStream[String, String]( 54 | ssc, 55 | PreferConsistent, 56 | Subscribe[String, String](_topics, kafkaParams) 57 | ) 58 | 59 | val datas = stream.map(line => { 60 | var node : JsonNode = JsonUtil.getJsonNode(line.value()) 61 | (node,line.value()) 62 | }).filter(x => x._1 != null && x._1.get("type") != null) 63 | //environment_id/container_uuid not null/"" 64 | .filter(x => { 65 | var environment_id = "" 66 | var container_uuid = "" 67 | try{ 68 | val data = x._1.get("data") 69 | environment_id = data.get("environment_id").asText() 70 | container_uuid = data.get("container_uuid").asText() 71 | }catch { case ex : Exception => ex.printStackTrace() } 72 | 73 | !"".equals(environment_id) && !"".equals(container_uuid) 74 | }) 75 | .map(node => (node._1.get("type").asText(),node._2)) 76 | 77 | // log 78 | ParseLogContainerSaveToEs2(datas) // log_container 79 | ParseLogNginxSaveToEs2(datas) // log_nginx 80 | ParseLogMySqlSaveToEs2(datas) // log_mysql 81 | ParseLogRedisSaveToEs2(datas) // log_redis 82 | 83 | /*datas.foreachRDD(x => { 84 | x.foreach(y => println(y)) 85 | })*/ 86 | 87 | ssc.start() 88 | ssc.awaitTermination() 89 | 90 | } 91 | 92 | def ParseLogContainerSaveToEs2(datas : DStream[(String,String)]): Unit ={ 93 | val container = datas.filter(_._1.equals("container")) 94 | container.map(_._2).saveJsonToEs("yun-{data.environment_id}-{data.container_uuid}-containerlogfile-container-{data.log_info.log_time:YYYY-MM-dd}/container-log",Map("es.batch.size.bytes"->"2097152","es.batch.size.entries"->"500","es.batch.write.refresh"->"false")) 95 | } 96 | 97 | /*def ParseLogContainerSaveToEs(datas : DStream[(String,String)]): Unit ={ 98 | datas.filter(_._1.equals("container")) 99 | .map(lines => { 100 | val logFileContainer = JsonUtil.getBean(lines._2, classOf[LogFileContainer]) 101 | val data = logFileContainer.getData 102 | val log_info = logFileContainer.getData.getLog_info 103 | // data.setTimestamp(DateUtil.formatToUTC_0(data.getTimestamp)) 104 | // val _logtimestamp = DateUtil.formatToUTC_0(log_info.getLog_time) 105 | // log_info.setLog_time(_logtimestamp) 106 | logFileContainer.setType(lines._1 + "-" + DateUtil.getyyyyMMdd(log_info.getLog_time)) 107 | JsonUtil.formatJson(logFileContainer) 108 | }).filter(_ != null).saveJsonToEs("yun-{data.environment_id}-{data.container_uuid}-containerlogfile-{type}/{type}") 109 | }*/ 110 | 111 | def ParseLogNginxSaveToEs2(datas : DStream[(String,String)]): Unit ={ 112 | val nginx = datas.filter(_._1.equals("nginx")) 113 | nginx.map(_._2).saveJsonToEs("yun-{data.environment_id}-{data.container_uuid}-applogfile-nginx-{data.log_info.log_time:YYYY-MM-dd}/nginx-log",Map("es.batch.size.bytes"->"2097152","es.batch.size.entries"->"500","es.batch.write.refresh"->"false")) 114 | } 115 | 116 | /* def ParseLogNginxSaveToEs(datas : DStream[(String,String)]): Unit ={ 117 | datas.filter(_._1.equals("nginx")) 118 | .map(lines => { 119 | val logFileNginx = JsonUtil.getBean(lines._2, classOf[LogFileNginx]) 120 | val data = logFileNginx.getData 121 | val log_info = logFileNginx.getData.getLog_info 122 | // data.setTimestamp(DateUtil.formatToUTC_0(data.getTimestamp)) 123 | // val _logtimestamp = DateUtil.formatToUTC_0(log_info.getLog_time) 124 | // log_info.setLog_time(_logtimestamp) 125 | logFileNginx.setType(lines._1 + "-" + DateUtil.getyyyyMMdd(log_info.getLog_time)) 126 | JsonUtil.formatJson(logFileNginx) 127 | }).filter(_ != null).saveJsonToEs("yun-{data.environment_id}-{data.container_uuid}-applogfile-{type}/{type}") 128 | }*/ 129 | 130 | def ParseLogMySqlSaveToEs2(datas : DStream[(String,String)]): Unit ={ 131 | val mysql = datas.filter(_._1.equals("mysql")) 132 | mysql.map(_._2).saveJsonToEs("yun-{data.environment_id}-{data.container_uuid}-applogfile-mysql-{data.log_info.log_time:YYYY-MM-dd}/mysql-log",Map("es.batch.size.bytes"->"2097152","es.batch.size.entries"->"500","es.batch.write.refresh"->"false")) 133 | } 134 | 135 | /* def ParseLogMySqlSaveToEs(datas : DStream[(String,String)]): Unit ={ 136 | datas.filter(_._1.equals("mysql")) 137 | .map(lines => { 138 | val logFileMySql = JsonUtil.getBean(lines._2, classOf[LogFileMySql]) 139 | val data = logFileMySql.getData 140 | val log_info = logFileMySql.getData.getLog_info 141 | // data.setTimestamp(DateUtil.formatToUTC_0(data.getTimestamp)) 142 | // val _logtimestamp = DateUtil.formatToUTC_0(log_info.getLog_time) 143 | // log_info.setLog_time(_logtimestamp) 144 | logFileMySql.setType(lines._1 + "-" + DateUtil.getyyyyMMdd(log_info.getLog_time)) 145 | JsonUtil.formatJson(logFileMySql) 146 | }).filter(_ != null).saveJsonToEs("yun-{data.environment_id}-{data.container_uuid}-applogfile-{type}/{type}") 147 | }*/ 148 | 149 | 150 | def ParseLogRedisSaveToEs2(datas : DStream[(String,String)]): Unit ={ 151 | val redis = datas.filter(_._1.equals("redis")) 152 | redis.map(_._2).saveJsonToEs("yun-{data.environment_id}-{data.container_uuid}-applogfile-redis-{data.log_info.log_time:YYYY-MM-dd}/redis-log",Map("es.batch.size.bytes"->"2097152","es.batch.size.entries"->"500","es.batch.write.refresh"->"false")) 153 | } 154 | 155 | } 156 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/scala/com/gh/yun/AppMysqlCap.scala: -------------------------------------------------------------------------------- 1 | package com.gh.yun 2 | 3 | import java.{lang, util} 4 | 5 | import com.gh.bean.alert.{KeyValue, AlertDataInfo, AlertData} 6 | import com.gh.utils._ 7 | import org.apache.kafka.clients.consumer.ConsumerRecord 8 | import org.apache.kafka.common.serialization.StringDeserializer 9 | import org.apache.log4j.{Logger, Level} 10 | import org.apache.spark.SparkConf 11 | import org.apache.spark.rdd.RDD 12 | import org.apache.spark.streaming._ 13 | import org.apache.spark.streaming.dstream.{DStream, InputDStream} 14 | import org.apache.spark.streaming.kafka010.ConsumerStrategies.Subscribe 15 | import org.apache.spark.streaming.kafka010.LocationStrategies.PreferConsistent 16 | import org.apache.spark.streaming.kafka010._ 17 | import org.codehaus.jackson.JsonNode 18 | import org.codehaus.jackson.node.NullNode 19 | 20 | import scala.collection.immutable.HashMap 21 | import scala.collection.mutable.ArrayBuffer 22 | 23 | /** 24 | * Created by GH-GAN on 2016/11/24. 25 | */ 26 | object AppMysqlCap { 27 | def main(args: Array[String]) { 28 | /* if (args.length < 4) { 29 | System.err.println("Usage: Kafka ") 30 | System.exit(1) 31 | }*/ 32 | Logger.getRootLogger.setLevel(Level.WARN) 33 | 34 | val sparkConf = new SparkConf().setAppName("capability-mysql").setMaster(ConfigUtil.sparkmaster) 35 | val ssc = new StreamingContext(sparkConf, Seconds(ConfigUtil.capStreamtime)) 36 | ssc.checkpoint(ConfigUtil.mysqlCapcheckpoint) 37 | 38 | var brokers = ConfigUtil.brokers 39 | val _topics = "capability-mysql".split(",").toSet 40 | val group = "capability-mysql-js" 41 | 42 | val kafkaParams = Map[String, Object]( 43 | "bootstrap.servers" -> brokers, 44 | "key.deserializer" -> classOf[StringDeserializer], 45 | "value.deserializer" -> classOf[StringDeserializer], 46 | "group.id" -> group, 47 | "auto.offset.reset" -> "latest", //当前偏移不在服务器上时,按最新开始 48 | "heartbeat.interval.ms" -> "6000", 49 | "session.timeout.ms" -> "20000", 50 | "max.partition.fetch.bytes" -> "1048576000", 51 | "max.poll.records" -> "5000000", // message.max.bytes 52 | "enable.auto.commit" -> (false: java.lang.Boolean) 53 | ) 54 | 55 | val stream = KafkaUtils.createDirectStream[String, String]( 56 | ssc, 57 | PreferConsistent, 58 | Subscribe[String, String](_topics, kafkaParams) 59 | ) 60 | 61 | val datas = parseLog2(stream) 62 | val mysql_group = datas.groupByKey() 63 | compute(mysql_group) 64 | 65 | ssc.start() 66 | ssc.awaitTermination() 67 | } 68 | 69 | //一条包含多条记录 70 | def parseLog2(stream : InputDStream[ConsumerRecord[String, String]]) : DStream[(String,(String,Double,Double))] ={ 71 | val datas = stream.map(line => { 72 | var node : JsonNode = JsonUtil.getJsonNode(line.value()) 73 | node 74 | }).filter(x => (x != null && x.get("type") != null)) 75 | .filter(x => { 76 | val _type = x.get("type").asText().trim 77 | var environment_id = "" 78 | var container_uuid = "" 79 | try{ 80 | val data = x.get("data") 81 | environment_id = data.get("environment_id").asText().trim 82 | container_uuid = data.get("container_uuid").asText().trim 83 | }catch { case ex : Exception => ex.printStackTrace() } 84 | 85 | !"".equals(_type) && !"".equals(environment_id) && !"".equals(container_uuid) && null != environment_id && null != container_uuid 86 | }) 87 | .flatMap(node => { 88 | val _type = node.get("type").asText() 89 | val environment_id = node.get("data").get("environment_id").asText() 90 | val container_uuid = node.get("data").get("container_uuid").asText() 91 | 92 | var container_name = "" 93 | var namespace = "" 94 | try { 95 | container_name = node.get("data").get("container_name").asText() 96 | namespace = node.get("data").get("namespace").asText() 97 | }catch { 98 | case ex : Exception => { 99 | println("--------> container_name/namespace is null") 100 | ex.printStackTrace() 101 | } 102 | } 103 | 104 | val _stats = node.get("data").get("stats") 105 | val arr = ArrayBuffer[(String,(String,Double,Double))]() 106 | for (i <- 0 to (_stats.size() - 1) ){ 107 | val stats = node.get("data").get("stats").get(i) 108 | val timestamp = stats.get("timestamp").asText() 109 | // println(DateUtil.toBase(timestamp)) 110 | val thread_connected = stats.get("thread_connected").asDouble() 111 | val max_connections = stats.get("max_connections").asDouble() 112 | arr.+=((environment_id+"#"+container_uuid+"#"+_type + "#" + container_name + "#" + namespace, (timestamp,thread_connected,max_connections))) 113 | } 114 | arr 115 | }) 116 | datas 117 | } 118 | 119 | def parseLog(stream : InputDStream[ConsumerRecord[String, String]]) : DStream[(String,(String,Double,Double))] ={ 120 | val datas = stream.map(line => { 121 | var node : JsonNode = JsonUtil.getJsonNode(line.value()) 122 | node 123 | }).filter(x => (x != null && x.get("type") != null)).map(node => { 124 | val _type = node.get("type").asText() 125 | val environment_id = node.get("data").get("environment_id").asText() 126 | val container_uuid = node.get("data").get("container_uuid").asText() 127 | 128 | val stats = node.get("data").get("stats").get(0) 129 | 130 | val timestamp = stats.get("timestamp").asText() 131 | val thread_connected = stats.get("thread_connected").asDouble() 132 | val max_connections = stats.get("max_connections").asDouble() 133 | 134 | (environment_id + "#" + container_uuid + "#" + _type, (timestamp,thread_connected,max_connections) ) 135 | }) 136 | datas 137 | } 138 | 139 | def compute(mysql_group : DStream[(String,Iterable[(String,Double,Double)])]): Unit ={ 140 | val warn = mysql_group.map(x => { 141 | val count = x._2.size 142 | 143 | // thread_connected 144 | val thread_connected_sum = x._2.map(_._2).reduce(_+_) 145 | val thread_connected_avg = thread_connected_sum / count 146 | 147 | val start_log = x._2.head 148 | val end_log = x._2.last 149 | // val start_time = DateUtil.df2.format(DateUtil.df_utc_base3.parse(start_log._1)) 150 | // val end_time = DateUtil.df2.format(DateUtil.df_utc_base3.parse(end_log._1)) 151 | val start_time = start_log._1 152 | val end_time = end_log._1 153 | val max_connections = start_log._3 154 | 155 | // thread_connected_avg / max_connections 156 | val con_threshold = thread_connected_avg / max_connections 157 | 158 | (x._1,start_time,end_time,con_threshold) 159 | }).map(x => aAlert(x)) // 阈值 160 | .filter(_._4.size() > 0) // 是否有告警信息 161 | 162 | warn.foreachRDD( record => { 163 | val alerts = record.map(line => { 164 | AlertInfoUtil.toWarnBean(AlertInfoUtil.SUCCESS,AlertInfoUtil.ALERT_TYPE_M,AlertInfoUtil.ALERT_DIM_A,line._1, line._2,line._3,line._4,line._5) 165 | }) 166 | if (!alerts.isEmpty()){ 167 | val collect = alerts.collect() 168 | if (collect.size > 0) new HttpUtil().alerts(collect) //告警 169 | } 170 | }) 171 | 172 | } 173 | 174 | def aAlert(line : (String,String,String,Double)): (String,String,String,util.ArrayList[KeyValue],String) ={ 175 | val list = new util.ArrayList[KeyValue]() 176 | // if (line._4 > 0.7) list.add(new KeyValue("connection",line._4.toString)) 177 | 178 | val gz = HttpUtil.gz_map.get("app_mysql_connection") 179 | // println("===================="+gz.getCondition+gz.getValue) 180 | gz.getCondition match { 181 | case "GTE" => if (line._4 >= gz.getValue) list.add(new KeyValue("connection",line._4.toString)) 182 | case "GT" => if(line._4 > gz.getValue) list.add(new KeyValue("connection",line._4.toString)) 183 | case "LTE" => if(line._4 <= gz.getValue) list.add(new KeyValue("connection",line._4.toString)) 184 | case "LT" => if(line._4 < gz.getValue) list.add(new KeyValue("connection",line._4.toString)) 185 | case "EQ" => if(line._4 == gz.getValue) list.add(new KeyValue("connection",line._4.toString)) 186 | } 187 | // list.add(new KeyValue("connection",line._4.toString)) 188 | (line._1,line._2,line._3,list,"") 189 | } 190 | 191 | } 192 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/scala/com/gh/yun/parse/ParseMysql.scala: -------------------------------------------------------------------------------- 1 | package com.gh.yun.parse 2 | 3 | import com.gh.bean.appcapability._ 4 | import com.gh.utils.{DateUtil, JsonUtil} 5 | import org.codehaus.jackson.JsonNode 6 | 7 | import scala.collection.mutable.ArrayBuffer 8 | 9 | /** 10 | * Created by GH-GAN on 2016/12/20. 11 | */ 12 | class ParseMysql { 13 | def statsToArray(_type: String,node : JsonNode): ArrayBuffer[String] ={ 14 | val arr = ArrayBuffer[String]() 15 | try{ 16 | var data = node.get("data") 17 | val environment_id = data.get("environment_id").asText().trim 18 | val container_uuid = data.get("container_uuid").asText().trim 19 | val container_name = data.get("container_name").asText() 20 | val namespace = data.get("namespace").asText() 21 | val data_timestamp = data.get("timestamp").asText() 22 | 23 | if (!"".equals(environment_id) && !"".equals(container_uuid)){ 24 | var _stats = data.get("stats") 25 | for (i <- 0 until _stats.size()) { 26 | val stats_i = _stats.get(i) 27 | 28 | val ams = new AppMysqlStats() 29 | val timestamp = stats_i.get("timestamp").asText() 30 | ams.setTimestamp(timestamp) 31 | ams.setConnections_total(stats_i.get("connections_total").asDouble()) 32 | ams.setCommand_query_total(stats_i.get("command_query_total").asDouble()) 33 | ams.setCommand_insert_total(stats_i.get("command_insert_total").asDouble()) 34 | ams.setCommand_update_total(stats_i.get("command_update_total").asDouble()) 35 | ams.setCommand_delete_total(stats_i.get("command_delete_total").asDouble()) 36 | ams.setCommands_total(stats_i.get("commands_total").asDouble()) 37 | ams.setHandlers_total(stats_i.get("handlers_total").asDouble()) 38 | ams.setConnection_errors_total(stats_i.get("connection_errors_total").asDouble()) 39 | ams.setBuffer_pool_pages(stats_i.get("buffer_pool_pages").asDouble()) 40 | ams.setThread_connected(stats_i.get("thread_connected").asDouble()) 41 | ams.setMax_connections(stats_i.get("max_connections").asDouble()) 42 | ams.setQuery_response_time_seconds(stats_i.get("query_response_time_seconds").asDouble()) 43 | ams.setRead_query_response_time_seconds(stats_i.get("read_query_response_time_seconds").asDouble()) 44 | ams.setWrite_query_response_time_seconds(stats_i.get("write_query_response_time_seconds").asDouble()) 45 | ams.setQueries_inside_innodb(stats_i.get("queries_inside_innodb").asDouble()) 46 | ams.setQueries_in_queue(stats_i.get("queries_in_queue").asDouble()) 47 | ams.setRead_views_open_inside_innodb(stats_i.get("read_views_open_inside_innodb").asDouble()) 48 | ams.setTable_statistics_rows_read_total(stats_i.get("table_statistics_rows_read_total").asDouble()) 49 | ams.setTable_statistics_rows_changed_total(stats_i.get("table_statistics_rows_changed_total").asDouble()) 50 | ams.setTable_statistics_rows_changed_x_indexes_total(stats_i.get("table_statistics_rows_changed_x_indexes_total").asDouble()) 51 | ams.setSql_lock_waits_total(stats_i.get("sql_lock_waits_total").asDouble()) 52 | ams.setExternal_lock_waits_total(stats_i.get("external_lock_waits_total").asDouble()) 53 | ams.setSql_lock_waits_seconds_total(stats_i.get("sql_lock_waits_seconds_total").asDouble()) 54 | ams.setExternal_lock_waits_seconds_total(stats_i.get("external_lock_waits_seconds_total").asDouble()) 55 | ams.setTable_io_waits_total(stats_i.get("table_io_waits_total").asDouble()) 56 | ams.setTable_io_waits_seconds_total(stats_i.get("table_io_waits_seconds_total").asDouble()) 57 | 58 | var amsd = new AppMySqlData2() 59 | amsd.setContainer_name(container_name) 60 | amsd.setContainer_uuid(container_uuid) 61 | amsd.setEnvironment_id(environment_id) 62 | amsd.setNamespace(namespace) 63 | amsd.setTimestamp(data_timestamp) 64 | amsd.setStats(ams) 65 | 66 | var ms = new AppMySql2() 67 | ms.setType(_type) 68 | ms.setData(amsd) 69 | 70 | // index yun-{data.environment_id}-{data.container_uuid}-app-mysql-{data.stats.timestamp:YYYY-MM-dd} 71 | // val index = "yun-" + environment_id + "-" + container_uuid + "-app-mysql-" + DateUtil.getYYYYMMdd(timestamp) 72 | arr.+=(JsonUtil.formatJson(ms)) 73 | } 74 | } 75 | }catch { 76 | case ex : Exception => ex.printStackTrace() 77 | } 78 | arr 79 | } 80 | 81 | def statsToArray2(_type: String,node : JsonNode): ArrayBuffer[(String,String)] ={ 82 | val arr = ArrayBuffer[(String,String)]() 83 | try{ 84 | var data = node.get("data") 85 | val environment_id = data.get("environment_id").asText().trim 86 | val container_uuid = data.get("container_uuid").asText().trim 87 | val container_name = data.get("container_name").asText() 88 | val namespace = data.get("namespace").asText() 89 | val data_timestamp = data.get("timestamp").asText() 90 | 91 | if (!"".equals(environment_id) && !"".equals(container_uuid)){ 92 | var _stats = data.get("stats") 93 | for (i <- 0 until _stats.size()) { 94 | val stats_i = _stats.get(i) 95 | 96 | val ams = new AppMysqlStats() 97 | val timestamp = stats_i.get("timestamp").asText() 98 | ams.setTimestamp(timestamp) 99 | ams.setConnections_total(stats_i.get("connections_total").asDouble()) 100 | ams.setCommand_query_total(stats_i.get("command_query_total").asDouble()) 101 | ams.setCommand_insert_total(stats_i.get("command_insert_total").asDouble()) 102 | ams.setCommand_update_total(stats_i.get("command_update_total").asDouble()) 103 | ams.setCommand_delete_total(stats_i.get("command_delete_total").asDouble()) 104 | ams.setCommands_total(stats_i.get("commands_total").asDouble()) 105 | ams.setHandlers_total(stats_i.get("handlers_total").asDouble()) 106 | ams.setConnection_errors_total(stats_i.get("connection_errors_total").asDouble()) 107 | ams.setBuffer_pool_pages(stats_i.get("buffer_pool_pages").asDouble()) 108 | ams.setThread_connected(stats_i.get("thread_connected").asDouble()) 109 | ams.setMax_connections(stats_i.get("max_connections").asDouble()) 110 | ams.setQuery_response_time_seconds(stats_i.get("query_response_time_seconds").asDouble()) 111 | ams.setRead_query_response_time_seconds(stats_i.get("read_query_response_time_seconds").asDouble()) 112 | ams.setWrite_query_response_time_seconds(stats_i.get("write_query_response_time_seconds").asDouble()) 113 | ams.setQueries_inside_innodb(stats_i.get("queries_inside_innodb").asDouble()) 114 | ams.setQueries_in_queue(stats_i.get("queries_in_queue").asDouble()) 115 | ams.setRead_views_open_inside_innodb(stats_i.get("read_views_open_inside_innodb").asDouble()) 116 | ams.setTable_statistics_rows_read_total(stats_i.get("table_statistics_rows_read_total").asDouble()) 117 | ams.setTable_statistics_rows_changed_total(stats_i.get("table_statistics_rows_changed_total").asDouble()) 118 | ams.setTable_statistics_rows_changed_x_indexes_total(stats_i.get("table_statistics_rows_changed_x_indexes_total").asDouble()) 119 | ams.setSql_lock_waits_total(stats_i.get("sql_lock_waits_total").asDouble()) 120 | ams.setExternal_lock_waits_total(stats_i.get("external_lock_waits_total").asDouble()) 121 | ams.setSql_lock_waits_seconds_total(stats_i.get("sql_lock_waits_seconds_total").asDouble()) 122 | ams.setExternal_lock_waits_seconds_total(stats_i.get("external_lock_waits_seconds_total").asDouble()) 123 | ams.setTable_io_waits_total(stats_i.get("table_io_waits_total").asDouble()) 124 | ams.setTable_io_waits_seconds_total(stats_i.get("table_io_waits_seconds_total").asDouble()) 125 | 126 | var amsd = new AppMySqlData2() 127 | amsd.setContainer_name(container_name) 128 | amsd.setContainer_uuid(container_uuid) 129 | amsd.setEnvironment_id(environment_id) 130 | amsd.setNamespace(namespace) 131 | amsd.setTimestamp(data_timestamp) 132 | amsd.setStats(ams) 133 | 134 | var ms = new AppMySql2() 135 | ms.setType(_type) 136 | ms.setData(amsd) 137 | 138 | // index yun-{data.environment_id}-{data.container_uuid}-app-mysql-{data.stats.timestamp:YYYY-MM-dd} 139 | val index = "yun-" + environment_id + "-" + container_uuid + "-app-mysql-" + DateUtil.getYYYYMMdd(timestamp) 140 | arr.+=((index, JsonUtil.formatJson(ms))) 141 | } 142 | } 143 | }catch { 144 | case ex : Exception => ex.printStackTrace() 145 | } 146 | arr 147 | } 148 | 149 | } 150 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/appcapability/AppMysqlStats.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.appcapability; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by GH-GAN on 2016/11/29. 7 | */ 8 | public class AppMysqlStats implements Serializable { 9 | String timestamp; 10 | Double connections_total; 11 | Double command_query_total; 12 | Double command_insert_total; 13 | Double command_update_total; 14 | Double command_delete_total; 15 | Double commands_total; 16 | Double handlers_total; 17 | Double connection_errors_total; 18 | Double buffer_pool_pages; 19 | Double thread_connected; 20 | Double max_connections; 21 | Double query_response_time_seconds; 22 | Double read_query_response_time_seconds; 23 | Double write_query_response_time_seconds; 24 | Double queries_inside_innodb; 25 | Double queries_in_queue; 26 | Double read_views_open_inside_innodb; 27 | Double table_statistics_rows_read_total; 28 | Double table_statistics_rows_changed_total; 29 | Double table_statistics_rows_changed_x_indexes_total; 30 | Double sql_lock_waits_total; 31 | Double external_lock_waits_total; 32 | Double sql_lock_waits_seconds_total; 33 | Double external_lock_waits_seconds_total; 34 | Double table_io_waits_total; 35 | Double table_io_waits_seconds_total; 36 | 37 | public String getTimestamp() { 38 | return timestamp; 39 | } 40 | 41 | public void setTimestamp(String timestamp) { 42 | this.timestamp = timestamp; 43 | } 44 | 45 | public Double getConnections_total() { 46 | return connections_total; 47 | } 48 | 49 | public void setConnections_total(Double connections_total) { 50 | this.connections_total = connections_total; 51 | } 52 | 53 | public Double getCommand_query_total() { 54 | return command_query_total; 55 | } 56 | 57 | public void setCommand_query_total(Double command_query_total) { 58 | this.command_query_total = command_query_total; 59 | } 60 | 61 | public Double getCommand_insert_total() { 62 | return command_insert_total; 63 | } 64 | 65 | public void setCommand_insert_total(Double command_insert_total) { 66 | this.command_insert_total = command_insert_total; 67 | } 68 | 69 | public Double getCommand_update_total() { 70 | return command_update_total; 71 | } 72 | 73 | public void setCommand_update_total(Double command_update_total) { 74 | this.command_update_total = command_update_total; 75 | } 76 | 77 | public Double getCommand_delete_total() { 78 | return command_delete_total; 79 | } 80 | 81 | public void setCommand_delete_total(Double command_delete_total) { 82 | this.command_delete_total = command_delete_total; 83 | } 84 | 85 | public Double getCommands_total() { 86 | return commands_total; 87 | } 88 | 89 | public void setCommands_total(Double commands_total) { 90 | this.commands_total = commands_total; 91 | } 92 | 93 | public Double getHandlers_total() { 94 | return handlers_total; 95 | } 96 | 97 | public void setHandlers_total(Double handlers_total) { 98 | this.handlers_total = handlers_total; 99 | } 100 | 101 | public Double getConnection_errors_total() { 102 | return connection_errors_total; 103 | } 104 | 105 | public void setConnection_errors_total(Double connection_errors_total) { 106 | this.connection_errors_total = connection_errors_total; 107 | } 108 | 109 | public Double getBuffer_pool_pages() { 110 | return buffer_pool_pages; 111 | } 112 | 113 | public void setBuffer_pool_pages(Double buffer_pool_pages) { 114 | this.buffer_pool_pages = buffer_pool_pages; 115 | } 116 | 117 | public Double getThread_connected() { 118 | return thread_connected; 119 | } 120 | 121 | public void setThread_connected(Double thread_connected) { 122 | this.thread_connected = thread_connected; 123 | } 124 | 125 | public Double getMax_connections() { 126 | return max_connections; 127 | } 128 | 129 | public void setMax_connections(Double max_connections) { 130 | this.max_connections = max_connections; 131 | } 132 | 133 | public Double getQuery_response_time_seconds() { 134 | return query_response_time_seconds; 135 | } 136 | 137 | public void setQuery_response_time_seconds(Double query_response_time_seconds) { 138 | this.query_response_time_seconds = query_response_time_seconds; 139 | } 140 | 141 | public Double getRead_query_response_time_seconds() { 142 | return read_query_response_time_seconds; 143 | } 144 | 145 | public void setRead_query_response_time_seconds(Double read_query_response_time_seconds) { 146 | this.read_query_response_time_seconds = read_query_response_time_seconds; 147 | } 148 | 149 | public Double getWrite_query_response_time_seconds() { 150 | return write_query_response_time_seconds; 151 | } 152 | 153 | public void setWrite_query_response_time_seconds(Double write_query_response_time_seconds) { 154 | this.write_query_response_time_seconds = write_query_response_time_seconds; 155 | } 156 | 157 | public Double getQueries_inside_innodb() { 158 | return queries_inside_innodb; 159 | } 160 | 161 | public void setQueries_inside_innodb(Double queries_inside_innodb) { 162 | this.queries_inside_innodb = queries_inside_innodb; 163 | } 164 | 165 | public Double getQueries_in_queue() { 166 | return queries_in_queue; 167 | } 168 | 169 | public void setQueries_in_queue(Double queries_in_queue) { 170 | this.queries_in_queue = queries_in_queue; 171 | } 172 | 173 | public Double getRead_views_open_inside_innodb() { 174 | return read_views_open_inside_innodb; 175 | } 176 | 177 | public void setRead_views_open_inside_innodb(Double read_views_open_inside_innodb) { 178 | this.read_views_open_inside_innodb = read_views_open_inside_innodb; 179 | } 180 | 181 | public Double getTable_statistics_rows_read_total() { 182 | return table_statistics_rows_read_total; 183 | } 184 | 185 | public void setTable_statistics_rows_read_total(Double table_statistics_rows_read_total) { 186 | this.table_statistics_rows_read_total = table_statistics_rows_read_total; 187 | } 188 | 189 | public Double getTable_statistics_rows_changed_total() { 190 | return table_statistics_rows_changed_total; 191 | } 192 | 193 | public void setTable_statistics_rows_changed_total(Double table_statistics_rows_changed_total) { 194 | this.table_statistics_rows_changed_total = table_statistics_rows_changed_total; 195 | } 196 | 197 | public Double getTable_statistics_rows_changed_x_indexes_total() { 198 | return table_statistics_rows_changed_x_indexes_total; 199 | } 200 | 201 | public void setTable_statistics_rows_changed_x_indexes_total(Double table_statistics_rows_changed_x_indexes_total) { 202 | this.table_statistics_rows_changed_x_indexes_total = table_statistics_rows_changed_x_indexes_total; 203 | } 204 | 205 | public Double getSql_lock_waits_total() { 206 | return sql_lock_waits_total; 207 | } 208 | 209 | public void setSql_lock_waits_total(Double sql_lock_waits_total) { 210 | this.sql_lock_waits_total = sql_lock_waits_total; 211 | } 212 | 213 | public Double getExternal_lock_waits_total() { 214 | return external_lock_waits_total; 215 | } 216 | 217 | public void setExternal_lock_waits_total(Double external_lock_waits_total) { 218 | this.external_lock_waits_total = external_lock_waits_total; 219 | } 220 | 221 | public Double getSql_lock_waits_seconds_total() { 222 | return sql_lock_waits_seconds_total; 223 | } 224 | 225 | public void setSql_lock_waits_seconds_total(Double sql_lock_waits_seconds_total) { 226 | this.sql_lock_waits_seconds_total = sql_lock_waits_seconds_total; 227 | } 228 | 229 | public Double getExternal_lock_waits_seconds_total() { 230 | return external_lock_waits_seconds_total; 231 | } 232 | 233 | public void setExternal_lock_waits_seconds_total(Double external_lock_waits_seconds_total) { 234 | this.external_lock_waits_seconds_total = external_lock_waits_seconds_total; 235 | } 236 | 237 | public Double getTable_io_waits_total() { 238 | return table_io_waits_total; 239 | } 240 | 241 | public void setTable_io_waits_total(Double table_io_waits_total) { 242 | this.table_io_waits_total = table_io_waits_total; 243 | } 244 | 245 | public Double getTable_io_waits_seconds_total() { 246 | return table_io_waits_seconds_total; 247 | } 248 | 249 | public void setTable_io_waits_seconds_total(Double table_io_waits_seconds_total) { 250 | this.table_io_waits_seconds_total = table_io_waits_seconds_total; 251 | } 252 | } 253 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/scala/com/gh/yun/AppRedisCap.scala: -------------------------------------------------------------------------------- 1 | package com.gh.yun 2 | 3 | import java.util 4 | 5 | import com.gh.bean.alert.{KeyValue, AlertDataInfo, AlertData} 6 | import com.gh.bean.containercapability.ContainerFileSystem 7 | import com.gh.utils._ 8 | import org.apache.kafka.clients.consumer.ConsumerRecord 9 | import org.apache.kafka.common.serialization.StringDeserializer 10 | import org.apache.log4j.{Level, Logger} 11 | import org.apache.spark.SparkConf 12 | import org.apache.spark.streaming._ 13 | import org.apache.spark.streaming.dstream.{DStream, InputDStream} 14 | import org.apache.spark.streaming.kafka010.ConsumerStrategies.Subscribe 15 | import org.apache.spark.streaming.kafka010.LocationStrategies.PreferConsistent 16 | import org.apache.spark.streaming.kafka010._ 17 | import org.codehaus.jackson.JsonNode 18 | 19 | import scala.collection.mutable.ArrayBuffer 20 | 21 | /** 22 | * Created by GH-GAN on 2016/11/24. 23 | */ 24 | object AppRedisCap { 25 | def main(args: Array[String]) { 26 | 27 | Logger.getRootLogger.setLevel(Level.WARN) 28 | 29 | val sparkConf = new SparkConf().setAppName("capability-redis").setMaster(ConfigUtil.sparkmaster) 30 | val ssc = new StreamingContext(sparkConf, Seconds(ConfigUtil.capStreamtime)) 31 | ssc.checkpoint(ConfigUtil.redisCapcheckpoint) 32 | 33 | var brokers = ConfigUtil.brokers 34 | val _topics = "capability-redis".split(",").toSet 35 | val group = "capability-redis-js" 36 | 37 | val kafkaParams = Map[String, Object]( 38 | "bootstrap.servers" -> brokers, 39 | "key.deserializer" -> classOf[StringDeserializer], 40 | "value.deserializer" -> classOf[StringDeserializer], 41 | "group.id" -> group, 42 | "auto.offset.reset" -> "latest", //当前偏移不在服务器上时,按最新开始 43 | "heartbeat.interval.ms" -> "6000", 44 | "session.timeout.ms" -> "20000", 45 | "max.partition.fetch.bytes" -> "1048576000", 46 | "max.poll.records" -> "5000000", // message.max.bytes 47 | "enable.auto.commit" -> (false: java.lang.Boolean) 48 | ) 49 | 50 | val stream = KafkaUtils.createDirectStream[String, String]( 51 | ssc, 52 | PreferConsistent, 53 | Subscribe[String, String](_topics, kafkaParams) 54 | ) 55 | 56 | val datas = parseLog2(stream) 57 | val redis_group = datas.groupByKey() 58 | compute(redis_group) 59 | 60 | ssc.start() 61 | ssc.awaitTermination() 62 | } 63 | 64 | def parseLog2(stream : InputDStream[ConsumerRecord[String, String]]) : DStream[(String,(String,Double,Double,Double,Double))] ={ 65 | val datas = stream.map(line => { 66 | var node : JsonNode = JsonUtil.getJsonNode(line.value()) 67 | node 68 | }).filter(x => (x != null && x.get("type") != null)) 69 | .filter(x => { 70 | val _type = x.get("type").asText().trim 71 | var environment_id = "" 72 | var container_uuid = "" 73 | try{ 74 | val data = x.get("data") 75 | environment_id = data.get("environment_id").asText().trim 76 | container_uuid = data.get("container_uuid").asText().trim 77 | }catch { case ex : Exception => ex.printStackTrace() } 78 | 79 | !"".equals(_type) && !"".equals(environment_id) && !"".equals(container_uuid) && null != environment_id && null != container_uuid 80 | }) 81 | .flatMap(node => { 82 | val _type = node.get("type").asText() 83 | val environment_id = node.get("data").get("environment_id").asText() 84 | val container_uuid = node.get("data").get("container_uuid").asText() 85 | 86 | var container_name = "" 87 | var namespace = "" 88 | try { 89 | container_name = node.get("data").get("container_name").asText() 90 | namespace = node.get("data").get("namespace").asText() 91 | }catch { 92 | case ex : Exception => { 93 | println("--------> container_name/namespace is null") 94 | ex.printStackTrace() 95 | } 96 | } 97 | 98 | val _stats = node.get("data").get("stats") 99 | val arr = ArrayBuffer[(String,(String,Double,Double,Double,Double))]() 100 | for (i <- 0 to (_stats.size() - 1) ){ 101 | val stats = node.get("data").get("stats").get(i) 102 | val timestamp = stats.get("timestamp").asText() 103 | val used_memory = stats.get("used_memory").asDouble() 104 | val max_memory = stats.get("max_memory").asDouble() 105 | val keyspace_hits = stats.get("keyspace_hits").asDouble() 106 | val keyspace_misses = stats.get("keyspace_misses").asDouble() 107 | arr.+=((environment_id+"#"+container_uuid+"#"+_type + "#" + container_name + "#" + namespace, (timestamp,used_memory,max_memory,keyspace_hits,keyspace_misses))) 108 | } 109 | arr 110 | }) 111 | datas 112 | } 113 | 114 | def parseLog(stream : InputDStream[ConsumerRecord[String, String]]) : DStream[(String,(String,Double,Double,Double,Double))] ={ 115 | val datas = stream.map(line => { 116 | var node : JsonNode = JsonUtil.getJsonNode(line.value()) 117 | node 118 | }).filter(x => (x != null && x.get("type") != null)).map(node => { 119 | val _type = node.get("type").asText() 120 | val environment_id = node.get("data").get("environment_id").asText() 121 | val container_uuid = node.get("data").get("container_uuid").asText() 122 | 123 | val stats = node.get("data").get("stats").get(0) 124 | 125 | val timestamp = stats.get("timestamp").asText() 126 | val used_memory = stats.get("used_memory").asDouble() 127 | val max_memory = stats.get("max_memory").asDouble() 128 | val keyspace_hits = stats.get("keyspace_hits").asDouble() 129 | val keyspace_misses = stats.get("keyspace_misses").asDouble() 130 | 131 | (environment_id+"#"+container_uuid+"#"+_type, (timestamp,used_memory,max_memory,keyspace_hits,keyspace_misses) ) 132 | }) 133 | datas 134 | } 135 | 136 | def compute(redis_group : DStream[(String,Iterable[(String,Double,Double,Double,Double)])]): Unit ={ 137 | val warn = redis_group.map(x => { 138 | val count = x._2.size 139 | 140 | val start_log = x._2.head 141 | val end_log = x._2.last 142 | val start_time = start_log._1 143 | val end_time = end_log._1 144 | 145 | // used_memory 146 | val used_memory_sum = x._2.map(_._2).reduce(_+_) 147 | val used_memory_avg = used_memory_sum / count 148 | 149 | //max_memory 150 | val max_memory = start_log._3 151 | 152 | var memory_threshold = - 1.0 153 | if (0 != max_memory && null != max_memory){ 154 | // 占用内存 155 | memory_threshold = used_memory_avg / max_memory 156 | } 157 | 158 | // 命中率 159 | val hit_threshold = ( end_log._4 - start_log._4 ) / ((end_log._4 + end_log._5) - (start_log._4 + start_log._5)) 160 | 161 | (x._1,start_time,end_time,memory_threshold,hit_threshold) 162 | }).map(aAlert(_)) 163 | .filter(_._4.size() > 0) // 是否有告警信息 164 | /* warn.filter(x => { 165 | HttpUtil.URL match { 166 | case "" => x._4 > 0.8 || x._5 > 0.6 167 | case "" => (x._4 > 0.8 || x._5 > 0.6) 168 | } 169 | })*/ 170 | 171 | warn.foreachRDD( record => { 172 | val alerts = record.map(line => { 173 | AlertInfoUtil.toWarnBean(AlertInfoUtil.SUCCESS,AlertInfoUtil.ALERT_TYPE_M,AlertInfoUtil.ALERT_DIM_A,line._1, line._2,line._3,line._4,line._5) 174 | }) 175 | if (!alerts.isEmpty()){ 176 | val collect = alerts.collect() 177 | if (collect.size > 0) new HttpUtil().alerts(collect) //告警 178 | } 179 | }) 180 | 181 | } 182 | 183 | def aAlert(line : (String,String,String,Double,Double)): (String,String,String,util.ArrayList[KeyValue],String) ={ 184 | val list = new util.ArrayList[KeyValue]() 185 | 186 | // if (line._4 > 0.8) list.add(new KeyValue("memory",line._4.toString)) 187 | // if (line._5 > 0.6) list.add(new KeyValue("hits",line._5.toString)) 188 | 189 | val memory = HttpUtil.gz_map.get("app_redis_memory") 190 | memory.getCondition match { 191 | case "GTE" => if(-1 != line._4 && line._4 >= memory.getValue) list.add(new KeyValue("memory",line._4.toString)) 192 | case "GT" => if(-1 != line._4 && line._4 > memory.getValue) list.add(new KeyValue("memory",line._4.toString)) 193 | case "LTE" => if(-1 != line._4 && line._4 <= memory.getValue) list.add(new KeyValue("memory",line._4.toString)) 194 | case "LT" => if(-1 != line._4 && line._4 < memory.getValue) list.add(new KeyValue("memory",line._4.toString)) 195 | case "EQ" => if(-1 != line._4 && line._4 == memory.getValue) list.add(new KeyValue("memory",line._4.toString)) 196 | } 197 | 198 | val hits = HttpUtil.gz_map.get("app_redis_hits") 199 | hits.getCondition match { 200 | case "GTE" => if(line._5 >= hits.getValue) list.add(new KeyValue("hits",line._5.toString)) 201 | case "GT" => if(line._5 > hits.getValue) list.add(new KeyValue("hits",line._5.toString)) 202 | case "LTE" => if(line._5 <= hits.getValue) list.add(new KeyValue("hits",line._5.toString)) 203 | case "LT" => if(line._5 < hits.getValue) list.add(new KeyValue("hits",line._5.toString)) 204 | case "EQ" => if(line._5 == hits.getValue) list.add(new KeyValue("hits",line._5.toString)) 205 | } 206 | 207 | // list.add(new KeyValue("memory",line._4.toString)) 208 | 209 | (line._1,line._2,line._3,list,"") 210 | } 211 | 212 | } 213 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/scala/com/gh/yun/parse/ParseRedis.scala: -------------------------------------------------------------------------------- 1 | package com.gh.yun.parse 2 | 3 | import com.gh.bean.appcapability._ 4 | import com.gh.utils.{DateUtil, JsonUtil} 5 | import org.codehaus.jackson.JsonNode 6 | 7 | import scala.collection.mutable.ArrayBuffer 8 | 9 | /** 10 | * Created by GH-GAN on 2016/12/20. 11 | */ 12 | class ParseRedis { 13 | def statsToArray(_type: String,node : JsonNode): ArrayBuffer[String] ={ 14 | val arr = ArrayBuffer[String]() 15 | try{ 16 | var data = node.get("data") 17 | val environment_id = data.get("environment_id").asText().trim 18 | val container_uuid = data.get("container_uuid").asText().trim 19 | val container_name = data.get("container_name").asText() 20 | val namespace = data.get("namespace").asText() 21 | val data_timestamp = data.get("timestamp").asText() 22 | 23 | if (!"".equals(environment_id) && !"".equals(container_uuid)){ 24 | var _stats = data.get("stats") 25 | for (i <- 0 until _stats.size()) { 26 | val stats_i = _stats.get(i) 27 | 28 | val ars = new AppRedisStats() 29 | val timestamp = stats_i.get("timestamp").asText() 30 | ars.setTimestamp(timestamp) 31 | ars.setUptime_in_seconds(stats_i.get("uptime_in_seconds").asDouble()) 32 | ars.setConnected_clients(stats_i.get("connected_clients").asDouble()) 33 | ars.setBlocked_clients(stats_i.get("blocked_clients").asDouble()) 34 | ars.setUsed_memory(stats_i.get("used_memory").asDouble()) 35 | ars.setUsed_memory_rss(stats_i.get("used_memory_rss").asDouble()) 36 | ars.setUsed_memory_peak(stats_i.get("used_memory_peak").asDouble()) 37 | ars.setUsed_memory_lua(stats_i.get("used_memory_lua").asDouble()) 38 | ars.setMax_memory(stats_i.get("max_memory").asDouble()) 39 | ars.setMem_fragmentation_ratio(stats_i.get("mem_fragmentation_ratio").asDouble()) 40 | ars.setRdb_changes_since_last_save(stats_i.get("rdb_changes_since_last_save").asDouble()) 41 | ars.setRdb_last_bgsave_time_sec(stats_i.get("rdb_last_bgsave_time_sec").asDouble()) 42 | ars.setRdb_current_bgsave_time_sec(stats_i.get("rdb_current_bgsave_time_sec").asDouble()) 43 | ars.setAof_enabled(stats_i.get("aof_enabled").asDouble()) 44 | ars.setAof_rewrite_in_progress(stats_i.get("aof_rewrite_in_progress").asDouble()) 45 | ars.setAof_rewrite_scheduled(stats_i.get("aof_rewrite_scheduled").asDouble()) 46 | ars.setAof_last_rewrite_time_sec(stats_i.get("aof_last_rewrite_time_sec").asDouble()) 47 | ars.setAof_current_rewrite_time_sec(stats_i.get("aof_current_rewrite_time_sec").asDouble()) 48 | ars.setTotal_connections_received(stats_i.get("total_connections_received").asDouble()) 49 | ars.setTotal_commands_processed(stats_i.get("total_commands_processed").asDouble()) 50 | ars.setTotal_net_input_bytes(stats_i.get("total_net_input_bytes").asDouble()) 51 | ars.setTotal_net_output_bytes(stats_i.get("total_net_output_bytes").asDouble()) 52 | ars.setRejected_connections(stats_i.get("rejected_connections").asDouble()) 53 | ars.setExpired_keys(stats_i.get("expired_keys").asDouble()) 54 | ars.setEvicted_keys(stats_i.get("evicted_keys").asDouble()) 55 | ars.setKeyspace_hits(stats_i.get("keyspace_hits").asDouble()) 56 | ars.setKeyspace_misses(stats_i.get("keyspace_misses").asDouble()) 57 | ars.setPubsub_channels(stats_i.get("pubsub_channels").asDouble()) 58 | ars.setPubsub_patterns(stats_i.get("pubsub_patterns").asDouble()) 59 | ars.setLoading(stats_i.get("loading").asDouble()) 60 | ars.setConnected_slaves(stats_i.get("connected_slaves").asDouble()) 61 | ars.setRepl_backlog_size(stats_i.get("repl_backlog_size").asDouble()) 62 | ars.setUsed_cpu_sys(stats_i.get("used_cpu_sys").asDouble()) 63 | ars.setUsed_cpu_user(stats_i.get("used_cpu_user").asDouble()) 64 | ars.setUsed_cpu_sys_children(stats_i.get("used_cpu_sys_children").asDouble()) 65 | ars.setUsed_cpu_user_children(stats_i.get("used_cpu_user_children").asDouble()) 66 | 67 | var ard2 = new AppRedisData2() 68 | ard2.setContainer_name(container_name) 69 | ard2.setContainer_uuid(container_uuid) 70 | ard2.setEnvironment_id(environment_id) 71 | ard2.setNamespace(namespace) 72 | ard2.setTimestamp(data_timestamp) 73 | ard2.setStats(ars) 74 | 75 | var ar2 = new AppRedis2() 76 | ar2.setType(_type) 77 | ar2.setData(ard2) 78 | 79 | // index yun-{data.environment_id}-{data.container_uuid}-app-redis-{data.stats.timestamp:YYYY-MM-dd} 80 | // val index = "yun-" + environment_id + "-" + container_uuid + "-app-redis-" + DateUtil.getYYYYMMdd(timestamp) 81 | arr.+=(JsonUtil.formatJson(ar2)) 82 | } 83 | } 84 | }catch { 85 | case ex : Exception => ex.printStackTrace() 86 | } 87 | arr 88 | } 89 | 90 | def statsToArray2(_type: String,node : JsonNode): ArrayBuffer[(String,String)] ={ 91 | val arr = ArrayBuffer[(String,String)]() 92 | try{ 93 | var data = node.get("data") 94 | val environment_id = data.get("environment_id").asText().trim 95 | val container_uuid = data.get("container_uuid").asText().trim 96 | val container_name = data.get("container_name").asText() 97 | val namespace = data.get("namespace").asText() 98 | val data_timestamp = data.get("timestamp").asText() 99 | 100 | if (!"".equals(environment_id) && !"".equals(container_uuid)){ 101 | var _stats = data.get("stats") 102 | for (i <- 0 until _stats.size()) { 103 | val stats_i = _stats.get(i) 104 | 105 | val ars = new AppRedisStats() 106 | val timestamp = stats_i.get("timestamp").asText() 107 | ars.setTimestamp(timestamp) 108 | ars.setUptime_in_seconds(stats_i.get("uptime_in_seconds").asDouble()) 109 | ars.setConnected_clients(stats_i.get("connected_clients").asDouble()) 110 | ars.setBlocked_clients(stats_i.get("blocked_clients").asDouble()) 111 | ars.setUsed_memory(stats_i.get("used_memory").asDouble()) 112 | ars.setUsed_memory_rss(stats_i.get("used_memory_rss").asDouble()) 113 | ars.setUsed_memory_peak(stats_i.get("used_memory_peak").asDouble()) 114 | ars.setUsed_memory_lua(stats_i.get("used_memory_lua").asDouble()) 115 | ars.setMax_memory(stats_i.get("max_memory").asDouble()) 116 | ars.setMem_fragmentation_ratio(stats_i.get("mem_fragmentation_ratio").asDouble()) 117 | ars.setRdb_changes_since_last_save(stats_i.get("rdb_changes_since_last_save").asDouble()) 118 | ars.setRdb_last_bgsave_time_sec(stats_i.get("rdb_last_bgsave_time_sec").asDouble()) 119 | ars.setRdb_current_bgsave_time_sec(stats_i.get("rdb_current_bgsave_time_sec").asDouble()) 120 | ars.setAof_enabled(stats_i.get("aof_enabled").asDouble()) 121 | ars.setAof_rewrite_in_progress(stats_i.get("aof_rewrite_in_progress").asDouble()) 122 | ars.setAof_rewrite_scheduled(stats_i.get("aof_rewrite_scheduled").asDouble()) 123 | ars.setAof_last_rewrite_time_sec(stats_i.get("aof_last_rewrite_time_sec").asDouble()) 124 | ars.setAof_current_rewrite_time_sec(stats_i.get("aof_current_rewrite_time_sec").asDouble()) 125 | ars.setTotal_connections_received(stats_i.get("total_connections_received").asDouble()) 126 | ars.setTotal_commands_processed(stats_i.get("total_commands_processed").asDouble()) 127 | ars.setTotal_net_input_bytes(stats_i.get("total_net_input_bytes").asDouble()) 128 | ars.setTotal_net_output_bytes(stats_i.get("total_net_output_bytes").asDouble()) 129 | ars.setRejected_connections(stats_i.get("rejected_connections").asDouble()) 130 | ars.setExpired_keys(stats_i.get("expired_keys").asDouble()) 131 | ars.setEvicted_keys(stats_i.get("evicted_keys").asDouble()) 132 | ars.setKeyspace_hits(stats_i.get("keyspace_hits").asDouble()) 133 | ars.setKeyspace_misses(stats_i.get("keyspace_misses").asDouble()) 134 | ars.setPubsub_channels(stats_i.get("pubsub_channels").asDouble()) 135 | ars.setPubsub_patterns(stats_i.get("pubsub_patterns").asDouble()) 136 | ars.setLoading(stats_i.get("loading").asDouble()) 137 | ars.setConnected_slaves(stats_i.get("connected_slaves").asDouble()) 138 | ars.setRepl_backlog_size(stats_i.get("repl_backlog_size").asDouble()) 139 | ars.setUsed_cpu_sys(stats_i.get("used_cpu_sys").asDouble()) 140 | ars.setUsed_cpu_user(stats_i.get("used_cpu_user").asDouble()) 141 | ars.setUsed_cpu_sys_children(stats_i.get("used_cpu_sys_children").asDouble()) 142 | ars.setUsed_cpu_user_children(stats_i.get("used_cpu_user_children").asDouble()) 143 | 144 | var ard2 = new AppRedisData2() 145 | ard2.setContainer_name(container_name) 146 | ard2.setContainer_uuid(container_uuid) 147 | ard2.setEnvironment_id(environment_id) 148 | ard2.setNamespace(namespace) 149 | ard2.setTimestamp(data_timestamp) 150 | ard2.setStats(ars) 151 | 152 | var ar2 = new AppRedis2() 153 | ar2.setType(_type) 154 | ar2.setData(ard2) 155 | 156 | // index yun-{data.environment_id}-{data.container_uuid}-app-redis-{data.stats.timestamp:YYYY-MM-dd} 157 | val index = "yun-" + environment_id + "-" + container_uuid + "-app-redis-" + DateUtil.getYYYYMMdd(timestamp) 158 | arr.+=((index, JsonUtil.formatJson(ar2))) 159 | } 160 | } 161 | }catch { 162 | case ex : Exception => ex.printStackTrace() 163 | } 164 | arr 165 | } 166 | 167 | } 168 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/appcapability/AppRedisStats.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.appcapability; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Created by GH-GAN on 2016/11/29. 7 | */ 8 | public class AppRedisStats implements Serializable { 9 | String timestamp; 10 | Double uptime_in_seconds; 11 | Double connected_clients; 12 | Double blocked_clients; 13 | Double used_memory; 14 | Double used_memory_rss; 15 | Double used_memory_peak; 16 | Double used_memory_lua; 17 | Double max_memory; 18 | Double mem_fragmentation_ratio; 19 | Double rdb_changes_since_last_save; 20 | Double rdb_last_bgsave_time_sec; 21 | Double rdb_current_bgsave_time_sec; 22 | Double aof_enabled; 23 | Double aof_rewrite_in_progress; 24 | Double aof_rewrite_scheduled; 25 | Double aof_last_rewrite_time_sec; 26 | Double aof_current_rewrite_time_sec; 27 | Double total_connections_received; 28 | Double total_commands_processed; 29 | Double total_net_input_bytes; 30 | Double total_net_output_bytes; 31 | Double rejected_connections; 32 | Double expired_keys; 33 | Double evicted_keys; 34 | Double keyspace_hits; 35 | Double keyspace_misses; 36 | Double pubsub_channels; 37 | Double pubsub_patterns; 38 | Double loading; 39 | Double connected_slaves; 40 | Double repl_backlog_size; 41 | Double used_cpu_sys; 42 | Double used_cpu_user; 43 | Double used_cpu_sys_children; 44 | Double used_cpu_user_children; 45 | 46 | public String getTimestamp() { 47 | return timestamp; 48 | } 49 | 50 | public void setTimestamp(String timestamp) { 51 | this.timestamp = timestamp; 52 | } 53 | 54 | public Double getUptime_in_seconds() { 55 | return uptime_in_seconds; 56 | } 57 | 58 | public void setUptime_in_seconds(Double uptime_in_seconds) { 59 | this.uptime_in_seconds = uptime_in_seconds; 60 | } 61 | 62 | public Double getConnected_clients() { 63 | return connected_clients; 64 | } 65 | 66 | public void setConnected_clients(Double connected_clients) { 67 | this.connected_clients = connected_clients; 68 | } 69 | 70 | public Double getBlocked_clients() { 71 | return blocked_clients; 72 | } 73 | 74 | public void setBlocked_clients(Double blocked_clients) { 75 | this.blocked_clients = blocked_clients; 76 | } 77 | 78 | public Double getUsed_memory() { 79 | return used_memory; 80 | } 81 | 82 | public void setUsed_memory(Double used_memory) { 83 | this.used_memory = used_memory; 84 | } 85 | 86 | public Double getUsed_memory_rss() { 87 | return used_memory_rss; 88 | } 89 | 90 | public void setUsed_memory_rss(Double used_memory_rss) { 91 | this.used_memory_rss = used_memory_rss; 92 | } 93 | 94 | public Double getUsed_memory_peak() { 95 | return used_memory_peak; 96 | } 97 | 98 | public void setUsed_memory_peak(Double used_memory_peak) { 99 | this.used_memory_peak = used_memory_peak; 100 | } 101 | 102 | public Double getUsed_memory_lua() { 103 | return used_memory_lua; 104 | } 105 | 106 | public void setUsed_memory_lua(Double used_memory_lua) { 107 | this.used_memory_lua = used_memory_lua; 108 | } 109 | 110 | public Double getMax_memory() { 111 | return max_memory; 112 | } 113 | 114 | public void setMax_memory(Double max_memory) { 115 | this.max_memory = max_memory; 116 | } 117 | 118 | public Double getMem_fragmentation_ratio() { 119 | return mem_fragmentation_ratio; 120 | } 121 | 122 | public void setMem_fragmentation_ratio(Double mem_fragmentation_ratio) { 123 | this.mem_fragmentation_ratio = mem_fragmentation_ratio; 124 | } 125 | 126 | public Double getRdb_changes_since_last_save() { 127 | return rdb_changes_since_last_save; 128 | } 129 | 130 | public void setRdb_changes_since_last_save(Double rdb_changes_since_last_save) { 131 | this.rdb_changes_since_last_save = rdb_changes_since_last_save; 132 | } 133 | 134 | public Double getRdb_last_bgsave_time_sec() { 135 | return rdb_last_bgsave_time_sec; 136 | } 137 | 138 | public void setRdb_last_bgsave_time_sec(Double rdb_last_bgsave_time_sec) { 139 | this.rdb_last_bgsave_time_sec = rdb_last_bgsave_time_sec; 140 | } 141 | 142 | public Double getRdb_current_bgsave_time_sec() { 143 | return rdb_current_bgsave_time_sec; 144 | } 145 | 146 | public void setRdb_current_bgsave_time_sec(Double rdb_current_bgsave_time_sec) { 147 | this.rdb_current_bgsave_time_sec = rdb_current_bgsave_time_sec; 148 | } 149 | 150 | public Double getAof_enabled() { 151 | return aof_enabled; 152 | } 153 | 154 | public void setAof_enabled(Double aof_enabled) { 155 | this.aof_enabled = aof_enabled; 156 | } 157 | 158 | public Double getAof_rewrite_in_progress() { 159 | return aof_rewrite_in_progress; 160 | } 161 | 162 | public void setAof_rewrite_in_progress(Double aof_rewrite_in_progress) { 163 | this.aof_rewrite_in_progress = aof_rewrite_in_progress; 164 | } 165 | 166 | public Double getAof_rewrite_scheduled() { 167 | return aof_rewrite_scheduled; 168 | } 169 | 170 | public void setAof_rewrite_scheduled(Double aof_rewrite_scheduled) { 171 | this.aof_rewrite_scheduled = aof_rewrite_scheduled; 172 | } 173 | 174 | public Double getAof_last_rewrite_time_sec() { 175 | return aof_last_rewrite_time_sec; 176 | } 177 | 178 | public void setAof_last_rewrite_time_sec(Double aof_last_rewrite_time_sec) { 179 | this.aof_last_rewrite_time_sec = aof_last_rewrite_time_sec; 180 | } 181 | 182 | public Double getAof_current_rewrite_time_sec() { 183 | return aof_current_rewrite_time_sec; 184 | } 185 | 186 | public void setAof_current_rewrite_time_sec(Double aof_current_rewrite_time_sec) { 187 | this.aof_current_rewrite_time_sec = aof_current_rewrite_time_sec; 188 | } 189 | 190 | public Double getTotal_connections_received() { 191 | return total_connections_received; 192 | } 193 | 194 | public void setTotal_connections_received(Double total_connections_received) { 195 | this.total_connections_received = total_connections_received; 196 | } 197 | 198 | public Double getTotal_commands_processed() { 199 | return total_commands_processed; 200 | } 201 | 202 | public void setTotal_commands_processed(Double total_commands_processed) { 203 | this.total_commands_processed = total_commands_processed; 204 | } 205 | 206 | public Double getTotal_net_input_bytes() { 207 | return total_net_input_bytes; 208 | } 209 | 210 | public void setTotal_net_input_bytes(Double total_net_input_bytes) { 211 | this.total_net_input_bytes = total_net_input_bytes; 212 | } 213 | 214 | public Double getTotal_net_output_bytes() { 215 | return total_net_output_bytes; 216 | } 217 | 218 | public void setTotal_net_output_bytes(Double total_net_output_bytes) { 219 | this.total_net_output_bytes = total_net_output_bytes; 220 | } 221 | 222 | public Double getRejected_connections() { 223 | return rejected_connections; 224 | } 225 | 226 | public void setRejected_connections(Double rejected_connections) { 227 | this.rejected_connections = rejected_connections; 228 | } 229 | 230 | public Double getExpired_keys() { 231 | return expired_keys; 232 | } 233 | 234 | public void setExpired_keys(Double expired_keys) { 235 | this.expired_keys = expired_keys; 236 | } 237 | 238 | public Double getEvicted_keys() { 239 | return evicted_keys; 240 | } 241 | 242 | public void setEvicted_keys(Double evicted_keys) { 243 | this.evicted_keys = evicted_keys; 244 | } 245 | 246 | public Double getKeyspace_hits() { 247 | return keyspace_hits; 248 | } 249 | 250 | public void setKeyspace_hits(Double keyspace_hits) { 251 | this.keyspace_hits = keyspace_hits; 252 | } 253 | 254 | public Double getKeyspace_misses() { 255 | return keyspace_misses; 256 | } 257 | 258 | public void setKeyspace_misses(Double keyspace_misses) { 259 | this.keyspace_misses = keyspace_misses; 260 | } 261 | 262 | public Double getPubsub_channels() { 263 | return pubsub_channels; 264 | } 265 | 266 | public void setPubsub_channels(Double pubsub_channels) { 267 | this.pubsub_channels = pubsub_channels; 268 | } 269 | 270 | public Double getPubsub_patterns() { 271 | return pubsub_patterns; 272 | } 273 | 274 | public void setPubsub_patterns(Double pubsub_patterns) { 275 | this.pubsub_patterns = pubsub_patterns; 276 | } 277 | 278 | public Double getLoading() { 279 | return loading; 280 | } 281 | 282 | public void setLoading(Double loading) { 283 | this.loading = loading; 284 | } 285 | 286 | public Double getConnected_slaves() { 287 | return connected_slaves; 288 | } 289 | 290 | public void setConnected_slaves(Double connected_slaves) { 291 | this.connected_slaves = connected_slaves; 292 | } 293 | 294 | public Double getRepl_backlog_size() { 295 | return repl_backlog_size; 296 | } 297 | 298 | public void setRepl_backlog_size(Double repl_backlog_size) { 299 | this.repl_backlog_size = repl_backlog_size; 300 | } 301 | 302 | public Double getUsed_cpu_sys() { 303 | return used_cpu_sys; 304 | } 305 | 306 | public void setUsed_cpu_sys(Double used_cpu_sys) { 307 | this.used_cpu_sys = used_cpu_sys; 308 | } 309 | 310 | public Double getUsed_cpu_user() { 311 | return used_cpu_user; 312 | } 313 | 314 | public void setUsed_cpu_user(Double used_cpu_user) { 315 | this.used_cpu_user = used_cpu_user; 316 | } 317 | 318 | public Double getUsed_cpu_sys_children() { 319 | return used_cpu_sys_children; 320 | } 321 | 322 | public void setUsed_cpu_sys_children(Double used_cpu_sys_children) { 323 | this.used_cpu_sys_children = used_cpu_sys_children; 324 | } 325 | 326 | public Double getUsed_cpu_user_children() { 327 | return used_cpu_user_children; 328 | } 329 | 330 | public void setUsed_cpu_user_children(Double used_cpu_user_children) { 331 | this.used_cpu_user_children = used_cpu_user_children; 332 | } 333 | } 334 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/scala/com/gh/yun/SaveToESCap.scala: -------------------------------------------------------------------------------- 1 | package com.gh.yun 2 | 3 | /** 4 | * Created by GH-GAN on 2016/6/3. 5 | */ 6 | 7 | import java.util 8 | 9 | import com.gh.bean.appcapability._ 10 | import com.gh.bean.containercapability._ 11 | import com.gh.bean.logfile.{LogFileRedis, LogFileMySql, LogFileNginx, LogFileContainer} 12 | import com.gh.utils.{ ConfigUtil, DateUtil, JsonUtil} 13 | import com.gh.yun.parse.{ParseMysql, ParseRedis, ParseContainer, ParseNginx} 14 | import org.apache.kafka.common.serialization.StringDeserializer 15 | import org.apache.log4j.{Level, Logger} 16 | import org.apache.spark.rdd.RDD 17 | import org.apache.spark.streaming.dstream.DStream 18 | import org.apache.spark.{SparkContext, SparkConf} 19 | import org.apache.spark.SparkContext 20 | import org.apache.spark.SparkContext._ 21 | import org.apache.spark.streaming.StreamingContext._ 22 | import org.elasticsearch.spark.rdd.EsSpark 23 | 24 | import org.elasticsearch.spark.streaming._ 25 | import org.apache.spark.sql.SparkSession 26 | import org.apache.spark.streaming.kafka010.ConsumerStrategies._ 27 | import org.apache.spark.streaming.kafka010.KafkaUtils 28 | import org.apache.spark.streaming.kafka010.LocationStrategies._ 29 | import org.apache.spark.streaming.{Seconds, StreamingContext} 30 | import org.codehaus.jackson.JsonNode 31 | import org.elasticsearch.spark._ 32 | 33 | import scala.collection.mutable.ArrayBuffer 34 | import scala.collection.{mutable, Seq} 35 | 36 | object SaveToESCap { 37 | def main(args: Array[String]) { 38 | Logger.getRootLogger.setLevel(Level.WARN) 39 | val conf = new SparkConf().setMaster(ConfigUtil.sparkmaster).setAppName("saveToesCap"); 40 | conf.set("es.index.auto.create", "true") 41 | conf.set("es.nodes", ConfigUtil.esnodes) 42 | conf.set("spark.streaming.kafka.consumer.poll.ms", "1024") 43 | 44 | val ssc = new StreamingContext(conf,Seconds(ConfigUtil.capStreamSaveTime)) 45 | ssc.checkpoint(ConfigUtil.saveCapcheckpoint) 46 | 47 | var brokers = ConfigUtil.brokers 48 | val _topics = "capability-mysql,capability-redis,capability-container,capability-nginx".split(",").toSet 49 | // val _topics = "capability-container".split(",").toSet 50 | val group = "saveToesCap" 51 | 52 | val kafkaParams = Map[String, Object]( 53 | "bootstrap.servers" -> brokers, 54 | "key.deserializer" -> classOf[StringDeserializer], 55 | "value.deserializer" -> classOf[StringDeserializer], 56 | "group.id" -> group, 57 | "auto.offset.reset" -> "latest", 58 | "heartbeat.interval.ms" -> "6000", 59 | "session.timeout.ms" -> "20000", 60 | "max.partition.fetch.bytes" -> "10485760", 61 | "max.poll.records" -> "10000", // message.max.bytes 62 | "enable.auto.commit" -> (false: java.lang.Boolean) 63 | ) 64 | 65 | val stream = KafkaUtils.createDirectStream[String, String]( 66 | ssc, 67 | PreferConsistent, 68 | Subscribe[String, String](_topics, kafkaParams) 69 | ) 70 | 71 | /* val datas = stream.map(line => { 72 | var node : JsonNode = JsonUtil.getJsonNode(line.value()) 73 | (node,line.value()) 74 | }).filter(x => x._1 != null && x._1.get("type") != null) 75 | .map(node => (node._1.get("type").asText(),node._2))*/ 76 | 77 | val datas = stream.map(line => { 78 | var node : JsonNode = JsonUtil.getJsonNode(line.value()) 79 | node 80 | }).filter(x => x != null && x.get("type") != null) 81 | .map(node => (node.get("type").asText(),node)) 82 | 83 | // cap 84 | ParseAppMysqlSaveToEs2(datas) // appmysql 85 | ParseAppRedisSaveToEs2(datas) // appredis 86 | ParseAppNginxSaveToEs2(datas) // appnginx 87 | ParseContainerSaveToEs2(datas) // container 88 | 89 | // datas.foreachRDD(x => { 90 | // x.foreach(y => println(y._1 + "============================================" + y._2)) 91 | // }) 92 | 93 | ssc.start() 94 | ssc.awaitTermination() 95 | 96 | } 97 | 98 | def ParseAppNginxSaveToEs(datas : DStream[(String,String)]): Unit ={ 99 | val nginx = datas.filter(_._1.equals("nginx")) 100 | nginx.flatMap(lines => { 101 | var node : JsonNode = JsonUtil.getJsonNode(lines._2) 102 | new ParseNginx().statsToArray(lines._1,node) 103 | }).saveJsonToEs("yun-{data.environment_id}-{data.container_uuid}-app-nginx-{data.stats.timestamp:YYYY-MM-dd}/nginx-cap") 104 | } 105 | 106 | def ParseAppNginxSaveToEs2(datas : DStream[(String,JsonNode)]): Unit ={ 107 | val nginx = datas.filter(_._1.equals("nginx")) 108 | nginx.flatMap(lines => { 109 | new ParseNginx().statsToArray(lines._1,lines._2) 110 | }).saveJsonToEs("yun-{data.environment_id}-{data.container_uuid}-app-nginx-{data.stats.timestamp:YYYY-MM-dd}/nginx-cap",Map("es.batch.size.bytes"->"2097152","es.batch.size.entries"->"500","es.batch.write.refresh"->"false")) 111 | } 112 | 113 | def ParseAppMysqlSaveToEs(datas : DStream[(String,String)]): Unit ={ 114 | val mysql = datas.filter(_._1.equals("mysql")) 115 | mysql.flatMap(lines => { 116 | val appMySql = JsonUtil.getBean(lines._2, classOf[AppMySql]) 117 | val _stats = appMySql.getData().getStats 118 | val arr = ArrayBuffer[(String)]() 119 | 120 | if (!"".equals(appMySql.getData.getContainer_uuid) && !"".equals(appMySql.getData.getEnvironment_id)){ 121 | for (i <- 0 to (_stats.size() - 1) ){ 122 | val stats_i = _stats.get(i) 123 | 124 | var ms = new AppMySql2() 125 | var amsd = new AppMySqlData2() 126 | amsd.setContainer_name(appMySql.getData.getContainer_name) 127 | amsd.setContainer_uuid(appMySql.getData.getContainer_uuid) 128 | amsd.setEnvironment_id(appMySql.getData.getEnvironment_id) 129 | amsd.setNamespace(appMySql.getData.getNamespace) 130 | amsd.setTimestamp(appMySql.getData.getTimestamp) 131 | amsd.setStats(stats_i) 132 | ms.setType(appMySql.getType) 133 | ms.setData(amsd) 134 | 135 | arr.+=(JsonUtil.formatJson(ms)) 136 | } 137 | } 138 | arr 139 | }).saveJsonToEs("yun-{data.environment_id}-{data.container_uuid}-app-mysql-{data.stats.timestamp:YYYY-MM-dd}/mysql-cap") 140 | } 141 | 142 | def ParseAppMysqlSaveToEs2(datas : DStream[(String,JsonNode)]): Unit ={ 143 | val mysql = datas.filter(_._1.equals("mysql")) 144 | val s = mysql.flatMap(lines => { 145 | new ParseMysql().statsToArray(lines._1,lines._2) 146 | }).saveJsonToEs("yun-{data.environment_id}-{data.container_uuid}-app-mysql-{data.stats.timestamp:YYYY-MM-dd}/mysql-cap",Map("es.batch.size.bytes"->"2097152","es.batch.size.entries"->"500","es.batch.write.refresh"->"false")) 147 | } 148 | 149 | def ParseAppRedisSaveToEs(datas : DStream[(String,String)]): Unit ={ 150 | val redis = datas.filter(_._1.equals("redis")) 151 | redis.flatMap(lines => { 152 | val appRedis = JsonUtil.getBean(lines._2, classOf[AppRedis]) 153 | val _stats = appRedis.getData.getStats 154 | val arr = ArrayBuffer[(String)]() 155 | 156 | if (!"".equals(appRedis.getData.getEnvironment_id) && !"".equals(appRedis.getData.getContainer_uuid)){ 157 | for (i <- 0 to (_stats.size() - 1) ){ 158 | val stats_i = _stats.get(i) 159 | var ar2 = new AppRedis2() 160 | var ard2 = new AppRedisData2 161 | ard2.setContainer_name(appRedis.getData.getContainer_name) 162 | ard2.setContainer_uuid(appRedis.getData.getContainer_uuid) 163 | ard2.setEnvironment_id(appRedis.getData.getEnvironment_id) 164 | ard2.setNamespace(appRedis.getData.getNamespace) 165 | ard2.setTimestamp(appRedis.getData.getTimestamp) 166 | ard2.setStats(stats_i) 167 | 168 | ar2.setType(appRedis.getType) 169 | ar2.setData(ard2) 170 | 171 | arr.+=(JsonUtil.formatJson(ar2)) 172 | } 173 | } 174 | arr 175 | }).saveJsonToEs("yun-{data.environment_id}-{data.container_uuid}-app-redis-{data.stats.timestamp:YYYY-MM-dd}/redis-cap") 176 | } 177 | 178 | def ParseAppRedisSaveToEs2(datas : DStream[(String,JsonNode)]): Unit ={ 179 | val redis = datas.filter(_._1.equals("redis")) 180 | redis.flatMap(lines => { 181 | new ParseRedis().statsToArray(lines._1,lines._2) 182 | }).saveJsonToEs("yun-{data.environment_id}-{data.container_uuid}-app-redis-{data.stats.timestamp:YYYY-MM-dd}/redis-cap",Map("es.batch.size.bytes"->"2097152","es.batch.size.entries"->"500","es.batch.write.refresh"->"false")) 183 | } 184 | 185 | def ParseContainerSaveToEs(datas : DStream[(String,String)]): Unit ={ 186 | val container = datas.filter(_._1.equals("container")) 187 | container.flatMap(lines => { 188 | val container = JsonUtil.getBean(lines._2, classOf[Container]) 189 | val datas = container.getData // containers 190 | 191 | val _containers = datas.flatMap(data => { // data 192 | val timestamp = data.getTimestamp 193 | val environment_id = data.getEnvironment_id.trim 194 | val container_uuid = data.getContainer_uuid.trim 195 | 196 | if (!"".equals(environment_id) && !"".equals(container_uuid)){ 197 | val _stats = data.getStats // 一个容器多条记录 198 | val ccs = _stats.flatMap(cont => { 199 | var container2 = new Container2() 200 | var cd2 = new ContainerData2() 201 | cd2.setStats(cont) 202 | cd2.setContainer_name(data.getContainer_name) 203 | cd2.setContainer_uuid(container_uuid) 204 | cd2.setEnvironment_id(environment_id) 205 | cd2.setNamespace(data.getNamespace) 206 | cd2.setTimestamp(data.getTimestamp) 207 | container2.setType(container.getType) 208 | container2.setData(cd2) 209 | Array(JsonUtil.formatJson(container2)) 210 | }) 211 | ccs 212 | }else{ 213 | Array[String]() 214 | } 215 | }) 216 | _containers 217 | }).saveJsonToEs("yun-{data.environment_id}-{data.container_uuid}-container-container-{data.stats.timestamp:YYYY-MM-dd}/container-cap") 218 | } 219 | 220 | def ParseContainerSaveToEs2(datas : DStream[(String,JsonNode)]): Unit ={ 221 | val container = datas.filter(_._1.equals("container")) 222 | val ss = container.flatMap(lines => { 223 | new ParseContainer().statsToArray(lines._1,lines._2) 224 | }).saveJsonToEs("yun-{data.environment_id}-{data.container_uuid}-container-container-{data.stats.timestamp:YYYY-MM-dd}/container-cap",Map("es.batch.size.bytes"->"2097152","es.batch.size.entries"->"500","es.batch.write.refresh"->"false")) 225 | 226 | } 227 | 228 | } 229 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/java/com/gh/bean/containercapability/ContainerStats.java: -------------------------------------------------------------------------------- 1 | package com.gh.bean.containercapability; 2 | 3 | import java.io.Serializable; 4 | import java.util.ArrayList; 5 | 6 | /** 7 | * Created by GH-GAN on 2016/11/29. 8 | */ 9 | public class ContainerStats implements Serializable { 10 | String timestamp; 11 | Double container_cpu_usage_seconds_total; 12 | Double container_cpu_user_seconds_total; 13 | Double container_cpu_system_seconds_total; 14 | Double container_memory_usage_bytes; 15 | Double container_memory_limit_bytes; 16 | Double container_memory_cache; 17 | Double container_memory_rss; 18 | Double container_memory_swap; 19 | Double container_network_receive_bytes_total; 20 | Double container_network_receive_packets_total; 21 | Double container_network_receive_packets_dropped_total; 22 | Double container_network_receive_errors_total; 23 | Double container_network_transmit_bytes_total; 24 | Double container_network_transmit_packets_total; 25 | Double container_network_transmit_packets_dropped_total; 26 | Double container_network_transmit_errors_total; 27 | ContainerFileSystem [] container_filesystem; 28 | Double container_diskio_service_bytes_async; 29 | Double container_diskio_service_bytes_read; 30 | Double container_diskio_service_bytes_sync; 31 | Double container_diskio_service_bytes_total; 32 | Double container_diskio_service_bytes_write; 33 | Double container_tasks_state_nr_sleeping; 34 | Double container_tasks_state_nr_running; 35 | Double container_tasks_state_nr_stopped; 36 | Double container_tasks_state_nr_uninterruptible; 37 | Double container_tasks_state_nr_io_wait; 38 | 39 | public String getTimestamp() { 40 | return timestamp; 41 | } 42 | 43 | public void setTimestamp(String timestamp) { 44 | this.timestamp = timestamp; 45 | } 46 | 47 | public Double getContainer_cpu_usage_seconds_total() { 48 | return container_cpu_usage_seconds_total; 49 | } 50 | 51 | public void setContainer_cpu_usage_seconds_total(Double container_cpu_usage_seconds_total) { 52 | this.container_cpu_usage_seconds_total = container_cpu_usage_seconds_total; 53 | } 54 | 55 | public Double getContainer_cpu_user_seconds_total() { 56 | return container_cpu_user_seconds_total; 57 | } 58 | 59 | public void setContainer_cpu_user_seconds_total(Double container_cpu_user_seconds_total) { 60 | this.container_cpu_user_seconds_total = container_cpu_user_seconds_total; 61 | } 62 | 63 | public Double getContainer_cpu_system_seconds_total() { 64 | return container_cpu_system_seconds_total; 65 | } 66 | 67 | public void setContainer_cpu_system_seconds_total(Double container_cpu_system_seconds_total) { 68 | this.container_cpu_system_seconds_total = container_cpu_system_seconds_total; 69 | } 70 | 71 | public Double getContainer_memory_usage_bytes() { 72 | return container_memory_usage_bytes; 73 | } 74 | 75 | public void setContainer_memory_usage_bytes(Double container_memory_usage_bytes) { 76 | this.container_memory_usage_bytes = container_memory_usage_bytes; 77 | } 78 | 79 | public Double getContainer_memory_limit_bytes() { 80 | return container_memory_limit_bytes; 81 | } 82 | 83 | public void setContainer_memory_limit_bytes(Double container_memory_limit_bytes) { 84 | this.container_memory_limit_bytes = container_memory_limit_bytes; 85 | } 86 | 87 | public Double getContainer_memory_cache() { 88 | return container_memory_cache; 89 | } 90 | 91 | public void setContainer_memory_cache(Double container_memory_cache) { 92 | this.container_memory_cache = container_memory_cache; 93 | } 94 | 95 | public Double getContainer_memory_rss() { 96 | return container_memory_rss; 97 | } 98 | 99 | public void setContainer_memory_rss(Double container_memory_rss) { 100 | this.container_memory_rss = container_memory_rss; 101 | } 102 | 103 | public Double getContainer_memory_swap() { 104 | return container_memory_swap; 105 | } 106 | 107 | public void setContainer_memory_swap(Double container_memory_swap) { 108 | this.container_memory_swap = container_memory_swap; 109 | } 110 | 111 | public Double getContainer_network_receive_bytes_total() { 112 | return container_network_receive_bytes_total; 113 | } 114 | 115 | public void setContainer_network_receive_bytes_total(Double container_network_receive_bytes_total) { 116 | this.container_network_receive_bytes_total = container_network_receive_bytes_total; 117 | } 118 | 119 | public Double getContainer_network_receive_packets_total() { 120 | return container_network_receive_packets_total; 121 | } 122 | 123 | public void setContainer_network_receive_packets_total(Double container_network_receive_packets_total) { 124 | this.container_network_receive_packets_total = container_network_receive_packets_total; 125 | } 126 | 127 | public Double getContainer_network_receive_packets_dropped_total() { 128 | return container_network_receive_packets_dropped_total; 129 | } 130 | 131 | public void setContainer_network_receive_packets_dropped_total(Double container_network_receive_packets_dropped_total) { 132 | this.container_network_receive_packets_dropped_total = container_network_receive_packets_dropped_total; 133 | } 134 | 135 | public Double getContainer_network_receive_errors_total() { 136 | return container_network_receive_errors_total; 137 | } 138 | 139 | public void setContainer_network_receive_errors_total(Double container_network_receive_errors_total) { 140 | this.container_network_receive_errors_total = container_network_receive_errors_total; 141 | } 142 | 143 | public Double getContainer_network_transmit_bytes_total() { 144 | return container_network_transmit_bytes_total; 145 | } 146 | 147 | public void setContainer_network_transmit_bytes_total(Double container_network_transmit_bytes_total) { 148 | this.container_network_transmit_bytes_total = container_network_transmit_bytes_total; 149 | } 150 | 151 | public Double getContainer_network_transmit_packets_total() { 152 | return container_network_transmit_packets_total; 153 | } 154 | 155 | public void setContainer_network_transmit_packets_total(Double container_network_transmit_packets_total) { 156 | this.container_network_transmit_packets_total = container_network_transmit_packets_total; 157 | } 158 | 159 | public Double getContainer_network_transmit_packets_dropped_total() { 160 | return container_network_transmit_packets_dropped_total; 161 | } 162 | 163 | public void setContainer_network_transmit_packets_dropped_total(Double container_network_transmit_packets_dropped_total) { 164 | this.container_network_transmit_packets_dropped_total = container_network_transmit_packets_dropped_total; 165 | } 166 | 167 | public Double getContainer_network_transmit_errors_total() { 168 | return container_network_transmit_errors_total; 169 | } 170 | 171 | public void setContainer_network_transmit_errors_total(Double container_network_transmit_errors_total) { 172 | this.container_network_transmit_errors_total = container_network_transmit_errors_total; 173 | } 174 | 175 | public ContainerFileSystem[] getContainer_filesystem() { 176 | return container_filesystem; 177 | } 178 | 179 | public void setContainer_filesystem(ContainerFileSystem[] container_filesystem) { 180 | this.container_filesystem = container_filesystem; 181 | } 182 | 183 | public Double getContainer_diskio_service_bytes_async() { 184 | return container_diskio_service_bytes_async; 185 | } 186 | 187 | public void setContainer_diskio_service_bytes_async(Double container_diskio_service_bytes_async) { 188 | this.container_diskio_service_bytes_async = container_diskio_service_bytes_async; 189 | } 190 | 191 | public Double getContainer_diskio_service_bytes_read() { 192 | return container_diskio_service_bytes_read; 193 | } 194 | 195 | public void setContainer_diskio_service_bytes_read(Double container_diskio_service_bytes_read) { 196 | this.container_diskio_service_bytes_read = container_diskio_service_bytes_read; 197 | } 198 | 199 | public Double getContainer_diskio_service_bytes_sync() { 200 | return container_diskio_service_bytes_sync; 201 | } 202 | 203 | public void setContainer_diskio_service_bytes_sync(Double container_diskio_service_bytes_sync) { 204 | this.container_diskio_service_bytes_sync = container_diskio_service_bytes_sync; 205 | } 206 | 207 | public Double getContainer_diskio_service_bytes_total() { 208 | return container_diskio_service_bytes_total; 209 | } 210 | 211 | public void setContainer_diskio_service_bytes_total(Double container_diskio_service_bytes_total) { 212 | this.container_diskio_service_bytes_total = container_diskio_service_bytes_total; 213 | } 214 | 215 | public Double getContainer_diskio_service_bytes_write() { 216 | return container_diskio_service_bytes_write; 217 | } 218 | 219 | public void setContainer_diskio_service_bytes_write(Double container_diskio_service_bytes_write) { 220 | this.container_diskio_service_bytes_write = container_diskio_service_bytes_write; 221 | } 222 | 223 | public Double getContainer_tasks_state_nr_sleeping() { 224 | return container_tasks_state_nr_sleeping; 225 | } 226 | 227 | public void setContainer_tasks_state_nr_sleeping(Double container_tasks_state_nr_sleeping) { 228 | this.container_tasks_state_nr_sleeping = container_tasks_state_nr_sleeping; 229 | } 230 | 231 | public Double getContainer_tasks_state_nr_running() { 232 | return container_tasks_state_nr_running; 233 | } 234 | 235 | public void setContainer_tasks_state_nr_running(Double container_tasks_state_nr_running) { 236 | this.container_tasks_state_nr_running = container_tasks_state_nr_running; 237 | } 238 | 239 | public Double getContainer_tasks_state_nr_stopped() { 240 | return container_tasks_state_nr_stopped; 241 | } 242 | 243 | public void setContainer_tasks_state_nr_stopped(Double container_tasks_state_nr_stopped) { 244 | this.container_tasks_state_nr_stopped = container_tasks_state_nr_stopped; 245 | } 246 | 247 | public Double getContainer_tasks_state_nr_uninterruptible() { 248 | return container_tasks_state_nr_uninterruptible; 249 | } 250 | 251 | public void setContainer_tasks_state_nr_uninterruptible(Double container_tasks_state_nr_uninterruptible) { 252 | this.container_tasks_state_nr_uninterruptible = container_tasks_state_nr_uninterruptible; 253 | } 254 | 255 | public Double getContainer_tasks_state_nr_io_wait() { 256 | return container_tasks_state_nr_io_wait; 257 | } 258 | 259 | public void setContainer_tasks_state_nr_io_wait(Double container_tasks_state_nr_io_wait) { 260 | this.container_tasks_state_nr_io_wait = container_tasks_state_nr_io_wait; 261 | } 262 | } 263 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/scala/com/gh/yun/parse/ParseContainer.scala: -------------------------------------------------------------------------------- 1 | package com.gh.yun.parse 2 | 3 | import com.gh.bean.appcapability.{AppNginx, AppNginxData, AppNginxStats} 4 | import com.gh.bean.containercapability._ 5 | import com.gh.utils.{DateUtil, JsonUtil} 6 | import org.codehaus.jackson.JsonNode 7 | 8 | import scala.collection.mutable.ArrayBuffer 9 | 10 | /** 11 | * Created by GH-GAN on 2016/12/20. 12 | */ 13 | class ParseContainer { 14 | def statsToArray(_type: String,node : JsonNode): ArrayBuffer[String] ={ 15 | val arr = ArrayBuffer[String]() 16 | try{ 17 | var data = node.get("data") 18 | for (i <- 0 until data.size()){ 19 | val data_i = data.get(i) 20 | 21 | val environment_id = data_i.get("environment_id").asText().trim 22 | val container_uuid = data_i.get("container_uuid").asText().trim 23 | val container_name = data_i.get("container_name").asText() 24 | val namespace = data_i.get("namespace").asText() 25 | val data_timestamp = data_i.get("timestamp").asText() 26 | 27 | if (!"".equals(environment_id) && !"".equals(container_uuid)){ 28 | var stats = data_i.get("stats") 29 | for (j <- 0 until stats.size()){ 30 | val stats_i = stats.get(j) 31 | val timestamp = stats_i.get("timestamp").asText() 32 | 33 | var cs = new ContainerStats() 34 | cs.setTimestamp(timestamp) 35 | cs.setContainer_cpu_usage_seconds_total(stats_i.get("container_cpu_usage_seconds_total").asDouble()) 36 | cs.setContainer_cpu_user_seconds_total(stats_i.get("container_cpu_user_seconds_total").asDouble()) 37 | cs.setContainer_cpu_system_seconds_total(stats_i.get("container_cpu_system_seconds_total").asDouble()) 38 | cs.setContainer_memory_usage_bytes(stats_i.get("container_memory_usage_bytes").asDouble()) 39 | cs.setContainer_memory_limit_bytes(stats_i.get("container_memory_limit_bytes").asDouble()) 40 | cs.setContainer_memory_cache(stats_i.get("container_memory_cache").asDouble()) 41 | cs.setContainer_memory_rss(stats_i.get("container_memory_rss").asDouble()) 42 | cs.setContainer_memory_swap(stats_i.get("container_memory_swap").asDouble()) 43 | cs.setContainer_network_receive_bytes_total(stats_i.get("container_network_receive_bytes_total").asDouble()) 44 | cs.setContainer_network_receive_packets_total(stats_i.get("container_network_receive_packets_total").asDouble()) 45 | cs.setContainer_network_receive_packets_dropped_total(stats_i.get("container_network_receive_packets_dropped_total").asDouble()) 46 | cs.setContainer_network_receive_errors_total(stats_i.get("container_network_receive_errors_total").asDouble()) 47 | cs.setContainer_network_transmit_bytes_total(stats_i.get("container_network_transmit_bytes_total").asDouble()) 48 | cs.setContainer_network_transmit_packets_total(stats_i.get("container_network_transmit_packets_total").asDouble()) 49 | cs.setContainer_network_transmit_packets_dropped_total(stats_i.get("container_network_transmit_packets_dropped_total").asDouble()) 50 | cs.setContainer_network_transmit_errors_total(stats_i.get("container_network_transmit_errors_total").asDouble()) 51 | 52 | val container_filesystem = stats_i.get("container_filesystem") 53 | val cfs = new Array[ContainerFileSystem](container_filesystem.size) 54 | 55 | for (z <- 0 until container_filesystem.size()){ 56 | val container_filesystem_z = container_filesystem.get(z) 57 | var cfs_z = new ContainerFileSystem() 58 | cfs_z.setContainer_filesystem_name(container_filesystem_z.get("container_filesystem_name").asText()) 59 | cfs_z.setContainer_filesystem_type(container_filesystem_z.get("container_filesystem_type").asText()) 60 | cfs_z.setContainer_filesystem_capacity(container_filesystem_z.get("container_filesystem_capacity").asDouble()) 61 | cfs_z.setContainer_filesystem_usage(container_filesystem_z.get("container_filesystem_usage").asDouble()) 62 | cfs(z) = cfs_z 63 | } 64 | 65 | cs.setContainer_filesystem(cfs) 66 | cs.setContainer_diskio_service_bytes_async(stats_i.get("container_diskio_service_bytes_async").asDouble()) 67 | cs.setContainer_diskio_service_bytes_read(stats_i.get("container_diskio_service_bytes_read").asDouble()) 68 | cs.setContainer_diskio_service_bytes_sync(stats_i.get("container_diskio_service_bytes_sync").asDouble()) 69 | cs.setContainer_diskio_service_bytes_total(stats_i.get("container_diskio_service_bytes_total").asDouble()) 70 | cs.setContainer_diskio_service_bytes_write(stats_i.get("container_diskio_service_bytes_write").asDouble()) 71 | cs.setContainer_tasks_state_nr_sleeping(stats_i.get("container_tasks_state_nr_sleeping").asDouble()) 72 | cs.setContainer_tasks_state_nr_running(stats_i.get("container_tasks_state_nr_running").asDouble()) 73 | cs.setContainer_tasks_state_nr_stopped(stats_i.get("container_tasks_state_nr_stopped").asDouble()) 74 | cs.setContainer_tasks_state_nr_uninterruptible(stats_i.get("container_tasks_state_nr_uninterruptible").asDouble()) 75 | cs.setContainer_tasks_state_nr_io_wait(stats_i.get("container_tasks_state_nr_io_wait").asDouble()) 76 | 77 | var cd2 = new ContainerData2() 78 | cd2.setContainer_name(container_name) 79 | cd2.setContainer_uuid(container_uuid) 80 | cd2.setEnvironment_id(environment_id) 81 | cd2.setNamespace(namespace) 82 | cd2.setTimestamp(data_timestamp) 83 | cd2.setStats(cs) 84 | 85 | var container2 = new Container2() 86 | container2.setType(_type) 87 | container2.setData(cd2) 88 | 89 | // yun-{data.environment_id}-{data.container_uuid}-container-container-{data.stats.timestamp:YYYY-MM-dd}/container-cap 90 | // val index = "yun-" + environment_id + "-" + container_uuid + "-container-container-" + DateUtil.getYYYYMMdd(timestamp) 91 | arr.+=(JsonUtil.formatJson(container2)) 92 | } 93 | } 94 | } 95 | }catch { 96 | case ex : Exception => ex.printStackTrace() 97 | } 98 | arr 99 | } 100 | 101 | def statsToArray2(_type: String,node : JsonNode): ArrayBuffer[(String,String)] ={ 102 | val arr = ArrayBuffer[(String,String)]() 103 | try{ 104 | var data = node.get("data") 105 | for (i <- 0 until data.size()){ 106 | val data_i = data.get(i) 107 | 108 | val environment_id = data_i.get("environment_id").asText().trim 109 | val container_uuid = data_i.get("container_uuid").asText().trim 110 | val container_name = data_i.get("container_name").asText() 111 | val namespace = data_i.get("namespace").asText() 112 | val data_timestamp = data_i.get("timestamp").asText() 113 | 114 | if (!"".equals(environment_id) && !"".equals(container_uuid)){ 115 | var stats = data_i.get("stats") 116 | for (j <- 0 until stats.size()){ 117 | val stats_i = stats.get(j) 118 | val timestamp = stats_i.get("timestamp").asText() 119 | 120 | var cs = new ContainerStats() 121 | cs.setTimestamp(timestamp) 122 | cs.setContainer_cpu_usage_seconds_total(stats_i.get("container_cpu_usage_seconds_total").asDouble()) 123 | cs.setContainer_cpu_user_seconds_total(stats_i.get("container_cpu_user_seconds_total").asDouble()) 124 | cs.setContainer_cpu_system_seconds_total(stats_i.get("container_cpu_system_seconds_total").asDouble()) 125 | cs.setContainer_memory_usage_bytes(stats_i.get("container_memory_usage_bytes").asDouble()) 126 | cs.setContainer_memory_limit_bytes(stats_i.get("container_memory_limit_bytes").asDouble()) 127 | cs.setContainer_memory_cache(stats_i.get("container_memory_cache").asDouble()) 128 | cs.setContainer_memory_rss(stats_i.get("container_memory_rss").asDouble()) 129 | cs.setContainer_memory_swap(stats_i.get("container_memory_swap").asDouble()) 130 | cs.setContainer_network_receive_bytes_total(stats_i.get("container_network_receive_bytes_total").asDouble()) 131 | cs.setContainer_network_receive_packets_total(stats_i.get("container_network_receive_packets_total").asDouble()) 132 | cs.setContainer_network_receive_packets_dropped_total(stats_i.get("container_network_receive_packets_dropped_total").asDouble()) 133 | cs.setContainer_network_receive_errors_total(stats_i.get("container_network_receive_errors_total").asDouble()) 134 | cs.setContainer_network_transmit_bytes_total(stats_i.get("container_network_transmit_bytes_total").asDouble()) 135 | cs.setContainer_network_transmit_packets_total(stats_i.get("container_network_transmit_packets_total").asDouble()) 136 | cs.setContainer_network_transmit_packets_dropped_total(stats_i.get("container_network_transmit_packets_dropped_total").asDouble()) 137 | cs.setContainer_network_transmit_errors_total(stats_i.get("container_network_transmit_errors_total").asDouble()) 138 | 139 | val container_filesystem = stats_i.get("container_filesystem") 140 | val cfs = new Array[ContainerFileSystem](container_filesystem.size) 141 | 142 | for (z <- 0 until container_filesystem.size()){ 143 | val container_filesystem_z = container_filesystem.get(z) 144 | var cfs_z = new ContainerFileSystem() 145 | cfs_z.setContainer_filesystem_name(container_filesystem_z.get("container_filesystem_name").asText()) 146 | cfs_z.setContainer_filesystem_type(container_filesystem_z.get("container_filesystem_type").asText()) 147 | cfs_z.setContainer_filesystem_capacity(container_filesystem_z.get("container_filesystem_capacity").asDouble()) 148 | cfs_z.setContainer_filesystem_usage(container_filesystem_z.get("container_filesystem_usage").asDouble()) 149 | cfs(z) = cfs_z 150 | } 151 | 152 | cs.setContainer_filesystem(cfs) 153 | cs.setContainer_diskio_service_bytes_async(stats_i.get("container_diskio_service_bytes_async").asDouble()) 154 | cs.setContainer_diskio_service_bytes_read(stats_i.get("container_diskio_service_bytes_read").asDouble()) 155 | cs.setContainer_diskio_service_bytes_sync(stats_i.get("container_diskio_service_bytes_sync").asDouble()) 156 | cs.setContainer_diskio_service_bytes_total(stats_i.get("container_diskio_service_bytes_total").asDouble()) 157 | cs.setContainer_diskio_service_bytes_write(stats_i.get("container_diskio_service_bytes_write").asDouble()) 158 | cs.setContainer_tasks_state_nr_sleeping(stats_i.get("container_tasks_state_nr_sleeping").asDouble()) 159 | cs.setContainer_tasks_state_nr_running(stats_i.get("container_tasks_state_nr_running").asDouble()) 160 | cs.setContainer_tasks_state_nr_stopped(stats_i.get("container_tasks_state_nr_stopped").asDouble()) 161 | cs.setContainer_tasks_state_nr_uninterruptible(stats_i.get("container_tasks_state_nr_uninterruptible").asDouble()) 162 | cs.setContainer_tasks_state_nr_io_wait(stats_i.get("container_tasks_state_nr_io_wait").asDouble()) 163 | 164 | var cd2 = new ContainerData2() 165 | cd2.setContainer_name(container_name) 166 | cd2.setContainer_uuid(container_uuid) 167 | cd2.setEnvironment_id(environment_id) 168 | cd2.setNamespace(namespace) 169 | cd2.setTimestamp(data_timestamp) 170 | cd2.setStats(cs) 171 | 172 | var container2 = new Container2() 173 | container2.setType(_type) 174 | container2.setData(cd2) 175 | 176 | // yun-{data.environment_id}-{data.container_uuid}-container-container-{data.stats.timestamp:YYYY-MM-dd}/container-cap 177 | val index = "yun-" + environment_id + "-" + container_uuid + "-container-container-" + DateUtil.getYYYYMMdd(timestamp) 178 | arr.+=((index,JsonUtil.formatJson(container2))) 179 | } 180 | } 181 | } 182 | }catch { 183 | case ex : Exception => ex.printStackTrace() 184 | } 185 | arr 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /SparkStreaming_2.0.2-Kafka_0.10.0.1-ES_5.1.1/src/main/scala/com/gh/yun/ContainerCap.scala: -------------------------------------------------------------------------------- 1 | package com.gh.yun 2 | 3 | import java.math.BigDecimal 4 | import java.text.SimpleDateFormat 5 | import java.util 6 | 7 | import com.gh.bean.alert.{AlertData, AlertDataInfo, KeyValue} 8 | import com.gh.bean.containercapability.ContainerFileSystem 9 | import com.gh.utils._ 10 | import org.apache.hadoop.fs.DF 11 | import org.apache.kafka.clients.consumer.ConsumerRecord 12 | import org.apache.kafka.common.serialization.StringDeserializer 13 | import org.apache.log4j.{Level, Logger} 14 | import org.apache.spark.SparkConf 15 | import org.apache.spark.streaming._ 16 | import org.apache.spark.streaming.dstream.{DStream, InputDStream} 17 | import org.apache.spark.streaming.kafka010.ConsumerStrategies.Subscribe 18 | import org.apache.spark.streaming.kafka010.LocationStrategies.PreferConsistent 19 | import org.apache.spark.streaming.kafka010._ 20 | import org.codehaus.jackson.JsonNode 21 | 22 | 23 | import scala.collection.mutable.ArrayBuffer 24 | 25 | /** 26 | * Created by GH-GAN on 2016/11/24. 27 | */ 28 | object ContainerCap { 29 | def main(args: Array[String]) { 30 | /* if (args.length < 4) { 31 | System.err.println("Usage: Kafka ") 32 | System.exit(1) 33 | }*/ 34 | Logger.getRootLogger.setLevel(Level.WARN) 35 | 36 | val sparkConf = new SparkConf().setAppName("capability-container").setMaster(ConfigUtil.sparkmaster) 37 | val ssc = new StreamingContext(sparkConf, Seconds(ConfigUtil.capStreamtime)) 38 | ssc.checkpoint(ConfigUtil.containerCapcheckpoint) 39 | 40 | var brokers = ConfigUtil.brokers 41 | val _topics = "capability-container".split(",").toSet 42 | val group = "capability-container-js" 43 | 44 | val kafkaParams = Map[String, Object]( 45 | "bootstrap.servers" -> brokers, 46 | "key.deserializer" -> classOf[StringDeserializer], 47 | "value.deserializer" -> classOf[StringDeserializer], 48 | "group.id" -> group, 49 | "auto.offset.reset" -> "latest", //当前偏移不在服务器上时,按最新开始 50 | "heartbeat.interval.ms" -> "6000", 51 | "session.timeout.ms" -> "20000", 52 | "max.partition.fetch.bytes" -> "1048576000", 53 | "max.poll.records" -> "5000000", // message.max.bytes 54 | "enable.auto.commit" -> (false: java.lang.Boolean) 55 | ) 56 | 57 | val stream = KafkaUtils.createDirectStream[String, String]( 58 | ssc, 59 | PreferConsistent, 60 | Subscribe[String, String](_topics, kafkaParams) 61 | ) 62 | 63 | val datas = parseLog(stream) 64 | val container_group = datas.groupByKey() 65 | compute(container_group) 66 | 67 | ssc.start() 68 | ssc.awaitTermination() 69 | } 70 | 71 | def parseLog(stream : InputDStream[ConsumerRecord[String, String]]) : DStream[(String,(String,Double,Double,Double,Double,Double,Array[ContainerFileSystem]))] ={ 72 | val datas = stream.map(line => { 73 | var node : JsonNode = JsonUtil.getJsonNode(line.value()) 74 | node 75 | }).filter(x => (x != null && x.get("type") != null)) 76 | .flatMap(node => { 77 | val _type = node.get("type").asText() 78 | 79 | val datas = node.get("data") 80 | val dataArr = ArrayBuffer[(String,(String,Double,Double,Double,Double,Double,Array[ContainerFileSystem]))]() 81 | for (i <- 0 to (datas.size() - 1) ){ 82 | // 容器 83 | val data_i = node.get("data").get(i) 84 | val environment_id = data_i.get("environment_id").asText() 85 | val container_uuid = data_i.get("container_uuid").asText() 86 | 87 | var container_name = "" 88 | var namespace = "" 89 | try { 90 | container_name = data_i.get("container_name").asText() 91 | namespace = data_i.get("namespace").asText() 92 | }catch { 93 | case ex : Exception => { 94 | println("--------> container_name/namespace is null") 95 | ex.printStackTrace() 96 | } 97 | } 98 | 99 | val statsArr = ArrayBuffer[(String,(String,Double,Double,Double,Double,Double,Array[ContainerFileSystem]))]() 100 | 101 | if ("".equals(environment_id) || "".equals(container_uuid)) statsArr 102 | else { 103 | val _stats = data_i.get("stats") 104 | for (i <- 0 to (_stats.size() - 1) ){ 105 | val stats_i = _stats.get(i) 106 | val timestamp = stats_i.get("timestamp").asText() 107 | 108 | val cpu = stats_i.get("container_cpu_usage_seconds_total").asDouble() 109 | val memory_usage = stats_i.get("container_memory_usage_bytes").asDouble() 110 | val memory_limit = stats_i.get("container_memory_limit_bytes").asDouble() 111 | 112 | val network_transmit = stats_i.get("container_network_transmit_bytes_total").asDouble() 113 | val network_receive = stats_i.get("container_network_receive_bytes_total").asDouble() 114 | 115 | val container_filesystem = stats_i.get("container_filesystem") 116 | val diskNum = container_filesystem.size() 117 | val disks = new Array[ContainerFileSystem](diskNum) 118 | for (i <- 0 to (diskNum - 1)){ 119 | val bean = JsonUtil.getBean(container_filesystem.get(i).toString,classOf[ContainerFileSystem]) 120 | disks(i) = bean 121 | } 122 | statsArr.+=( (environment_id+"#"+container_uuid+"#"+_type + "#" + container_name + "#" + namespace,(timestamp,cpu,memory_usage,memory_limit,network_transmit,network_receive,disks)) ) 123 | } 124 | } 125 | dataArr.++=(statsArr) 126 | } 127 | dataArr 128 | }) 129 | datas 130 | } 131 | 132 | def compute(container_group : DStream[(String,Iterable[(String,Double,Double,Double,Double,Double,Array[ContainerFileSystem])])]): Unit ={ 133 | val warn = container_group.map(x => { 134 | val count = x._2.size 135 | 136 | val start_log = x._2.head 137 | val end_log = x._2.last 138 | val start_time = start_log._1.toString 139 | val end_time = end_log._1.toString 140 | // 1s = 1000 ms = 1000000 us = 1000000000 ns => 1 ms = 1000000 ns 141 | // println("passe:"+end_time+"------------start_time:"+start_time) 142 | // val intervalInMs = df_utc_223231.parse(end_time).getTime() - df_utc_223231.parse(start_time).getTime() // 毫秒 143 | val df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); 144 | var t1 = df.parse(start_time).getTime 145 | var t2 = df.parse(end_time).getTime 146 | val intervalInMs = t2 - t1 // 毫秒 147 | val intervalInSec = intervalInMs / 1000 // 秒 148 | val intervalNs = intervalInMs * 1000000 // 纳秒 149 | 150 | // cpu 151 | var cpu = (end_log._2 - start_log._2) / intervalNs 152 | // cpu = new BigDecimal(cpu).toString 153 | 154 | //memory 155 | var memory = - 1.0 156 | if (null != start_log._4 && 0 != start_log._4){ 157 | val memory_sum = x._2.map(_._3).reduce(_+_) 158 | memory = memory_sum / count / start_log._4 159 | } 160 | 161 | // disk 162 | val _disks = end_log._7.map(x => (x.getContainer_filesystem_usage / x.getContainer_filesystem_capacity,x)) 163 | 164 | // network_tx 165 | var network_tx = (end_log._5 - start_log._5) / intervalInSec // byte/s 166 | network_tx = network_tx / 1024 / 1024 167 | 168 | // network_rx 169 | var network_rx = (end_log._6 - start_log._6) / intervalInSec // byte/s 170 | network_rx = network_rx / 1024 / 1024 171 | 172 | (x._1,start_time,end_time,cpu,memory,_disks,network_tx,network_rx) 173 | }).map(aAlert(_)) // 条件判断 174 | .filter(_._4.size() > 0) // 是否有告警信息 175 | 176 | // 超过阈值 177 | warn.foreachRDD( record => { 178 | val alerts = record.map(line => { 179 | AlertInfoUtil.toWarnBean(AlertInfoUtil.SUCCESS,AlertInfoUtil.ALERT_TYPE_M,AlertInfoUtil.ALERT_DIM_C,line._1, line._2, line._3, line._4,line._5) 180 | }) 181 | if (!alerts.isEmpty()){ 182 | val collect = alerts.collect() 183 | if (collect.size > 0) new HttpUtil().alerts(collect) //告警 184 | } 185 | }) 186 | 187 | } 188 | 189 | def aAlert(line : (String,String,String,Double,Double,Array[(Double,ContainerFileSystem)],Double,Double)): (String,String,String,util.ArrayList[KeyValue],String) ={ 190 | var msg : String = "" 191 | val list = new util.ArrayList[KeyValue]() 192 | 193 | /* if (line._4 >= 0.9) list.add(new KeyValue("cpu",line._4.toString)) 194 | if (-1 != line._5 && line._5 >= 0.7) list.add(new KeyValue("memory",line._5.toString)) 195 | val disks = line._6.filter(_._1 > 0.8) 196 | if (disks.size > 0 ) { 197 | list.add(new KeyValue("disk",disks(0)._1.toString)) 198 | msg = disks.map(x => x._2.getContainer_filesystem_name + ",").toString.trim 199 | msg = msg.substring(0,msg.length - 2) 200 | } 201 | if (line._7 > 2) list.add(new KeyValue("network_tx",line._7.toString)) 202 | if (line._8 > 2) list.add(new KeyValue("network_rx",line._8.toString))*/ 203 | 204 | 205 | val cpu = HttpUtil.gz_map.get("container_cpu") 206 | cpu.getCondition match { 207 | case "GTE" => if(line._4 >= cpu.getValue) list.add(new KeyValue("cpu",new BigDecimal(line._4).toString)) 208 | case "GT" => if(line._4 > cpu.getValue) list.add(new KeyValue("cpu",new BigDecimal(line._4).toString)) 209 | case "LTE" => if(line._4 <= cpu.getValue) list.add(new KeyValue("cpu",new BigDecimal(line._4).toString)) 210 | case "LT" => if(line._4 < cpu.getValue) list.add(new KeyValue("cpu",new BigDecimal(line._4).toString)) 211 | case "EQ" => if(line._4 == cpu.getValue) list.add(new KeyValue("cpu",new BigDecimal(line._4).toString)) 212 | } 213 | 214 | val memory = HttpUtil.gz_map.get("container_memory") 215 | memory.getCondition match { 216 | case "GTE" => if(-1 != line._5 && line._5 >= memory.getValue) list.add(new KeyValue("memory",line._5.toString)) 217 | case "GT" => if(-1 != line._5 && line._5 > memory.getValue) list.add(new KeyValue("memory",line._5.toString)) 218 | case "LTE" => if(-1 != line._5 && line._5 <= memory.getValue) list.add(new KeyValue("memory",line._5.toString)) 219 | case "LT" => if(-1 != line._5 && line._5 < memory.getValue) list.add(new KeyValue("memory",line._5.toString)) 220 | case "EQ" => if(-1 != line._5 && line._5 == memory.getValue) list.add(new KeyValue("memory",line._5.toString)) 221 | } 222 | 223 | 224 | val disk = HttpUtil.gz_map.get("container_disk") 225 | val disks = line._6.filter(x => { 226 | var flag = false 227 | disk.getCondition match { 228 | case "GTE" => if(x._1 >= disk.getValue) flag = true 229 | case "GT" => if(x._1 > disk.getValue) flag = true 230 | case "LTE" => if(x._1 <= disk.getValue) flag = true 231 | case "LT" => if(x._1 < disk.getValue) flag = true 232 | case "EQ" => if(x._1 == disk.getValue) flag = true 233 | } 234 | flag 235 | }) 236 | if (disks.size > 0 ) { 237 | list.add(new KeyValue("disk",disks(0)._1.toString)) 238 | msg = disks.map(x => x._2.getContainer_filesystem_name + ",").toString.trim 239 | msg = msg.substring(0,msg.length - 2) 240 | } 241 | 242 | 243 | val network_tx = HttpUtil.gz_map.get("container_network_tx") 244 | network_tx.getCondition match { 245 | case "GTE" => if(line._7 >= network_tx.getValue) list.add(new KeyValue("network_tx",line._7.toString)) 246 | case "GT" => if(line._7 > network_tx.getValue) list.add(new KeyValue("network_tx",line._7.toString)) 247 | case "LTE" => if(line._7 <= network_tx.getValue) list.add(new KeyValue("network_tx",line._7.toString)) 248 | case "LT" => if(line._7 < network_tx.getValue) list.add(new KeyValue("network_tx",line._7.toString)) 249 | case "EQ" => if(line._7 == network_tx.getValue) list.add(new KeyValue("network_tx",line._7.toString)) 250 | } 251 | 252 | val network_rx = HttpUtil.gz_map.get("container_network_rx") 253 | network_rx.getCondition match { 254 | case "GTE" => if(line._8 >= network_rx.getValue) list.add(new KeyValue("network_rx",line._8.toString)) 255 | case "GT" => if(line._8 > network_rx.getValue) list.add(new KeyValue("network_rx",line._8.toString)) 256 | case "LTE" => if(line._8 <= network_rx.getValue) list.add(new KeyValue("network_rx",line._8.toString)) 257 | case "LT" => if(line._8 < network_rx.getValue) list.add(new KeyValue("network_rx",line._8.toString)) 258 | case "EQ" => if(line._8 == network_rx.getValue) list.add(new KeyValue("network_rx",line._8.toString)) 259 | } 260 | 261 | //test 262 | // list.add(new KeyValue("cpu",line._4.toString)) 263 | // list.add(new KeyValue("network_tx",line._7.toString)) 264 | 265 | (line._1,line._2,line._3,list,msg) 266 | } 267 | 268 | } 269 | --------------------------------------------------------------------------------