├── .gitignore ├── src └── main │ └── java │ └── cn │ └── itcast │ └── logMonitor │ ├── mail │ ├── MailCenterConstant.java │ ├── MailAuthenticator.java │ ├── MessageSender.java │ └── MailInfo.java │ ├── spout │ ├── StringScheme.java │ └── RandomSpout.java │ ├── bolt │ ├── SaveMessage2MySql.java │ ├── PrepareRecordBolt.java │ └── FilterBolt.java │ ├── domain │ ├── User.java │ ├── App.java │ ├── Rule.java │ ├── Record.java │ └── Message.java │ ├── dao │ ├── DataSourceUtil.java │ └── LogMonitorDao.java │ ├── sms │ └── SMSBase.java │ ├── LogMonitorTopologyMain.java │ └── utils │ ├── DateUtils.java │ └── MonitorHandler.java └── pom.xml /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | target/ 3 | out/ 4 | LogMonitor.iml -------------------------------------------------------------------------------- /src/main/java/cn/itcast/logMonitor/mail/MailCenterConstant.java: -------------------------------------------------------------------------------- 1 | package cn.itcast.logMonitor.mail; 2 | 3 | /** 4 | * @author y15079 5 | * @create 2018-05-06 22:16 6 | * @desc 7 | **/ 8 | public class MailCenterConstant { 9 | public final static String PROTOCOL = "smtp"; 10 | // 设置发件人使用的SMTP服务器、用户名、密码 11 | public final static String SMTP_SERVER = "smtp.163.com"; 12 | public final static String FROM_ADDRESS = "wjymaoxiangyi@163.com"; 13 | public final static String USER = "wjymaoxiangyi@163.com"; 14 | public final static String PWD = "striver"; 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/cn/itcast/logMonitor/mail/MailAuthenticator.java: -------------------------------------------------------------------------------- 1 | package cn.itcast.logMonitor.mail; 2 | 3 | import javax.mail.Authenticator; 4 | import javax.mail.PasswordAuthentication; 5 | 6 | /** 7 | * @author y15079 8 | * @create 2018-05-06 22:14 9 | * @desc 10 | **/ 11 | public class MailAuthenticator extends Authenticator { 12 | String userName; 13 | String userPassword; 14 | 15 | public MailAuthenticator() { 16 | super(); 17 | } 18 | 19 | public MailAuthenticator(String user, String pwd) { 20 | this.userName = user; 21 | this.userPassword = pwd; 22 | } 23 | 24 | public PasswordAuthentication getPasswordAuthentication(){ 25 | return new PasswordAuthentication(userName, userPassword); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/cn/itcast/logMonitor/spout/StringScheme.java: -------------------------------------------------------------------------------- 1 | package cn.itcast.logMonitor.spout; 2 | 3 | import org.apache.storm.spout.Scheme; 4 | import org.apache.storm.tuple.Fields; 5 | import org.apache.storm.tuple.Values; 6 | 7 | import java.nio.ByteBuffer; 8 | import java.nio.charset.Charset; 9 | import java.util.List; 10 | 11 | /** 12 | * @author y15079 13 | * @create 2018-05-06 23:34 14 | * @desc 15 | **/ 16 | public class StringScheme implements Scheme { 17 | Charset charset = Charset.forName("utf-8"); 18 | 19 | public List deserialize(ByteBuffer byteBuffer) { 20 | try { 21 | return new Values(charset.decode(byteBuffer).toString()); 22 | } catch (Exception e) { 23 | throw new RuntimeException(e); 24 | } 25 | } 26 | public Fields getOutputFields() { 27 | return new Fields("line"); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/cn/itcast/logMonitor/bolt/SaveMessage2MySql.java: -------------------------------------------------------------------------------- 1 | package cn.itcast.logMonitor.bolt; 2 | 3 | import cn.itcast.logMonitor.domain.Record; 4 | import cn.itcast.logMonitor.utils.MonitorHandler; 5 | import com.sun.org.apache.regexp.internal.RE; 6 | import org.apache.log4j.Logger; 7 | import org.apache.storm.topology.BasicOutputCollector; 8 | import org.apache.storm.topology.OutputFieldsDeclarer; 9 | import org.apache.storm.topology.base.BaseBasicBolt; 10 | import org.apache.storm.tuple.Tuple; 11 | 12 | /** 13 | * @author y15079 14 | * @create 2018-05-06 23:20 15 | * @desc 16 | **/ 17 | public class SaveMessage2MySql extends BaseBasicBolt{ 18 | private static Logger logger = Logger.getLogger(SaveMessage2MySql.class); 19 | 20 | @Override 21 | public void execute(Tuple input, BasicOutputCollector basicOutputCollector) { 22 | Record record = (Record) input.getValueByField("record"); 23 | MonitorHandler.save(record); 24 | } 25 | 26 | @Override 27 | public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { 28 | 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/cn/itcast/logMonitor/domain/User.java: -------------------------------------------------------------------------------- 1 | package cn.itcast.logMonitor.domain; 2 | 3 | /** 4 | * Describe: 用户信息 5 | */ 6 | public class User { 7 | private int id;//用户编号 8 | private String name;//用户名称 9 | private String mobile;//用户手机 10 | private String email;//用户邮箱 11 | private int isValid;//用户是否可用 12 | 13 | public int getId() { 14 | return id; 15 | } 16 | 17 | public void setId(int id) { 18 | this.id = id; 19 | } 20 | 21 | public String getName() { 22 | return name; 23 | } 24 | 25 | public void setName(String name) { 26 | this.name = name; 27 | } 28 | 29 | public String getMobile() { 30 | return mobile; 31 | } 32 | 33 | public void setMobile(String mobile) { 34 | this.mobile = mobile; 35 | } 36 | 37 | public String getEmail() { 38 | return email; 39 | } 40 | 41 | public void setEmail(String email) { 42 | this.email = email; 43 | } 44 | 45 | public int getIsValid() { 46 | return isValid; 47 | } 48 | 49 | public void setIsValid(int isValid) { 50 | this.isValid = isValid; 51 | } 52 | 53 | @Override 54 | public String toString() { 55 | return "User{" + 56 | "id=" + id + 57 | ", name='" + name + '\'' + 58 | ", mobile='" + mobile + '\'' + 59 | ", email='" + email + '\'' + 60 | ", isValid=" + isValid + 61 | '}'; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/cn/itcast/logMonitor/domain/App.java: -------------------------------------------------------------------------------- 1 | package cn.itcast.logMonitor.domain; 2 | 3 | /** 4 | * Describe: 请补充类描述 5 | */ 6 | public class App { 7 | private int id;//应用编号 8 | private String name;//应用名称 9 | private int isOnline;//应用是否在线 10 | private int typeId;//应用所属类别 11 | private String userId;//应用的负责人，多个用户用逗号分开 12 | 13 | public int getId() { 14 | return id; 15 | } 16 | 17 | public void setId(int id) { 18 | this.id = id; 19 | } 20 | 21 | public String getName() { 22 | return name; 23 | } 24 | 25 | public void setName(String name) { 26 | this.name = name; 27 | } 28 | 29 | public int getIsOnline() { 30 | return isOnline; 31 | } 32 | 33 | public void setIsOnline(int isOnline) { 34 | this.isOnline = isOnline; 35 | } 36 | 37 | public int getTypeId() { 38 | return typeId; 39 | } 40 | 41 | public void setTypeId(int typeId) { 42 | this.typeId = typeId; 43 | } 44 | 45 | public String getUserId() { 46 | return userId; 47 | } 48 | 49 | public void setUserId(String userId) { 50 | this.userId = userId; 51 | } 52 | 53 | @Override 54 | public String toString() { 55 | return "App{" + 56 | "id=" + id + 57 | ", name='" + name + '\'' + 58 | ", isOnline=" + isOnline + 59 | ", typeId=" + typeId + 60 | ", userId='" + userId + '\'' + 61 | '}'; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/cn/itcast/logMonitor/domain/Rule.java: -------------------------------------------------------------------------------- 1 | package cn.itcast.logMonitor.domain; 2 | 3 | /** 4 | * Describe: 请补充类描述 5 | */ 6 | public class Rule { 7 | 8 | private int id;//规则编号 9 | private String name;//规则名称 10 | private String keyword;//规则过滤的关键字 11 | private int isValid;//规则是否可用 12 | private int appId;//规则所属的应用 13 | 14 | public int getId() { 15 | return id; 16 | } 17 | 18 | public void setId(int id) { 19 | this.id = id; 20 | } 21 | 22 | public String getName() { 23 | return name; 24 | } 25 | 26 | public void setName(String name) { 27 | this.name = name; 28 | } 29 | 30 | public String getKeyword() { 31 | return keyword; 32 | } 33 | 34 | public void setKeyword(String keyword) { 35 | this.keyword = keyword; 36 | } 37 | 38 | public int getIsValid() { 39 | return isValid; 40 | } 41 | 42 | public void setIsValid(int isValid) { 43 | this.isValid = isValid; 44 | } 45 | 46 | public int getAppId() { 47 | return appId; 48 | } 49 | 50 | public void setAppId(int appId) { 51 | this.appId = appId; 52 | } 53 | 54 | @Override 55 | public String toString() { 56 | return "RuleField{" + 57 | "id=" + id + 58 | ", name='" + name + '\'' + 59 | ", keyword='" + keyword + '\'' + 60 | ", isValid=" + isValid + 61 | ", appId=" + appId + 62 | '}'; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/main/java/cn/itcast/logMonitor/bolt/PrepareRecordBolt.java: -------------------------------------------------------------------------------- 1 | package cn.itcast.logMonitor.bolt; 2 | 3 | import cn.itcast.logMonitor.domain.Message; 4 | import cn.itcast.logMonitor.domain.Record; 5 | import cn.itcast.logMonitor.utils.MonitorHandler; 6 | import org.apache.commons.beanutils.BeanUtils; 7 | import org.apache.log4j.Logger; 8 | import org.apache.storm.topology.BasicOutputCollector; 9 | import org.apache.storm.topology.OutputFieldsDeclarer; 10 | import org.apache.storm.topology.base.BaseBasicBolt; 11 | import org.apache.storm.tuple.Fields; 12 | import org.apache.storm.tuple.Tuple; 13 | import org.apache.storm.tuple.Values; 14 | 15 | /** 16 | * @author y15079 17 | * @create 2018-05-06 23:13 18 | * @desc 将触发信息发送邮件或短息，并保存到mysql数据库中 19 | * //BaseRichBolt 需要手动调ack方法，BaseBasicBolt由storm框架自动调ack方法 20 | **/ 21 | public class PrepareRecordBolt extends BaseBasicBolt{ 22 | private static Logger logger = Logger.getLogger(PrepareRecordBolt.class); 23 | 24 | @Override 25 | public void execute(Tuple input, BasicOutputCollector basicOutputCollector) { 26 | Message message = (Message) input.getValueByField("message"); 27 | String appId = input.getStringByField("appId"); 28 | //将触发规则的信息进行通知 29 | MonitorHandler.notifly(appId, message); 30 | Record record = new Record(); 31 | try { 32 | BeanUtils.copyProperties(record,message); 33 | basicOutputCollector.emit(new Values(record)); 34 | } catch (Exception e) { 35 | e.printStackTrace(); 36 | } 37 | } 38 | 39 | @Override 40 | public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { 41 | outputFieldsDeclarer.declare(new Fields("record")); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/cn/itcast/logMonitor/dao/DataSourceUtil.java: -------------------------------------------------------------------------------- 1 | package cn.itcast.logMonitor.dao; 2 | 3 | 4 | import cn.itcast.logMonitor.domain.Record; 5 | import com.mchange.v2.c3p0.ComboPooledDataSource; 6 | import org.apache.log4j.Logger; 7 | import org.springframework.jdbc.core.JdbcTemplate; 8 | 9 | import javax.sql.DataSource; 10 | import java.util.Date; 11 | 12 | /** 13 | * @author y15079 14 | * @create 2018-05-06 21:56 15 | * @desc 16 | **/ 17 | public class DataSourceUtil { 18 | private static Logger logger = Logger.getLogger(DataSourceUtil.class); 19 | private static DataSource dataSource; 20 | 21 | static { 22 | dataSource = new ComboPooledDataSource("logMonitor"); 23 | } 24 | 25 | public static synchronized DataSource getDataSource(){ 26 | if (dataSource == null){ 27 | dataSource = new ComboPooledDataSource(); 28 | } 29 | return dataSource; 30 | } 31 | 32 | public static void main(String[] args) { 33 | JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource); 34 | //String sql = "SELECT `id`,`name`,`keyword`,`isValid`,`appId` FROM `log_monitor`.`log_monitor_rule` WHERE isValid =1"; 35 | //System.out.println(jdbcTemplate.query(sql, new BeanPropertyRowMapper(RuleField.class))); 36 | 37 | Record record = new Record(); 38 | record.setAppId(1111); 39 | record.setRuleId(1); 40 | record.setIsEmail(1); 41 | record.setIsPhone(1); 42 | record.setIsColse(0); 43 | String sql = "INSERT INTO `log_monitor`.`log_monitor_rule_record`" + 44 | " (`appId`,`ruleId`,`isEmail`,`isPhone`,`isColse`,`noticeInfo`,`updataDate`) " + 45 | "VALUES ( ?,?,?,?,?,?,?)"; 46 | jdbcTemplate.update(sql, record.getAppId(), record.getRuleId() , record.getIsEmail(), record.getIsPhone(), 0, record.getLine(),new Date()); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/cn/itcast/logMonitor/bolt/FilterBolt.java: -------------------------------------------------------------------------------- 1 | package cn.itcast.logMonitor.bolt; 2 | 3 | import cn.itcast.logMonitor.domain.Message; 4 | import cn.itcast.logMonitor.utils.MonitorHandler; 5 | import org.apache.log4j.Logger; 6 | import org.apache.storm.topology.BasicOutputCollector; 7 | import org.apache.storm.topology.OutputFieldsDeclarer; 8 | import org.apache.storm.topology.base.BaseBasicBolt; 9 | import org.apache.storm.tuple.Fields; 10 | import org.apache.storm.tuple.Tuple; 11 | import org.apache.storm.tuple.Values; 12 | 13 | 14 | /** 15 | * @author y15079 16 | * @create 2018-05-06 18:22 17 | * @desc 过滤规则信息 18 | * 19 | * BaseRichBolt 需要手动调ack方法，BaseBasicBolt由storm框架自动调ack方法 20 | **/ 21 | public class FilterBolt extends BaseBasicBolt{ 22 | private static Logger logger = Logger.getLogger(FilterBolt.class); 23 | 24 | @Override 25 | public void execute(Tuple input, BasicOutputCollector basicOutputCollector) { 26 | //获取KafkaSpout发送出来的数据 27 | String line = input.getString(0); 28 | //获取kafka发送的数据，是一个byte数组 29 | //byte[] value = (byte[]) input.getValue(0); 30 | //将数组转化成字符串 31 | //String line = new String(value); 32 | //对数据进行解析 33 | // appid content 34 | //1 error: Caused by: java.lang.NoClassDefFoundError: com/starit/gejie/dao/SysNameDao 35 | 36 | //把line转换为Message对象 37 | Message message = MonitorHandler.parser(line); 38 | if (message == null){ 39 | return; 40 | } 41 | //对日志message进行规制判定，看看是否触发规则 42 | if (MonitorHandler.trigger(message)){ 43 | basicOutputCollector.emit(new Values(message.getAppId(), message)); 44 | } 45 | //定时更新规则信息 46 | MonitorHandler.scheduleLoad(); 47 | } 48 | 49 | @Override 50 | public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) { 51 | outputFieldsDeclarer.declare(new Fields("appId", "message")); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/cn/itcast/logMonitor/domain/Record.java: -------------------------------------------------------------------------------- 1 | package cn.itcast.logMonitor.domain; 2 | 3 | /** 4 | * Describe: 触发报警之后的记录 5 | */ 6 | public class Record { 7 | private int id;//告警信息编号 8 | private int appId;//告警信息所属的应用 9 | private int ruleId;//告警信息所属的规则 10 | private int isEmail;//告警信息是否通过邮件告警 11 | private int isPhone;//告警信息是否通过短信告警 12 | private int isClose;//告警信息是否处理完毕 13 | private String line;//原始日志信息 14 | 15 | public int getId() { 16 | return id; 17 | } 18 | 19 | public void setId(int id) { 20 | this.id = id; 21 | } 22 | 23 | public int getAppId() { 24 | return appId; 25 | } 26 | 27 | public void setAppId(int appId) { 28 | this.appId = appId; 29 | } 30 | 31 | public int getRuleId() { 32 | return ruleId; 33 | } 34 | 35 | public void setRuleId(int ruleId) { 36 | this.ruleId = ruleId; 37 | } 38 | 39 | public int getIsEmail() { 40 | return isEmail; 41 | } 42 | 43 | public void setIsEmail(int isEmail) { 44 | this.isEmail = isEmail; 45 | } 46 | 47 | public int getIsPhone() { 48 | return isPhone; 49 | } 50 | 51 | public void setIsPhone(int isPhone) { 52 | this.isPhone = isPhone; 53 | } 54 | 55 | public int getIsColse() { 56 | return isClose; 57 | } 58 | 59 | public void setIsColse(int isColse) { 60 | this.isClose = isColse; 61 | } 62 | 63 | public String getLine() { 64 | return line; 65 | } 66 | 67 | public void setLine(String line) { 68 | this.line = line; 69 | } 70 | 71 | @Override 72 | public String toString() { 73 | return "Record{" + 74 | "id=" + id + 75 | ", appId=" + appId + 76 | ", ruleId=" + ruleId + 77 | ", isEmail=" + isEmail + 78 | ", isPhone=" + isPhone + 79 | ", isClose=" + isClose + 80 | ", line='" + line + '\'' + 81 | '}'; 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/main/java/cn/itcast/logMonitor/domain/Message.java: -------------------------------------------------------------------------------- 1 | package cn.itcast.logMonitor.domain; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Describe: 请补充类描述 7 | */ 8 | public class Message implements Serializable { 9 | private static final long serialVersionUID = 7241117393416877510L; 10 | private String appId;//消息所属服务器编号 11 | private String line;//消息内容 12 | private String ruleId;//规则编号 13 | private String keyword;//规则中的关键词 14 | private int isEmail;//是否已发送邮件 15 | private int isPhone;//是否已发送短信 16 | private String appName;//应用的名称 17 | 18 | public static long getSerialVersionUID() { 19 | return serialVersionUID; 20 | } 21 | 22 | public String getAppName() { 23 | return appName; 24 | } 25 | 26 | public void setAppName(String appName) { 27 | this.appName = appName; 28 | } 29 | 30 | public int getIsEmail() { 31 | return isEmail; 32 | } 33 | 34 | public void setIsEmail(int isEmail) { 35 | this.isEmail = isEmail; 36 | } 37 | 38 | public int getIsPhone() { 39 | return isPhone; 40 | } 41 | 42 | public void setIsPhone(int isPhone) { 43 | this.isPhone = isPhone; 44 | } 45 | 46 | public String getRuleId() { 47 | return ruleId; 48 | } 49 | 50 | public void setRuleId(String ruleId) { 51 | this.ruleId = ruleId; 52 | } 53 | 54 | public String getAppId() { 55 | return appId; 56 | } 57 | 58 | public void setAppId(String appId) { 59 | this.appId = appId; 60 | } 61 | 62 | public String getLine() { 63 | return line; 64 | } 65 | 66 | public void setLine(String line) { 67 | this.line = line; 68 | } 69 | 70 | public String getKeyword() { 71 | return keyword; 72 | } 73 | 74 | public void setKeyword(String keyword) { 75 | this.keyword = keyword; 76 | } 77 | 78 | @Override 79 | public String toString() { 80 | return "Message{" + 81 | "appId='" + appId + '\'' + 82 | ", line='" + line + '\'' + 83 | ", ruleId='" + ruleId + '\'' + 84 | ", keyword='" + keyword + '\'' + 85 | ", isEmail=" + isEmail + 86 | ", isPhone=" + isPhone + 87 | '}'; 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/main/java/cn/itcast/logMonitor/sms/SMSBase.java: -------------------------------------------------------------------------------- 1 | package cn.itcast.logMonitor.sms; 2 | 3 | import org.apache.commons.lang.StringUtils; 4 | import org.apache.log4j.Logger; 5 | 6 | import java.io.BufferedReader; 7 | import java.io.IOException; 8 | import java.io.InputStreamReader; 9 | import java.net.HttpURLConnection; 10 | import java.net.MalformedURLException; 11 | import java.net.URL; 12 | import java.net.URLEncoder; 13 | 14 | 15 | public class SMSBase { 16 | private static Logger logger = Logger.getLogger(SMSBase.class); 17 | private static final String USER_ID = "wangsenfeng"; 18 | private static final String PASSWORD = "wangsenfeng"; 19 | 20 | public static boolean sendSms(String mobile, String content) { 21 | HttpURLConnection httpconn = null; 22 | String result = ""; 23 | try { 24 | StringBuilder sb = new StringBuilder(); 25 | sb.append("http://service.winic.org:8009/sys_port/gateway/index.asp?"); 26 | //以下是参数 27 | sb.append("id=").append(URLEncoder.encode(USER_ID, "gb2312")); 28 | sb.append("&pwd=").append(PASSWORD); 29 | sb.append("&to=").append(mobile); 30 | sb.append("&content=").append(URLEncoder.encode(content, "gb2312")); 31 | sb.append("&time=").append(""); 32 | 33 | URL url = new URL(sb.toString()); 34 | httpconn = (HttpURLConnection) url.openConnection(); 35 | BufferedReader rd = new BufferedReader(new InputStreamReader(httpconn.getInputStream())); 36 | result = rd.readLine(); 37 | System.out.println("===================================="+result); 38 | rd.close(); 39 | } catch (MalformedURLException e) { 40 | e.printStackTrace(); 41 | } catch (IOException e) { 42 | e.printStackTrace(); 43 | } finally { 44 | if (httpconn != null) { 45 | httpconn.disconnect(); 46 | } 47 | httpconn = null; 48 | } 49 | if (StringUtils.isNotBlank(result)) { 50 | if (result.substring(0, 3).equals("000")) { 51 | return true; 52 | } 53 | } 54 | return false; 55 | } 56 | 57 | public static void main(String[] args) throws Exception { 58 | System.out.println(sendSms("15652306418", "传智播客日志监控平台-系统1发生异常，需要处理！")); 59 | // String result = "000/Send:1/Consumption:.1/Tmoney:1.8/sid:1112144311751362"; 60 | // if (StringUtils.isNotBlank(result)) { 61 | // if (result.substring(0, 3).equals("000")) { 62 | // System.out.println(true); 63 | // } 64 | // } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/cn/itcast/logMonitor/LogMonitorTopologyMain.java: -------------------------------------------------------------------------------- 1 | package cn.itcast.logMonitor; 2 | 3 | import cn.itcast.logMonitor.bolt.FilterBolt; 4 | import cn.itcast.logMonitor.bolt.PrepareRecordBolt; 5 | import cn.itcast.logMonitor.bolt.SaveMessage2MySql; 6 | import cn.itcast.logMonitor.spout.RandomSpout; 7 | import cn.itcast.logMonitor.spout.StringScheme; 8 | import org.apache.log4j.Logger; 9 | import org.apache.storm.Config; 10 | import org.apache.storm.LocalCluster; 11 | import org.apache.storm.StormSubmitter; 12 | import org.apache.storm.kafka.BrokerHosts; 13 | import org.apache.storm.kafka.KafkaSpout; 14 | import org.apache.storm.kafka.SpoutConfig; 15 | import org.apache.storm.kafka.ZkHosts; 16 | import org.apache.storm.topology.TopologyBuilder; 17 | import org.apache.storm.tuple.Fields; 18 | import org.apache.storm.utils.Utils; 19 | 20 | /** 21 | * @author y15079 22 | * @create 2018-05-06 18:22 23 | * @desc 日志监控系统驱动类 24 | **/ 25 | public class LogMonitorTopologyMain { 26 | private static Logger logger = Logger.getLogger(LogMonitorTopologyMain.class); 27 | 28 | public static void main(String[] args) throws Exception { 29 | // 使用TopologyBuilder进行构建驱动类 30 | TopologyBuilder builder = new TopologyBuilder(); 31 | // 设置kafka的zookeeper集群 32 | /* BrokerHosts hosts = new ZkHosts("zk01:2181,zk02:2181,zk03:2181"); 33 | // 初始化配置信息 34 | SpoutConfig spoutConfig = new SpoutConfig(hosts, "logmonitor", "/aaa", "log_monitor"); 35 | // 在topology中设置spout 36 | builder.setSpout("kafka-spout", new KafkaSpout(spoutConfig),3);*/ 37 | 38 | builder.setSpout("kafka-spout", new RandomSpout(new StringScheme()), 2); 39 | builder.setBolt("filter-bolt", new FilterBolt(), 3).shuffleGrouping("kafka-spou"); 40 | builder.setBolt("prepareRecord-bolt", new PrepareRecordBolt(), 2).fieldsGrouping("filter-bolt", new Fields("appId")); 41 | builder.setBolt("saveMessage-bolt", new SaveMessage2MySql(), 2).shuffleGrouping("prepareRecord-bolt"); 42 | 43 | //启动topology的配置信息 44 | Config topologConf = new Config(); 45 | //TOPOLOGY_DEBUG(setDebug), 当它被设置成true的话， storm会记录下每个组件所发射的每条消息。 46 | //这在本地环境调试topology很有用， 但是在线上这么做的话会影响性能的。 47 | topologConf.setDebug(true); 48 | //storm的运行有两种模式: 本地模式和分布式模式. 49 | if (args != null && args.length > 0) { 50 | //定义你希望集群分配多少个工作进程给你来执行这个topology 51 | topologConf.setNumWorkers(2); 52 | //向集群提交topology 53 | StormSubmitter.submitTopologyWithProgressBar(args[0], topologConf, builder.createTopology()); 54 | } else { 55 | topologConf.setMaxTaskParallelism(3); 56 | LocalCluster cluster = new LocalCluster(); 57 | cluster.submitTopology("word-count", topologConf, builder.createTopology()); 58 | Utils.sleep(10000000); 59 | cluster.shutdown(); 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/cn/itcast/logMonitor/dao/LogMonitorDao.java: -------------------------------------------------------------------------------- 1 | package cn.itcast.logMonitor.dao; 2 | 3 | import cn.itcast.logMonitor.domain.App; 4 | import cn.itcast.logMonitor.domain.Record; 5 | import cn.itcast.logMonitor.domain.Rule; 6 | import cn.itcast.logMonitor.domain.User; 7 | import org.apache.log4j.Logger; 8 | import org.springframework.jdbc.core.BeanPropertyRowMapper; 9 | import org.springframework.jdbc.core.JdbcTemplate; 10 | 11 | import java.util.Date; 12 | import java.util.List; 13 | 14 | 15 | /** 16 | * @author y15079 17 | * @create 2018-05-06 22:33 18 | * @desc 19 | **/ 20 | public class LogMonitorDao { 21 | private static Logger logger = Logger.getLogger(LogMonitorDao.class); 22 | private JdbcTemplate jdbcTemplate; 23 | 24 | public LogMonitorDao() { 25 | jdbcTemplate = new JdbcTemplate(DataSourceUtil.getDataSource()); 26 | } 27 | 28 | /** 29 | * 查询所有规则信息 30 | * 31 | * @return 32 | */ 33 | public List getRuleList(){ 34 | String sql = "SELECT `id`,`name`,`keyword`,`isValid`,`appId` FROM `log_monitor`.`log_monitor_rule` WHERE isValid =1"; 35 | return jdbcTemplate.query(sql, new BeanPropertyRowMapper(Rule.class)); 36 | } 37 | 38 | /** 39 | * 查询所有应用的信息 40 | * 41 | * @return 42 | */ 43 | public List getAppList(){ 44 | String sql = "SELECT `id`,`name`,`isOnline`,`typeId`,`userId` FROM `log_monitor`.`log_monitor_app` WHERE isOnline =1"; 45 | return jdbcTemplate.query(sql, new BeanPropertyRowMapper(App.class) ); 46 | } 47 | 48 | /** 49 | * 查询所有用户的信息 50 | * 51 | * @return 52 | */ 53 | public List getUserList() { 54 | String sql = "SELECT `id`,`name`,`mobile`,`email`,`isValid` FROM `log_monitor`.`log_monitor_user` WHERE isValid =1"; 55 | return jdbcTemplate.query(sql, new BeanPropertyRowMapper(User.class)); 56 | } 57 | 58 | /** 59 | * 插入触发规则的信息 60 | * 61 | * @param record 62 | */ 63 | public void saveRecord(Record record) { 64 | String sql = "INSERT INTO `log_monitor`.`log_monitor_rule_record`" + 65 | " (`appId`,`ruleId`,`isEmail`,`isPhone`,`isColse`,`noticeInfo`,`updataDate`) " + 66 | "VALUES ( ?,?,?,?,?,?,?)"; 67 | jdbcTemplate.update(sql, record.getAppId(), record.getRuleId(), record.getIsEmail(), record.getIsPhone(), 0, record.getLine(), new Date()); 68 | } 69 | 70 | public static void main(String[] args) { 71 | System.out.println("start....."); 72 | //测试数据库连接及相关方法是否正常 73 | LogMonitorDao logMonitorDao = new LogMonitorDao(); 74 | //打印所有的规则信息 75 | List rules = logMonitorDao.getRuleList(); 76 | for (Rule rule:rules){ 77 | System.out.println("rule: "+rule); 78 | } 79 | System.out.println(); 80 | System.out.println(); 81 | //打印所有的应用信息 82 | List apps= logMonitorDao.getAppList(); 83 | for (App app:apps){ 84 | System.out.println("app: "+app); 85 | } 86 | System.out.println(); 87 | System.out.println(); 88 | //打印所有的用户信息 89 | List users= logMonitorDao.getUserList(); 90 | for (User user:users){ 91 | System.out.println("user: "+user); 92 | } 93 | System.out.println("end....."); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/main/java/cn/itcast/logMonitor/mail/MessageSender.java: -------------------------------------------------------------------------------- 1 | package cn.itcast.logMonitor.mail; 2 | 3 | import org.apache.commons.lang.StringUtils; 4 | import org.apache.log4j.Logger; 5 | 6 | import javax.mail.*; 7 | import javax.mail.internet.InternetAddress; 8 | import javax.mail.internet.MimeBodyPart; 9 | import javax.mail.internet.MimeMessage; 10 | import javax.mail.internet.MimeMultipart; 11 | import java.io.UnsupportedEncodingException; 12 | import java.util.Date; 13 | import java.util.Properties; 14 | 15 | /** 16 | * @author y15079 17 | * @create 2018-05-06 22:19 18 | * @desc 19 | **/ 20 | public class MessageSender { 21 | private static final Logger logger = Logger.getLogger(MessageSender.class); 22 | 23 | //发送邮件-邮件内容为文本格式 24 | public static boolean sendMail(MailInfo mailInfo) { 25 | try { 26 | Message mailMessage = generateBaseInfo(mailInfo); 27 | mailMessage.setText(mailInfo.getMailContent());// 设置邮件消息的主要内容 28 | Transport.send(mailMessage); // 发送邮件 29 | logger.info("【 TEXT 邮件发送完毕，成功时间： " + System.currentTimeMillis() + " 】"); 30 | return true; 31 | } catch (Exception e) { 32 | e.printStackTrace(); 33 | } 34 | return false; 35 | } 36 | 37 | //邮件内容为html格式 38 | public static boolean sendHtmlMail(MailInfo mailInfo) { 39 | try { 40 | Message mailMessage = generateBaseInfo(mailInfo); 41 | Multipart mainPart = new MimeMultipart();// MiniMultipart类是一个容器类，包含MimeBodyPart类型的对象 42 | BodyPart html = new MimeBodyPart();// 创建一个包含HTML内容的MimeBodyPart 43 | html.setContent(mailInfo.getMailContent(), "text/html; charset=utf-8");// 设置HTML内容 44 | mainPart.addBodyPart(html); 45 | mailMessage.setContent(mainPart);// 将MiniMultipart对象设置为邮件内容 46 | Transport.send(mailMessage);// 发送邮件 47 | logger.info("【 HTML 邮件发送完毕，成功时间： " + System.currentTimeMillis() + " 】"); 48 | System.out.println("send ok!"); 49 | return true; 50 | } catch (Exception e) { 51 | e.printStackTrace(); 52 | } 53 | return false; 54 | } 55 | 56 | public static Message generateBaseInfo(MailInfo mailInfo) throws Exception{ 57 | // 判断是否需要身份认证 58 | MailAuthenticator authenticator = null; 59 | Properties props = mailInfo.getProperties(); 60 | // 如果需要身份认证，则创建一个密码验证器 61 | if (mailInfo.isAuthValidate()){ 62 | authenticator = new MailAuthenticator(mailInfo.getUserName(), mailInfo.getUserPassword()); 63 | } 64 | // 根据邮件会话属性和密码验证器构造一个发送邮件的session 65 | Session sendMailSession = Session.getDefaultInstance(props, authenticator); 66 | Message mailMessage = null; 67 | mailMessage = new MimeMessage(sendMailSession); // 根据session创建一个邮件消息 68 | Address from = new InternetAddress(mailInfo.getFromAddress(), mailInfo.getFromUserName()); // 创建邮件发送者地址 69 | mailMessage.setFrom(from);// 设置邮件消息的发送者 70 | if (mailInfo.getToAddress() != null && mailInfo.getToAddress().contains(",")){ 71 | mailMessage.setRecipients(Message.RecipientType.TO, InternetAddress.parse(mailInfo.getToAddress())); // Message.RecipientType.TO属性表示接收者的类型为TO 72 | }else { 73 | Address to = new InternetAddress(mailInfo.getToAddress()); //创建邮件的接收者地址，并设置到邮件消息中 74 | mailMessage.setRecipient(Message.RecipientType.TO, to); // Message.RecipientType.TO属性表示接收者的类型为TO 75 | } 76 | if (StringUtils.isNotBlank(mailInfo.getCcAddress())){ 77 | if (mailInfo.getCcAddress().contains(",")){ 78 | mailMessage.setRecipients(Message.RecipientType.CC, InternetAddress.parse(mailInfo.getCcAddress())); // Message.RecipientType.CC属性表示接收者的类型为CC 79 | }else { 80 | Address cc = new InternetAddress(mailInfo.getCcAddress()); // 创建邮件的抄送者地址，并设置到邮件消息中 81 | mailMessage.setRecipient(Message.RecipientType.CC, cc); // Message.RecipientType.CC属性表示接收者的类型为CC 82 | } 83 | } 84 | mailMessage.setSubject(mailInfo.getMailSubject()); // 设置邮件消息的主题 85 | mailMessage.setSentDate(new Date()); // 设置邮件消息发送的时间 86 | return mailMessage; 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/main/java/cn/itcast/logMonitor/mail/MailInfo.java: -------------------------------------------------------------------------------- 1 | package cn.itcast.logMonitor.mail; 2 | 3 | import java.util.List; 4 | import java.util.Properties; 5 | 6 | /** 7 | * @author y15079 8 | * @create 2018-05-06 22:18 9 | * @desc 10 | **/ 11 | public class MailInfo { 12 | private String mailServerHost; // 发送邮件的服务器的IP 13 | private String mailServerPort = "25"; // 发送邮件的服务器端口 14 | private String userName; // 登陆邮件发送服务器的用户名 15 | private String userPassword; // 登陆邮件发送服务器的密码 16 | private String fromAddress; // 邮件发送者的地址 17 | private String toAddress; // 邮件接收者的地址 18 | private String ccAddress; // 邮件抄送者的地址 19 | private String fromUserName = "日志监控平台"; // 邮件发送者的名称，显示在他人邮件的发件人 20 | private String mailSubject; // 邮件主题 21 | private String mailContent; // 邮件的文本内容 22 | private boolean authValidate = true; // 是否需要身份验证 23 | private Properties properties; // 邮件会话属性 24 | 25 | public MailInfo() { 26 | } 27 | 28 | public MailInfo(String title, String content, List receiver, List ccList) { 29 | this.mailServerHost = MailCenterConstant.SMTP_SERVER; 30 | this.userName = MailCenterConstant.USER; 31 | this.userPassword = MailCenterConstant.PWD; 32 | this.fromAddress = MailCenterConstant.FROM_ADDRESS; 33 | this.toAddress = listToStringFormat(receiver); 34 | this.ccAddress = ccList==null?"":listToStringFormat(ccList); 35 | this.mailSubject = title; 36 | this.mailContent = content; 37 | } 38 | 39 | private synchronized String listToStringFormat(List list) { 40 | StringBuilder stringBuilder = new StringBuilder(); 41 | for (int i = 0; i < list.size(); i++) { 42 | if (i == list.size() - 1) { 43 | stringBuilder.append(list.get(i)); 44 | } else { 45 | stringBuilder.append(list.get(i)).append(","); 46 | } 47 | } 48 | return stringBuilder.toString(); 49 | } 50 | 51 | public String getMailServerHost() { 52 | return mailServerHost; 53 | } 54 | 55 | public void setMailServerHost(String mailServerHost) { 56 | this.mailServerHost = mailServerHost; 57 | } 58 | 59 | public String getMailServerPort() { 60 | return mailServerPort; 61 | } 62 | 63 | public void setMailServerPort(String mailServerPort) { 64 | this.mailServerPort = mailServerPort; 65 | } 66 | 67 | public String getUserName() { 68 | return userName; 69 | } 70 | 71 | public void setUserName(String userName) { 72 | this.userName = userName; 73 | } 74 | 75 | public String getUserPassword() { 76 | return userPassword; 77 | } 78 | 79 | public void setUserPassword(String userPassword) { 80 | this.userPassword = userPassword; 81 | } 82 | 83 | public String getFromAddress() { 84 | return fromAddress; 85 | } 86 | 87 | public void setFromAddress(String fromAddress) { 88 | this.fromAddress = fromAddress; 89 | } 90 | 91 | public String getToAddress() { 92 | return toAddress; 93 | } 94 | 95 | public void setToAddress(String toAddress) { 96 | this.toAddress = toAddress; 97 | } 98 | 99 | public String getCcAddress() { 100 | return ccAddress; 101 | } 102 | 103 | public void setCcAddress(String ccAddress) { 104 | this.ccAddress = ccAddress; 105 | } 106 | 107 | public String getMailSubject() { 108 | return mailSubject; 109 | } 110 | 111 | public void setMailSubject(String mailSubject) { 112 | this.mailSubject = mailSubject; 113 | } 114 | 115 | public String getMailContent() { 116 | return mailContent; 117 | } 118 | 119 | public void setMailContent(String mailContent) { 120 | this.mailContent = mailContent; 121 | } 122 | 123 | public String getFromUserName() { 124 | return fromUserName; 125 | } 126 | 127 | public void setFromUserName(String fromUserName) { 128 | this.fromUserName = fromUserName; 129 | } 130 | 131 | public boolean isAuthValidate() { 132 | return authValidate; 133 | } 134 | 135 | public void setAuthValidate(boolean authValidate) { 136 | this.authValidate = authValidate; 137 | } 138 | 139 | public Properties getProperties() { 140 | Properties p = new Properties(); 141 | p.put("mail.smtp.host", this.mailServerHost); 142 | p.put("mail.smtp.port", this.mailServerPort); 143 | p.put("mail.smtp.auth", authValidate ? "true" : "false"); 144 | p.put("mail.smtp.starttls.enable", "true"); 145 | return p; 146 | } 147 | 148 | public void setProperties(Properties properties) { 149 | this.properties = properties; 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /src/main/java/cn/itcast/logMonitor/utils/DateUtils.java: -------------------------------------------------------------------------------- 1 | package cn.itcast.logMonitor.utils; 2 | 3 | import java.text.NumberFormat; 4 | import java.text.ParseException; 5 | import java.text.SimpleDateFormat; 6 | import java.util.Calendar; 7 | import java.util.Date; 8 | 9 | public class DateUtils { 10 | 11 | /** 12 | * 根据 formatter格式返回系统日期 13 | * 14 | * @param formatter 15 | * @return 16 | */ 17 | public static String getDateTime(String formatter) { 18 | SimpleDateFormat df = new SimpleDateFormat(formatter); 19 | return df.format(new Date()); 20 | } 21 | 22 | public static String getDateTime() { 23 | return DateUtils.getDateTime("yyyy-MM-dd HH:mm:ss"); 24 | } 25 | 26 | public static String getDate() { 27 | return DateUtils.getDateTime("yyyy-MM-dd"); 28 | } 29 | 30 | public static String getDate(String formatter) { 31 | return DateUtils.getDateTime(formatter); 32 | } 33 | 34 | public static String removeTime(String dateTime) { 35 | return dateTime.substring(0, dateTime.indexOf(" ")); 36 | } 37 | 38 | /** 39 | * 获取指定时间之前minute的时间 例如:minute = 30, 2014-07-15 12:00:00 -> 2014-07-15 11:30:00 40 | * @param time 41 | * @return 42 | */ 43 | public static String getBeforeMinute(String time, int minute){ 44 | String result = time; 45 | SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 46 | try { 47 | Date myDate = formatter.parse( time ); 48 | Calendar c = Calendar.getInstance(); 49 | c.setTime(myDate); 50 | c.add(Calendar.MINUTE, -minute); 51 | myDate = c.getTime(); 52 | result = formatter.format(myDate); 53 | } catch (ParseException e) { 54 | e.printStackTrace(); 55 | } 56 | return result; 57 | } 58 | 59 | /** 60 | * 截取日期 yyyyMMdd 61 | * 62 | * @param date 63 | * @return 64 | */ 65 | public static String splitDate(String date) { 66 | return date.substring(0, date.indexOf(" ")).replace("-", ""); 67 | } 68 | 69 | /** 70 | * 替换{}中的变量 71 | * 72 | * @param data 73 | * @param key 74 | * @param newData 75 | * @return 76 | */ 77 | public static String replaceParentheses(String data, String key, String newData) { 78 | return data.replaceAll("\\{" + key + "\\}", newData); 79 | } 80 | 81 | public static String replaceParentheses(String data, String key) { 82 | return data.replaceAll("\\{" + key + "\\}", ""); 83 | } 84 | 85 | /** 86 | * 格式化double，不使用科学计数法 87 | * 88 | * @param doubleValue 89 | * @param fractionDigits 90 | * @return 91 | */ 92 | public static String formatDouble(String doubleValue, int fractionDigits) { 93 | NumberFormat nf = NumberFormat.getInstance(); 94 | nf.setGroupingUsed(false); 95 | nf.setMaximumFractionDigits(fractionDigits); 96 | return nf.format(Double.parseDouble(doubleValue)); 97 | } 98 | 99 | public static String formatDouble(double doubleValue, int fractionDigits) { 100 | NumberFormat nf = NumberFormat.getInstance(); 101 | nf.setGroupingUsed(false); 102 | nf.setMaximumFractionDigits(fractionDigits); 103 | return nf.format(doubleValue); 104 | } 105 | 106 | public static String formatDouble(String doubleValue) { 107 | NumberFormat nf = NumberFormat.getInstance(); 108 | nf.setGroupingUsed(false); 109 | nf.setMaximumFractionDigits(2); 110 | return nf.format(Double.parseDouble(doubleValue)); 111 | } 112 | 113 | public static String formatDouble(double doubleValue) { 114 | NumberFormat nf = NumberFormat.getInstance(); 115 | nf.setGroupingUsed(false); 116 | nf.setMaximumFractionDigits(2); 117 | return nf.format(doubleValue); 118 | } 119 | 120 | public static String getInt(Object str) { 121 | return Integer.toString(Integer.parseInt(str.toString().replaceAll("\\.\\d+", ""))); 122 | } 123 | 124 | public static String getYesterday(String formatter) { 125 | SimpleDateFormat df = new SimpleDateFormat(formatter); 126 | Calendar calendar = Calendar.getInstance(); 127 | calendar.add(Calendar.DATE, -1); 128 | return df.format(calendar.getTime()); 129 | } 130 | 131 | public static void main(String[] args) { 132 | System.out.print(getDate()); 133 | } 134 | 135 | 136 | } 137 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | cn.itcast 8 | logmonitor 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 13 | 14 | org.apache.storm 15 | storm-core 16 | 1.2.1 17 | 18 | 19 | org.apache.storm 20 | storm-kafka 21 | 1.2.1 22 | 23 | 24 | org.slf4j 25 | slf4j-nop 26 | 27 | 28 | org.slf4j 29 | slf4j-jdk14 30 | 31 | 32 | 33 | 34 | org.clojure 35 | clojure 36 | 1.5.1 37 | 38 | 39 | org.apache.kafka 40 | kafka_2.11 41 | 0.8.2.1 42 | 43 | 44 | jmxtools 45 | com.sun.jdmk 46 | 47 | 48 | jmxri 49 | com.sun.jmx 50 | 51 | 52 | jms 53 | javax.jms 54 | 55 | 56 | org.apache.zookeeper 57 | zookeeper 58 | 59 | 60 | org.slf4j 61 | slf4j-log4j12 62 | 63 | 64 | org.slf4j 65 | slf4j-api 66 | 67 | 68 | 69 | 70 | junit 71 | junit 72 | 4.11 73 | 74 | 75 | org.apache.zookeeper 76 | zookeeper 77 | 3.4.6 78 | 79 | 80 | org.slf4j 81 | slf4j-log4j12 82 | 83 | 84 | org.slf4j 85 | slf4j-api 86 | 87 | 88 | 89 | 90 | com.google.code.gson 91 | gson 92 | 2.4 93 | 94 | 95 | redis.clients 96 | jedis 97 | 2.9.0 98 | 99 | 100 | javax.mail 101 | mail 102 | 1.4.7 103 | 104 | 105 | com.mchange 106 | c3p0 107 | 0.9.5.2 108 | 109 | 110 | mysql 111 | mysql-connector-java 112 | 5.1.38 113 | 114 | 115 | commons-beanutils 116 | commons-beanutils 117 | 1.8.2 118 | 119 | 120 | org.springframework 121 | spring-jdbc 122 | 4.3.14.RELEASE 123 | 124 | 125 | org.springframework 126 | spring-core 127 | 4.3.14.RELEASE 128 | 129 | 130 | org.springframework 131 | spring-context 132 | 4.3.14.RELEASE 133 | 134 | 135 | 136 | 137 | 138 | maven-assembly-plugin 139 | 140 | 141 | jar-with-dependencies 142 | 143 | 144 | 145 | com.itcast.storm.wordcount.WordCountTopologyMain 146 | 147 | 148 | 149 | 150 | 151 | make-assembly 152 | package 153 | 154 | single 155 | 156 | 157 | 158 | 159 | 160 | org.apache.maven.plugins 161 | maven-compiler-plugin 162 | 163 | 1.8 164 | 1.8 165 | 166 | 167 | 168 | 169 | -------------------------------------------------------------------------------- /src/main/java/cn/itcast/logMonitor/spout/RandomSpout.java: -------------------------------------------------------------------------------- 1 | package cn.itcast.logMonitor.spout; 2 | 3 | import org.apache.storm.spout.Scheme; 4 | import org.apache.storm.spout.SpoutOutputCollector; 5 | import org.apache.storm.task.TopologyContext; 6 | import org.apache.storm.topology.OutputFieldsDeclarer; 7 | import org.apache.storm.topology.base.BaseRichSpout; 8 | 9 | import java.nio.ByteBuffer; 10 | import java.util.ArrayList; 11 | import java.util.List; 12 | import java.util.Map; 13 | import java.util.Random; 14 | 15 | /** 16 | * @author y15079 17 | * @create 2018-05-06 23:26 18 | * @desc 随机产生消息发送出去 19 | **/ 20 | public class RandomSpout extends BaseRichSpout { 21 | private SpoutOutputCollector collector; 22 | private TopologyContext context; 23 | private List list ; 24 | private final Scheme scheme; 25 | 26 | public RandomSpout(final Scheme scheme) { 27 | super(); 28 | this.scheme = scheme; 29 | } 30 | 31 | public void open(Map conf, TopologyContext context, 32 | SpoutOutputCollector collector) { 33 | this.context = context; 34 | this.collector = collector; 35 | list = new ArrayList(); 36 | list.add("1$$$$$error: Caused by: java.lang.NoClassDefFoundError: com/starit/gejie/dao/SysNameDao"); 37 | list.add("2$$$$$java.sql.SQLException: You have an error in your SQL syntax;"); 38 | list.add("1$$$$$error Unable to connect to any of the specified MySQL hosts."); 39 | list.add("1$$$$$error:Servlet.service() for servlet action threw exception java.lang.NullPointerException"); 40 | list.add("1$$$$$error:Exception in thread main java.lang.ArrayIndexOutOfBoundsException: 2"); 41 | list.add("1$$$$$error:NoSuchMethodError: com/starit/."); 42 | list.add("2$$$$$error:java.lang.NoClassDefFoundError: org/coffeesweet/test01/Test01"); 43 | list.add("1$$$$$error:java.lang.NoClassDefFoundError: org/coffeesweet/test01/Test01"); 44 | list.add("1$$$$$error:Java.lang.IllegalStateException"); 45 | list.add("1$$$$$error:Java.lang.IllegalMonitorStateException"); 46 | list.add("1$$$$$error:Java.lang.NegativeArraySizeException"); 47 | list.add("2$$$$$error:java.sql.SQLException: You have an error in your SQL syntax;"); 48 | list.add("1$$$$$error:Java.lang.TypeNotPresentException "); 49 | list.add("1$$$$$error:Java.lang.UnsupprotedOperationException "); 50 | list.add("1$$$$$error Java.lang.IndexOutOfBoundsException"); 51 | list.add("1$$$$$error Java.lang.ClassNotFoundException"); 52 | list.add("2$$$$$error java.lang.ExceptionInInitializerError "); 53 | list.add("1$$$$$error:java.lang.IncompatibleClassChangeError "); 54 | list.add("1$$$$$error:java.lang.LinkageError "); 55 | list.add("1$$$$$error:java.lang.OutOfMemoryError "); 56 | list.add("1$$$$$error java.lang.StackOverflowError"); 57 | list.add("2$$$$$error: java.lang.UnsupportedClassVersionError"); 58 | list.add("1$$$$$error java.lang.ClassCastException"); 59 | list.add("1$$$$$error: java.lang.CloneNotSupportedException"); 60 | list.add("1$$$$$error: java.lang.EnumConstantNotPresentException "); 61 | list.add("1$$$$$error java.lang.IllegalMonitorStateException "); 62 | list.add("2$$$$$error java.lang.IllegalStateException "); 63 | list.add("1$$$$$error java.lang.IndexOutOfBoundsException "); 64 | list.add("1$$$$$error java.lang.NumberFormatException "); 65 | list.add("1$$$$$error java.lang.RuntimeException "); 66 | list.add("1$$$$$error java.lang.TypeNotPresentException "); 67 | list.add("2$$$$$error MetaSpout.java:9: variable i might not have been initialized"); 68 | list.add("1$$$$$error MyEvaluator.java:1: class Test1 is public, should be declared in a file named Test1.java "); 69 | list.add("1$$$$$error Main.java:5: cannot find symbol "); 70 | list.add("1$$$$$error NoClassDefFoundError: asa wrong name: ASA "); 71 | list.add("1$$$$$error Test1.java:54: 'void' type not allowed here"); 72 | list.add("2$$$$$error Test5.java:8: missing return statement"); 73 | list.add("1$$$$$error:Next.java:66: cannot find symbol "); 74 | list.add("1$$$$$error symbol  : method createTempFile(java.lang.String,java.lang.String,java.lang.String) "); 75 | list.add("1$$$$$error invalid method declaration; return type required"); 76 | list.add("1$$$$$error array required, but java.lang.String found"); 77 | list.add("2$$$$$error Exception in thread main java.lang.NumberFormatException: null 20. ."); 78 | list.add("1$$$$$error non-static method cannot be referenced from a static context"); 79 | list.add("1$$$$$error Main.java:5: non-static method fun1() cannot be referenced from a static context"); 80 | list.add("1$$$$$error continue outside of  loop"); 81 | list.add("1$$$$$error MyAbstract.java:6: missing method body, or declare abstract"); 82 | list.add("2$$$$$error Main.java:6: Myabstract is abstract; cannot be instantiated"); 83 | list.add("1$$$$$error MyInterface.java:2: interface methods cannot have body "); 84 | list.add("1$$$$$error Myabstract is abstract; cannot be instantiated"); 85 | list.add("1$$$$$error asa.java:3: modifier static not allowed here"); 86 | list.add("1$$$$$error possible loss of precision  found: long required:byte  var=varlong"); 87 | list.add("2$$$$$error  java.lang.NegativeArraySizeException "); 88 | list.add("1$$$$$error java.lang.ArithmeticException:  by zero"); 89 | list.add("1$$$$$error java.lang.ArithmeticException"); 90 | list.add("1$$$$$error java.lang.ArrayIndexOutOfBoundsException"); 91 | list.add("1$$$$$error java.lang.ClassNotFoundException"); 92 | list.add("2$$$$$error java.lang.IllegalArgumentException"); 93 | list.add("1$$$$$error fatal error C1010: unexpected end of file while looking for precompiled header directive"); 94 | list.add("1$$$$$error fatal error C1083: Cannot open include file: R…….h: No such file or directory"); 95 | list.add("1$$$$$error C2011:C……clas type redefinition"); 96 | list.add("1$$$$$error C2018: unknown character 0xa3"); 97 | list.add("2$$$$$error C2057: expected constant expression"); 98 | list.add("1$$$$$error C2065: IDD_MYDIALOG : undeclared identifier IDD_MYDIALOG"); 99 | list.add("1$$$$$error C2082: redefinition of formal parameter bReset"); 100 | list.add("1$$$$$error C2143: syntax error: missing : before "); 101 | list.add("1$$$$$error C2146: syntax error : missing ';' before identifier dc"); 102 | list.add("2$$$$$error C2196: case value '69' already used"); 103 | list.add("1$$$$$error C2509: 'OnTimer' : member function not declared in 'CHelloView'"); 104 | list.add("1$$$$$error C2555: 'B::f1': overriding virtual function differs from 'A::f1' only by return type or calling convention"); 105 | list.add("1$$$$$error C2511: 'reset': overloaded member function 'void (int)' not found in 'B'"); 106 | list.add("1$$$$$error C2660: 'SetTimer' : function does not take 2 parameters"); 107 | list.add("2$$$$$error warning C4035: 'f……': no return value"); 108 | list.add("1$$$$$error warning C4553: '= =' : operator has no effect; did you intend '='"); 109 | list.add("1$$$$$error C4716: 'CMyApp::InitInstance' : must return a value"); 110 | list.add("1$$$$$error LINK : fatal error LNK1168: cannot open Debug/P1.exe for writing"); 111 | list.add("1$$$$$error LNK2001: unresolved external symbol public: virtual _ _thiscall C (void)"); 112 | list.add("2$$$$$error java.lang.IllegalArgumentException: Path index.jsp does not start with"); 113 | list.add("1$$$$$error org.apache.struts.action.ActionServlet.process(ActionServlet.java:148"); 114 | list.add("1$$$$$error org.apache.jasper.JasperException: Exception in JSP"); 115 | list.add("1$$$$$error The server encountered an internal error () that prevented it from fulfilling this request"); 116 | list.add("1$$$$$error org.apache.jasper.servlet.JspServletWrapper.handleJspException(JspServletWrapper.java:467"); 117 | list.add("2$$$$$error javax.servlet.http.HttpServlet.service(HttpServlet.java:803)"); 118 | list.add("1$$$$$error javax.servlet.jsp.JspException: Cannot find message resources under key org.apache.struts.action.MESSAGE"); 119 | list.add("1$$$$$error Stacktrace: org.apache.jasper.servlet.JspServletWrapper.handleJspException(JspServletWrapper.java:467)"); 120 | list.add("1$$$$$error javax.servlet.ServletException: Cannot find bean org.apache.struts.taglib.html.BEAN in any scope"); 121 | list.add("1$$$$$error no data found"); 122 | list.add("2$$$$$error exception in thread main org.hibernate.MappingException: Unknown entity:."); 123 | list.add("1$$$$$error using namespace std;"); 124 | list.add("1$$$$$error C2065: 'cout' : undeclared identifier"); 125 | list.add("1$$$$$error main already defined in aaa.obj"); 126 | list.add("1$$$$$error syntax error : missing ';' before '}'"); 127 | list.add("2$$$$$error cout : undeclared identifier"); 128 | list.add("1$$$$$error weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.run(WebAp "); 129 | list.add("1$$$$$error Caused by: java.lang.reflect.InvocationTargetException"); 130 | list.add("1$$$$$error Caused by: java.lang.NoClassDefFoundError: com/starit/gejie/dao/SysNameDao"); 131 | list.add("1$$$$$error at com.starit.gejie.Util.Trans.BL_getSysNamesByType(Trans.java:220)"); 132 | 133 | } 134 | 135 | /** 136 | * 发送消息 137 | */ 138 | public void nextTuple() { 139 | final Random rand = new Random(); 140 | String msg = list.get(rand.nextInt(90)).toString(); 141 | this.collector.emit(this.scheme.deserialize(ByteBuffer.wrap(msg.getBytes()))); 142 | try { 143 | Thread.sleep(10000); 144 | } catch (InterruptedException e) { 145 | e.printStackTrace(); 146 | } 147 | } 148 | 149 | public void declareOutputFields(final OutputFieldsDeclarer declarer) { 150 | declarer.declare(this.scheme.getOutputFields()); 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /src/main/java/cn/itcast/logMonitor/utils/MonitorHandler.java: -------------------------------------------------------------------------------- 1 | package cn.itcast.logMonitor.utils; 2 | 3 | 4 | import cn.itcast.logMonitor.dao.LogMonitorDao; 5 | import cn.itcast.logMonitor.domain.*; 6 | import cn.itcast.logMonitor.mail.MailInfo; 7 | import cn.itcast.logMonitor.mail.MessageSender; 8 | import cn.itcast.logMonitor.sms.SMSBase; 9 | import org.apache.commons.lang.StringUtils; 10 | import org.apache.log4j.Logger; 11 | 12 | import java.util.ArrayList; 13 | import java.util.HashMap; 14 | import java.util.List; 15 | import java.util.Map; 16 | 17 | /** 18 | * @author y15079 19 | * @create 2018-05-06 18:30 20 | * @desc 日志监控的核心类，包括了日志监控系统所有的核心处理。 21 | **/ 22 | public class MonitorHandler { 23 | 24 | private static Logger logger = Logger.getLogger(MonitorHandler.class); 25 | 26 | //定义一个map，其中appId为Key，以该appId下的所有rule为Value 27 | private static Map> ruleMap; 28 | 29 | //定义一个map,其中appId为Key，以该appId下的所有user为Value 30 | private static Map> userMap; 31 | 32 | //定义一个list，用来封装所有的应用信息 33 | private static List appList; 34 | 35 | //定义一个list，用来封装所有的用户信息 36 | private static List userList; 37 | 38 | //定时加载配置文件的标识 39 | private static boolean reloaded = false; 40 | 41 | //定时加载下一配置文件的标识 42 | private static long nextReload = 0l; 43 | 44 | static { 45 | load(); 46 | } 47 | 48 | /** 49 | * 解析输入的日志，将数据按照一定的规则进行分割。 50 | * 判断日志是否合法，主要校验日志所属应用的appId是否存在 51 | * 52 | * @param line 一条日志 53 | * @return 54 | */ 55 | public static Message parser(String line){ 56 | //日志内容分为两个部分：由5个$$$$$符号作为分隔符，第一部分为appid，第二部分为日志内容。 57 | String[] messageArr = line.split("\\$\\$\\$\\$\\$"); 58 | //对日志进行校验 59 | if (messageArr.length != 2){ 60 | return null; 61 | } 62 | if (StringUtils.isBlank(messageArr[0]) || StringUtils.isBlank(messageArr[1])){ 63 | return null; 64 | } 65 | 66 | //检验当前日志所属的appid是否是经过授权的。 67 | if (appIdisValid(messageArr[0].trim())){ 68 | Message message = new Message(); 69 | message.setAppId(messageArr[0].trim()); 70 | message.setLine(messageArr[1]); 71 | return message; 72 | } 73 | return null; 74 | } 75 | 76 | private static boolean appIdisValid(String appId){ 77 | try { 78 | for (App app: appList){ 79 | if (app.getId() == Integer.parseInt(appId)){ 80 | return true; 81 | } 82 | } 83 | } catch (NumberFormatException e) { 84 | return false; 85 | } 86 | return false; 87 | } 88 | 89 | /** 90 | * 对日志进行规制判定，看看是否触发规则 91 | * @param message 92 | * @return 93 | */ 94 | public static boolean trigger(Message message){ 95 | //如果规则模型为空，需要初始化加载规则模型 96 | if (ruleMap == null){ 97 | load(); 98 | } 99 | 100 | //从规则模型中获取当前appid配置的规则 101 | System.out.println(message.getAppId()); 102 | List keywordByAppIdList = ruleMap.get(message.getAppId()); 103 | for (Rule rule: keywordByAppIdList){ 104 | //如果日志中包含过滤的关键词，即为匹配成功 105 | if (message.getLine().contains(rule.getKeyword())){ 106 | message.setRuleId(rule.getId() + ""); 107 | message.setKeyword(rule.getKeyword()); 108 | return true; 109 | } 110 | } 111 | return false; 112 | } 113 | 114 | /** 115 | * 加载数据模型，主要是用户列表、应用管理表、组合规则模型、组合用户模型。 116 | */ 117 | public static synchronized void load(){ 118 | if (userList == null) { 119 | userList = loadUserList(); 120 | } 121 | if (appList == null) { 122 | appList = loadAppList(); 123 | } 124 | if (ruleMap == null) { 125 | ruleMap = loadRuleMap(); 126 | } 127 | if (userMap == null) { 128 | userMap = loadUserMap(); 129 | } 130 | } 131 | 132 | /** 133 | * 访问数据库获取所有有效的app列表 134 | * @return 135 | */ 136 | private static List loadAppList() { 137 | return new LogMonitorDao().getAppList(); 138 | } 139 | 140 | /** 141 | * 访问数据库获取所有有效用户的列表 142 | * @return 143 | */ 144 | private static List loadUserList() { 145 | return new LogMonitorDao().getUserList(); 146 | } 147 | 148 | /** 149 | * 封装应用与用户对应的map 150 | * @return 151 | */ 152 | private static Map> loadUserMap(){ 153 | //以应用的appId为key，以应用的所有负责人的userList对象为value。 154 | //HashMap> 155 | HashMap> map = new HashMap>(); 156 | for (App app: appList){ 157 | String userIds = app.getUserId(); 158 | List userListInApp = map.get(app.getId()); 159 | if (userListInApp == null){ 160 | userListInApp = new ArrayList(); 161 | map.put(app.getId() + "", userListInApp); 162 | } 163 | String[] userIdArr = userIds.split(","); 164 | for (String userId:userIdArr){ 165 | userListInApp.add(queryUserById(userId)); 166 | } 167 | map.put(app.getId() + "", userListInApp); 168 | } 169 | return map; 170 | } 171 | 172 | /** 173 | * 封装应用与规则的map 174 | * @return 175 | */ 176 | private static Map> loadRuleMap(){ 177 | Map> map = new HashMap>(); 178 | LogMonitorDao logMonitorDao = new LogMonitorDao(); 179 | List ruleList = logMonitorDao.getRuleList(); 180 | //将代表rule的list转化成一个map，转化的逻辑是， 181 | // 从rule.getAppId作为map的key，然后将rule对象作为value传入map 182 | //Map 一个appid的规则信息，保存在一个list中。 183 | for (Rule rule: ruleList){ 184 | List ruleListByAppId = map.get(rule.getAppId()+""); 185 | if (ruleListByAppId == null){ 186 | ruleListByAppId = new ArrayList(); 187 | map.put(rule.getAppId() + "" , ruleListByAppId); 188 | } 189 | ruleListByAppId.add(rule); 190 | map.put(rule.getAppId() + "", ruleListByAppId); 191 | } 192 | return map; 193 | } 194 | 195 | /** 196 | * 通过用户编号获取用户的JavaBean 197 | * @param userId 198 | * @return 199 | */ 200 | private static User queryUserById(String userId){ 201 | for (User user: userList){ 202 | if (user.getId() == Integer.parseInt(userId)){ 203 | return user; 204 | } 205 | } 206 | return null; 207 | } 208 | 209 | /** 210 | * 通过app编号，获取当前app的所有负责人列表 211 | * @param appId 212 | * @return 213 | */ 214 | public static List getUserIdsByAppId(String appId){ 215 | return userMap.get(appId); 216 | } 217 | 218 | /** 219 | * 告警模块，用来发送邮件和短信 220 | * 短信功能由于短信资源匮乏，目前默认返回已发送。 221 | * @param appId 222 | * @param message 223 | */ 224 | public static void notifly(String appId, Message message){ 225 | //通过appId获取应用负责人的对象 226 | List users = getUserIdsByAppId(appId); 227 | //发送邮件 228 | if (sendMail(appId, users, message)){ 229 | message.setIsEmail(1); 230 | } 231 | //发送短信 232 | if (sendSMS(appId, users, message)){ 233 | message.setIsPhone(1); 234 | } 235 | } 236 | 237 | /** 238 | * 发送短信的模块 239 | * 由于短信资源匮乏，目前该功能不开启，默认true，即短信发送成功。 240 | * 目前发送短信功能使用的是外部接口，外面接口的并发性没法保证，会影响storm程序运行的效率。 241 | * 后期可以改造为将短信数据发送到外部的消息队里中，然后创建一个worker去发送短信。 242 | * @param appId 243 | * @param users 244 | * @param message 245 | * @return 246 | */ 247 | private static boolean sendSMS(String appId, List users, Message message){ 248 | List mobileList = new ArrayList(); 249 | for (User user : users) { 250 | mobileList.add(user.getMobile()); 251 | } 252 | for (App app : appList) { 253 | if (app.getId() == Integer.parseInt(appId.trim())) { 254 | message.setAppName(app.getName()); 255 | break; 256 | } 257 | } 258 | String content = "系统【" + message.getAppName() + "】在 " + DateUtils.getDateTime() + " 触发规则 " + message.getRuleId() + ",关键字：" + message.getKeyword(); 259 | return SMSBase.sendSms(listToStringFormat(mobileList), content); 260 | } 261 | 262 | private static boolean sendMail(String appId, List userList, Message message){ 263 | List receiver = new ArrayList(); 264 | for (User user: userList){ 265 | receiver.add(user.getEmail()); 266 | } 267 | for (App app: appList){ 268 | if (app.getId() == Integer.parseInt(appId.trim())){ 269 | message.setAppName(app.getName()); 270 | break; 271 | } 272 | } 273 | if (receiver.size() >= 1){ 274 | String date = DateUtils.getDateTime(); 275 | String content = "系统【" + message.getAppName() + "】在 " + date + " 触发规则 " + message.getRuleId() + " ，过滤关键字为：" + message.getKeyword() + " 错误内容：" + message.getLine(); 276 | MailInfo mailInfo = new MailInfo("系统运行日志监控", content, receiver, null); 277 | return MessageSender.sendMail(mailInfo); 278 | } 279 | return false; 280 | } 281 | 282 | /** 283 | * 保存触发规则的信息，将触发信息写入到mysql数据库中。 284 | * @param record 285 | */ 286 | public static void save(Record record){ 287 | new LogMonitorDao().saveRecord(record); 288 | } 289 | 290 | /** 291 | * 将list转换为String 292 | * @param list 293 | * @return 294 | */ 295 | private static String listToStringFormat(List list){ 296 | StringBuilder stringBuilder = new StringBuilder(); 297 | for (int i = 0; i < list.size(); i++){ 298 | if (i == list.size() -1 ){ 299 | stringBuilder.append(list.get(i)); 300 | }else { 301 | stringBuilder.append(list.get(i)).append(","); 302 | } 303 | } 304 | return stringBuilder.toString(); 305 | } 306 | 307 | /** 308 | * 配置scheduleLoad重新加载底层数据模型。 309 | */ 310 | public static synchronized void reloadDataModel(){ 311 | if (reloaded){ 312 | long start = System.currentTimeMillis(); 313 | userList = loadUserList(); 314 | appList = loadAppList(); 315 | ruleMap = loadRuleMap(); 316 | userMap = loadUserMap(); 317 | reloaded = false; 318 | nextReload = 0l; 319 | logger.info("配置文件reload完成，时间："+DateUtils.getDateTime()+" 耗时："+ (System.currentTimeMillis()-start)); 320 | } 321 | } 322 | 323 | /** 324 | * 定时加载配置信息 325 | * 配合reloadDataModel模块一起使用。 326 | * 主要实现原理如下： 327 | * 1，获取分钟的数据值，当分钟数据是10的倍数，就会触发reloadDataModel方法，简称reload时间。 328 | * 2，reloadDataModel方式是线程安全的，在当前worker中只有一个线程能够操作。 329 | * 3，为了保证当前线程操作完毕之后，其他线程不再重复操作，设置了一个标识符reloaded。 330 | * 在非reload时间段时，reloaded一直被置为true； 331 | * 在reload时间段时，第一个线程进入reloadDataModel后，加载完毕之后会将reloaded置为false。 332 | */ 333 | public static void scheduleLoad(){ 334 | // String date = DateUtils.getDateTime(); 335 | // int now = Integer.parseInt(date.split(":")[1]); 336 | // if (now % 10 == 0) {//每10分钟加载一次 337 | // //1,2,3,4,5,6 338 | // reloadDataModel(); 339 | // }else { 340 | // reloaded = true; 341 | // } 342 | 343 | if (System.currentTimeMillis() == nextReload){ 344 | reloadDataModel(); 345 | } 346 | } 347 | } 348 | --------------------------------------------------------------------------------