├── src └── main │ ├── resources │ └── application.yml │ └── java │ └── com │ └── xiaochuan │ └── wang │ └── stormtraffic │ ├── bolt │ ├── TimedCarCountBolt.java │ ├── PeakAlertBolt.java │ ├── AlertBolt.java │ ├── CarCountBolt.java │ └── FilterBolt.java │ ├── config │ ├── TrafficPersistenceBolt.java │ ├── AppConfig.java │ └── DbConfig.java │ ├── spout │ └── TrafficKafkaSpoutBuilder.java │ ├── TrafficApplication.java │ ├── traffic │ └── TrafficRecord.java │ └── topology │ └── TrafficKPITopologyBuilder.java ├── scripts ├── README.md └── trafficdatamocker.py ├── .gitignore ├── LICENSE.TXT ├── README.md └── pom.xml /src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | alertCars: 2 | - 苏A10001 3 | - 沪C10003 4 | - 浙B10002 5 | 6 | localModeEnabled: true -------------------------------------------------------------------------------- /scripts/README.md: -------------------------------------------------------------------------------- 1 | 2 | # 交通数据模拟脚本 3 | 4 | 执行trafficdatamocker.py自动生成交通数据并发送到kafka队列。 5 | 6 | 使用方法： 7 | 8 | * 确认安装了python 2.7或以上版本 9 | * 安装依赖库 10 | * pip install kafka-python 11 | * 配置kafka broker和topic，请修改脚本中的`brokers`和`topic` 12 | * 执行脚本 13 | `python trafficdatamocker.py` -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | 4 | # Log file 5 | *.log 6 | 7 | # BlueJ files 8 | *.ctxt 9 | 10 | # Mobile Tools for Java (J2ME) 11 | .mtj.tmp/ 12 | 13 | # Package Files # 14 | *.jar 15 | *.war 16 | *.ear 17 | *.zip 18 | *.tar.gz 19 | *.rar 20 | 21 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 22 | hs_err_pid* 23 | 24 | # IDEA 25 | *.iml 26 | .idea/ 27 | target/ 28 | logs/ 29 | 30 | -------------------------------------------------------------------------------- /LICENSE.TXT: -------------------------------------------------------------------------------- 1 | Copyright 2020 Xiaochuan Wang 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /src/main/java/com/xiaochuan/wang/stormtraffic/bolt/TimedCarCountBolt.java: -------------------------------------------------------------------------------- 1 | package com.xiaochuan.wang.stormtraffic.bolt; 2 | 3 | import org.apache.storm.topology.base.BaseWindowedBolt; 4 | import org.apache.storm.windowing.TupleWindow; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.time.LocalDateTime; 9 | 10 | /** 11 | * @author: wangxiaochuan 12 | * @Description: 另外一种计算时间窗口内汽车数量的实现，基于storm自身的窗口功能 13 | * @Date: Created in 10:26 2018/1/8 14 | * @Modified By: 15 | */ 16 | public class TimedCarCountBolt extends BaseWindowedBolt { 17 | private static final Logger LOG = LoggerFactory.getLogger(TimedCarCountBolt.class); 18 | @Override 19 | public void execute(TupleWindow inputWindow) { 20 | LOG.info("[{}] Count is: {}", LocalDateTime.now(), inputWindow.get().size()); 21 | 22 | 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/xiaochuan/wang/stormtraffic/config/TrafficPersistenceBolt.java: -------------------------------------------------------------------------------- 1 | package com.xiaochuan.wang.stormtraffic.config; 2 | 3 | import org.apache.storm.jdbc.bolt.JdbcInsertBolt; 4 | import org.apache.storm.jdbc.common.ConnectionProvider; 5 | import org.apache.storm.jdbc.mapper.JdbcMapper; 6 | 7 | /** 8 | * @author: wangxiaochuan 9 | * @Description: 数据库存储bolt，将统计结果插入数据库 10 | * @Date: Created in 17:39 2018/1/11 11 | * @Modified By: 12 | */ 13 | public class TrafficPersistenceBolt extends JdbcInsertBolt { 14 | public TrafficPersistenceBolt(ConnectionProvider connectionProvider, JdbcMapper jdbcMapper) { 15 | super(connectionProvider, jdbcMapper); 16 | } 17 | 18 | public TrafficPersistenceBolt(DbConfig config) { 19 | super(config.getConnectionProvider(), config.getJdbcMapper()); 20 | this.withInsertQuery("insert into carStatistics (time, count) values (?, ?)"); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/com/xiaochuan/wang/stormtraffic/config/AppConfig.java: -------------------------------------------------------------------------------- 1 | package com.xiaochuan.wang.stormtraffic.config; 2 | 3 | import org.yaml.snakeyaml.Yaml; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | * @author: wangxiaochuan 9 | * @Description: 10 | * @Date: Created in 16:17 2018/1/9 11 | * @Modified By: 12 | */ 13 | public class AppConfig { 14 | private List alertCars; 15 | private boolean localModeEnabled; 16 | 17 | public static AppConfig from(String ymlPath) { 18 | Yaml yaml = new Yaml(); 19 | return yaml.loadAs(AppConfig.class.getClassLoader().getResourceAsStream(ymlPath), AppConfig.class); 20 | } 21 | 22 | public List getAlertCars() { 23 | return alertCars; 24 | } 25 | 26 | public void setAlertCars(List alertCars) { 27 | this.alertCars = alertCars; 28 | } 29 | 30 | public boolean isLocalModeEnabled() { 31 | return localModeEnabled; 32 | } 33 | 34 | public void setLocalModeEnabled(boolean localModeEnabled) { 35 | this.localModeEnabled = localModeEnabled; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /scripts/trafficdatamocker.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf8 -*- 2 | import kafka 3 | import time 4 | import datetime 5 | import random 6 | 7 | brokers = ['nbot18.dg.163.org:9092'] 8 | topic = 'traffic' 9 | 10 | producer = kafka.KafkaProducer(bootstrap_servers=brokers) 11 | 12 | # data schema: time, car plate, speed, longitude, latitude 13 | data = "{},{},{},{},{}" 14 | 15 | 16 | def gen_car(): 17 | head1, head2, num = ('苏', '沪', '浙'), "ABC", (10001, 10010) 18 | h1 = head1[random.randint(0, len(head1)-1)] 19 | h2 = head2[random.randint(0, len(head2)-1)] 20 | h3 = random.randint(num[0], num[1]) 21 | return '{}{}{}'.format(h1, h2, h3) 22 | 23 | 24 | def gen_speed(): 25 | return random.randint(0, 80) 26 | 27 | 28 | def gen_longitude(): 29 | return random.uniform(120, 122) 30 | 31 | 32 | def gen_latitude(): 33 | return random.uniform(80, 86) 34 | 35 | 36 | key = 0 37 | while 1: 38 | dt = datetime.datetime.now().isoformat() 39 | key += 1 40 | value = data.format(dt, gen_car(), gen_speed(), gen_longitude(), gen_latitude()) 41 | print(value) 42 | producer.send(topic, value=value.encode()) 43 | producer.flush() 44 | 45 | if 0 != random.randint(0, 3): 46 | time.sleep(1) 47 | -------------------------------------------------------------------------------- /src/main/java/com/xiaochuan/wang/stormtraffic/bolt/PeakAlertBolt.java: -------------------------------------------------------------------------------- 1 | package com.xiaochuan.wang.stormtraffic.bolt; 2 | 3 | import org.apache.storm.topology.base.BaseWindowedBolt; 4 | import org.apache.storm.windowing.TupleWindow; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | /** 9 | * @author: wangxiaochuan 10 | * @Description: 车流量峰值告警 11 | * 单位时间内容超过执行数量的汽车，发出告警通知 12 | * @Date: Created in 11:20 2018/1/8 13 | * @Modified By: 14 | */ 15 | public class PeakAlertBolt extends BaseWindowedBolt { 16 | private static final Logger LOG = LoggerFactory.getLogger(PeakAlertBolt.class); 17 | private final int intervalSecs; 18 | private final int threshold; 19 | 20 | /** 21 | * 构造函数 22 | * @param intervalSecs 时间窗口，单位秒 23 | * @param checkIntervalSecs 检测间隔，单位秒 24 | * @param threshold 阈值 25 | */ 26 | public PeakAlertBolt(int intervalSecs, int checkIntervalSecs, int threshold) { 27 | super(); 28 | this.intervalSecs = intervalSecs; 29 | this.threshold = threshold; 30 | this.withWindow(Duration.seconds(intervalSecs), Duration.seconds(checkIntervalSecs)); 31 | } 32 | 33 | @Override 34 | public void execute(TupleWindow inputWindow) { 35 | int carCount = inputWindow.get().size(); 36 | if (carCount >= this.threshold) { 37 | LOG.info("Alert !!! Last {} seconds, car count: {} over threshold: {} .", 38 | this.intervalSecs, carCount, this.threshold); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/com/xiaochuan/wang/stormtraffic/spout/TrafficKafkaSpoutBuilder.java: -------------------------------------------------------------------------------- 1 | package com.xiaochuan.wang.stormtraffic.spout; 2 | 3 | import org.apache.storm.kafka.spout.KafkaSpout; 4 | import org.apache.storm.kafka.spout.KafkaSpoutConfig; 5 | 6 | import java.util.Arrays; 7 | import java.util.List; 8 | 9 | /** 10 | * KafkaSpout构建器 11 | */ 12 | public class TrafficKafkaSpoutBuilder { 13 | private List brokers; 14 | private String topic; 15 | 16 | public TrafficKafkaSpoutBuilder brokers(List v) { 17 | brokers = v; 18 | return this; 19 | } 20 | 21 | public TrafficKafkaSpoutBuilder topic(String v) { 22 | topic = v; 23 | return this; 24 | } 25 | 26 | 27 | public KafkaSpout build() { 28 | /** 配置kafka 29 | * 1. 需要设置consumer group, 注意一个partition中的消息只能被同一group中的一个consumer消费 30 | * 2. 起始消费策略：根据业务需要配置 31 | */ 32 | String allBrokers = String.join(",", brokers); 33 | KafkaSpoutConfig conf = KafkaSpoutConfig 34 | .builder(allBrokers, topic) 35 | .setGroupId("wangxiaochuan-storm") 36 | .setFirstPollOffsetStrategy(KafkaSpoutConfig.FirstPollOffsetStrategy.LATEST) 37 | .build(); 38 | return new KafkaSpout(conf); 39 | } 40 | 41 | public static void main(String[] args) { 42 | KafkaSpout spout = new TrafficKafkaSpoutBuilder() 43 | .brokers(Arrays.asList("nbot18.dg.163.org:9092")) 44 | .topic("traffic") 45 | .build(); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/com/xiaochuan/wang/stormtraffic/config/DbConfig.java: -------------------------------------------------------------------------------- 1 | package com.xiaochuan.wang.stormtraffic.config; 2 | 3 | import com.google.common.collect.Maps; 4 | import org.apache.storm.jdbc.common.Column; 5 | import org.apache.storm.jdbc.common.ConnectionProvider; 6 | import org.apache.storm.jdbc.common.HikariCPConnectionProvider; 7 | import org.apache.storm.jdbc.mapper.JdbcMapper; 8 | import org.apache.storm.jdbc.mapper.SimpleJdbcMapper; 9 | 10 | import java.sql.Types; 11 | import java.util.Arrays; 12 | import java.util.List; 13 | import java.util.Map; 14 | 15 | /** 16 | * @author: wangxiaochuan 17 | * @Description: 18 | * @Date: Created in 17:50 2018/1/11 19 | * @Modified By: 20 | */ 21 | public class DbConfig { 22 | private AppConfig config; 23 | private ConnectionProvider connectionProvider; 24 | private JdbcMapper jdbcMapper; 25 | public DbConfig(AppConfig config) { 26 | this.config = config; 27 | 28 | Map hikariConfigMap = Maps.newHashMap(); 29 | hikariConfigMap.put("dataSourceClassName","com.mysql.jdbc.jdbc2.optional.MysqlDataSource"); 30 | hikariConfigMap.put("dataSource.url", "jdbc:mysql://localhost:3306/test"); 31 | hikariConfigMap.put("dataSource.user","tester"); 32 | hikariConfigMap.put("dataSource.password","password"); 33 | 34 | this.connectionProvider = new HikariCPConnectionProvider(hikariConfigMap); 35 | 36 | List columns = Arrays.asList( 37 | new Column("time", Types.TIMESTAMP), 38 | new Column("count", Types.BIGINT)); 39 | this.jdbcMapper = new SimpleJdbcMapper(columns); 40 | } 41 | 42 | public ConnectionProvider getConnectionProvider() { 43 | return connectionProvider; 44 | } 45 | 46 | public JdbcMapper getJdbcMapper() { 47 | return jdbcMapper; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/com/xiaochuan/wang/stormtraffic/bolt/AlertBolt.java: -------------------------------------------------------------------------------- 1 | package com.xiaochuan.wang.stormtraffic.bolt; 2 | 3 | import com.xiaochuan.wang.stormtraffic.traffic.TrafficRecord; 4 | import org.apache.storm.task.TopologyContext; 5 | import org.apache.storm.topology.BasicOutputCollector; 6 | import org.apache.storm.topology.IBasicBolt; 7 | import org.apache.storm.topology.OutputFieldsDeclarer; 8 | import org.apache.storm.tuple.Tuple; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.util.List; 13 | import java.util.Map; 14 | 15 | /** 16 | * @author: wangxiaochuan 17 | * @Description: 告警模块，实时检测指定车辆的出现 18 | * @Date: Created in 11:29 2018/1/5 19 | * @Modified By: 20 | */ 21 | public class AlertBolt implements IBasicBolt { 22 | private static final Logger LOG = LoggerFactory.getLogger(AlertBolt.class); 23 | private final List cars; 24 | private final List notifyMails; 25 | 26 | public AlertBolt(List cars, List notifyMails) { 27 | this.cars = cars; 28 | this.notifyMails = notifyMails; 29 | } 30 | 31 | @Override 32 | public void prepare(Map stormConf, TopologyContext context) { 33 | 34 | } 35 | 36 | @Override 37 | public void execute(Tuple input, BasicOutputCollector collector) { 38 | TrafficRecord record = (TrafficRecord)input.getValueByField(TrafficRecord.getFIELD()); 39 | if (cars.contains(record.getCarPlate())) { 40 | LOG.info("Found dangerous car: {} !!!!!!!!", record.getCarPlate()); 41 | } 42 | } 43 | 44 | @Override 45 | public void cleanup() { 46 | 47 | } 48 | 49 | @Override 50 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 51 | 52 | } 53 | 54 | @Override 55 | public Map getComponentConfiguration() { 56 | return null; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/com/xiaochuan/wang/stormtraffic/TrafficApplication.java: -------------------------------------------------------------------------------- 1 | package com.xiaochuan.wang.stormtraffic; 2 | 3 | 4 | import com.xiaochuan.wang.stormtraffic.bolt.AlertBolt; 5 | import com.xiaochuan.wang.stormtraffic.config.AppConfig; 6 | import com.xiaochuan.wang.stormtraffic.topology.TrafficKPITopologyBuilder; 7 | import com.xiaochuan.wang.stormtraffic.traffic.TrafficRecord; 8 | import org.apache.storm.Config; 9 | import org.apache.storm.LocalCluster; 10 | import org.apache.storm.StormSubmitter; 11 | import org.apache.storm.generated.AlreadyAliveException; 12 | import org.apache.storm.generated.AuthorizationException; 13 | import org.apache.storm.generated.InvalidTopologyException; 14 | import org.apache.storm.generated.StormTopology; 15 | import org.slf4j.Logger; 16 | import org.slf4j.LoggerFactory; 17 | 18 | public class TrafficApplication { 19 | 20 | 21 | public static void main(String[] args) throws InvalidTopologyException, AuthorizationException, AlreadyAliveException { 22 | final Logger LOG = LoggerFactory.getLogger(AlertBolt.class); 23 | AppConfig trafficConfig = AppConfig.from("application.yml"); 24 | 25 | // 创建拓扑（spout & bolt） 26 | StormTopology topology = TrafficKPITopologyBuilder.create(trafficConfig); 27 | 28 | // 拓扑相关配置 29 | Config conf = new Config(); 30 | conf.put(Config.TOPOLOGY_WORKERS, 4); 31 | conf.put(Config.TOPOLOGY_DEBUG, false); 32 | conf.put(Config.TOPOLOGY_NAME, "wangxiaochuan-traffic"); 33 | // 该超时时间需要大于统计窗口的时间 34 | conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 300); 35 | conf.registerSerialization(TrafficRecord.class); 36 | 37 | if (trafficConfig.isLocalModeEnabled()) { 38 | LOG.info("Submit task as local mode !!"); 39 | // local模式提交，测试用 40 | LocalCluster cluster = new LocalCluster(); 41 | cluster.submitTopology("demo", conf, topology); 42 | } else { 43 | // 将拓扑提交到集群 44 | StormSubmitter.submitTopology("mytopology", conf, topology); 45 | } 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/com/xiaochuan/wang/stormtraffic/bolt/CarCountBolt.java: -------------------------------------------------------------------------------- 1 | package com.xiaochuan.wang.stormtraffic.bolt; 2 | 3 | import com.xiaochuan.wang.stormtraffic.traffic.TrafficRecord; 4 | import org.apache.storm.task.TopologyContext; 5 | import org.apache.storm.topology.BasicOutputCollector; 6 | import org.apache.storm.topology.IBasicBolt; 7 | import org.apache.storm.topology.OutputFieldsDeclarer; 8 | import org.apache.storm.tuple.Fields; 9 | import org.apache.storm.tuple.Tuple; 10 | import org.apache.storm.tuple.Values; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import java.util.Map; 15 | 16 | /** 17 | * @author: wangxiaochuan 18 | * @Description: 按照时间间隔统计汽车数量 19 | * 该实现仅适用于bolt并发度为1的场景，当并发度大于1时，统计的仅是该job内接受的消息的数量， 20 | * 没有进行合并，如果要计算全局数量，可借助于redis等第三方组件实现，或者使用package的另 21 | * 外一个类：TimedCarCountBolt 22 | * @Date: Created in 9:45 2018/1/5 23 | * @Modified By: 24 | */ 25 | public class CarCountBolt implements IBasicBolt { 26 | private static final Logger LOG = LoggerFactory.getLogger(CarCountBolt.class); 27 | private TopologyContext context; 28 | private Map stormConf; 29 | 30 | // 统计时间窗口，例如3600，即统计每小时汽车数量 31 | private long intervalMs; 32 | private long startTime = System.currentTimeMillis(); 33 | private long count; 34 | 35 | public CarCountBolt(long intervalSeconds) { 36 | this.context = context; 37 | this.intervalMs = intervalSeconds * 1000L; 38 | } 39 | 40 | public void prepare(Map stormConf, TopologyContext context) { 41 | this.stormConf = stormConf; 42 | this.context = context; 43 | } 44 | 45 | // 由于使用了IBasicBolt，不需要手动ACK，如果使用IRichBolt，请手动ACK确认消息处理成功 46 | public void execute(Tuple input, BasicOutputCollector collector) { 47 | TrafficRecord record = (TrafficRecord)input.getValueByField(TrafficRecord.getFIELD()); 48 | LOG.info("{}: {}", count, record.getCarPlate()); 49 | 50 | long currentTime = System.currentTimeMillis(); 51 | if (currentTime - startTime > intervalMs) { 52 | collector.emit(new Values(currentTime, count)); 53 | LOG.info("Total count is {} in current time window", count); 54 | count = 0; 55 | startTime = currentTime; 56 | } 57 | count++; 58 | } 59 | 60 | public void cleanup() { 61 | 62 | } 63 | 64 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 65 | declarer.declare(new Fields("time", "count")); 66 | } 67 | 68 | public Map getComponentConfiguration() { 69 | return null; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/com/xiaochuan/wang/stormtraffic/bolt/FilterBolt.java: -------------------------------------------------------------------------------- 1 | package com.xiaochuan.wang.stormtraffic.bolt; 2 | 3 | import com.xiaochuan.wang.stormtraffic.traffic.TrafficRecord; 4 | import org.apache.storm.task.TopologyContext; 5 | import org.apache.storm.topology.BasicOutputCollector; 6 | import org.apache.storm.topology.IBasicBolt; 7 | import org.apache.storm.topology.OutputFieldsDeclarer; 8 | import org.apache.storm.tuple.Fields; 9 | import org.apache.storm.tuple.Tuple; 10 | import org.apache.storm.tuple.Values; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import java.util.Map; 15 | 16 | /** 17 | * @author: wangxiaochuan 18 | * @Description: 交通数据清洗，过滤掉无效的数据 19 | * @Date: Created in 11:47 2018/1/5 20 | * @Modified By: 21 | */ 22 | public class FilterBolt implements IBasicBolt { 23 | private static final Logger LOG = LoggerFactory.getLogger(FilterBolt.class); 24 | private String[] patterns; 25 | 26 | public FilterBolt(String...patterns) { 27 | this.patterns = patterns; 28 | } 29 | 30 | @Override 31 | public void prepare(Map stormConf, TopologyContext context) { 32 | 33 | } 34 | 35 | @Override 36 | public void execute(Tuple input, BasicOutputCollector collector) { 37 | String component = input.getSourceComponent(); 38 | if (component.equals("traffic-kafka")) { 39 | TrafficRecord record = TrafficRecord.of(input.getStringByField("value")); 40 | if (record != null && record.getCarPlate() != null) { 41 | boolean checkResult = false; 42 | for (String pattern : patterns) { 43 | if (record.getCarPlate().contains(pattern)) { 44 | checkResult = true; 45 | break; 46 | } 47 | } 48 | 49 | if (checkResult) { 50 | collector.emit(new Values(record)); 51 | } else { 52 | LOG.info("Ignored car: {}", record.getCarPlate()); 53 | } 54 | } else { 55 | LOG.info("Invalid record:", record.getRaw()); 56 | } 57 | } else if (component.equals("db")) { 58 | int a = 0; 59 | System.out.println("111" + a); 60 | } 61 | } 62 | 63 | @Override 64 | public void cleanup() { 65 | 66 | } 67 | 68 | @Override 69 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 70 | declarer.declare(new Fields(TrafficRecord.getFIELD())); 71 | } 72 | 73 | @Override 74 | public Map getComponentConfiguration() { 75 | return null; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/com/xiaochuan/wang/stormtraffic/traffic/TrafficRecord.java: -------------------------------------------------------------------------------- 1 | package com.xiaochuan.wang.stormtraffic.traffic; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.time.LocalDateTime; 7 | import java.time.format.DateTimeParseException; 8 | 9 | /** 10 | * @author: wangxiaochuan 11 | * @Description: 交通数据记录，为了支持序列化，需要注册到Config中 12 | * @Date: Created in 16:22 2018/1/8 13 | * @Modified By: 14 | */ 15 | public class TrafficRecord { 16 | private static final Logger LOG = LoggerFactory.getLogger(TrafficRecord.class); 17 | private static final String FIELD = "traffic.record"; 18 | 19 | private LocalDateTime localDateTime; 20 | private String carPlate; 21 | private float carSpeed; 22 | private float longitude; 23 | private float latitude; 24 | private String raw; 25 | 26 | public static String getFIELD() { 27 | return FIELD; 28 | } 29 | 30 | public LocalDateTime getLocalDateTime() { 31 | return localDateTime; 32 | } 33 | 34 | public void setLocalDateTime(LocalDateTime localDateTime) { 35 | this.localDateTime = localDateTime; 36 | } 37 | 38 | public String getCarPlate() { 39 | return carPlate; 40 | } 41 | 42 | public void setCarPlate(String carPlate) { 43 | this.carPlate = carPlate; 44 | } 45 | 46 | public float getCarSpeed() { 47 | return carSpeed; 48 | } 49 | 50 | public void setCarSpeed(float carSpeed) { 51 | this.carSpeed = carSpeed; 52 | } 53 | 54 | public float getLongitude() { 55 | return longitude; 56 | } 57 | 58 | public void setLongitude(float longitude) { 59 | this.longitude = longitude; 60 | } 61 | 62 | public float getLatitude() { 63 | return latitude; 64 | } 65 | 66 | public void setLatitude(float latitude) { 67 | this.latitude = latitude; 68 | } 69 | 70 | public String getRaw() { 71 | return raw; 72 | } 73 | 74 | public void setRaw(String raw) { 75 | this.raw = raw; 76 | } 77 | 78 | @Override 79 | public String toString() { 80 | return this.raw; 81 | } 82 | 83 | public static TrafficRecord of(String raw) { 84 | String[] items = raw.split(","); 85 | if (items.length < 5) { 86 | return null; 87 | } 88 | TrafficRecord record = new TrafficRecord(); 89 | 90 | try { 91 | record.setLocalDateTime(LocalDateTime.parse(items[0])); 92 | record.setCarPlate(items[1]); 93 | record.setCarSpeed(Float.parseFloat(items[2])); 94 | record.setLongitude(Float.parseFloat(items[3])); 95 | record.setLatitude(Float.parseFloat(items[4])); 96 | record.setRaw(raw); 97 | } catch (DateTimeParseException e) { 98 | LOG.error("{} parse time error, {}", raw, e.getMessage()); 99 | record = null; 100 | } catch (NumberFormatException e) { 101 | LOG.error("{} parse error, {}", raw, e.getMessage()); 102 | record = null; 103 | } 104 | 105 | return record; 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/main/java/com/xiaochuan/wang/stormtraffic/topology/TrafficKPITopologyBuilder.java: -------------------------------------------------------------------------------- 1 | package com.xiaochuan.wang.stormtraffic.topology; 2 | 3 | import com.xiaochuan.wang.stormtraffic.bolt.*; 4 | import com.xiaochuan.wang.stormtraffic.config.AppConfig; 5 | import com.xiaochuan.wang.stormtraffic.config.DbConfig; 6 | import com.xiaochuan.wang.stormtraffic.config.TrafficPersistenceBolt; 7 | import com.xiaochuan.wang.stormtraffic.spout.TrafficKafkaSpoutBuilder; 8 | import org.apache.storm.generated.StormTopology; 9 | import org.apache.storm.kafka.spout.KafkaSpout; 10 | import org.apache.storm.topology.TopologyBuilder; 11 | import org.apache.storm.topology.base.BaseWindowedBolt; 12 | 13 | import java.util.Arrays; 14 | import java.util.List; 15 | 16 | /** 17 | * @author: wangxiaochuan 18 | * @Description: 19 | * @Date: Created in 10:07 2018/1/5 20 | * @Modified By: 21 | */ 22 | public class TrafficKPITopologyBuilder { 23 | public static StormTopology create(AppConfig config) { 24 | KafkaSpout kafkaSpout = new TrafficKafkaSpoutBuilder() 25 | .brokers(Arrays.asList("nbot18.dg.163.org:9092")) 26 | .topic("traffic") 27 | .build(); 28 | 29 | TopologyBuilder builder = new TopologyBuilder(); 30 | 31 | /** 添加kafka数据源 */ 32 | String spoutName = "traffic-kafka"; 33 | builder.setSpout(spoutName, kafkaSpout) 34 | .setDebug(false) 35 | .setNumTasks(1) 36 | .setMaxTaskParallelism(2); 37 | 38 | 39 | 40 | String filterBolt = FilterBolt.class.getSimpleName(); 41 | builder.setBolt(filterBolt, new FilterBolt("苏", "沪", "浙A")) 42 | .shuffleGrouping(spoutName); 43 | 44 | 45 | /** 添加bolt统计车辆数, 并关联到kafka spout 46 | 注意该bolt并行度为2，即2个executor，所以单个executor中的bolt统计数量 47 | 并不是全部的汽车数量 */ 48 | String countBolt1 = "countBolt1"; 49 | builder.setBolt(countBolt1, new CarCountBolt(60), 1) 50 | .shuffleGrouping(filterBolt) 51 | .setDebug(false); 52 | 53 | /** 添加基于时间窗口的车辆统计bolt */ 54 | String countBolt2 = "countBolt2"; 55 | builder.setBolt(countBolt2, new TimedCarCountBolt() 56 | .withTumblingWindow(BaseWindowedBolt.Duration.seconds(60)), 1) 57 | .shuffleGrouping(filterBolt); 58 | 59 | /** 添加告警bolt, 检查指定汽车是否被检测到 */ 60 | List dangerousCars = config.getAlertCars(); 61 | List mails = Arrays.asList("wangxiaochuan01@163.com"); 62 | builder.setBolt(AlertBolt.class.getSimpleName(), new AlertBolt(dangerousCars, mails)) 63 | .shuffleGrouping(filterBolt); 64 | 65 | /** 添加告警bolt, 检查单位时间内汽车数量是否达到阈值 66 | * 过去30秒内汽车数超过40则告警，每5秒检测一次 67 | */ 68 | builder.setBolt(PeakAlertBolt.class.getSimpleName(), 69 | new PeakAlertBolt(30, 5, 40)) 70 | .shuffleGrouping(filterBolt); 71 | 72 | 73 | String dbBoltName = "db"; 74 | builder.setBolt(dbBoltName, new TrafficPersistenceBolt(new DbConfig(new AppConfig()))) 75 | .setDebug(true) 76 | .shuffleGrouping(countBolt1); 77 | 78 | StormTopology topology = builder.createTopology(); 79 | 80 | return topology; 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Storm-traffic 2 | 使用storm接入交通数据，实时计算速度流量等交通模型 3 | 4 | ## 功能 5 | * Kafka数据接入 6 | * 实时数据清洗和过滤 7 | * 实时告警 8 | * 检测指定车辆是否出现 9 | * 峰值告警，检测一定时间内车辆数是否超过阈值 10 | * 车辆信息统计 11 | * 工具：交通数据模拟脚本, 参考[Here](./scripts/README.md) 12 | 13 | 说明：告警后的通知功能不在该项目范围内，通知功能应作为单独的微服务部署并提供相应的接口。 14 | 15 | 16 | ## 环境 17 | 18 | * Window 7 19 | * Storm 1.1.1 20 | * Zookeeper 3.4.9 21 | * Kafka 2.12 22 | * Java 8 23 | * Python 3.6(官方推荐版本2.6.6，3.x也可以工作) 24 | 25 | ## Zookeeper 安装 26 | Storm cluster 使用zookeeper来管理集群，同时由于kafka也使用zookeeper来进行分布式协调， 27 | 简化起见，我们对storm和kafka使用同一套zookeeper，作为开发环境，我们不考虑zookeeper 28 | 的单点故障，所以不搭建zookeeper的集群 29 | * 安装方法: 30 | * 参考我的另一个repo：[zookeeper-gym](https://github.com/wang1365/zookeeper-gym) 31 | * 官方文档：[zookeeper](https://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_singleAndDevSetup) 32 | 33 | ## Storm安装 34 | 35 | * 下载 storm release并解压 36 | 安装文件：[downloads](http://storm.apache.org/downloads.html) 37 | 注意：不要下载源码，源码中在storm的根目录下缺少lib文件夹，storm命令无法运行。 38 | 39 | * 环境变量设置 40 | * 为了方便的使用storm命令行，在windows系统变量中新建STORM_HOME环境变量，并将storm的bin路径添加到系统PATH中 41 | * 注意JAVA_HOME环境变量中不能包含空格！！！ 42 | 43 | * Storm配置 44 | Storm配置文件为`%STORM_HOME%\conf\storm.yaml`, 打开该文件修改配置如下： 45 | ```yaml 46 | # zookeeper集群地址列表，如果zookeeper不是使用的默认端口， 47 | # 还需要配置storm.zookeeper.ports 48 | storm.zookeeper.servers: 49 | - "localhost" 50 | 51 | # nimbus所在主机名称 52 | nimbus.host: "localhost" 53 | 54 | # nimbus和supervisor的数据存放路径 55 | storm.local.dir: "/storm_local" 56 | 57 | # supervisor所在机器的所有工作进程可用的端口列表 58 | supervisor.slots.ports: 59 | - 6700 60 | - 6701 61 | ``` 62 | * 启动storm 63 | 生产环境中，一般zookeeper、nimbus、supervisor会分别部署在不同的主机上，简化起见，我们在同一台机器 64 | 上启动部署这些进程： 65 | * 启动zookeeper：`zkServer` 66 | * 启动nimbus：`storm numbus` 67 | * 启动supervisor：`storm supervisor` 68 | * 启动storm ui：`storm ui` 69 | 70 | ## Storm概念说明 71 | * zookeeper 72 | zookeeper本身不属于Storm的范围，它是作为一个分布式协调框架帮助storm管理集群中的节点。在启动nimbus和supervisor后， 73 | 可以发现在zookeeper节点/storm/nimbuses和/storm/supervisor下分别多了对应的子节点，并且这些子节点都是临时节点。 74 | 75 | ``` 76 | [zk: localhost:2181(CONNECTED) 14] ls2 /storm/nimbuses 77 | [HIH-D-8204.hz.ntes.domain:6627] 78 | cZxid = 0xcb 79 | ctime = Fri Oct 27 10:11:04 CST 2017 80 | mZxid = 0xcb 81 | mtime = Fri Oct 27 10:11:04 CST 2017 82 | pZxid = 0x460 83 | cversion = 3 84 | dataVersion = 0 85 | aclVersion = 0 86 | ephemeralOwner = 0x0 87 | dataLength = 1 88 | numChildren = 1 89 | 90 | [zk: localhost:2181(CONNECTED) 15] ls2 /storm/nimbuses/HIH-D-8204.hz.ntes.domain:6627 91 | [] 92 | cZxid = 0x460 93 | ctime = Fri Oct 27 11:13:57 CST 2017 94 | mZxid = 0x460 95 | mtime = Fri Oct 27 11:13:57 CST 2017 96 | pZxid = 0x460 97 | cversion = 0 98 | dataVersion = 0 99 | aclVersion = 0 100 | ephemeralOwner = 0x15f5b97a5af000d 101 | dataLength = 79 102 | numChildren = 0 103 | ``` 104 | 105 | * nimbus 106 | nimbus进程部署在storm集群的主控节点上，负责在集群中分发代码，对工作节点分配任务，并监控主机故障 107 | * supervisor 108 | supervisor进程部署在storm集群的每一个工作节点，负责监听工作节点上已经分配的任务，管理工作进程，给工作进程 109 | 分配任务 110 | 111 | ## Kafka安装 112 | * 下载并解压：[https://kafka.apache.org/downloads](https://kafka.apache.org/downloads) 113 | * 环境变量配置，`KAFKA_HOME`以及将`%KAFKA_HOME%\bin\windows`加入到系统`PATH` 114 | * 根据实际情况修改kafka server的配置文件：`%KAFKA_HOME%\config\server.properties`, 115 | 注意其中的zookeeper的配置如`zookeeper.connect=localhost:2181`要根据实际的情况配置。 116 | * 启动kafka server：`kafka-server-start.bat %KAFKA_HOME%\config\server.properties` 117 | * 测试kafka server是否正常工作 118 | * 启动一个consumer client：`kafka-console-consumer.bat --zookeeper localhost --topic mytopic` 119 | * 启动一个producer client：`kafka-console-producer.bat --broker-list localhost:9092 --topic mytopic` 120 | * 在producer中发送一条消息，然后观察consumer中是否可以收到 121 | 122 | ### Intellij IDEA 调试 123 | * 请把storm-core和storm-core的provided注释掉，否则会出现ClassNotFound异常; 集群提交时需要保留provided 124 | * application.yml中把localModeEnabled设置为true，集群提交设置为false -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.xiaochuan.wang 8 | stormtraffic 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 2.1.0 13 | 1.8 14 | UTF-8 15 | 1.2.9 16 | 1.7.21 17 | 18 | 19 | 20 | 21 | commons-lang 22 | commons-lang 23 | 2.6 24 | 25 | 26 | org.apache.storm 27 | storm-core 28 | ${storm.version} 29 | 30 | 31 | 32 | org.apache.storm 33 | storm-kafka-client 34 | ${storm.version} 35 | 36 | 37 | 38 | org.slf4j 39 | slf4j-api 40 | ${slf4j.version} 41 | compile 42 | 43 | 44 | ch.qos.logback 45 | logback-core 46 | ${logback.version} 47 | 48 | 49 | ch.qos.logback 50 | logback-classic 51 | ${logback.version} 52 | 53 | 54 | org.yaml 55 | snakeyaml 56 | 1.26 57 | 58 | 59 | org.apache.storm 60 | storm-jdbc 61 | 1.1.1 62 | 63 | 64 | mysql 65 | mysql-connector-java 66 | 8.0.28 67 | 68 | 69 | 70 | 71 | 72 | maven-assembly-plugin 73 | 74 | 75 | jar-with-dependencies 76 | 77 | 78 | 79 | com.xiaochuan.wang.stormtraffic.TrafficApplication 80 | 81 | 82 | 83 | 84 | 85 | make-assembly 86 | package 87 | 88 | single 89 | 90 | 91 | 92 | 93 | 94 | org.apache.maven.plugins 95 | maven-compiler-plugin 96 | 97 | 1.8 98 | 1.8 99 | 100 | 101 | 102 | 103 | --------------------------------------------------------------------------------