├── .idea └── uiDesigner.xml ├── drools_kie_demo ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── canthny │ └── drools_kie_demo │ ├── domain │ ├── Assets.java │ └── UserInfo.java │ └── test │ └── DroolKieTestDemo.java ├── tanghao-bigdata-drools ├── README.md ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── tanghao │ │ │ └── bigdata │ │ │ └── drools │ │ │ ├── DroolsStarter.java │ │ │ ├── controller │ │ │ └── TestPayRule.java │ │ │ ├── domain │ │ │ ├── DefeatMapReduceObject.java │ │ │ ├── MobilePaymentInfo.java │ │ │ ├── PhoneNoBlackObject.java │ │ │ ├── request │ │ │ │ ├── BaseRequest.java │ │ │ │ └── MobilePaymentRequest.java │ │ │ └── response │ │ │ │ └── RuleResponse.java │ │ │ ├── enums │ │ │ └── EnumRuleResponseLevel.java │ │ │ ├── kafka │ │ │ ├── config │ │ │ │ └── MobilePaymentInfoConfig.java │ │ │ └── listener │ │ │ │ └── MobilePaymentInfoListener.java │ │ │ ├── mongodb │ │ │ ├── config │ │ │ │ └── SpringMongoConfig.java │ │ │ └── service │ │ │ │ ├── MobilePaymentInfoService.java │ │ │ │ └── impl │ │ │ │ └── MobilePaymentInfoServiceImpl.java │ │ │ ├── redis │ │ │ └── RedisConfig.java │ │ │ ├── service │ │ │ ├── MobilePaymentRuleService.java │ │ │ └── PhoneNoBlacklistService.java │ │ │ └── util │ │ │ └── DateUtil.java │ └── resources │ │ ├── META-INF │ │ └── kmodule.xml │ │ ├── application.properties │ │ ├── com.tanghao.bigdata.drools.rules │ │ └── pay_rule.drl │ │ ├── drools │ │ ├── pay_rule.drl │ │ └── pay_rule2.drl │ │ └── mapreduce │ │ ├── mobile_payment_info_map.js │ │ └── mobile_payment_info_reduce.js │ └── test │ └── java │ └── com │ └── tanghao │ └── bigdata │ └── drools │ └── test │ ├── TestDrools.java │ ├── TestKafkaProduct.java │ ├── TestMongodb.java │ ├── TestMongodbJob.java │ └── TestRuleService.java └── tanghao-flink-demo ├── pom.xml └── src ├── README.md └── main ├── java └── com │ └── tanghao │ └── flink │ └── demo │ ├── GetPayInfoJob.java │ ├── TestAction.java │ ├── api │ ├── datastream │ │ ├── FilterApiTest.java │ │ ├── FlatMapApiTest.java │ │ ├── KeyByApiTest.java │ │ ├── MapApiTest.java │ │ └── UnionAndSplitStreamTest.java │ ├── others │ │ └── BroadcastTableTest.java │ └── window │ │ ├── GlobalWindowTest.java │ │ ├── SessionWindowTest.java │ │ ├── SlidingWindowTest.java │ │ ├── TumblingWindowAggregateFunctionTest.java │ │ ├── TumblingWindowProcessWindowFunctionTest.java │ │ └── TumblingWindowUseEventTimeTest.java │ ├── common │ ├── ConstantKafka.java │ ├── Message.java │ ├── MessageDeserializationSchema.java │ ├── TradeCountMinuteResult.java │ ├── TransTypeInfo.java │ └── UserConsumerAmountPerMinute.java │ ├── source │ ├── InnerSource.java │ └── KafkaSource.java │ └── test │ ├── BroadcastMsgProducerTest.java │ └── MsgProducerTest.java └── resources └── log4j.properties /.idea/uiDesigner.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | -------------------------------------------------------------------------------- /drools_kie_demo/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.canthny 8 | drools_kie_demo 9 | 1.0.0 10 | 11 | 12 | 13 | junit 14 | junit 15 | 3.8.1 16 | test 17 | 18 | 19 | 20 | org.drools 21 | drools-core 22 | 7.6.0.Final 23 | 24 | 25 | org.drools 26 | drools-compiler 27 | 7.6.0.Final 28 | 29 | 30 | org.kie.server 31 | kie-server 32 | 7.6.0.Final 33 | 34 | 35 | org.slf4j 36 | slf4j-api 37 | 1.7.22 38 | 39 | 40 | 41 | com.alibaba 42 | fastjson 43 | 1.2.51 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | nexus-aliyun 59 | Nexus aliyun 60 | http://maven.aliyun.com/nexus/content/groups/public 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /drools_kie_demo/src/main/java/com/canthny/drools_kie_demo/domain/Assets.java: -------------------------------------------------------------------------------- 1 | package com.canthny.drools_kie_demo.domain; 2 | 3 | /** 4 | * This class was automatically generated by the data modeler tool. 5 | */ 6 | 7 | public class Assets implements java.io.Serializable { 8 | 9 | static final long serialVersionUID = 1L; 10 | 11 | private java.lang.Long money; 12 | private java.lang.Long stock; 13 | private java.lang.Long fund; 14 | 15 | public Assets() { 16 | } 17 | 18 | public java.lang.Long getMoney() { 19 | return this.money; 20 | } 21 | 22 | public void setMoney(java.lang.Long money) { 23 | this.money = money; 24 | } 25 | 26 | public java.lang.Long getStock() { 27 | return this.stock; 28 | } 29 | 30 | public void setStock(java.lang.Long stock) { 31 | this.stock = stock; 32 | } 33 | 34 | public java.lang.Long getFund() { 35 | return this.fund; 36 | } 37 | 38 | public void setFund(java.lang.Long fund) { 39 | this.fund = fund; 40 | } 41 | 42 | public Assets(java.lang.Long money, java.lang.Long stock, 43 | java.lang.Long fund) { 44 | this.money = money; 45 | this.stock = stock; 46 | this.fund = fund; 47 | } 48 | 49 | } -------------------------------------------------------------------------------- /drools_kie_demo/src/main/java/com/canthny/drools_kie_demo/domain/UserInfo.java: -------------------------------------------------------------------------------- 1 | package com.canthny.drools_kie_demo.domain; 2 | 3 | /** 4 | * This class was automatically generated by the data modeler tool. 5 | */ 6 | 7 | public class UserInfo implements java.io.Serializable { 8 | 9 | static final long serialVersionUID = 1L; 10 | 11 | private java.lang.Long id; 12 | private java.lang.String name; 13 | private java.lang.Integer age; 14 | private java.lang.String accountNo; 15 | 16 | private java.lang.String risk_type; 17 | 18 | private com.canthny.drools_kie_demo.domain.Assets assets; 19 | 20 | public UserInfo() { 21 | } 22 | 23 | public java.lang.Long getId() { 24 | return this.id; 25 | } 26 | 27 | public void setId(java.lang.Long id) { 28 | this.id = id; 29 | } 30 | 31 | public java.lang.String getName() { 32 | return this.name; 33 | } 34 | 35 | public void setName(java.lang.String name) { 36 | this.name = name; 37 | } 38 | 39 | public java.lang.Integer getAge() { 40 | return this.age; 41 | } 42 | 43 | public void setAge(java.lang.Integer age) { 44 | this.age = age; 45 | } 46 | 47 | public java.lang.String getAccountNo() { 48 | return this.accountNo; 49 | } 50 | 51 | public void setAccountNo(java.lang.String accountNo) { 52 | this.accountNo = accountNo; 53 | } 54 | 55 | public java.lang.String getRisk_type() { 56 | return this.risk_type; 57 | } 58 | 59 | public void setRisk_type(java.lang.String risk_type) { 60 | this.risk_type = risk_type; 61 | } 62 | 63 | public com.canthny.drools_kie_demo.domain.Assets getAssets() { 64 | return this.assets; 65 | } 66 | 67 | public void setAssets(com.canthny.drools_kie_demo.domain.Assets assets) { 68 | this.assets = assets; 69 | } 70 | 71 | public UserInfo(java.lang.Long id, java.lang.String name, 72 | java.lang.Integer age, java.lang.String accountNo, 73 | java.lang.String risk_type, 74 | com.canthny.drools_kie_demo.domain.Assets assets) { 75 | this.id = id; 76 | this.name = name; 77 | this.age = age; 78 | this.accountNo = accountNo; 79 | this.risk_type = risk_type; 80 | this.assets = assets; 81 | } 82 | 83 | } -------------------------------------------------------------------------------- /drools_kie_demo/src/main/java/com/canthny/drools_kie_demo/test/DroolKieTestDemo.java: -------------------------------------------------------------------------------- 1 | package com.canthny.drools_kie_demo.test; 2 | 3 | import com.alibaba.fastjson.JSONObject; 4 | import com.canthny.drools_kie_demo.domain.Assets; 5 | import com.canthny.drools_kie_demo.domain.UserInfo; 6 | import org.kie.api.KieServices; 7 | import org.kie.api.command.Command; 8 | import org.kie.api.command.KieCommands; 9 | import org.kie.api.runtime.ExecutionResults; 10 | import org.kie.server.api.marshalling.MarshallingFormat; 11 | import org.kie.server.api.model.ServiceResponse; 12 | import org.kie.server.client.KieServicesClient; 13 | import org.kie.server.client.KieServicesConfiguration; 14 | import org.kie.server.client.KieServicesFactory; 15 | import org.kie.server.client.RuleServicesClient; 16 | 17 | import java.util.LinkedList; 18 | import java.util.List; 19 | 20 | /** 21 | * Description： kie-server动态部署demo测试 22 | * Created By tanghao on 2019/7/22 23 | */ 24 | public class DroolKieTestDemo { 25 | 26 | public static void main(String[] args) { 27 | KieServicesConfiguration kieServicesConfiguration = KieServicesFactory.newRestConfiguration("http://192.168.8.132:8080/kie-server/services/rest/server","kieserver","kieserver1!"); 28 | kieServicesConfiguration.setMarshallingFormat(MarshallingFormat.JSON); 29 | KieServicesClient kieServicesClient = KieServicesFactory.newKieServicesClient(kieServicesConfiguration); 30 | RuleServicesClient ruleServicesClient = kieServicesClient.getServicesClient(RuleServicesClient.class); 31 | UserInfo userInfo = new UserInfo(); 32 | userInfo.setId(1L); 33 | userInfo.setName("canthny"); 34 | userInfo.setAge(31); 35 | Assets assets = new Assets(); 36 | assets.setMoney(20L); 37 | assets.setFund(30L); 38 | assets.setStock(50L); 39 | userInfo.setAssets(assets); 40 | KieCommands kieCommands = KieServices.Factory.get().getCommands(); 41 | List> commands = new LinkedList<>(); 42 | commands.add(kieCommands.newInsert(userInfo, "userInfo")); 43 | commands.add(kieCommands.newFireAllRules()); 44 | //kie-server中发布的规则，通过/kie-server/services/rest/server/containers获取容器id,以及在kie-web端设置的kiebase的id 45 | ServiceResponse results = ruleServicesClient.executeCommandsWithResults("drools-kie-demo_1.0.0", 46 | kieCommands.newBatchExecution(commands, "session1")); 47 | UserInfo value = (UserInfo) results.getResult().getValue("userInfo"); 48 | System.out.println(JSONObject.toJSON(value).toString()); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/README.md: -------------------------------------------------------------------------------- 1 | # drool规则引擎测试-手机缴费风险控制Demo 2 | ## 1-单笔金额超限； 3 | ## 2-手机号在黑名单中； 4 | ## 3-充值金额不能大于近3天平均值的3倍 5 | 6 | # 启动服务 7 | //mongodb启动命令中的ip跟着本机的动态ip改变
8 | cd /usr/local/mongodb/bin
9 | nohup ./mongod -bind_ip 192.168.0.102 &
10 | //kafka的server.properties中advertised.listeners=PLAINTEXT://192.168.1.113:9092修改一下当前ip
11 | cd /home/canthny/kafka_2.11-0.11.0.3
12 | nohup bin/zookeeper-server-start.sh config/zookeeper.properties &
13 | nohup bin/kafka-server-start.sh config/server.properties &
14 | //redis启动本地windows的
15 | 16 | //tanghao-bigdata-drools工程中配置mongodb和redis的ip为本地或虚拟机的ip
17 | //tanghao-flink-demo工程中配置kafka的地址
18 | # 数据准备 19 | ## 1、插入3天缴费数据十条，金额随机，执行TestMongodb中的testSave()方法插入数据 20 | cd /usr/local/mongodb/bin
21 | ./mongo 192.168.0.102:27017/rule进入命令行查询数据
22 | 23 | Mongodb常用命令
24 | //进入库
25 | use rule
26 | //创建集合
27 | db.createCollection("mobilePaymentInfo")
28 | //查看所有集合
29 | show collections
30 | //查看当前库指定集合所有数据
31 | db.mobilePaymentInfo.find()
32 | //向当前库指定集合插入文档
33 | db.mobilePaymentInfo.insert({"_class" : "com.tanghao.bigdata.drools.mongodb.domain.MobilePaymentInfo", "pay_order_no" : "2018092700001", "account_no" : "1234312412345234", "bank_card_no" : "62260113241234", "mobile" : "18709858763", "amount" : "12.00", "time" : "20180927 16:57:00" })
34 | //删除文档下所有数据
35 | db.mobilePaymentInfo.remove({})
36 | //根据日期查询
37 | db.mobilePaymentInfo.find({time:{'$gte':ISODate("2018-09-27"),'$lt':ISODate("2018-09-28")}})
38 | ## 2、插入手机号黑名单数据，执行TestMongodb中的insertPhoneNoBlackList()方法 39 | 40 | ## 3、计算3天内用户缴费平均金额，执行TestMongodb中的testMapReduce()方法，并将计算结果插入新表用户3日缴费平均金额中 41 | _id即为用户的account_no，value为3日缴费平均金额 42 | 43 | 44 | # 单元测试 45 | *规则引擎测试：tanghao-bigdata-drools工程TestDrools单元测试中的testCase1\2\3\4方法
46 | *规则执行顺序(pay_rule.drl文件中的salience属性)：单笔限额>手机号黑名单>充值平均值
47 | testCase1()——单笔金额超限，打印风险，金额自行修改。
48 | testCase2()——风控判断手机号是否在黑名单中，需要事先建立黑名单表数据，TestMongodb类insertPhoneNoBlackList中key插入黑名单数据，修改手机号运行单元测试即可。
49 | testCase3()——判断当前单笔充值额度大于用户最近一个月充值额度平均值的3倍，需要事先进行MapReduce操作，产生一段时间内平均金额数据，插入中间表，在TestMongodb类testMapReduce方法中，自行调整日期参数即可，上节数据准备中第三步已经做过可忽略，也可以自行尝试。
50 | testCase4()——一小时内支付次数或支付金额限制，需要先跑流计算测试，将数据插入缓存中，流程暂未串起来，后续可以写个消息生产者将流计算和规则引擎串起来
51 | 52 | *流计算测试：tanghao-flink-demo工程GetPayInfoJob中main方法 53 | //flink demo测试，kafka生产消息 54 | cd /home/canthny/kafka_2.11-0.11.0.3
55 | bin/kafka-console-producer.sh --broker-list localhost:9092 --topic CountPerHourByAccountNo2
56 | 57 | bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic CountPerHourByAccountNo --from-beginning 58 | {"accountId":"0716","amount":"130","eventTime":"2018/09/30 02:51:00"} 59 | {"accountId":"0718","amount":"130","eventTime":"2018/09/30 02:51:01"} 60 | {"accountId":"0718","amount":"130","eventTime":"2018/09/30 02:51:06"} 61 | {"accountId":"0718","amount":"130","eventTime":"2018/09/30 02:51:07"} 62 | {"accountId":"0716","amount":"140","eventTime":"2018/09/30 02:51:02"} 63 | {"accountId":"0716","amount":"150","eventTime":"2018/09/30 02:51:06"} 64 | {"accountId":"0716","amount":"150","eventTime":"2018/09/30 02:51:12"} 65 | 66 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | org.springframework.boot 7 | spring-boot-starter-parent 8 | 1.5.8.RELEASE 9 | 10 | 4.0.0 11 | 12 | tanghao.drools.demo 13 | tanghao-bigdata-drools 14 | 1.0-SNAPSHOT 15 | 16 | 6.5.0.Final 17 | 18 | 19 | 20 | 21 | org.kie 22 | kie-api 23 | ${drools.version} 24 | 25 | 26 | org.drools 27 | drools-core 28 | ${drools.version} 29 | 30 | 31 | org.drools 32 | drools-compiler 33 | ${drools.version} 34 | 35 | 36 | org.drools 37 | drools-decisiontables 38 | ${drools.version} 39 | 40 | 41 | org.drools 42 | drools-templates 43 | ${drools.version} 44 | 45 | 46 | org.springframework.boot 47 | spring-boot-starter-web 48 | 49 | 50 | 51 | 52 | org.springframework.boot 53 | spring-boot-starter 54 | 55 | 56 | 57 | org.springframework.boot 58 | spring-boot-starter-test 59 | test 60 | 61 | 62 | 63 | 64 | org.springframework.boot 65 | spring-boot-starter-data-mongodb 66 | 67 | 68 | 69 | 70 | org.springframework.boot 71 | spring-boot-starter-data-redis 72 | 73 | 74 | 75 | 76 | org.springframework.kafka 77 | spring-kafka 78 | 79 | 80 | 81 | com.alibaba 82 | fastjson 83 | 1.2.34 84 | 85 | 86 | 87 | 88 | 89 | 90 | org.springframework.boot 91 | spring-boot-maven-plugin 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | nexus-aliyun 107 | Nexus aliyun 108 | http://maven.aliyun.com/nexus/content/groups/public 109 | 110 | 111 | 112 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/java/com/tanghao/bigdata/drools/DroolsStarter.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.bigdata.drools; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.context.annotation.ComponentScan; 6 | 7 | /** 8 | * @Author： Canthny 9 | * @Description： 启动类 10 | * @Date： Created in 2018/9/11 16:43 11 | */ 12 | @SpringBootApplication 13 | public class DroolsStarter { 14 | 15 | public static void main(String[] args) { 16 | SpringApplication.run(DroolsStarter.class,args); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/java/com/tanghao/bigdata/drools/controller/TestPayRule.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.bigdata.drools.controller; 2 | 3 | import com.tanghao.bigdata.drools.domain.request.MobilePaymentRequest; 4 | import com.tanghao.bigdata.drools.domain.response.RuleResponse; 5 | import com.tanghao.bigdata.drools.util.DateUtil; 6 | import org.kie.api.KieServices; 7 | import org.kie.api.runtime.KieContainer; 8 | import org.kie.api.runtime.KieSession; 9 | import org.springframework.web.bind.annotation.RequestMapping; 10 | import org.springframework.web.bind.annotation.RequestMethod; 11 | import org.springframework.web.bind.annotation.RestController; 12 | 13 | import java.util.Date; 14 | 15 | /** 16 | * @Author： Canthny 17 | * @Description： 测试支付风控拦截 18 | * @Date： Created in 2018/9/11 15:18 19 | */ 20 | @RequestMapping(value = "/test/rules") 21 | @RestController 22 | public class TestPayRule { 23 | 24 | @RequestMapping(value = "/pay", method = RequestMethod.GET) 25 | public RuleResponse testPayRule(){ 26 | 27 | KieContainer kc = KieServices.Factory.get().getKieClasspathContainer(); 28 | KieSession ksession = kc.newKieSession("pay_rule"); 29 | MobilePaymentRequest payRequest = new MobilePaymentRequest(); 30 | payRequest.setAccountNo("1"); 31 | payRequest.setAmount(51); 32 | payRequest.setOutTradeDate(DateUtil.convertDateToString("yyyyMMdd",new Date())); 33 | payRequest.setMobile("18709858763"); 34 | payRequest.setBankCardNo("62260113241234"); 35 | ksession.insert(payRequest); 36 | ksession.fireAllRules(); 37 | ksession.dispose(); 38 | return payRequest.getResponse(); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/java/com/tanghao/bigdata/drools/domain/DefeatMapReduceObject.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.bigdata.drools.domain; 2 | 3 | import org.springframework.data.mongodb.core.mapping.Document; 4 | import org.springframework.data.mongodb.core.mapping.Field; 5 | 6 | import java.io.Serializable; 7 | 8 | /** 9 | * @Author： Canthny 10 | * @Description： mapreduce后默认输出对象 11 | * @Date： Created in 2018/9/30 0:35 12 | */ 13 | public class DefeatMapReduceObject implements Serializable { 14 | 15 | private static final long serialVersionUID = -5403346308235899592L; 16 | 17 | private String id; 18 | 19 | private Integer value; 20 | 21 | public String getId() { 22 | return id; 23 | } 24 | 25 | public void setId(String id) { 26 | this.id = id; 27 | } 28 | 29 | public Integer getValue() { 30 | return value; 31 | } 32 | 33 | public void setValue(Integer value) { 34 | this.value = value; 35 | } 36 | 37 | @Override 38 | public String toString() { 39 | return "DefeatMapReduceObject{" + 40 | "id='" + id + '\'' + 41 | ", value=" + value + 42 | '}'; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/java/com/tanghao/bigdata/drools/domain/MobilePaymentInfo.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.bigdata.drools.domain; 2 | 3 | import com.alibaba.fastjson.JSONObject; 4 | import org.springframework.data.mongodb.core.mapping.Document; 5 | import org.springframework.data.mongodb.core.mapping.Field; 6 | 7 | import java.io.Serializable; 8 | import java.util.Date; 9 | 10 | /** 11 | * @Author： Canthny 12 | * @Description： 手机缴费信息对象 13 | * @Date： Created in 2018/9/27 14:35 14 | */ 15 | @Document(collection="MOBILE_PAYMENT_INFO") 16 | public class MobilePaymentInfo implements Serializable{ 17 | private static final long serialVersionUID = -4612172600889702761L; 18 | 19 | @Field("pay_order_no") 20 | private String payOrderNo; 21 | 22 | @Field("account_no") 23 | private String accountNo; 24 | 25 | @Field("bank_card_no") 26 | private String bankCardNo; 27 | 28 | @Field("mobile") 29 | private String mobile; 30 | 31 | @Field("amount") 32 | private Integer amount; 33 | 34 | @Field("time") 35 | private Date time; 36 | 37 | public String getPayOrderNo() { 38 | return payOrderNo; 39 | } 40 | 41 | public void setPayOrderNo(String payOrderNo) { 42 | this.payOrderNo = payOrderNo; 43 | } 44 | 45 | public String getAccountNo() { 46 | return accountNo; 47 | } 48 | 49 | public void setAccountNo(String accountNo) { 50 | this.accountNo = accountNo; 51 | } 52 | 53 | public String getBankCardNo() { 54 | return bankCardNo; 55 | } 56 | 57 | public void setBankCardNo(String bankCardNo) { 58 | this.bankCardNo = bankCardNo; 59 | } 60 | 61 | public String getMobile() { 62 | return mobile; 63 | } 64 | 65 | public void setMobile(String mobile) { 66 | this.mobile = mobile; 67 | } 68 | 69 | public Integer getAmount() { 70 | return amount; 71 | } 72 | 73 | public void setAmount(Integer amount) { 74 | this.amount = amount; 75 | } 76 | 77 | public Date getTime() { 78 | return time; 79 | } 80 | 81 | public void setTime(Date time) { 82 | this.time = time; 83 | } 84 | 85 | @Override 86 | public String toString() { 87 | return "MobilePaymentInfo{" + 88 | "payOrderNo='" + payOrderNo + '\'' + 89 | ", accountNo='" + accountNo + '\'' + 90 | ", bankCardNo='" + bankCardNo + '\'' + 91 | ", mobile='" + mobile + '\'' + 92 | ", amount='" + amount + '\'' + 93 | ", time='" + time + '\'' + 94 | '}'; 95 | } 96 | 97 | public static void main(String[] args) { 98 | MobilePaymentInfo info = new MobilePaymentInfo(); 99 | info.setAccountNo("1234312412345235"); 100 | info.setAmount(120); 101 | info.setBankCardNo("62260113241234"); 102 | info.setMobile("18709858763"); 103 | info.setPayOrderNo("2018092700002"); 104 | info.setTime(new Date()); 105 | 106 | String demo = JSONObject.toJSONString(info); 107 | System.out.println(demo); 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/java/com/tanghao/bigdata/drools/domain/PhoneNoBlackObject.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.bigdata.drools.domain; 2 | 3 | import org.springframework.data.mongodb.core.mapping.Document; 4 | import org.springframework.data.mongodb.core.mapping.Field; 5 | 6 | import java.io.Serializable; 7 | 8 | /** 9 | * @Author： Canthny 10 | * @Description： 手机号黑名单 11 | * @Date： Created in 2018/9/29 16:06 12 | */ 13 | @Document(collection="PHONE_NO_BLACK_LIST") 14 | public class PhoneNoBlackObject implements Serializable { 15 | 16 | private static final long serialVersionUID = 9180593713475107020L; 17 | 18 | @Field("phone_no") 19 | private String phoneNo; 20 | 21 | public String getPhoneNo() { 22 | return phoneNo; 23 | } 24 | 25 | public void setPhoneNo(String phoneNo) { 26 | this.phoneNo = phoneNo; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/java/com/tanghao/bigdata/drools/domain/request/BaseRequest.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.bigdata.drools.domain.request; 2 | 3 | import com.tanghao.bigdata.drools.domain.response.RuleResponse; 4 | 5 | import java.io.Serializable; 6 | 7 | /** 8 | * @Author： Canthny 9 | * @Description： 一次支付请求参数 10 | * @Date： Created in 2018/9/11 15:17 11 | */ 12 | public class BaseRequest implements Serializable{ 13 | 14 | 15 | private static final long serialVersionUID = 3849610254845766744L; 16 | private RuleResponse response; 17 | 18 | public void setResponse(String code,String level,String desc){ 19 | this.response = new RuleResponse(code,level,desc); 20 | } 21 | 22 | public RuleResponse getResponse() { 23 | return response; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/java/com/tanghao/bigdata/drools/domain/request/MobilePaymentRequest.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.bigdata.drools.domain.request; 2 | 3 | import org.springframework.data.mongodb.core.mapping.Field; 4 | 5 | /** 6 | * @Author： Canthny 7 | * @Description： 手机充值请求对象 8 | * @Date： Created in 2018/9/29 19:44 9 | */ 10 | public class MobilePaymentRequest extends BaseRequest{ 11 | 12 | private static final long serialVersionUID = -5099820860840607608L; 13 | 14 | private String accountNo; 15 | 16 | private String bankCardNo; 17 | 18 | private String mobile; 19 | 20 | private Integer amount; 21 | 22 | private String outTradeDate; 23 | 24 | public String getAccountNo() { 25 | return accountNo; 26 | } 27 | 28 | public void setAccountNo(String accountNo) { 29 | this.accountNo = accountNo; 30 | } 31 | 32 | public String getBankCardNo() { 33 | return bankCardNo; 34 | } 35 | 36 | public void setBankCardNo(String bankCardNo) { 37 | this.bankCardNo = bankCardNo; 38 | } 39 | 40 | public String getMobile() { 41 | return mobile; 42 | } 43 | 44 | public void setMobile(String mobile) { 45 | this.mobile = mobile; 46 | } 47 | 48 | public Integer getAmount() { 49 | return amount; 50 | } 51 | 52 | public void setAmount(Integer amount) { 53 | this.amount = amount; 54 | } 55 | 56 | public String getOutTradeDate() { 57 | return outTradeDate; 58 | } 59 | 60 | public void setOutTradeDate(String outTradeDate) { 61 | this.outTradeDate = outTradeDate; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/java/com/tanghao/bigdata/drools/domain/response/RuleResponse.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.bigdata.drools.domain.response; 2 | 3 | import com.tanghao.bigdata.drools.enums.EnumRuleResponseLevel; 4 | 5 | import java.io.Serializable; 6 | 7 | /** 8 | * @Author： Canthny 9 | * @Description： 风控响应对象 10 | * @Date： Created in 2018/9/11 15:22 11 | */ 12 | public class RuleResponse implements Serializable{ 13 | 14 | private static final long serialVersionUID = 8501708883775504510L; 15 | 16 | public RuleResponse(String code, String level, String msg){ 17 | this.code = code; 18 | this.level = level; 19 | this.msg = msg; 20 | } 21 | 22 | private String code; 23 | 24 | private String level; 25 | 26 | private String msg; 27 | 28 | public String getCode() { 29 | return code; 30 | } 31 | 32 | public void setCode(String code) { 33 | this.code = code; 34 | } 35 | 36 | public String getLevel() { 37 | return level; 38 | } 39 | 40 | public void setLevel(String level) { 41 | this.level = level; 42 | } 43 | 44 | public String getMsg() { 45 | return msg; 46 | } 47 | 48 | public void setMsg(String msg) { 49 | this.msg = msg; 50 | } 51 | 52 | public static Boolean checkIsNull(RuleResponse response){ 53 | if (null==response){ 54 | return true; 55 | }else{ 56 | return false; 57 | } 58 | } 59 | 60 | @Override 61 | public String toString() { 62 | return "RuleResponse{" + 63 | "code='" + code + '\'' + 64 | ", level='" + EnumRuleResponseLevel.getDescByCode(level) + '\'' + 65 | ", msg='" + msg + '\'' + 66 | '}'; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/java/com/tanghao/bigdata/drools/enums/EnumRuleResponseLevel.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.bigdata.drools.enums; 2 | 3 | /** 4 | * @Author： Canthny 5 | * @Description： 风控响应级别 6 | * @Date： Created in 2018/9/11 15:23 7 | */ 8 | public enum EnumRuleResponseLevel { 9 | 10 | NONE("0","无风险"), 11 | LOW("1","低风险"), 12 | MIDDLE("2","中风险 "), 13 | HIGH("3","高风险"), 14 | SUPER("4","超高风险"); 15 | 16 | private String code; 17 | 18 | private String desc; 19 | 20 | public String getCode() { 21 | return code; 22 | } 23 | 24 | public void setCode(String code) { 25 | this.code = code; 26 | } 27 | 28 | public String getDesc() { 29 | return desc; 30 | } 31 | 32 | public void setDesc(String desc) { 33 | this.desc = desc; 34 | } 35 | 36 | EnumRuleResponseLevel(String code, String desc){ 37 | this.code = code; 38 | this.desc = desc; 39 | } 40 | 41 | public static String getDescByCode(String code){ 42 | if (code == null) { 43 | return null; 44 | } 45 | for (EnumRuleResponseLevel level : values()) { 46 | if (level.getCode().equals(code)) { 47 | return level.getDesc(); 48 | } 49 | } 50 | return null; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/java/com/tanghao/bigdata/drools/kafka/config/MobilePaymentInfoConfig.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.bigdata.drools.kafka.config; 2 | 3 | import com.tanghao.bigdata.drools.kafka.listener.MobilePaymentInfoListener; 4 | import org.springframework.beans.factory.annotation.Autowired; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.context.annotation.Configuration; 7 | import org.springframework.kafka.core.ConsumerFactory; 8 | import org.springframework.kafka.listener.KafkaMessageListenerContainer; 9 | import org.springframework.kafka.listener.config.ContainerProperties; 10 | import org.springframework.stereotype.Component; 11 | 12 | /** 13 | * @Author： Canthny 14 | * @Description： 手机充值消息配置 15 | * @Date： Created in 2018/9/29 10:24 16 | */ 17 | @Configuration 18 | @Component 19 | public class MobilePaymentInfoConfig { 20 | 21 | @Autowired 22 | MobilePaymentInfoListener mobilePaymentInfoListener; 23 | 24 | @Bean 25 | ContainerProperties containerProperties(){ 26 | //配置监听topic 27 | ContainerProperties containerProperties = new ContainerProperties("mobile_pay_info"); 28 | containerProperties.setMessageListener(mobilePaymentInfoListener); 29 | return containerProperties; 30 | } 31 | @Bean 32 | KafkaMessageListenerContainer messageListenerContainer(ConsumerFactory consumerFactory, 33 | ContainerProperties containerProperties){ 34 | return new KafkaMessageListenerContainer(consumerFactory, containerProperties); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/java/com/tanghao/bigdata/drools/kafka/listener/MobilePaymentInfoListener.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.bigdata.drools.kafka.listener; 2 | 3 | import com.tanghao.bigdata.drools.domain.MobilePaymentInfo; 4 | import com.tanghao.bigdata.drools.mongodb.service.MobilePaymentInfoService; 5 | import org.apache.kafka.clients.consumer.ConsumerRecord; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.kafka.listener.MessageListener; 8 | import org.springframework.stereotype.Component; 9 | import com.alibaba.fastjson.JSON; 10 | 11 | /** 12 | * @Author： Canthny 13 | * @Description： 手机充值消息监听者 14 | * @Date： Created in 2018/9/29 10:27 15 | */ 16 | @Component 17 | public class MobilePaymentInfoListener implements MessageListener{ 18 | 19 | @Autowired 20 | MobilePaymentInfoService mobilePaymentInfoService; 21 | 22 | 23 | @Override 24 | public void onMessage(ConsumerRecord stringStringConsumerRecord) { 25 | String info = stringStringConsumerRecord.value(); 26 | System.out.println(info); 27 | MobilePaymentInfo mobilePaymentInfo = JSON.parseObject(info,MobilePaymentInfo.class); 28 | mobilePaymentInfoService.saveMobilePaymentInfo(mobilePaymentInfo); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/java/com/tanghao/bigdata/drools/mongodb/config/SpringMongoConfig.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.bigdata.drools.mongodb.config; 2 | 3 | import com.mongodb.MongoClientURI; 4 | import org.springframework.beans.factory.annotation.Value; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.context.annotation.Configuration; 7 | import org.springframework.data.mongodb.MongoDbFactory; 8 | import org.springframework.data.mongodb.core.MongoTemplate; 9 | import org.springframework.data.mongodb.core.SimpleMongoDbFactory; 10 | import org.springframework.data.mongodb.core.convert.DbRefResolver; 11 | import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver; 12 | import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper; 13 | import org.springframework.data.mongodb.core.convert.MappingMongoConverter; 14 | import org.springframework.data.mongodb.core.mapping.MongoMappingContext; 15 | 16 | /** 17 | * @Author： Canthny 18 | * @Description： mongodb配置 19 | * @Date： Created in 2018/9/29 11:21 20 | */ 21 | @Configuration 22 | public class SpringMongoConfig{ 23 | 24 | private @Value("${spring.data.mongodb.uri}") String uri; 25 | 26 | public @Bean 27 | MongoDbFactory mongoDbFactory() throws Exception { 28 | return new SimpleMongoDbFactory(new MongoClientURI(uri)); 29 | } 30 | 31 | public @Bean 32 | MongoTemplate mongoTemplate() throws Exception { 33 | 34 | // MappingMongoConverter converter = 35 | // new MappingMongoConverter(mongoDbFactory(), new MongoMappingContext()); 36 | 37 | DbRefResolver dbRefResolver = new DefaultDbRefResolver(mongoDbFactory()); 38 | MappingMongoConverter converter = new MappingMongoConverter(dbRefResolver, new MongoMappingContext()); 39 | //这里是为了去掉插入mongodb时自动添加的_class类 40 | converter.setTypeMapper(new DefaultMongoTypeMapper(null)); 41 | MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory(), converter); 42 | 43 | return mongoTemplate; 44 | 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/java/com/tanghao/bigdata/drools/mongodb/service/MobilePaymentInfoService.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.bigdata.drools.mongodb.service; 2 | 3 | import com.tanghao.bigdata.drools.domain.MobilePaymentInfo; 4 | 5 | /** 6 | * @Author： Canthny 7 | * @Description： 手机缴费信息接口 8 | * @Date： Created in 2018/9/27 14:46 9 | */ 10 | public interface MobilePaymentInfoService { 11 | 12 | public void saveMobilePaymentInfo(MobilePaymentInfo info); 13 | 14 | } 15 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/java/com/tanghao/bigdata/drools/mongodb/service/impl/MobilePaymentInfoServiceImpl.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.bigdata.drools.mongodb.service.impl; 2 | 3 | import com.tanghao.bigdata.drools.domain.MobilePaymentInfo; 4 | import com.tanghao.bigdata.drools.mongodb.service.MobilePaymentInfoService; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.data.mongodb.core.MongoOperations; 7 | import org.springframework.stereotype.Component; 8 | 9 | /** 10 | * @Author： Canthny 11 | * @Description： 手机缴费信息接口实现类 12 | * @Date： Created in 2018/9/27 14:47 13 | */ 14 | @Component("mobilePaymentInfoService") 15 | public class MobilePaymentInfoServiceImpl implements MobilePaymentInfoService{ 16 | 17 | @Autowired 18 | MongoOperations mongoTemplate; 19 | 20 | public void saveMobilePaymentInfo(MobilePaymentInfo info) { 21 | mongoTemplate.save(info); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/java/com/tanghao/bigdata/drools/redis/RedisConfig.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.bigdata.drools.redis; 2 | 3 | import org.springframework.context.annotation.Bean; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.data.redis.connection.RedisConnectionFactory; 6 | import org.springframework.data.redis.core.RedisTemplate; 7 | import org.springframework.data.redis.serializer.RedisSerializer; 8 | import org.springframework.data.redis.serializer.StringRedisSerializer; 9 | import org.springframework.util.Assert; 10 | 11 | import java.nio.charset.Charset; 12 | 13 | /** 14 | * @Author： Canthny 15 | * @Description： redis模板类序列化问题配置 16 | * @Date： Created in 2018/9/29 21:40 17 | */ 18 | @Configuration 19 | public class RedisConfig { 20 | 21 | /** 22 | * redisTemplate 序列化使用的jdkSerializeable, 存储二进制字节码, 所以自定义序列化类 23 | * @return 24 | */ 25 | /** 26 | * redisTemplate默认jdkSerializeable，key会产生\xac\xed\x00\x05t\x00\x13这样的字符，所以设置成StringRedisSerializer 27 | * @return 28 | */ 29 | @Bean 30 | public RedisTemplate redisTemplate(RedisConnectionFactory factory) { 31 | RedisTemplate redisTemplate = new RedisTemplate(); 32 | RedisSerializer stringSerializer = new StringRedisSerializer(); 33 | redisTemplate.setKeySerializer(stringSerializer); 34 | redisTemplate.setValueSerializer(stringSerializer); 35 | redisTemplate.setHashKeySerializer(stringSerializer); 36 | redisTemplate.setHashValueSerializer(new IntegerRedisSerializer()); 37 | redisTemplate.setConnectionFactory(factory); 38 | return redisTemplate; 39 | } 40 | 41 | public class IntegerRedisSerializer implements RedisSerializer { 42 | 43 | private final Charset charset; 44 | 45 | public IntegerRedisSerializer() { 46 | this(Charset.forName("UTF8")); 47 | } 48 | 49 | public IntegerRedisSerializer(Charset charset) { 50 | Assert.notNull(charset, "Charset must not be null!"); 51 | this.charset = charset; 52 | } 53 | 54 | public Integer deserialize(byte[] bytes) { 55 | return (bytes == null ? null : Integer.parseInt(new String(bytes, charset))); 56 | } 57 | 58 | public byte[] serialize(Integer num) { 59 | return (num == null ? null : String.valueOf(num).getBytes(charset)); 60 | } 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/java/com/tanghao/bigdata/drools/service/MobilePaymentRuleService.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.bigdata.drools.service; 2 | 3 | import com.mongodb.BasicDBObject; 4 | import com.tanghao.bigdata.drools.domain.DefeatMapReduceObject; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.data.mongodb.core.MongoTemplate; 7 | import org.springframework.data.mongodb.core.query.BasicQuery; 8 | import org.springframework.data.mongodb.core.query.Query; 9 | import org.springframework.data.redis.core.RedisTemplate; 10 | import org.springframework.stereotype.Component; 11 | 12 | import javax.annotation.PostConstruct; 13 | import java.util.List; 14 | 15 | /** 16 | * @Author： Canthny 17 | * @Description： 缴费规则服务 18 | * @Date： Created in 2018/9/30 0:12 19 | */ 20 | @Component 21 | public class MobilePaymentRuleService { 22 | 23 | @Autowired 24 | private RedisTemplate redisTemplate; 25 | 26 | @Autowired 27 | private MongoTemplate mongoTemplate; 28 | 29 | private static MobilePaymentRuleService mobilePaymentRuleService; 30 | 31 | @PostConstruct 32 | public void init() { 33 | mobilePaymentRuleService = this; 34 | mobilePaymentRuleService.redisTemplate = this.redisTemplate; 35 | mobilePaymentRuleService.mongoTemplate = this.mongoTemplate; 36 | } 37 | 38 | public static Integer getAvgAmountPerMonthByAccountNo(String accountNo){ 39 | Integer amount = (Integer)mobilePaymentRuleService.redisTemplate.opsForHash().get("AvgAmountPerMonthByAccountNo",accountNo); 40 | if(null==amount){ 41 | BasicDBObject queryObject=new BasicDBObject(); 42 | queryObject.put("_id",accountNo); 43 | BasicDBObject fieldsObject=new BasicDBObject();//指定返回的字段fieldsObject.put("name", true); 44 | fieldsObject.put("_id", true); 45 | fieldsObject.put("value", true); 46 | Query query = new BasicQuery(queryObject,fieldsObject); 47 | List list= mobilePaymentRuleService.mongoTemplate.find(query,DefeatMapReduceObject.class,"ACCOUNT_MOBILEPAYMENT_DAY_AVG"); 48 | amount = list.get(0).getValue(); 49 | mobilePaymentRuleService.redisTemplate.opsForHash().put("AvgAmountPerMonthByAccountNo",list.get(0).getId(),list.get(0).getValue()); 50 | } 51 | //TODO 这里有很多空判断NullPointException，当空的时候应该返回什么呢，成功or失败，还是返回金额在rule文件里再去判断 52 | 53 | return amount; 54 | } 55 | 56 | /** 57 | * 从缓存中获取流计算结果，该用户一小时内充值次数 58 | * @param accountNo 59 | * @return 60 | */ 61 | public static Integer getCountPerHourByAccountNo(String accountNo){ 62 | Integer amount = (Integer)mobilePaymentRuleService.redisTemplate.opsForHash().get("CountPerHourByAccountNo",accountNo); 63 | return null==amount?0:amount; 64 | 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/java/com/tanghao/bigdata/drools/service/PhoneNoBlacklistService.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.bigdata.drools.service; 2 | 3 | import com.mongodb.BasicDBObject; 4 | import com.tanghao.bigdata.drools.domain.PhoneNoBlackObject; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.data.mongodb.core.MongoTemplate; 7 | import org.springframework.data.mongodb.core.query.BasicQuery; 8 | import org.springframework.data.mongodb.core.query.Query; 9 | import org.springframework.data.redis.core.RedisTemplate; 10 | import org.springframework.data.redis.serializer.RedisSerializer; 11 | import org.springframework.data.redis.serializer.StringRedisSerializer; 12 | import org.springframework.stereotype.Component; 13 | import org.springframework.util.CollectionUtils; 14 | 15 | import javax.annotation.PostConstruct; 16 | import java.util.HashSet; 17 | import java.util.List; 18 | import java.util.Set; 19 | 20 | /** 21 | * @Author： Canthny 22 | * @Description： 手机号黑名单 23 | * @Date： Created in 2018/9/29 15:54 24 | */ 25 | @Component 26 | public class PhoneNoBlacklistService { 27 | 28 | @Autowired 29 | private RedisTemplate redisTemplate; 30 | 31 | @Autowired 32 | private MongoTemplate mongoTemplate; 33 | 34 | private static PhoneNoBlacklistService phoneNoBlacklistService; 35 | 36 | @PostConstruct 37 | public void init() { 38 | phoneNoBlacklistService = this; 39 | phoneNoBlacklistService.redisTemplate = this.redisTemplate; 40 | phoneNoBlacklistService.mongoTemplate = this.mongoTemplate; 41 | } 42 | 43 | public static boolean isBlack(String phoneNo){ 44 | Set resultSet = (HashSet)phoneNoBlacklistService.redisTemplate.opsForSet().members("PHONE_NO_BLACK_LIST"); 45 | if(CollectionUtils.isEmpty(resultSet)) { 46 | resultSet = getPhoneNoBlackListFromMongoDb(); 47 | phoneNoBlacklistService.redisTemplate.opsForSet().add("PHONE_NO_BLACK_LIST",resultSet.toArray()); 48 | } 49 | if(resultSet.contains(phoneNo)){ 50 | return true; 51 | }else{ 52 | return false; 53 | } 54 | } 55 | 56 | private static Set getPhoneNoBlackListFromMongoDb(){ 57 | BasicDBObject fieldsObject=new BasicDBObject();//指定返回的字段fieldsObject.put("name", true); 58 | fieldsObject.put("phone_no", true); 59 | Query query = new BasicQuery(new BasicDBObject(),fieldsObject); 60 | List list= phoneNoBlacklistService.mongoTemplate.find(query,PhoneNoBlackObject.class,"PHONE_NO_BLACK_LIST"); 61 | Set phoneNoBlackListSet = new HashSet(); 62 | for(PhoneNoBlackObject info:list){ 63 | phoneNoBlackListSet.add(info.getPhoneNo()); 64 | } 65 | 66 | return phoneNoBlackListSet; 67 | } 68 | } -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/java/com/tanghao/bigdata/drools/util/DateUtil.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.bigdata.drools.util; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.sql.Timestamp; 7 | import java.text.ParseException; 8 | import java.text.SimpleDateFormat; 9 | import java.util.Calendar; 10 | import java.util.Date; 11 | 12 | /** 13 | * Date Utility Class used to convert Strings to Dates and Timestamps 14 | * 15 | * @author Matt Raible Modified by 16 | * Dan Kibler to correct time 17 | * pattern. Minutes should be mm not MM (MM is month). 18 | */ 19 | public class DateUtil { 20 | 21 | private static Logger log = LoggerFactory.getLogger(DateUtil.class); 22 | private static final String TIME_PATTERN = "HH:mm"; 23 | private static final String TIME_PATTERN_ALL = "yyyy-MM-dd HH:mm:ss.S"; 24 | private static final String TIME_YYYY_MM_DD_HH_MM_SS = "yyyy-MM-dd HH:mm:ss"; 25 | private static final String TIME_YYYY_MM_DD = "yyyy-MM-dd"; 26 | private static final String TIME_FORM_ALL = "yyyyMMddHHmmss"; 27 | private static final String TIME_YYYY_MMDD_HHMMSS_SSS = "yyyyMMddHHmmssSSS"; 28 | private static final String TIME_FORM = "HHmmss"; 29 | private static final String DATE_FORM = "yyyyMMdd"; 30 | 31 | /** 32 | * Checkstyle rule: utility classes should not have public constructor 33 | */ 34 | public DateUtil() { 35 | } 36 | 37 | /** 38 | * Timestamp和String之间转换的函数： 39 | */ 40 | public static String converTimestampToString(Timestamp obj) { 41 | return converTimestampToString(TIME_YYYY_MM_DD, obj); 42 | } 43 | 44 | /** 45 | * 自定义 转换模式将Timestamp 输出 46 | */ 47 | public static String converTimestampToString(String formatPattern, Timestamp obj) { 48 | SimpleDateFormat df = new SimpleDateFormat(formatPattern); 49 | String str = df.format(obj); 50 | return str; 51 | } 52 | 53 | /** 54 | * String转化为Timestamp: 55 | */ 56 | public static Timestamp converStringToTimestamp(String str) { 57 | Timestamp ts = Timestamp.valueOf(str); 58 | return ts; 59 | } 60 | 61 | /** 62 | * Return default datePattern (MM/dd/yyyy) 63 | * 64 | * @return a string representing the date pattern on the UI 65 | */ 66 | private static String getDatePattern() { 67 | String defaultDatePattern = "MM/dd/yyyy"; 68 | // try { 69 | // defaultDatePattern = ResourceBundle.getBundle(Constants.BUNDLE_KEY, 70 | // locale).getString( 71 | // "date.format"); 72 | // } catch (MissingResourceException mse) { 73 | // defaultDatePattern = "MM/dd/yyyy"; 74 | // } 75 | 76 | return defaultDatePattern; 77 | } 78 | 79 | public static String getDateTimePattern() { 80 | return DateUtil.getDatePattern() + " HH:mm:ss.S"; 81 | } 82 | 83 | /** 84 | * This method attempts to convert an Oracle-formatted date in the form 85 | * dd-MMM-yyyy to mm/dd/yyyy. 86 | * 87 | * @param aDate 88 | * date from database as a string 89 | * @return formatted string for the ui 90 | */ 91 | public static String getDate(Date aDate) { 92 | SimpleDateFormat df; 93 | String returnValue = ""; 94 | 95 | if (aDate != null) { 96 | df = new SimpleDateFormat(getDatePattern()); 97 | returnValue = df.format(aDate); 98 | } 99 | 100 | return (returnValue); 101 | } 102 | 103 | /** 104 | * This method generates a string representation of a date/time in the 105 | * format you specify on input 106 | * 107 | * @param aMask 108 | * the date pattern the string is in 109 | * @param strDate 110 | * a string representation of a date 111 | * @return a converted Date object 112 | * @see SimpleDateFormat 113 | * @throws ParseException 114 | * when String doesn't match the expected format 115 | */ 116 | public static Date convertStringToDate(String aMask, String strDate) { 117 | SimpleDateFormat df; 118 | Date date = null; 119 | df = new SimpleDateFormat(aMask); 120 | 121 | if (log.isDebugEnabled()) { 122 | log.debug("converting '" + strDate + "' to date with mask '" + aMask + "'"); 123 | } 124 | try { 125 | date = df.parse(strDate); 126 | } catch (ParseException pe) { 127 | log.error("Could not convert '" + strDate + "' to a date, throwing exception"); 128 | log.error("", pe); 129 | } 130 | 131 | return date; 132 | } 133 | 134 | /** 135 | * This method returns the current date time in the format: MM/dd/yyyy HH:MM 136 | * a 137 | * 138 | * @param theTime 139 | * the current time 140 | * @return the current date/time 141 | */ 142 | public static String getTimeNow(Date theTime) { 143 | return getDateTime(TIME_PATTERN, theTime); 144 | } 145 | 146 | /** 147 | * This method returns the current date in the format: MM/dd/yyyy 148 | * 149 | * @return the current date 150 | * @throws ParseException 151 | * when String doesn't match the expected format 152 | */ 153 | public static Calendar getCalendarToday() { 154 | Calendar cal = Calendar.getInstance(); 155 | return cal; 156 | } 157 | 158 | /** 159 | * This method generates a string representation of a date's date/time in 160 | * the format you specify on input 161 | * 162 | * @param aMask 163 | * the date pattern the string is in 164 | * @param aDate 165 | * a date object 166 | * @return a formatted string representation of the date 167 | * 168 | * @see SimpleDateFormat 169 | */ 170 | private static String getDateTime(String aMask, Date aDate) { 171 | SimpleDateFormat df = null; 172 | String returnValue = ""; 173 | 174 | if (aDate == null) { 175 | log.info("aDate is null!"); 176 | } else { 177 | df = new SimpleDateFormat(aMask); 178 | returnValue = df.format(aDate); 179 | } 180 | 181 | return (returnValue); 182 | } 183 | 184 | /** 185 | * This method generates a string representation of a date based on the 186 | * System Property 'dateFormat' in the format you specify on input 187 | * 188 | * @param aDate 189 | * A date to convert 190 | * @return a string representation of the date 191 | */ 192 | public static String convertDateToString(Date aDate) { 193 | return getDateTime(getDatePattern(), aDate); 194 | } 195 | 196 | /** 197 | * This method converts a String to a date using the datePattern 198 | * 199 | * @param strDate 200 | * the date to convert (in format MM/dd/yyyy) 201 | * @return a date object 202 | */ 203 | public static Date convertStringToDate(String strDate) { 204 | return convertStringToDate(getDatePattern(), strDate); 205 | } 206 | 207 | /** 208 | * 209 | * @param aDate 210 | * @return string YYYY_MM_DD 211 | */ 212 | public static String convertDateToString(String pattern, Date aDate) { 213 | return getDateTime(pattern, aDate); 214 | } 215 | 216 | /** 217 | * 取得从startDate开始的前(正)/后(负)day天 218 | * 219 | * @param startDate 220 | * @param day 221 | * @return date 222 | */ 223 | public static Date getRelativeDate(Date startDate, int day) { 224 | long myTime = (startDate.getTime() / 1000) + day * 24 * 60 * 60; 225 | startDate.setTime(myTime * 1000); 226 | return startDate; 227 | } 228 | 229 | /** 230 | * 取得从startDate开始的前(正)/后(负)minute分 231 | * 232 | * @param startDate 233 | * @param minute 234 | * @return date 235 | */ 236 | public static Date getRelativeMinute(Date startDate, int minute) { 237 | Calendar calendar = Calendar.getInstance(); 238 | try { 239 | calendar.setTime(startDate); 240 | calendar.add(Calendar.MINUTE, minute); 241 | return calendar.getTime(); 242 | } catch (Exception e) { 243 | log.error("", e); 244 | return startDate; 245 | } 246 | } 247 | 248 | /** 249 | * 根据日期获取星期几 250 | * 251 | * @param date 252 | * java.util.Date对象,不能为null 253 | * @return int 254 | */ 255 | public static int getDay(Date date) { 256 | Calendar cal = Calendar.getInstance(); 257 | cal.setTime(date); 258 | return cal.get(Calendar.DAY_OF_WEEK) - 1; 259 | } 260 | 261 | /** 262 | * 统计两个时间差，返回的是天数(即24小时算一天，少于24小时就为0，用这个的时候最好把小时、分钟等去掉) 263 | * 264 | * @param beginStr 265 | * 开始时间 266 | * @param endStr 267 | * @param format 268 | * @return day 269 | */ 270 | public static int countDays(String beginStr, String endStr, String format) { 271 | Date end = convertStringToDate(endStr, format); 272 | Date begin = convertStringToDate(beginStr, format); 273 | long times = end.getTime() - begin.getTime(); 274 | return (int) (times / 60 / 60 / 1000 / 24); 275 | } 276 | 277 | /** 278 | * 获取8位数字的今天日期格式 279 | * @return 280 | */ 281 | public static String getTodayYYYYMMDD() { 282 | return convertDateToString(DATE_FORM, new Date()); 283 | } 284 | 285 | /** 286 | * 计算当前日期到结束日期的天小时分钟 287 | * 288 | * @param endDate 289 | * @return string 290 | */ 291 | public static String getCountDays(Date endDate) { 292 | Date now = new Date(); 293 | String returnString = ""; 294 | try { 295 | long l = endDate.getTime() - now.getTime(); 296 | if (l < 0) { 297 | return "00天00小时00分"; 298 | } 299 | long day = l / (24 * 60 * 60 * 1000); 300 | if (day != 0) { 301 | returnString = day + "天"; 302 | } 303 | long hour = (l / (60 * 60 * 1000) - day * 24); 304 | long min = ((l / (60 * 1000)) - day * 24 * 60 - hour * 60); 305 | returnString = returnString + hour + "小时" + min + "分"; 306 | } catch (Exception e) { 307 | return returnString; 308 | } 309 | return returnString; 310 | } 311 | 312 | /** 313 | * 判断当前时间是否大于传入时间加传入天数 314 | * 315 | * @param date 316 | * 传入日期 317 | * @param day 318 | * 增加天数 319 | * @return true or false 320 | */ 321 | public static boolean compareDate(Date date, int day) { 322 | Date newDate = getRelativeDate(date, day); 323 | return new Date().after(newDate); 324 | } 325 | 326 | public static int countDays(Date begin, Date end) { 327 | if (begin == null || end == null) { 328 | return -9999; 329 | } 330 | long times = end.getTime() - begin.getTime(); 331 | return (int) (times / 60 / 60 / 1000 / 24); 332 | } 333 | 334 | /** 335 | * 336 | * 功能：天数向上取整，velocity使用
337 | * 338 | * @param in 339 | * @return int 340 | */ 341 | public static int getWholeDay(String in) { 342 | Double double1 = Double.parseDouble(in); 343 | if (double1 < 0) { 344 | return 0; 345 | } 346 | return (int) Math.ceil(double1); 347 | } 348 | 349 | public static Timestamp getCurrentTimestamp() { 350 | Date today = new Date(); 351 | return new Timestamp(today.getTime()); 352 | } 353 | 354 | /** 355 | * 得到一个时间延后或前移几天的时间,nowdate为时间,delay为前移或后延的天数 356 | * @param nowdate yyyyMMdd 357 | * @param delay 358 | * @return 359 | */ 360 | public static String getNextDay(String nowdate, int delay) { 361 | try { 362 | SimpleDateFormat format = new SimpleDateFormat(DATE_FORM); 363 | Date d = format.parse(nowdate); 364 | return getNextDay(d, delay); 365 | } catch (ParseException e) { 366 | log.error("", e); 367 | return null; 368 | } 369 | } 370 | 371 | public static String getNextDay(Date nowdate, int delay) { 372 | SimpleDateFormat format = new SimpleDateFormat(DATE_FORM); 373 | return format.format(getRelativeDate(nowdate, delay)); 374 | } 375 | 376 | /** 377 | * 378 | * 当期时间的前后几小时 379 | * 380 | * @param hour 381 | * @return 382 | */ 383 | public static String getTimeByHour(int hour) { 384 | Calendar calendar = Calendar.getInstance(); 385 | calendar.set(Calendar.HOUR_OF_DAY, calendar.get(Calendar.HOUR_OF_DAY) + hour); 386 | return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(calendar.getTime()); 387 | // return new SimpleDateFormat("HH:mm:ss").format(calendar.getTime()); 388 | } 389 | 390 | /** 391 | * 根据字符串日期获取对应的星期 392 | * 393 | * @param time 394 | * @return 395 | * @throws ParseException 396 | */ 397 | public static String getWeek(String time) throws ParseException { 398 | final String[] dayNames = { "周日", "周一", "周二", "周三", "周四", "周五", "周六" }; 399 | SimpleDateFormat sdfInput = new SimpleDateFormat("yyyy-MM-dd HH:mm"); 400 | Calendar calendar = Calendar.getInstance(); 401 | Date date = new Date(); 402 | date = sdfInput.parse(time); 403 | calendar.setTime(date); 404 | int dayOfWeek = calendar.get(Calendar.DAY_OF_WEEK) - 1; 405 | if (dayOfWeek < 0) { 406 | dayOfWeek = 0; 407 | } 408 | return dayNames[dayOfWeek]; 409 | } 410 | } 411 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/resources/META-INF/kmodule.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.data.mongodb.uri=mongodb://192.168.177.128:27017/rule 2 | 3 | #spring.kafka.bootstrap-servers=192.168.0.102:9092 4 | #spring.kafka.consumer.group-id=mobile_pay_info_into_db 5 | #spring.kafka.consumer.auto-offset-reset=latest 6 | 7 | # Redis缓存配置 8 | spring.redis.database=0 9 | spring.redis.host=localhost 10 | spring.redis.port=6379 11 | spring.redis.password= 12 | spring.redis.timeout=0 -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/resources/com.tanghao.bigdata.drools.rules/pay_rule.drl: -------------------------------------------------------------------------------- 1 | import com.tanghao.bigdata.drools.domain.request.MobilePaymentRequest; 2 | import com.tanghao.bigdata.drools.domain.response.RuleResponse; 3 | import com.tanghao.bigdata.drools.enums.EnumRuleResponseLevel; 4 | import com.tanghao.bigdata.drools.service.PhoneNoBlacklistService; 5 | import com.tanghao.bigdata.drools.service.MobilePaymentRuleService; 6 | import com.tanghao.bigdata.drools.util.DateUtil; 7 | 8 | rule "pay_rule1" 9 | //执行顺序配置，数字越大优先级越高 10 | salience 4 11 | //对于自己规则操作引起的update重新匹配，不会重复执行，对于其他规则操作引起的update重新匹配会执行一次 12 | //当两个规则都满足，并且then中都会update的情况下就会进入死循环这时要使用lock-on-active属性保证规则只会执行一次 13 | //这里通过塞入响应对象以及前置空判断保证规则不会同时满足，只会执行一次 14 | no-loop true 15 | when 16 | $pay : MobilePaymentRequest(RuleResponse.checkIsNull(response) && amount > 300); 17 | then 18 | $pay.setResponse("9999",EnumRuleResponseLevel.LOW.getCode(),"金额单笔金额超300了"); 19 | System.out.println("金额单笔金额超300了"); 20 | update($pay); 21 | end 22 | 23 | rule "pay_rule2" 24 | salience 3 25 | no-loop true 26 | when 27 | $pay : MobilePaymentRequest(RuleResponse.checkIsNull(response) && PhoneNoBlacklistService.isBlack(mobile)); 28 | then 29 | $pay.setResponse("9999",EnumRuleResponseLevel.LOW.getCode(),"手机号在黑名单中"); 30 | System.out.println("手机号在黑名单中"); 31 | update($pay); 32 | end 33 | 34 | rule "pay_rule3" 35 | salience 2 36 | no-loop true 37 | when 38 | $pay : MobilePaymentRequest(RuleResponse.checkIsNull(response) && amount > 3*MobilePaymentRuleService.getAvgAmountPerMonthByAccountNo(accountNo)); 39 | then 40 | $pay.setResponse("9999",EnumRuleResponseLevel.MIDDLE.getCode(),"大于上月平均充值金额的3倍"); 41 | System.out.println("大于上月平均充值金额的3倍"); 42 | update($pay); 43 | end 44 | 45 | rule "pay_rule4" 46 | salience 1 47 | no-loop true 48 | when 49 | $pay : MobilePaymentRequest(RuleResponse.checkIsNull(response) && 10 < MobilePaymentRuleService.getCountPerHourByAccountNo(accountNo)); 50 | then 51 | $pay.setResponse("9999",EnumRuleResponseLevel.HIGH.getCode(),"1小时内最多充值10次！"); 52 | update($pay); 53 | end -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/resources/drools/pay_rule.drl: -------------------------------------------------------------------------------- 1 | import com.tanghao.bigdata.drools.domain.request.MobilePaymentRequest; 2 | import com.tanghao.bigdata.drools.domain.response.RuleResponse; 3 | import com.tanghao.bigdata.drools.enums.EnumRuleResponseLevel; 4 | import com.tanghao.bigdata.drools.service.PhoneNoBlacklistService; 5 | import com.tanghao.bigdata.drools.service.MobilePaymentRuleService; 6 | import com.tanghao.bigdata.drools.util.DateUtil; 7 | 8 | rule "pay_rule1" 9 | //执行顺序配置，数字越大优先级越高 10 | salience 4 11 | //对于自己规则操作引起的update重新匹配，不会重复执行，对于其他规则操作引起的update重新匹配会执行一次 12 | //当两个规则都满足，并且then中都会update的情况下就会进入死循环这时要使用lock-on-active属性保证规则只会执行一次 13 | //这里通过塞入响应对象以及前置空判断保证规则不会同时满足，只会执行一次 14 | no-loop true 15 | when 16 | $pay : MobilePaymentRequest(RuleResponse.checkIsNull(response) && amount > 300); 17 | then 18 | $pay.setResponse("9999",EnumRuleResponseLevel.LOW.getCode(),"金额单笔金额超300了"); 19 | System.out.println("金额单笔金额超300了"); 20 | update($pay); 21 | end -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/resources/drools/pay_rule2.drl: -------------------------------------------------------------------------------- 1 | import com.tanghao.bigdata.drools.domain.request.MobilePaymentRequest; 2 | import com.tanghao.bigdata.drools.domain.response.RuleResponse; 3 | import com.tanghao.bigdata.drools.enums.EnumRuleResponseLevel; 4 | import com.tanghao.bigdata.drools.service.PhoneNoBlacklistService; 5 | import com.tanghao.bigdata.drools.service.MobilePaymentRuleService; 6 | import com.tanghao.bigdata.drools.util.DateUtil; 7 | 8 | rule "pay_rule2" 9 | //执行顺序配置，数字越大优先级越高 10 | salience 3 11 | //对于自己规则操作引起的update重新匹配，不会重复执行，对于其他规则操作引起的update重新匹配会执行一次 12 | //当两个规则都满足，并且then中都会update的情况下就会进入死循环这时要使用lock-on-active属性保证规则只会执行一次 13 | //这里通过塞入响应对象以及前置空判断保证规则不会同时满足，只会执行一次 14 | no-loop true 15 | when 16 | $pay : MobilePaymentRequest(RuleResponse.checkIsNull(response) && amount > 290); 17 | then 18 | $pay.setResponse("9999",EnumRuleResponseLevel.LOW.getCode(),"金额单笔金额超290了"); 19 | System.out.println("金额单笔金额超290了"); 20 | update($pay); 21 | end -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/resources/mapreduce/mobile_payment_info_map.js: -------------------------------------------------------------------------------- 1 | function() {emit(this.account_no,this.amount);} -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/main/resources/mapreduce/mobile_payment_info_reduce.js: -------------------------------------------------------------------------------- 1 | function(key,values) {return Array.avg(values)} -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/test/java/com/tanghao/bigdata/drools/test/TestDrools.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.bigdata.drools.test; 2 | 3 | import com.tanghao.bigdata.drools.DroolsStarter; 4 | import com.tanghao.bigdata.drools.domain.request.MobilePaymentRequest; 5 | import com.tanghao.bigdata.drools.util.DateUtil; 6 | import org.junit.Test; 7 | import org.junit.runner.RunWith; 8 | import org.kie.api.KieServices; 9 | import org.kie.api.io.ResourceType; 10 | import org.kie.api.runtime.KieContainer; 11 | import org.kie.api.runtime.KieSession; 12 | import org.kie.internal.KnowledgeBase; 13 | import org.kie.internal.KnowledgeBaseFactory; 14 | import org.kie.internal.builder.KnowledgeBuilder; 15 | import org.kie.internal.builder.KnowledgeBuilderFactory; 16 | import org.kie.internal.io.ResourceFactory; 17 | import org.kie.internal.runtime.StatefulKnowledgeSession; 18 | import org.springframework.boot.test.context.SpringBootTest; 19 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 20 | 21 | import java.util.Date; 22 | 23 | /** 24 | * @Author： Canthny 25 | * @Description： drools单元测试类 26 | * @Date： Created in 2018/9/29 19:51 27 | */ 28 | @RunWith(SpringJUnit4ClassRunner.class) 29 | @SpringBootTest(classes = DroolsStarter.class) 30 | public class TestDrools { 31 | 32 | /** 33 | *测试case1：金额超限 34 | */ 35 | @Test 36 | public void testCase1(){ 37 | 38 | KnowledgeBuilder knowledgeBuilder = KnowledgeBuilderFactory.newKnowledgeBuilder(); 39 | knowledgeBuilder.add(ResourceFactory.newClassPathResource("drools/pay_rule.drl"), ResourceType.DRL); 40 | knowledgeBuilder.add(ResourceFactory.newClassPathResource("drools/pay_rule2.drl"), ResourceType.DRL); 41 | KnowledgeBase knowledgeBase = KnowledgeBaseFactory.newKnowledgeBase(); 42 | knowledgeBase.addKnowledgePackages(knowledgeBuilder.getKnowledgePackages()); 43 | StatefulKnowledgeSession knowledgeSession = knowledgeBase.newStatefulKnowledgeSession(); 44 | MobilePaymentRequest payRequest = new MobilePaymentRequest(); 45 | payRequest.setAccountNo("3691529391467418"); 46 | payRequest.setAmount(351);//设置为大于单笔的最大限额，pay_rule.drl中规则pay_rule1配的是300 47 | payRequest.setOutTradeDate(DateUtil.convertDateToString("yyyyMMdd",new Date())); 48 | payRequest.setMobile("18709858763"); 49 | payRequest.setBankCardNo("62260113241234"); 50 | knowledgeSession.insert(payRequest);//具体的java类对象添加到workingMemory中。 51 | knowledgeSession.fireAllRules(); 52 | knowledgeSession.dispose(); 53 | System.out.println(payRequest.getResponse()); 54 | 55 | // KieContainer kc = KieServices.Factory.get().getKieClasspathContainer(); 56 | // KieSession ksession = kc.newKieSession("pay_rule"); 57 | // MobilePaymentRequest payRequest = new MobilePaymentRequest(); 58 | // payRequest.setAccountNo("3691529391467418"); 59 | // payRequest.setAmount(351);//设置为大于单笔的最大限额，pay_rule.drl中规则pay_rule1配的是300 60 | // payRequest.setOutTradeDate(DateUtil.convertDateToString("yyyyMMdd",new Date())); 61 | // payRequest.setMobile("18709858763"); 62 | // payRequest.setBankCardNo("62260113241234"); 63 | // ksession.insert(payRequest); 64 | // ksession.fireAllRules(); 65 | // ksession.dispose(); 66 | // System.out.println(payRequest.getResponse()); 67 | } 68 | 69 | /** 70 | * 该银行卡号在黑名单内（银行卡黑名单表） 71 | * 手机号在黑名单内（手机号黑名单表） 72 | */ 73 | @Test 74 | public void testCase2(){ 75 | KieContainer kc = KieServices.Factory.get().getKieClasspathContainer(); 76 | KieSession ksession = kc.newKieSession("pay_rule"); 77 | MobilePaymentRequest payRequest = new MobilePaymentRequest(); 78 | payRequest.setAccountNo("3691529391467418"); 79 | payRequest.setAmount(49);//金额不要超过单笔限额 80 | payRequest.setOutTradeDate(DateUtil.convertDateToString("yyyyMMdd",new Date())); 81 | payRequest.setMobile("18709858763");//手机号在黑名单中 82 | payRequest.setBankCardNo("62260113241234"); 83 | ksession.insert(payRequest); 84 | ksession.fireAllRules(); 85 | ksession.dispose(); 86 | System.out.println(payRequest.getResponse()); 87 | } 88 | 89 | /** 90 | * 当前单笔充值额度大于用户最近一个月充值额度平均值的3倍(定时批处理，月初统计上个月充值平均额度) 91 | */ 92 | @Test 93 | public void testCase3(){ 94 | KieContainer kc = KieServices.Factory.get().getKieClasspathContainer(); 95 | KieSession ksession = kc.newKieSession("pay_rule"); 96 | MobilePaymentRequest payRequest = new MobilePaymentRequest(); 97 | payRequest.setAccountNo("3691529391467418"); 98 | payRequest.setAmount(280);//设置为大于月平均值的3倍，且不超过单笔最大限额 99 | payRequest.setOutTradeDate(DateUtil.convertDateToString("yyyyMMdd",new Date())); 100 | payRequest.setMobile("18709858764");//手机号不在黑名单 101 | payRequest.setBankCardNo("62260113241234"); 102 | ksession.insert(payRequest); 103 | ksession.fireAllRules(); 104 | ksession.dispose(); 105 | System.out.println(payRequest.getResponse()); 106 | } 107 | 108 | /** 109 | * 一个小时内充值次数不超过5笔（实时流计算，保留用户一小时充值笔数） 110 | */ 111 | @Test 112 | public void testCase4(){ 113 | KieContainer kc = KieServices.Factory.get().getKieClasspathContainer(); 114 | KieSession ksession = kc.newKieSession("pay_rule"); 115 | MobilePaymentRequest payRequest = new MobilePaymentRequest(); 116 | payRequest.setAccountNo("3691529391467418"); 117 | payRequest.setAmount(220);//设置为小于月平均值的3倍，且不超过单笔最大限额 118 | payRequest.setOutTradeDate(DateUtil.convertDateToString("yyyyMMdd",new Date())); 119 | payRequest.setMobile("18709858764");//手机号不在黑名单 120 | payRequest.setBankCardNo("62260113241234"); 121 | ksession.insert(payRequest); 122 | ksession.fireAllRules(); 123 | ksession.dispose(); 124 | System.out.println(payRequest.getResponse()); 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/test/java/com/tanghao/bigdata/drools/test/TestKafkaProduct.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.bigdata.drools.test; 2 | 3 | import com.alibaba.fastjson.JSONObject; 4 | import com.tanghao.bigdata.drools.DroolsStarter; 5 | import com.tanghao.bigdata.drools.domain.MobilePaymentInfo; 6 | import org.junit.Test; 7 | import org.junit.runner.RunWith; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.boot.test.context.SpringBootTest; 10 | import org.springframework.kafka.core.KafkaTemplate; 11 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 12 | 13 | import java.util.Date; 14 | 15 | /** 16 | * @Author： Canthny 17 | * @Description： kafka生产者测试 18 | * @Date： Created in 2018/9/29 15:02 19 | */ 20 | @RunWith(SpringJUnit4ClassRunner.class) 21 | @SpringBootTest(classes = DroolsStarter.class) 22 | public class TestKafkaProduct { 23 | 24 | @Autowired 25 | KafkaTemplate kafkaTemplate; 26 | 27 | @Test 28 | public void testSendMobilePaymentInfo() { 29 | MobilePaymentInfo info = new MobilePaymentInfo(); 30 | info.setAccountNo("3691529391467418"); 31 | info.setAmount((int)(Math.random()*100)); 32 | info.setBankCardNo("62260113241234"); 33 | info.setMobile("18709858763"); 34 | info.setPayOrderNo("2018090100001"); 35 | info.setTime(new Date()); 36 | 37 | String infoStr = JSONObject.toJSONString(info); 38 | kafkaTemplate.send("mobile_pay_info",infoStr); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/test/java/com/tanghao/bigdata/drools/test/TestMongodb.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.bigdata.drools.test; 2 | 3 | import com.alibaba.fastjson.JSONObject; 4 | import com.mongodb.BasicDBObject; 5 | import com.mongodb.QueryBuilder; 6 | import com.tanghao.bigdata.drools.DroolsStarter; 7 | import com.tanghao.bigdata.drools.domain.DefeatMapReduceObject; 8 | import com.tanghao.bigdata.drools.domain.MobilePaymentInfo; 9 | import com.tanghao.bigdata.drools.domain.PhoneNoBlackObject; 10 | import com.tanghao.bigdata.drools.mongodb.service.MobilePaymentInfoService; 11 | import com.tanghao.bigdata.drools.service.PhoneNoBlacklistService; 12 | import com.tanghao.bigdata.drools.util.DateUtil; 13 | import org.junit.Test; 14 | import org.junit.runner.RunWith; 15 | import org.springframework.beans.factory.annotation.Autowired; 16 | import org.springframework.boot.test.context.SpringBootTest; 17 | import org.springframework.data.mongodb.core.MongoOperations; 18 | import org.springframework.data.mongodb.core.mapreduce.MapReduceOptions; 19 | import org.springframework.data.mongodb.core.query.BasicQuery; 20 | import org.springframework.data.mongodb.core.query.Criteria; 21 | import org.springframework.data.mongodb.core.query.Query; 22 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 23 | 24 | import java.text.ParseException; 25 | import java.text.SimpleDateFormat; 26 | import java.util.*; 27 | 28 | /** 29 | * @Author： Canthny 30 | * @Description： mongodb测试类 31 | * @Date： Created in 2018/9/27 14:54 32 | */ 33 | @RunWith(SpringJUnit4ClassRunner.class) 34 | @SpringBootTest(classes = DroolsStarter.class) 35 | public class TestMongodb { 36 | 37 | @Autowired 38 | MobilePaymentInfoService mobilePaymentInfoService; 39 | 40 | @Autowired 41 | MongoOperations mongoTemplate; 42 | 43 | /** 44 | * 缴费数据插入 45 | * @throws ParseException 46 | */ 47 | @Test 48 | public void testSave() throws ParseException { 49 | for(int i=0;i<10;i++){ 50 | MobilePaymentInfo info = new MobilePaymentInfo(); 51 | info.setAccountNo("3691529391467418"); 52 | info.setAmount((int)(Math.random()*100)); 53 | info.setBankCardNo("62260113241234"); 54 | info.setMobile("18709858763"); 55 | info.setPayOrderNo("2018090300001"); 56 | // SimpleDateFormat format = new SimpleDateFormat( "yyyy-MM-dd" ); 57 | // info.setTime(format.parse("2018-09-26")); 58 | if(i%3==0){ 59 | info.setTime(DateUtil.getRelativeDate(new Date(),-1)); 60 | }else if(i%3==1){ 61 | info.setTime(new Date()); 62 | }else if(i%3==2){ 63 | info.setTime(DateUtil.getRelativeDate(new Date(),1)); 64 | } 65 | mobilePaymentInfoService.saveMobilePaymentInfo(info); 66 | } 67 | } 68 | 69 | /** 70 | * 插入黑名单数据 71 | */ 72 | @Test 73 | public void insertPhoneNoBlackList(){ 74 | String phoneNo = "18709858763"; 75 | PhoneNoBlackObject obj = new PhoneNoBlackObject(); 76 | obj.setPhoneNo(phoneNo); 77 | mongoTemplate.save(obj); 78 | 79 | 80 | 81 | } 82 | 83 | /** 84 | * mongodb mapreduce批处理任务，可以通过定时器去执行，也可以通过crontab定时任务方式去执行脚本 85 | * 计算三天内用户缴费平均值 86 | * @throws ParseException 87 | */ 88 | @Test 89 | public void testMapReduce() throws ParseException { 90 | Query query = new Query(); 91 | SimpleDateFormat format = new SimpleDateFormat( "yyyy-MM-dd" ); 92 | Criteria criteria = Criteria.where("time").gte(DateUtil.getRelativeDate(new Date(),-1)).lte(DateUtil.getRelativeDate(new Date(),1)); 93 | query.addCriteria(criteria); 94 | MapReduceOptions mapReduceOptions = new MapReduceOptions(); 95 | mapReduceOptions.outputCollection("ACCOUNT_MOBILEPAYMENT_DAY_AVG"); 96 | mongoTemplate.mapReduce(query, "MOBILE_PAYMENT_INFO","classpath:mapreduce/mobile_payment_info_map.js", "classpath:mapreduce/mobile_payment_info_reduce.js", 97 | mapReduceOptions, HashMap.class); 98 | } 99 | 100 | /** 101 | * mongodb查询，指定查询条件和查询结果字段 102 | */ 103 | @Test 104 | public void findPhonenNoBlackList(){ 105 | BasicDBObject fieldsObject=new BasicDBObject();//指定返回的字段fieldsObject.put("name", true); 106 | fieldsObject.put("phone_no", true); 107 | Query query = new BasicQuery(new BasicDBObject(),fieldsObject); 108 | List list= mongoTemplate.find(query,PhoneNoBlackObject.class,"PHONE_NO_BLACK_LIST"); 109 | Set phoneNoBlackListSet = new HashSet(); 110 | for(PhoneNoBlackObject info:list){ 111 | phoneNoBlackListSet.add(info.getPhoneNo()); 112 | } 113 | System.out.println(phoneNoBlackListSet.toArray()); 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/test/java/com/tanghao/bigdata/drools/test/TestMongodbJob.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.bigdata.drools.test; 2 | 3 | /** 4 | * @Author： Canthny 5 | * @Description： 6 | * @Date： Created in 2018/9/29 15:29 7 | */ 8 | public class TestMongodbJob { 9 | 10 | 11 | } 12 | -------------------------------------------------------------------------------- /tanghao-bigdata-drools/src/test/java/com/tanghao/bigdata/drools/test/TestRuleService.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.bigdata.drools.test; 2 | 3 | import com.tanghao.bigdata.drools.DroolsStarter; 4 | import com.tanghao.bigdata.drools.service.MobilePaymentRuleService; 5 | import com.tanghao.bigdata.drools.service.PhoneNoBlacklistService; 6 | import org.junit.Test; 7 | import org.junit.runner.RunWith; 8 | import org.springframework.boot.test.context.SpringBootTest; 9 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 10 | 11 | /** 12 | * @Author： Canthny 13 | * @Description： 规则服务类测试 14 | * @Date： Created in 2018/9/30 0:49 15 | */ 16 | @RunWith(SpringJUnit4ClassRunner.class) 17 | @SpringBootTest(classes = DroolsStarter.class) 18 | public class TestRuleService { 19 | 20 | @Test 21 | public void testPhoneNoBlacklistService(){ 22 | System.out.println(PhoneNoBlacklistService.isBlack("18709858763")); 23 | } 24 | 25 | @Test 26 | public void testMobilePaymentRuleService(){ 27 | System.out.println(MobilePaymentRuleService.getAvgAmountPerMonthByAccountNo("3691529391467418")); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /tanghao-flink-demo/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | tanghao.flink.demo 8 | tanghao-flink-demo 9 | 1.0-SNAPSHOT 10 | jar 11 | 12 | tanghao flink demo 13 | 14 | 15 | UTF-8 16 | 1.10.0 17 | 18 | 19 | 20 | 21 | org.apache.flink 22 | flink-connector-kafka-0.11_2.11 23 | ${flink.version} 24 | 25 | 26 | org.apache.flink 27 | flink-java 28 | ${flink.version} 29 | 30 | 31 | org.apache.flink 32 | flink-streaming-java_2.11 33 | ${flink.version} 34 | 35 | 36 | org.apache.flink 37 | flink-clients_2.11 38 | ${flink.version} 39 | 40 | 41 | log4j 42 | log4j 43 | 1.2.17 44 | 45 | 46 | org.slf4j 47 | slf4j-api 48 | 1.7.25 49 | 50 | 51 | org.slf4j 52 | slf4j-log4j12 53 | 1.7.25 54 | 55 | 56 | org.apache.flink 57 | flink-json 58 | 1.6.0 59 | 60 | 61 | com.alibaba 62 | fastjson 63 | 1.2.62 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | org.apache.maven.plugins 78 | maven-shade-plugin 79 | 3.0.0 80 | 81 | 82 | package 83 | 84 | shade 85 | 86 | 87 | 88 | 89 | com.google.code.findbugs:jsr305 90 | 91 | 92 | 93 | 94 | 95 | 96 | 98 | *:* 99 | 100 | META-INF/*.SF 101 | META-INF/*.DSA 102 | META-INF/*.RSA 103 | 104 | 105 | 106 | 107 | 108 | com.tanghao.flink.demo.GetPayInfoJob 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | -------------------------------------------------------------------------------- /tanghao-flink-demo/src/README.md: -------------------------------------------------------------------------------- 1 | 执行计划：https://flink.apache.org/visualizer/ -------------------------------------------------------------------------------- /tanghao-flink-demo/src/main/java/com/tanghao/flink/demo/GetPayInfoJob.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.flink.demo; 2 | 3 | import org.apache.flink.api.common.functions.MapFunction; 4 | import org.apache.flink.api.common.functions.ReduceFunction; 5 | import org.apache.flink.api.common.serialization.SimpleStringSchema; 6 | import org.apache.flink.api.java.tuple.Tuple; 7 | import org.apache.flink.api.java.tuple.Tuple2; 8 | import org.apache.flink.api.java.tuple.Tuple3; 9 | import org.apache.flink.formats.json.JsonNodeDeserializationSchema; 10 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; 11 | import org.apache.flink.streaming.api.datastream.DataStream; 12 | import org.apache.flink.streaming.api.datastream.KeyedStream; 13 | import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; 14 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 15 | import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor; 16 | import org.apache.flink.streaming.api.windowing.assigners.SlidingEventTimeWindows; 17 | import org.apache.flink.streaming.api.windowing.assigners.SlidingProcessingTimeWindows; 18 | import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows; 19 | import org.apache.flink.streaming.api.windowing.assigners.TumblingProcessingTimeWindows; 20 | import org.apache.flink.streaming.api.windowing.time.Time; 21 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011; 22 | 23 | import java.text.ParseException; 24 | import java.text.SimpleDateFormat; 25 | import java.util.Date; 26 | import java.util.Properties; 27 | 28 | /** 29 | * @Author： Canthny 30 | * @Description： 获取支付信息日志流计算job 31 | * @Date： Created in 2018/9/13 2:06 32 | */ 33 | public class GetPayInfoJob { 34 | 35 | public static void main(String[] args) throws Exception { 36 | StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 37 | 38 | Properties properties = new Properties(); 39 | properties.setProperty("bootstrap.servers", "192.168.8.145:9082"); 40 | 41 | 42 | // DataStream stream = env.addSource(new FlinkKafkaConsumer011("20180913", new SimpleStringSchema(), properties)); 43 | SingleOutputStreamOperator stream= env.addSource(new FlinkKafkaConsumer011("CountPerHourByAccountNo2", new JsonNodeDeserializationSchema(), properties)); 44 | // SingleOutputStreamOperator> stream = env.addSource(new FlinkKafkaConsumer011("20180913", new JsonNodeDeserializationSchema(), properties)); 45 | 46 | SingleOutputStreamOperator objectNodeOperator = stream.assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor(Time.seconds(15)) { 47 | @Override 48 | public long extractTimestamp(ObjectNode element) { 49 | SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); 50 | Date eventTime = null; 51 | try { 52 | eventTime = format.parse(element.get("eventTime").asText()); 53 | } catch (ParseException e) { 54 | e.printStackTrace(); 55 | } 56 | return eventTime.getTime(); 57 | } 58 | }); 59 | 60 | KeyedStream, Tuple> keyBy = objectNodeOperator.map(new MapFunction>() { 61 | public Tuple2 map(ObjectNode jsonNodes) throws Exception { 62 | System.out.println(jsonNodes.get("accountId").asText() + "==map====" + jsonNodes.get("amount").asLong()); 63 | return new Tuple2(jsonNodes.get("accountId").asText(), jsonNodes.get("amount").asLong()); 64 | } 65 | }).keyBy(0); 66 | 67 | testTumblingEventTimeWindows(keyBy); 68 | 69 | // stream.keyBy(0).timeWindow(Time.minutes(1)).sum(1); 70 | stream.print(); 71 | env.execute(); 72 | } 73 | 74 | private static void testTumblingEventTimeWindows(KeyedStream, Tuple> keyBy) { 75 | keyBy.window(TumblingEventTimeWindows.of(Time.seconds(5))).reduce(new ReduceFunction>() 76 | { 77 | public Tuple2 reduce(Tuple2 v1, Tuple2 v2) { 78 | Tuple2 test = new Tuple2(v1.f0, v1.f1 + v2.f1); 79 | System.out.println(test); 80 | return test; 81 | } 82 | }); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /tanghao-flink-demo/src/main/java/com/tanghao/flink/demo/TestAction.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.flink.demo; 2 | 3 | import org.apache.flink.formats.json.JsonNodeDeserializationSchema; 4 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; 5 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; 6 | import org.apache.flink.streaming.util.serialization.JSONKeyValueDeserializationSchema; 7 | 8 | /** 9 | * @Author： Canthny 10 | * @Description： 测试一些api 11 | * @Date： Created in 2018/9/13 15:15 12 | */ 13 | public class TestAction { 14 | 15 | public static void main(String[] args) { 16 | try{ 17 | // ObjectMapper mapper = new ObjectMapper(); 18 | // ObjectNode initialKey = mapper.createObjectNode(); 19 | // initialKey.put("index", 4); 20 | // byte[] serializedKey = mapper.writeValueAsBytes(initialKey); 21 | // 22 | // ObjectNode initialValue = mapper.createObjectNode(); 23 | // initialValue.put("word", "world"); 24 | // byte[] serializedValue = mapper.writeValueAsBytes(initialValue); 25 | // 26 | // JSONKeyValueDeserializationSchema schema = new JSONKeyValueDeserializationSchema(false); 27 | // ObjectNode deserializedValue = schema.deserialize(serializedKey, serializedValue, "", 0, 28 | // 0); 29 | // System.out.println(deserializedValue); 30 | // 31 | // String testjson = "{\"accountId\":\"0716\",\"amount\":\"100\",\"eventTime\":\"2018/09/13 14:56:00\"}"; 32 | // JsonNodeDeserializationSchema schema1 = new JsonNodeDeserializationSchema(); 33 | // ObjectNode node = schema1.deserialize(testjson.getBytes()); 34 | // System.out.println(node); 35 | }catch (Exception e){ 36 | System.out.println(e); 37 | } 38 | 39 | 40 | 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /tanghao-flink-demo/src/main/java/com/tanghao/flink/demo/api/datastream/FilterApiTest.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.flink.demo.api.datastream; 2 | 3 | import org.apache.flink.api.common.functions.FilterFunction; 4 | import org.apache.flink.streaming.api.datastream.DataStreamSource; 5 | import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; 6 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 7 | 8 | import java.util.Arrays; 9 | 10 | /** 11 | * Description： FilterApi测试类 12 | * Created By tanghao on 2020/4/1 13 | */ 14 | public class FilterApiTest { 15 | 16 | public static void main(String[] args) { 17 | StreamExecutionEnvironment streamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment(); 18 | String[] strArray={"0","1","2","3","4","5","6","7","8","9","10"}; 19 | DataStreamSource streamSource = streamExecutionEnvironment.fromCollection(Arrays.asList(strArray)); 20 | SingleOutputStreamOperator streamAfterFilter = streamSource.filter(new FilterFunction() { 21 | public boolean filter(String s) throws Exception { 22 | if(Integer.parseInt(s) % 2==0){ 23 | return true; 24 | }else{ 25 | return false; 26 | } 27 | } 28 | }); 29 | 30 | streamAfterFilter.print("双数流"); 31 | 32 | SingleOutputStreamOperator streamAfterFilter2 = streamSource.filter(new FilterFunction() { 33 | public boolean filter(String s) throws Exception { 34 | if(Integer.parseInt(s) % 2==0){ 35 | return false; 36 | }else{ 37 | return true; 38 | } 39 | } 40 | }); 41 | streamAfterFilter2.print("单数流"); 42 | 43 | 44 | try { 45 | streamExecutionEnvironment.execute(); 46 | } catch (Exception e) { 47 | e.printStackTrace(); 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /tanghao-flink-demo/src/main/java/com/tanghao/flink/demo/api/datastream/FlatMapApiTest.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.flink.demo.api.datastream; 2 | 3 | import org.apache.flink.api.common.functions.FlatMapFunction; 4 | import org.apache.flink.streaming.api.datastream.DataStreamSource; 5 | import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; 6 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 7 | import org.apache.flink.util.Collector; 8 | 9 | import java.util.Arrays; 10 | 11 | /** 12 | * Description： FlatMapApi测试类 13 | * Created By tanghao on 2020/4/1 14 | */ 15 | public class FlatMapApiTest { 16 | 17 | public static void main(String[] args) { 18 | StreamExecutionEnvironment streamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment(); 19 | String[] strArray={"blue,green,red","green,green,red","blue,blue","red"}; 20 | DataStreamSource streamSource = streamExecutionEnvironment.fromCollection(Arrays.asList(strArray)); 21 | SingleOutputStreamOperator stream = streamSource.flatMap(new FlatMapFunction() { 22 | 23 | /** flatMap相对于Map和filter更灵活，不仅仅一对一，也可以过滤也可以一对多 */ 24 | public void flatMap(String s, Collector collector) throws Exception { 25 | String[] array = s.split(","); 26 | for (int i=0;i list = new ArrayList(); 23 | list.add(new Message("1","1001",30L)); 24 | list.add(new Message("2","2001",21L)); 25 | list.add(new Message("3","2001",65L)); 26 | list.add(new Message("4","1001",1L)); 27 | list.add(new Message("5","2001",21L)); 28 | DataStreamSource streamSource = streamExecutionEnvironment.fromCollection(list); 29 | KeyedStream keyedStream= streamSource.keyBy(new KeySelector() { 30 | public String getKey(Message message) throws Exception { 31 | return message.getTransCode(); 32 | } 33 | }); 34 | 35 | /** 根据类型统计每种类型的交易金额总和 */ 36 | SingleOutputStreamOperator reduceResult = keyedStream.reduce(new ReduceFunction() { 37 | public Message reduce(Message message, Message message2) throws Exception { 38 | Long sum = message.getAmount() + message2.getAmount(); 39 | String id = message.getId() + "_" +message2.getId(); 40 | return new Message(id,message.getTransCode(),sum); 41 | } 42 | }); 43 | 44 | reduceResult.print(); 45 | 46 | /** 聚合函数，滚动输出当前key下最新的一条聚合操作后的记录 */ 47 | // keyedStream.sum("amount").print(); 48 | keyedStream.max("amount").print(); 49 | // keyedStream.maxBy("amount").print(); 50 | // keyedStream.min("amount").print(); 51 | // keyedStream.minBy("amount").print(); 52 | 53 | try { 54 | streamExecutionEnvironment.execute(); 55 | } catch (Exception e) { 56 | e.printStackTrace(); 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /tanghao-flink-demo/src/main/java/com/tanghao/flink/demo/api/datastream/MapApiTest.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.flink.demo.api.datastream; 2 | 3 | import org.apache.commons.collections.CollectionUtils; 4 | import org.apache.flink.api.common.functions.MapFunction; 5 | import org.apache.flink.streaming.api.datastream.DataStreamSource; 6 | import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; 7 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 8 | 9 | import java.util.Arrays; 10 | import java.util.HashMap; 11 | import java.util.Map; 12 | 13 | /** 14 | * Description： MapApi测试 15 | * Created By tanghao on 2020/3/31 16 | */ 17 | public class MapApiTest { 18 | 19 | public static void main(String[] args) { 20 | StreamExecutionEnvironment streamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment(); 21 | String[] strArray={"deposit,10","withdraw,10","deposit,3","withdraw,10"}; 22 | DataStreamSource streamSource = streamExecutionEnvironment.fromCollection(Arrays.asList(strArray)); 23 | SingleOutputStreamOperator mapStream = streamSource.map(new MapFunction>() { 24 | public Map map(String s) throws Exception { 25 | String[] array = s.split(","); 26 | Map map = new HashMap(); 27 | for (int i=0;i list1 = new ArrayList(); 25 | list1.add(new Message("1","1001",30L)); 26 | list1.add(new Message("4","1001",1L)); 27 | 28 | DataStreamSource streamSource1 = streamExecutionEnvironment.fromCollection(list1); 29 | 30 | List list2 = new ArrayList(); 31 | list2.add(new Message("2","2001",21L)); 32 | list2.add(new Message("3","2001",65L)); 33 | list2.add(new Message("5","2001",21L)); 34 | DataStreamSource streamSource2 = streamExecutionEnvironment.fromCollection(list2); 35 | 36 | DataStream streamSource = streamSource1.union(streamSource2); 37 | // streamSource.print(); 38 | 39 | /** SplitStream is deprecated,now use Side Outputs instead*/ 40 | final OutputTag outputTag1001 = new OutputTag("side-output-1001"){}; 41 | final OutputTag outputTag2001 = new OutputTag("side-output-2001"){}; 42 | 43 | SingleOutputStreamOperator sideOutStream = streamSource.process(new ProcessFunction() { 44 | @Override 45 | public void processElement(Message value, Context ctx, Collector out) throws Exception { 46 | out.collect(value); 47 | if(value.getTransCode().equals("1001")){ 48 | ctx.output(outputTag1001,value); 49 | }else if(value.getTransCode().equals("2001")){ 50 | ctx.output(outputTag2001,value); 51 | } 52 | } 53 | }); 54 | DataStream sideOutputStream1001 = sideOutStream.getSideOutput(outputTag1001); 55 | DataStream sideOutputStream2001 = sideOutStream.getSideOutput(outputTag2001); 56 | 57 | sideOutStream.print("all"); 58 | sideOutputStream1001.print("1001"); 59 | sideOutputStream2001.print("2001"); 60 | 61 | try { 62 | streamExecutionEnvironment.execute(); 63 | } catch (Exception e) { 64 | e.printStackTrace(); 65 | } 66 | 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /tanghao-flink-demo/src/main/java/com/tanghao/flink/demo/api/others/BroadcastTableTest.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.flink.demo.api.others; 2 | 3 | import com.alibaba.fastjson.JSONObject; 4 | import com.tanghao.flink.demo.common.ConstantKafka; 5 | import com.tanghao.flink.demo.common.Message; 6 | import com.tanghao.flink.demo.common.TransTypeInfo; 7 | import org.apache.flink.api.common.functions.MapFunction; 8 | import org.apache.flink.api.common.serialization.SimpleStringSchema; 9 | import org.apache.flink.api.common.state.BroadcastState; 10 | import org.apache.flink.api.common.state.MapStateDescriptor; 11 | import org.apache.flink.api.common.state.ReadOnlyBroadcastState; 12 | import org.apache.flink.api.common.typeinfo.BasicTypeInfo; 13 | import org.apache.flink.api.java.tuple.Tuple2; 14 | import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; 15 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 16 | import org.apache.flink.streaming.api.functions.co.BroadcastProcessFunction; 17 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011; 18 | import org.apache.flink.util.Collector; 19 | 20 | import java.util.Properties; 21 | 22 | /** 23 | * Description： 广播维表动态更新测试，对应的数据源为BroadcastMsgProducerTest产生的数据 24 | * 场景：上游的消息中仅包含id和code之类的数据要转化成对应的类目名称，从字典表或者动态广播流中获取映射关系，并且支持类目名称可更新或者开关控制 25 | * Created By tanghao on 2020/4/8 26 | */ 27 | public class BroadcastTableTest { 28 | 29 | public static void main(String[] args) { 30 | StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 31 | Properties properties = new Properties(); 32 | properties.setProperty("bootstrap.servers", "192.168.8.145:9082"); 33 | 34 | /** 动态配置交易类型信息流，可以修改transCode对应的交易名称，也可以通过isOpen开关来展示是否要转换transName */ 35 | SingleOutputStreamOperator stream= env.addSource(new FlinkKafkaConsumer011(ConstantKafka.TRANS_TYPE_TOPIC, new SimpleStringSchema(), properties)).map(new MapFunction() { 36 | public TransTypeInfo map(String s) throws Exception { 37 | return JSONObject.parseObject(s,TransTypeInfo.class); 38 | } 39 | }); 40 | 41 | /** 业务消息中仅包含transCode交易编码，因此需要从广播表中获取交易编码对应的transName */ 42 | SingleOutputStreamOperator messageStream= env.addSource(new FlinkKafkaConsumer011(ConstantKafka.TRADE_MESSAGE_TOPIC, new SimpleStringSchema(), properties)).map(new MapFunction() { 43 | public Message map(String s) throws Exception { 44 | return JSONObject.parseObject(s,Message.class); 45 | } 46 | }); 47 | 48 | /** 存放transCode和transName的映射关系 */ 49 | final MapStateDescriptor TRANS_TYPE_MAP = new MapStateDescriptor( 50 | "TRANS_TYPE_MAP", 51 | BasicTypeInfo.STRING_TYPE_INFO, 52 | BasicTypeInfo.STRING_TYPE_INFO); 53 | 54 | /** 将业务消息流和交易类型码流connect，并通过BroadcastProcessFunction设置业务消息处理方法和广播表更新方法 */ 55 | SingleOutputStreamOperator> resStream = messageStream.connect(stream.broadcast(TRANS_TYPE_MAP)).process(new BroadcastProcessFunction>() { 56 | @Override 57 | public void processElement(Message value, ReadOnlyContext ctx, Collector> out) throws Exception { 58 | ReadOnlyBroadcastState broadcastState = 59 | ctx.getBroadcastState(TRANS_TYPE_MAP); 60 | String transName = broadcastState.get(value.getTransCode()); 61 | out.collect(Tuple2.of(value,transName)); 62 | } 63 | 64 | @Override 65 | public void processBroadcastElement(TransTypeInfo value, Context ctx, Collector> out) throws Exception { 66 | BroadcastState broadcastState = 67 | ctx.getBroadcastState(TRANS_TYPE_MAP); 68 | //判断开关，添加或者删除映射关系记录 69 | if(value.isOpen()){ 70 | broadcastState.put(value.getTransCode(), value.getTransName()); 71 | }else{ 72 | broadcastState.remove(value.getTransCode()); 73 | } 74 | } 75 | }); 76 | 77 | resStream.print(); 78 | 79 | /** 80 | * 从MessageKafkaProducerTest中producer交易类型信息的isOpen开关，输出结果如下： 81 | * 5> (Message{id='af02aa86-927d-47d2-af56-a4f06d5151c8', transCode='2001', amount=101, tradeTime=Wed Apr 08 17:14:06 CST 2020, userId=209},提现) 82 | * 5> (Message{id='0eca95a8-5d79-4e99-a992-a33c9ba20937', transCode='2001', amount=101, tradeTime=Wed Apr 08 17:14:21 CST 2020, userId=111},null) 83 | * 5> (Message{id='93731fd3-8ecb-4aaa-8ace-078b451de178', transCode='2001', amount=101, tradeTime=Wed Apr 08 17:15:06 CST 2020, userId=141},提现) 84 | */ 85 | 86 | try { 87 | env.execute(); 88 | } catch (Exception e) { 89 | e.printStackTrace(); 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /tanghao-flink-demo/src/main/java/com/tanghao/flink/demo/api/window/GlobalWindowTest.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.flink.demo.api.window; 2 | 3 | /** 4 | * Description： TODO 5 | * Created By tanghao on 2020/4/7 6 | */ 7 | public class GlobalWindowTest { 8 | } 9 | -------------------------------------------------------------------------------- /tanghao-flink-demo/src/main/java/com/tanghao/flink/demo/api/window/SessionWindowTest.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.flink.demo.api.window; 2 | 3 | /** 4 | * Description： TODO 5 | * Created By tanghao on 2020/4/7 6 | */ 7 | public class SessionWindowTest { 8 | } 9 | -------------------------------------------------------------------------------- /tanghao-flink-demo/src/main/java/com/tanghao/flink/demo/api/window/SlidingWindowTest.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.flink.demo.api.window; 2 | 3 | import com.alibaba.fastjson.JSONObject; 4 | import com.tanghao.flink.demo.common.ConstantKafka; 5 | import com.tanghao.flink.demo.common.Message; 6 | import com.tanghao.flink.demo.common.UserConsumerAmountPerMinute; 7 | import org.apache.flink.api.common.functions.AggregateFunction; 8 | import org.apache.flink.api.common.functions.FilterFunction; 9 | import org.apache.flink.api.common.functions.MapFunction; 10 | import org.apache.flink.api.common.functions.ReduceFunction; 11 | import org.apache.flink.api.common.serialization.SimpleStringSchema; 12 | import org.apache.flink.api.java.functions.KeySelector; 13 | import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; 14 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 15 | import org.apache.flink.streaming.api.functions.aggregation.AggregationFunction; 16 | import org.apache.flink.streaming.api.windowing.time.Time; 17 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011; 18 | 19 | import java.text.DateFormat; 20 | import java.text.SimpleDateFormat; 21 | import java.util.*; 22 | 23 | /** 24 | * Description： 滑动窗口场景——每5s刷新一次最近1分钟消费总金额用户排名top3 25 | * Created By tanghao on 2020/4/7 26 | */ 27 | public class SlidingWindowTest { 28 | 29 | public static void main(String[] args) { 30 | StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 31 | Properties properties = new Properties(); 32 | properties.setProperty("bootstrap.servers", "192.168.8.145:9082"); 33 | 34 | SingleOutputStreamOperator messageStream= env.addSource(new FlinkKafkaConsumer011(ConstantKafka.TRADE_MESSAGE_TOPIC, new SimpleStringSchema(), properties)).map(new MapFunction() { 35 | public Message map(String s) throws Exception { 36 | return JSONObject.parseObject(s,Message.class); 37 | } 38 | }); 39 | 40 | SingleOutputStreamOperator> result = messageStream.filter(new FilterFunction() { 41 | public boolean filter(Message value) throws Exception { 42 | if(value.getTransCode().equals("3001")){ 43 | return true; 44 | }else{ 45 | return false; 46 | } 47 | } 48 | }).keyBy(new KeySelector() { 49 | public String getKey(Message value) throws Exception { 50 | return value.getTransCode(); 51 | } 52 | }).timeWindow(Time.minutes(1),Time.seconds(5)).aggregate(new AggregateFunction, List>() { 53 | public Map createAccumulator() { 54 | return new HashMap(); 55 | } 56 | 57 | public Map add(Message value, Map accumulator) { 58 | UserConsumerAmountPerMinute userConsumerAmountPerMinute = accumulator.get(value.getUserId()); 59 | if(null == userConsumerAmountPerMinute){ 60 | userConsumerAmountPerMinute = new UserConsumerAmountPerMinute(); 61 | DateFormat format = new SimpleDateFormat("YYYY-MM-dd HH:mm"); 62 | String currentMinute = format.format(new Date()); 63 | userConsumerAmountPerMinute.setAmount(value.getAmount()); 64 | userConsumerAmountPerMinute.setUserId(value.getUserId()); 65 | userConsumerAmountPerMinute.setCurrentMinute(currentMinute); 66 | accumulator.put(value.getUserId(),userConsumerAmountPerMinute); 67 | }else{ 68 | userConsumerAmountPerMinute.setAmount(userConsumerAmountPerMinute.getAmount()+value.getAmount()); 69 | } 70 | return accumulator; 71 | } 72 | 73 | public List getResult(Map accumulator) { 74 | List list = new ArrayList(accumulator.values()); 75 | list.sort(new UserConsumerAmountPerMinuteComparator()); 76 | return list; 77 | } 78 | 79 | public Map merge(Map a, Map b) { 80 | return null; 81 | } 82 | }); 83 | 84 | result.print(); 85 | 86 | try { 87 | env.execute(); 88 | } catch (Exception e) { 89 | e.printStackTrace(); 90 | } 91 | } 92 | 93 | private static class UserConsumerAmountPerMinuteComparator implements Comparator{ 94 | public int compare(UserConsumerAmountPerMinute o1, UserConsumerAmountPerMinute o2) { 95 | return o1.getAmount().compareTo(o2.getAmount()); 96 | } 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /tanghao-flink-demo/src/main/java/com/tanghao/flink/demo/api/window/TumblingWindowAggregateFunctionTest.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.flink.demo.api.window; 2 | 3 | import com.alibaba.fastjson.JSONObject; 4 | import com.tanghao.flink.demo.common.ConstantKafka; 5 | import com.tanghao.flink.demo.common.Message; 6 | import com.tanghao.flink.demo.common.TradeCountMinuteResult; 7 | import com.tanghao.flink.demo.common.UserConsumerAmountPerMinute; 8 | import org.apache.commons.lang3.StringUtils; 9 | import org.apache.flink.api.common.functions.*; 10 | import org.apache.flink.api.common.serialization.SimpleStringSchema; 11 | import org.apache.flink.api.common.state.*; 12 | import org.apache.flink.api.common.typeinfo.BasicTypeInfo; 13 | import org.apache.flink.api.java.functions.KeySelector; 14 | import org.apache.flink.configuration.Configuration; 15 | import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; 16 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 17 | import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction; 18 | import org.apache.flink.streaming.api.windowing.time.Time; 19 | import org.apache.flink.streaming.api.windowing.windows.TimeWindow; 20 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011; 21 | import org.apache.flink.util.Collector; 22 | import org.apache.flink.util.OutputTag; 23 | 24 | import java.io.IOException; 25 | import java.text.DateFormat; 26 | import java.text.SimpleDateFormat; 27 | import java.util.*; 28 | 29 | /** 30 | * Description： 滚动窗口测试 31 | * 场景：根据transCode统计每分钟充值、提现、消费等各类型交易总金额；并且根据userId统计用户分钟消费金额。 32 | * Created By tanghao on 2020/4/7 33 | */ 34 | public class TumblingWindowAggregateFunctionTest { 35 | 36 | public static void main(String[] args) { 37 | StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 38 | Properties properties = new Properties(); 39 | properties.setProperty("bootstrap.servers", "192.168.8.145:9082"); 40 | 41 | SingleOutputStreamOperator messageStream= env.addSource(new FlinkKafkaConsumer011(ConstantKafka.TRADE_MESSAGE_TOPIC, new SimpleStringSchema(), properties)).map(new MapFunction() { 42 | public Message map(String s) throws Exception { 43 | return JSONObject.parseObject(s,Message.class); 44 | } 45 | }); 46 | 47 | //用户分钟消费额子流标记 48 | final OutputTag userConsumerAmountPerMin = new OutputTag("side-output-user-consumer-amount-per-min"){}; 49 | 50 | //按交易类型分区 51 | SingleOutputStreamOperator res = messageStream.keyBy(new KeySelector() { 52 | public String getKey(Message message) throws Exception { 53 | return message.getTransCode(); 54 | } 55 | }).timeWindow(Time.minutes(1)).aggregate(new MySumAggregate()); 56 | res.print(); 57 | res.getSideOutput(userConsumerAmountPerMin).print(); 58 | /** 59 | * 输出结果：各类型分钟交易总额，用户分钟消费总额 60 | * 4> TradeCountMinuteResult{transCode='1001', sumAmount=4998, currentMinute='2020-04-09 15:52'} 61 | * 3> TradeCountMinuteResult{transCode='3001', sumAmount=4023, currentMinute='2020-04-09 15:52'} 62 | * 6> TradeCountMinuteResult{transCode='2001', sumAmount=6190, currentMinute='2020-04-09 15:52'} 63 | * 3> UserConsumerAmountPerMinute{userId=754, amount=599, currentMinute='2020-04-09 15:52'} 64 | * 3> UserConsumerAmountPerMinute{userId=578, amount=725, currentMinute='2020-04-09 15:52'} 65 | * 3> UserConsumerAmountPerMinute{userId=931, amount=622, currentMinute='2020-04-09 15:52'} 66 | */ 67 | try { 68 | env.execute(); 69 | } catch (Exception e) { 70 | e.printStackTrace(); 71 | } 72 | } 73 | 74 | /** 75 | * 这里禁止使用RichFunction,暂时不清楚为何这样设计，说是这样使用状态不正确 76 | */ 77 | private static class MySumAggregate extends RichAggregateFunction { 78 | 79 | private transient MapState firstMinUserConsumerAmountMap; 80 | private transient ValueState totalTradeAmountValue; 81 | 82 | @Override 83 | public void open(Configuration parameters) throws Exception { 84 | MapStateDescriptor firstMinUserConsumerAmount = new MapStateDescriptor("firstMinUserConsumerAmount", BasicTypeInfo.LONG_TYPE_INFO,BasicTypeInfo.LONG_TYPE_INFO); 85 | StateTtlConfig ttlConfig = StateTtlConfig 86 | .newBuilder(org.apache.flink.api.common.time.Time.seconds(1)) 87 | .setUpdateType(StateTtlConfig.UpdateType.OnCreateAndWrite) 88 | .setStateVisibility(StateTtlConfig.StateVisibility.NeverReturnExpired) 89 | .build(); 90 | firstMinUserConsumerAmount.enableTimeToLive(ttlConfig); 91 | firstMinUserConsumerAmountMap = getRuntimeContext().getMapState(firstMinUserConsumerAmount); 92 | 93 | ValueStateDescriptor totalTradeAmount = new ValueStateDescriptor("totalTradeAmount", BasicTypeInfo.LONG_TYPE_INFO); 94 | StateTtlConfig ttlConfig2 = StateTtlConfig 95 | .newBuilder(org.apache.flink.api.common.time.Time.seconds(1)) 96 | .setUpdateType(StateTtlConfig.UpdateType.OnCreateAndWrite) 97 | .setStateVisibility(StateTtlConfig.StateVisibility.NeverReturnExpired) 98 | .build(); 99 | totalTradeAmount.enableTimeToLive(ttlConfig2); 100 | totalTradeAmountValue = getRuntimeContext().getState(totalTradeAmount); 101 | } 102 | 103 | public TradeCountMinuteResult createAccumulator() { 104 | return new TradeCountMinuteResult(); 105 | } 106 | 107 | public TradeCountMinuteResult add(Message value, TradeCountMinuteResult accumulator) { 108 | DateFormat format = new SimpleDateFormat("YYYY-MM-dd HH:mm"); 109 | String currentMinute = format.format(new Date()); 110 | if(currentMinute.equals("2020-04-10 17:42")){ 111 | Long userId = value.getUserId(); 112 | Long userSum = null; 113 | try { 114 | userSum = firstMinUserConsumerAmountMap.get(userId); 115 | } catch (Exception e) { 116 | e.printStackTrace(); 117 | } 118 | if(null==userSum){ 119 | userSum = value.getAmount(); 120 | }else{ 121 | userSum += value.getAmount(); 122 | } 123 | try { 124 | firstMinUserConsumerAmountMap.put(userId,userSum); 125 | } catch (Exception e) { 126 | e.printStackTrace(); 127 | } 128 | } 129 | try { 130 | Long totalTradeAmount = totalTradeAmountValue.value(); 131 | if(null==totalTradeAmount){ 132 | totalTradeAmount = value.getAmount(); 133 | }else{ 134 | totalTradeAmount += value.getAmount(); 135 | } 136 | totalTradeAmountValue.update(totalTradeAmount); 137 | } catch (IOException e) { 138 | e.printStackTrace(); 139 | } 140 | 141 | if(StringUtils.isEmpty(accumulator.getTransCode())){ 142 | accumulator.setTransCode(value.getTransCode()); 143 | } 144 | if(StringUtils.isEmpty(accumulator.getCurrentMinute())){ 145 | accumulator.setCurrentMinute(currentMinute); 146 | } 147 | Long sum = null==accumulator.getSumAmount()?0L:accumulator.getSumAmount(); 148 | sum += value.getAmount(); 149 | accumulator.setSumAmount(sum); 150 | return accumulator; 151 | } 152 | 153 | public TradeCountMinuteResult getResult(TradeCountMinuteResult accumulator) { 154 | return accumulator; 155 | } 156 | 157 | public TradeCountMinuteResult merge(TradeCountMinuteResult a, TradeCountMinuteResult b) { 158 | return null; 159 | } 160 | } 161 | 162 | } 163 | -------------------------------------------------------------------------------- /tanghao-flink-demo/src/main/java/com/tanghao/flink/demo/api/window/TumblingWindowProcessWindowFunctionTest.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.flink.demo.api.window; 2 | 3 | import com.alibaba.fastjson.JSONObject; 4 | import com.tanghao.flink.demo.common.ConstantKafka; 5 | import com.tanghao.flink.demo.common.Message; 6 | import com.tanghao.flink.demo.common.TradeCountMinuteResult; 7 | import com.tanghao.flink.demo.common.UserConsumerAmountPerMinute; 8 | import org.apache.flink.api.common.functions.MapFunction; 9 | import org.apache.flink.api.common.serialization.SimpleStringSchema; 10 | import org.apache.flink.api.java.functions.KeySelector; 11 | import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; 12 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 13 | import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction; 14 | import org.apache.flink.streaming.api.windowing.time.Time; 15 | import org.apache.flink.streaming.api.windowing.windows.TimeWindow; 16 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011; 17 | import org.apache.flink.util.Collector; 18 | import org.apache.flink.util.OutputTag; 19 | 20 | import java.text.DateFormat; 21 | import java.text.SimpleDateFormat; 22 | import java.util.*; 23 | 24 | /** 25 | * Description： 滚动窗口测试 26 | * 场景：根据transCode统计每分钟充值、提现、消费等各类型交易总金额；并且根据userId统计用户分钟消费金额。 27 | * Created By tanghao on 2020/4/7 28 | */ 29 | public class TumblingWindowProcessWindowFunctionTest { 30 | 31 | public static void main(String[] args) { 32 | StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 33 | Properties properties = new Properties(); 34 | properties.setProperty("bootstrap.servers", "192.168.8.145:9082"); 35 | 36 | SingleOutputStreamOperator messageStream= env.addSource(new FlinkKafkaConsumer011(ConstantKafka.TRADE_MESSAGE_TOPIC, new SimpleStringSchema(), properties)).map(new MapFunction() { 37 | public Message map(String s) throws Exception { 38 | return JSONObject.parseObject(s,Message.class); 39 | } 40 | }); 41 | 42 | //用户分钟消费额子流标记 43 | final OutputTag userConsumerAmountPerMin = new OutputTag("side-output-user-consumer-amount-per-min"){}; 44 | 45 | //按交易类型分区 46 | SingleOutputStreamOperator res = messageStream.keyBy(new KeySelector() { 47 | public String getKey(Message message) throws Exception { 48 | return message.getTransCode(); 49 | } 50 | }).timeWindow(Time.minutes(1)).process(new ProcessWindowFunction() { 51 | public void process(String s, Context context, Iterable elements, Collector out) throws Exception { 52 | DateFormat format = new SimpleDateFormat("YYYY-MM-dd HH:mm"); 53 | String currentMinute = format.format(new Date()); 54 | Long sum = 0L; 55 | Map userConsumerAmountPerMinMap = new HashMap(); 56 | for(Message message:elements){ 57 | /** 之前已经keyBy根据交易类型分区过，每个分区都是统一类型的交易数据，直接累加即可 */ 58 | sum += message.getAmount(); 59 | /** 若交易类型是消费，则要统计用户该分钟内的交易总额，输出到用户维度的子流中 */ 60 | if(message.getTransCode().equals("3001")){ 61 | if(null==userConsumerAmountPerMinMap.get(message.getUserId())){ 62 | userConsumerAmountPerMinMap.put(message.getUserId(),message.getAmount()); 63 | }else{ 64 | long temp = userConsumerAmountPerMinMap.get(message.getUserId()); 65 | temp += message.getAmount(); 66 | userConsumerAmountPerMinMap.put(message.getUserId(),temp); 67 | } 68 | } 69 | } 70 | /** 主数据流输出各类型交易额汇总信息*/ 71 | TradeCountMinuteResult tradeCountMinuteResult = new TradeCountMinuteResult(); 72 | tradeCountMinuteResult.setTransCode(s); 73 | tradeCountMinuteResult.setSumAmount(sum); 74 | tradeCountMinuteResult.setCurrentMinute(currentMinute); 75 | out.collect(tradeCountMinuteResult); 76 | /** 用户分钟消费额统计，输出到子数据流中。可以用于第1分钟消费金额top3用户统计，交给redis排序即可 */ 77 | for (Map.Entry m : userConsumerAmountPerMinMap.entrySet()) { 78 | Long userId = m.getKey(); 79 | Long amount = m.getValue(); 80 | UserConsumerAmountPerMinute userConsumerAmountPerMinute = new UserConsumerAmountPerMinute(); 81 | userConsumerAmountPerMinute.setAmount(amount); 82 | userConsumerAmountPerMinute.setUserId(userId); 83 | userConsumerAmountPerMinute.setCurrentMinute(currentMinute); 84 | context.output(userConsumerAmountPerMin,userConsumerAmountPerMinute); 85 | } 86 | } 87 | }); 88 | res.print(); 89 | System.out.println("执行计划:"+env.getExecutionPlan()); 90 | res.getSideOutput(userConsumerAmountPerMin).print(); 91 | /** 92 | * 输出结果：各类型分钟交易总额，用户分钟消费总额 93 | * 4> TradeCountMinuteResult{transCode='1001', sumAmount=4998, currentMinute='2020-04-09 15:52'} 94 | * 3> TradeCountMinuteResult{transCode='3001', sumAmount=4023, currentMinute='2020-04-09 15:52'} 95 | * 6> TradeCountMinuteResult{transCode='2001', sumAmount=6190, currentMinute='2020-04-09 15:52'} 96 | * 3> UserConsumerAmountPerMinute{userId=754, amount=599, currentMinute='2020-04-09 15:52'} 97 | * 3> UserConsumerAmountPerMinute{userId=578, amount=725, currentMinute='2020-04-09 15:52'} 98 | * 3> UserConsumerAmountPerMinute{userId=931, amount=622, currentMinute='2020-04-09 15:52'} 99 | */ 100 | try { 101 | env.execute(); 102 | } catch (Exception e) { 103 | e.printStackTrace(); 104 | } 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /tanghao-flink-demo/src/main/java/com/tanghao/flink/demo/api/window/TumblingWindowUseEventTimeTest.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.flink.demo.api.window; 2 | 3 | import com.alibaba.fastjson.JSONObject; 4 | import com.tanghao.flink.demo.common.ConstantKafka; 5 | import com.tanghao.flink.demo.common.Message; 6 | import org.apache.flink.api.common.functions.MapFunction; 7 | import org.apache.flink.api.common.serialization.SimpleStringSchema; 8 | import org.apache.flink.api.java.functions.KeySelector; 9 | import org.apache.flink.streaming.api.TimeCharacteristic; 10 | import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; 11 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 12 | import org.apache.flink.streaming.api.functions.timestamps.AscendingTimestampExtractor; 13 | import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows; 14 | import org.apache.flink.streaming.api.windowing.time.Time; 15 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011; 16 | 17 | import java.util.Properties; 18 | 19 | /** 20 | * Description： 使用事件时间 21 | * Created By tanghao on 2020/4/9 22 | */ 23 | public class TumblingWindowUseEventTimeTest { 24 | public static void main(String[] args) { 25 | StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 26 | env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); 27 | 28 | Properties properties = new Properties(); 29 | properties.setProperty("bootstrap.servers", "192.168.8.145:9082"); 30 | SingleOutputStreamOperator messageStream= env.addSource(new FlinkKafkaConsumer011(ConstantKafka.TRADE_MESSAGE_TOPIC, new SimpleStringSchema(), properties)).map(new MapFunction() { 31 | public Message map(String s) throws Exception { 32 | return JSONObject.parseObject(s,Message.class); 33 | } 34 | }).assignTimestampsAndWatermarks(new AscendingTimestampExtractor() { 35 | @Override 36 | public long extractAscendingTimestamp(Message element) { 37 | return element.getTradeTime().getTime(); 38 | } 39 | }); 40 | 41 | SingleOutputStreamOperator res = messageStream.keyBy(new KeySelector() { 42 | public String getKey(Message message) throws Exception { 43 | return message.getTransCode(); 44 | } 45 | }).window(TumblingEventTimeWindows.of(Time.minutes(1))).sum("amount"); 46 | res.print(); 47 | try { 48 | env.execute(); 49 | } catch (Exception e) { 50 | e.printStackTrace(); 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /tanghao-flink-demo/src/main/java/com/tanghao/flink/demo/common/ConstantKafka.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.flink.demo.common; 2 | 3 | /** 4 | * Description： kafka常量 5 | * Created By tanghao on 2020/4/7 6 | */ 7 | public class ConstantKafka { 8 | 9 | public static String KAFKA_SINGLE = "192.168.8.145:9082"; 10 | 11 | public static String TRADE_MESSAGE_TOPIC = "trade_message"; 12 | 13 | public static String TRANS_TYPE_TOPIC = "trans_type"; 14 | } 15 | -------------------------------------------------------------------------------- /tanghao-flink-demo/src/main/java/com/tanghao/flink/demo/common/Message.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.flink.demo.common; 2 | 3 | import java.io.Serializable; 4 | import java.util.Date; 5 | import java.util.Random; 6 | import java.util.UUID; 7 | 8 | /** 9 | * Description： 消息POJO类 10 | * Created By tanghao on 2020/4/1 11 | */ 12 | public class Message implements Serializable { 13 | private String id; 14 | 15 | private String transCode;//1001-充值，2001-提现，3001-消费 16 | 17 | private Long amount; 18 | 19 | private Date tradeTime; 20 | 21 | private Long userId; 22 | 23 | /** 符合POJO类的规则： 24 | * The class is public and standalone (no non-static inner class) 25 | * The class has a public no-argument constructor 26 | * All non-static, non-transient fields in the class (and all superclasses) are either public (and non-final) or have a public getter- and a setter- method that follows the Java beans naming conventions for getters and setters. 27 | * 这里关注第二条，必须有无参构造器 28 | * */ 29 | public Message(){ 30 | 31 | } 32 | 33 | public Message(String transCode, Long amount) { 34 | this.id = generatorId(); 35 | this.transCode = transCode; 36 | this.amount = amount; 37 | this.tradeTime = new Date(); 38 | this.userId = generatorUserId(); 39 | } 40 | 41 | 42 | public Message(String id, String transCode, Long amount) { 43 | this.id = id; 44 | this.transCode = transCode; 45 | this.amount = amount; 46 | this.tradeTime = new Date(); 47 | this.userId = generatorUserId(); 48 | } 49 | 50 | public Message(String id, String transCode, Long amount, Date tradeTime) { 51 | this.id = id; 52 | this.transCode = transCode; 53 | this.amount = amount; 54 | this.tradeTime = tradeTime; 55 | this.userId = generatorUserId(); 56 | } 57 | 58 | public String getId() { 59 | return id; 60 | } 61 | 62 | public void setId(String id) { 63 | this.id = id; 64 | } 65 | 66 | public String getTransCode() { 67 | return transCode; 68 | } 69 | 70 | public void setTransCode(String transCode) { 71 | this.transCode = transCode; 72 | } 73 | 74 | public Long getAmount() { 75 | return amount; 76 | } 77 | 78 | public void setAmount(Long amount) { 79 | this.amount = amount; 80 | } 81 | 82 | public Date getTradeTime() { 83 | return tradeTime; 84 | } 85 | 86 | public void setTradeTime(Date tradeTime) { 87 | this.tradeTime = tradeTime; 88 | } 89 | 90 | private String generatorId(){ 91 | return UUID.randomUUID().toString(); 92 | } 93 | 94 | private long generatorUserId(){ 95 | Random random = new Random(); 96 | return random.nextInt(1000); 97 | } 98 | 99 | public Long getUserId() { 100 | return userId; 101 | } 102 | 103 | public void setUserId(Long userId) { 104 | this.userId = userId; 105 | } 106 | 107 | @Override 108 | public String toString() { 109 | return "Message{" + 110 | "id='" + id + '\'' + 111 | ", transCode='" + transCode + '\'' + 112 | ", amount=" + amount + 113 | ", tradeTime=" + tradeTime + 114 | ", userId=" + userId + 115 | '}'; 116 | } 117 | } -------------------------------------------------------------------------------- /tanghao-flink-demo/src/main/java/com/tanghao/flink/demo/common/MessageDeserializationSchema.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.flink.demo.common; 2 | 3 | import org.apache.flink.api.common.serialization.DeserializationSchema; 4 | import org.apache.flink.api.common.typeinfo.TypeInformation; 5 | 6 | import java.io.IOException; 7 | 8 | /** 9 | * Description： TODO 10 | * Created By tanghao on 2020/4/8 11 | */ 12 | public class MessageDeserializationSchema implements DeserializationSchema { 13 | public Message deserialize(byte[] bytes) throws IOException { 14 | return null; 15 | } 16 | 17 | public boolean isEndOfStream(Message message) { 18 | return false; 19 | } 20 | 21 | public TypeInformation getProducedType() { 22 | return null; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /tanghao-flink-demo/src/main/java/com/tanghao/flink/demo/common/TradeCountMinuteResult.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.flink.demo.common; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Description： 交易统计小时结果 7 | * Created By tanghao on 2020/4/9 8 | */ 9 | public class TradeCountMinuteResult implements Serializable { 10 | 11 | private String transCode; 12 | 13 | private Long sumAmount; 14 | 15 | private String currentMinute; 16 | 17 | public String getTransCode() { 18 | return transCode; 19 | } 20 | 21 | public void setTransCode(String transCode) { 22 | this.transCode = transCode; 23 | } 24 | 25 | public Long getSumAmount() { 26 | return sumAmount; 27 | } 28 | 29 | public void setSumAmount(Long sumAmount) { 30 | this.sumAmount = sumAmount; 31 | } 32 | 33 | public String getCurrentMinute() { 34 | return currentMinute; 35 | } 36 | 37 | public void setCurrentMinute(String currentMinute) { 38 | this.currentMinute = currentMinute; 39 | } 40 | 41 | @Override 42 | public String toString() { 43 | return "TradeCountMinuteResult{" + 44 | "transCode='" + transCode + '\'' + 45 | ", sumAmount=" + sumAmount + 46 | ", currentMinute='" + currentMinute + '\'' + 47 | '}'; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /tanghao-flink-demo/src/main/java/com/tanghao/flink/demo/common/TransTypeInfo.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.flink.demo.common; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Description： 交易类型信息 7 | * Created By tanghao on 2020/4/8 8 | */ 9 | public class TransTypeInfo implements Serializable { 10 | 11 | private String transCode; 12 | 13 | private String transName; 14 | 15 | private boolean isOpen; 16 | 17 | public String getTransCode() { 18 | return transCode; 19 | } 20 | 21 | public void setTransCode(String transCode) { 22 | this.transCode = transCode; 23 | } 24 | 25 | public String getTransName() { 26 | return transName; 27 | } 28 | 29 | public void setTransName(String transName) { 30 | this.transName = transName; 31 | } 32 | 33 | public boolean isOpen() { 34 | return isOpen; 35 | } 36 | 37 | public void setOpen(boolean open) { 38 | isOpen = open; 39 | } 40 | 41 | @Override 42 | public String toString() { 43 | return "TransTypeInfo{" + 44 | "transCode='" + transCode + '\'' + 45 | ", transName='" + transName + '\'' + 46 | ", isOpen=" + isOpen + 47 | '}'; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /tanghao-flink-demo/src/main/java/com/tanghao/flink/demo/common/UserConsumerAmountPerMinute.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.flink.demo.common; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * Description： 用户每分钟消费金额统计对象 7 | * Created By tanghao on 2020/4/9 8 | */ 9 | public class UserConsumerAmountPerMinute implements Serializable { 10 | 11 | private Long userId; 12 | 13 | private Long amount; 14 | 15 | private String currentMinute; 16 | 17 | public Long getUserId() { 18 | return userId; 19 | } 20 | 21 | public void setUserId(Long userId) { 22 | this.userId = userId; 23 | } 24 | 25 | public Long getAmount() { 26 | return amount; 27 | } 28 | 29 | public void setAmount(Long amount) { 30 | this.amount = amount; 31 | } 32 | 33 | public String getCurrentMinute() { 34 | return currentMinute; 35 | } 36 | 37 | public void setCurrentMinute(String currentMinute) { 38 | this.currentMinute = currentMinute; 39 | } 40 | 41 | @Override 42 | public String toString() { 43 | return "UserConsumerAmountPerMinute{" + 44 | "userId=" + userId + 45 | ", amount=" + amount + 46 | ", currentMinute='" + currentMinute + '\'' + 47 | '}'; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /tanghao-flink-demo/src/main/java/com/tanghao/flink/demo/source/InnerSource.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.flink.demo.source; 2 | 3 | import org.apache.flink.api.common.functions.MapFunction; 4 | import org.apache.flink.streaming.api.datastream.DataStreamSource; 5 | import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; 6 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 7 | 8 | import java.util.Arrays; 9 | import java.util.HashMap; 10 | import java.util.Map; 11 | 12 | /** 13 | * Description： 内置数据源 14 | * Created By tanghao on 2020/3/31 15 | */ 16 | public class InnerSource { 17 | 18 | public static void main(String[] args) { 19 | StreamExecutionEnvironment streamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment(); 20 | 21 | //从文件读流 22 | fromFile(streamExecutionEnvironment); 23 | //从本地集合中读流 24 | // fromCollection(streamExecutionEnvironment); 25 | } 26 | 27 | private static void fromFile(StreamExecutionEnvironment streamExecutionEnvironment){ 28 | DataStreamSource streamSource = streamExecutionEnvironment.readTextFile("D://test.txt","UTF-8"); 29 | streamSource.print(); 30 | try { 31 | streamExecutionEnvironment.execute(); 32 | } catch (Exception e) { 33 | e.printStackTrace(); 34 | } 35 | } 36 | 37 | private static void fromCollection(StreamExecutionEnvironment streamExecutionEnvironment){ 38 | String[] strArray={"deposit,10","withdraw,10","deposit,3","withdraw,10"}; 39 | DataStreamSource streamSource = streamExecutionEnvironment.fromCollection(Arrays.asList(strArray)); 40 | streamSource.print(); 41 | try { 42 | streamExecutionEnvironment.execute(); 43 | } catch (Exception e) { 44 | e.printStackTrace(); 45 | } 46 | } 47 | 48 | 49 | } 50 | -------------------------------------------------------------------------------- /tanghao-flink-demo/src/main/java/com/tanghao/flink/demo/source/KafkaSource.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.flink.demo.source; 2 | 3 | import com.tanghao.flink.demo.common.ConstantKafka; 4 | import com.tanghao.flink.demo.common.Message; 5 | import org.apache.flink.api.common.serialization.SimpleStringSchema; 6 | import org.apache.flink.formats.json.JsonNodeDeserializationSchema; 7 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; 8 | import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; 9 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 10 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011; 11 | import org.apache.flink.streaming.connectors.kafka.KafkaDeserializationSchema; 12 | import org.apache.flink.streaming.connectors.kafka.internals.KafkaDeserializationSchemaWrapper; 13 | 14 | import java.util.Properties; 15 | 16 | /** 17 | * Description： TODO 18 | * Created By tanghao on 2020/4/8 19 | */ 20 | public class KafkaSource { 21 | 22 | public static void main(String[] args) { 23 | StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 24 | 25 | Properties properties = new Properties(); 26 | properties.setProperty("bootstrap.servers", "192.168.8.145:9082"); 27 | SingleOutputStreamOperator stream= env.addSource(new FlinkKafkaConsumer011(ConstantKafka.TRADE_MESSAGE_TOPIC, new SimpleStringSchema(), properties)); 28 | stream.print(); 29 | 30 | try { 31 | env.execute(); 32 | } catch (Exception e) { 33 | e.printStackTrace(); 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /tanghao-flink-demo/src/main/java/com/tanghao/flink/demo/test/BroadcastMsgProducerTest.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.flink.demo.test; 2 | 3 | import com.alibaba.fastjson.JSONObject; 4 | import com.tanghao.flink.demo.common.ConstantKafka; 5 | import com.tanghao.flink.demo.common.Message; 6 | import com.tanghao.flink.demo.common.TransTypeInfo; 7 | import org.apache.kafka.clients.producer.KafkaProducer; 8 | import org.apache.kafka.clients.producer.ProducerRecord; 9 | 10 | import java.util.Properties; 11 | 12 | /** 13 | * Description： kafka生成交易消息 14 | * Created By tanghao on 2020/4/7 15 | */ 16 | public class BroadcastMsgProducerTest { 17 | 18 | public static void main(String[] args) throws InterruptedException { 19 | Properties pro = new Properties(); 20 | pro.setProperty("bootstrap.servers", "192.168.8.145:9082"); 21 | pro.setProperty("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 22 | pro.setProperty("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 23 | KafkaProducer producer = new KafkaProducer(pro); 24 | sendTransTypeInfo(producer); 25 | Thread.sleep(300);//更新广播表有个时间间隔 26 | sendMessage(producer); 27 | producer.close(); 28 | } 29 | 30 | /** 31 | * 发送维表配置的交易类型信息 32 | * isOpen字段标识开关，如果打开可以在接收的消息中根据transCode去TRANS_TYPE_MAP得到transName 33 | * @param producer 34 | */ 35 | private static void sendTransTypeInfo(KafkaProducer producer){ 36 | TransTypeInfo transTypeInfo = new TransTypeInfo(); 37 | transTypeInfo.setOpen(true); 38 | transTypeInfo.setTransCode("2001"); 39 | transTypeInfo.setTransName("提现"); 40 | producer.send(new ProducerRecord(ConstantKafka.TRANS_TYPE_TOPIC, JSONObject.toJSONString(transTypeInfo))); 41 | } 42 | 43 | private static void sendMessage(KafkaProducer producer){ 44 | Message msg = new Message("2001",101L); 45 | producer.send(new ProducerRecord(ConstantKafka.TRADE_MESSAGE_TOPIC, JSONObject.toJSONString(msg))); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /tanghao-flink-demo/src/main/java/com/tanghao/flink/demo/test/MsgProducerTest.java: -------------------------------------------------------------------------------- 1 | package com.tanghao.flink.demo.test; 2 | 3 | import com.alibaba.fastjson.JSONObject; 4 | import com.tanghao.flink.demo.common.ConstantKafka; 5 | import com.tanghao.flink.demo.common.Message; 6 | import org.apache.kafka.clients.producer.KafkaProducer; 7 | import org.apache.kafka.clients.producer.ProducerRecord; 8 | 9 | import java.util.Arrays; 10 | import java.util.List; 11 | import java.util.Properties; 12 | import java.util.Random; 13 | 14 | /** 15 | * Description： 消息生成器 16 | * Created By tanghao on 2020/4/8 17 | */ 18 | public class MsgProducerTest { 19 | private static final String[] transCodes = {"1001","2001","3001"}; 20 | 21 | public static void main(String[] args) throws InterruptedException { 22 | Properties pro = new Properties(); 23 | pro.setProperty("bootstrap.servers", "192.168.8.145:9082"); 24 | pro.setProperty("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 25 | pro.setProperty("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 26 | KafkaProducer producer = new KafkaProducer(pro); 27 | for(int i=0;i<30;i++){ 28 | Message msg = new Message(transCodes[i%3],(long)new Random().nextInt(1000)); 29 | System.out.println(msg); 30 | producer.send(new ProducerRecord(ConstantKafka.TRADE_MESSAGE_TOPIC, JSONObject.toJSONString(msg))); 31 | Thread.sleep(500); 32 | } 33 | producer.close(); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /tanghao-flink-demo/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, console 2 | 3 | log4j.appender.console=org.apache.log4j.ConsoleAppender 4 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 5 | log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n 6 | --------------------------------------------------------------------------------