├── .gitignore ├── README.MD ├── collections ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── gmall │ │ └── data │ │ └── collection │ │ ├── canal │ │ └── SimpleCanalClientExample.java │ │ ├── flow │ │ ├── Csv2JsonDemo02.java │ │ ├── Csv2JsonDemo03.java │ │ ├── CsvToJsonDemo.java │ │ ├── GenLogData.scala │ │ └── SparkDemo.scala │ │ └── pojo │ │ └── OdsScEvent.java │ └── resources │ ├── canal │ └── conf │ │ ├── canal.properties │ │ └── example │ │ └── instance.properties │ ├── log4j.properties │ └── sales │ ├── application.properties │ ├── gmall2020-mock-db-2020-11-27.jar │ ├── gmall2020_12_08.sql │ └── start_data.sh ├── common ├── pom.xml └── src │ ├── main │ ├── resources │ │ └── application.properties │ └── scala │ │ └── com │ │ └── gmall │ │ └── data │ │ └── common │ │ ├── config │ │ ├── Config.scala │ │ ├── KafkaConfig.scala │ │ └── RedisConfig.scala │ │ ├── entity │ │ ├── dim │ │ │ └── DimUserInfo.scala │ │ ├── dwd │ │ │ └── DwdOrderDetail.scala │ │ └── ods │ │ │ ├── Model.scala │ │ │ ├── OdsModel.scala │ │ │ ├── SqlType.scala │ │ │ ├── flow │ │ │ ├── DwdBaseLog.scala │ │ │ ├── DwdDisplayLog.scala │ │ │ ├── DwdPageLog.scala │ │ │ ├── DwdStartLog.scala │ │ │ ├── DwdUserBaseLog.scala │ │ │ ├── OdsBaseLog.scala │ │ │ └── OdsUserActionLog.scala │ │ │ └── gmall2021 │ │ │ ├── ActivityInfo.scala │ │ │ ├── ActivityRule.scala │ │ │ ├── ActivitySku.scala │ │ │ ├── BaseAttrInfo.scala │ │ │ ├── BaseAttrValue.scala │ │ │ ├── BaseCategory1.scala │ │ │ ├── BaseCategory2.scala │ │ │ ├── BaseCategory3.scala │ │ │ ├── BaseCategoryView.scala │ │ │ ├── BaseDic.scala │ │ │ ├── BaseFrontendParam.scala │ │ │ ├── BaseProvince.scala │ │ │ ├── BaseRegion.scala │ │ │ ├── BaseSaleAttr.scala │ │ │ ├── BaseTrademark.scala │ │ │ ├── CartInfo.scala │ │ │ ├── CmsBanner.scala │ │ │ ├── CommentInfo.scala │ │ │ ├── CouponInfo.scala │ │ │ ├── CouponRange.scala │ │ │ ├── CouponUse.scala │ │ │ ├── FavorInfo.scala │ │ │ ├── FinancialSkuCost.scala │ │ │ ├── OrderDetail.scala │ │ │ ├── OrderDetailActivity.scala │ │ │ ├── OrderDetailCoupon.scala │ │ │ ├── OrderInfo.scala │ │ │ ├── OrderRefundInfo.scala │ │ │ ├── OrderStatusLog.scala │ │ │ ├── PaymentInfo.scala │ │ │ ├── RefundPayment.scala │ │ │ ├── SeckillGoods.scala │ │ │ ├── SkuAttrValue.scala │ │ │ ├── SkuImage.scala │ │ │ ├── SkuInfo.scala │ │ │ ├── SkuSaleAttrValue.scala │ │ │ ├── SpuImage.scala │ │ │ ├── SpuInfo.scala │ │ │ ├── SpuPoster.scala │ │ │ ├── SpuSaleAttr.scala │ │ │ ├── SpuSaleAttrValue.scala │ │ │ ├── UserAddress.scala │ │ │ ├── UserInfo.scala │ │ │ ├── WareInfo.scala │ │ │ ├── WareOrderTask.scala │ │ │ ├── WareOrderTaskDetail.scala │ │ │ └── WareSku.scala │ │ ├── sink │ │ ├── SinkFactory.scala │ │ └── kafka │ │ │ └── EventKafkaSerializationSchema.scala │ │ ├── source │ │ ├── SourceFactory.scala │ │ └── binlog │ │ │ └── OdsModelFactory.scala │ │ ├── transform │ │ ├── Convert.scala │ │ ├── Format.scala │ │ ├── Merger.scala │ │ └── Sink.scala │ │ └── utils │ │ ├── CacheUtil.scala │ │ ├── Constants.scala │ │ ├── GsonUtil.scala │ │ ├── HBaseUtil.scala │ │ ├── JedisConnectionPool.scala │ │ ├── JedisWrapper.scala │ │ ├── LoggerUtil.scala │ │ ├── StringUtil.scala │ │ ├── TimeUtil.scala │ │ └── Util.scala │ └── test │ └── java │ ├── FlinkRedisConnectorTest.scala │ ├── KafkaConsumerDemo.scala │ └── KafkaConsumerTest.scala ├── docs ├── 0项目简介.md ├── 1数据产品需求.md ├── 2实时数仓架构.md ├── 3实时数仓分层设计.assets │ ├── 224955E00-1.jpg │ ├── 224955I30-0.jpg │ ├── 224955O15-2.jpg │ ├── 640-20210518140232661.png │ ├── 640-20210518140232689.png │ ├── 640-20210518140233105.png │ ├── 640.png │ ├── p44562.png │ ├── p58877.png │ └── p59651.png ├── 3实时数仓分层设计.md ├── 3实时数仓设计.assets │ ├── 640 │ ├── 640-20210515184302228 │ ├── 640-20210515184302234 │ ├── 640-20210515184302235 │ ├── 640-20210515184302239 │ ├── 640-20210515184302246 │ ├── 640-20210515184302246-1075382. │ ├── 640-20210515184302251 │ ├── 640-20210515184302255.png │ ├── 640-20210515184302298.png │ ├── 640-20210515184302307.png │ ├── 640-20210515184302375.png │ └── Untitled.md ├── 4业务数据采集.md ├── 5用户行为数据采集.md ├── 6业务数据DWD层.md ├── 7流量数据DWD层.md ├── 8实时指标.assets │ ├── 640-20210516224001799 │ ├── 640-20210516225935573.png │ ├── 640-20210516232619115 │ ├── 640-20210516232619151.png │ ├── 640-20210516232619160.png │ ├── 640-20210516232730698.png │ ├── 640-20210516233805869.png │ ├── FFBekRjksJE8Q4rtGN28.png │ ├── Qktv9k4uWadLTLiXqNOz.png │ ├── U0PhVDw2bYXZmBKdBrjR.png │ ├── ZoCJhNbfM6TwIqBvP7t4.png │ ├── append-mode.png │ ├── image-20210516235107922.png │ ├── image-20210518232430888.png │ ├── image-20210518232504987.png │ ├── image-20210518233251037.png │ ├── image-20210519223858298.png │ ├── image-20210522004340330.png │ ├── image-20210522004416835.png │ ├── pbfzmZFLAeBF4O6cFtF2.png │ ├── query-groupBy-cnt.png │ ├── query-groupBy-window-cnt.png │ └── stream-query-stream.png ├── 8实时指标.md ├── 9实时OLAP.assets │ ├── 2bd1edeecda8a379bde481389ddff425.png │ ├── 640.jpeg │ ├── 640.png │ ├── Data-Cube.png │ ├── OLAP.png │ ├── image-20210523193954290.png │ ├── image-20210523194032047.png │ ├── image-20210524001623967.png │ ├── kylin_diagram.png │ └── v2-6a455b3039e28e98b7592f66910a0793_1440w.jpg ├── 9实时OLAP.md ├── Apache Kylin与ClickHouse 的对比.assets │ ├── 640 │ ├── 640-20210523225447666.png │ ├── 640-20210523225447678.png │ ├── 640-20210523225447718.png │ ├── 640-20210523225447735 │ ├── 640-20210523225447783.png │ ├── 640-20210523225447860.png │ └── 640-20210523225447981.png ├── DWD层数据准备.assets │ ├── async_io.svg │ ├── image-20210407224537117.png │ ├── image-20210409233246633.png │ ├── image-20210410000728103.png │ ├── image-20210410000759718.png │ ├── image-20210413225611694.png │ ├── image-20210413230248772.png │ ├── image-20210413230458950.png │ ├── image-20210413230514762.png │ ├── image-20210413231328430.png │ ├── image-20210418152453150.png │ ├── session-window-join.svg │ └── tumbling-window-join.svg ├── 业务数据采集.assets │ ├── image-20210329235325068.png │ ├── image-20210329235403290.png │ ├── image-20210401220353400.png │ ├── image-20210404232758656.png │ ├── image-20210404233127046.png │ └── image-20210404233331763.png ├── 产品需求.assets │ ├── image-20210405165204052.png │ ├── image-20210405165428593.png │ ├── image-20210405205643466.png │ ├── image-20210405205804475.png │ ├── image-20210405205922293.png │ ├── image-20210405210015951.png │ ├── image-20210405210345486.png │ ├── image-20210405210615660.png │ ├── image-20210405210828060.png │ └── image-20210405211124781.png ├── 基于docker部署canal环境.md ├── 如何从0到1构建指标体系.assets │ ├── 2sVUM4J4nNKHOdqCLVJi.png │ ├── 66eVXE58NipBmJO4Zsvu.png │ ├── FFBekRjksJE8Q4rtGN28-20210516233233469.png │ ├── FltPuhpqlLUUPeLzk7cJ.png │ ├── IODeeazNNoOlmLJ4S0Lu.png │ ├── KkQIpy08fKwfnLJgKZQI.png │ ├── Qktv9k4uWadLTLiXqNOz-20210516233233452.png │ ├── Rv2rBKjEaRKnQ8xuHMmW.png │ ├── TAOT112YZQUkjsxPBEv1.jpg │ ├── U0PhVDw2bYXZmBKdBrjR-20210516233233322.png │ ├── WtbMmcHjgmkuFiFLhP8g.png │ ├── ZoCJhNbfM6TwIqBvP7t4-20210516233233415.png │ ├── gDwC1ZSk527aLIdCTUbY.png │ ├── hpipN9GYAin3xPDvJG3L.png │ ├── itMgHpcqrkHHogkaO7A1.png │ ├── pbfzmZFLAeBF4O6cFtF2-20210516233233439.png │ ├── r82Fof1uKLtZa7gtsbe4.png │ └── u7LlkgKdFht23fDZvEAO.png ├── 如何从0到1构建指标体系.md ├── 实时数仓架构.assets │ ├── image-20210328225453095.png │ ├── image-20210328231618797.png │ ├── image-20210328232802457.png │ ├── image-20210328233222184.png │ ├── image-20210328233335006.png │ ├── image-20210328233720452.png │ └── image-20210411153254131.png ├── 数据埋点需求.assets │ └── link-solid.svg ├── 数据埋点需求.md ├── 用户行为数据采集.assets │ └── 640.png └── 项目简介.assets │ └── image-20210402102703175.png ├── dwd2dws ├── pom.xml └── src │ └── main │ └── scala │ └── com │ └── gmall │ └── data │ ├── App.scala │ └── Sql.scala ├── ods2dim ├── pom.xml └── src │ └── main │ ├── resources │ ├── log4j.properties │ └── logback.xml │ └── scala │ └── com │ └── gmall │ └── data │ └── dim │ ├── App.scala │ ├── Step.scala │ ├── StreamTopology.scala │ └── sink │ ├── Category3InfoSink.scala │ ├── ProvinceInfoSink.scala │ ├── SkuInfoSink.scala │ ├── SpuInfoSink.scala │ ├── TradeInfoSink.scala │ └── UserInfoSink.scala ├── ods2dwd ├── pom.xml └── src │ ├── main │ ├── resources │ │ ├── event_mapping.txt │ │ ├── log4j.properties │ │ └── logback.xml │ └── scala │ │ └── com │ │ └── gmall │ │ └── data │ │ └── dwd │ │ ├── App.scala │ │ ├── FlowTopology.scala │ │ ├── FlowTopologyV2.scala │ │ ├── Step.scala │ │ ├── StreamTopology.scala │ │ └── transform │ │ ├── DwdLogJoinDimEvent.scala │ │ ├── EventTypeSplitFunc.scala │ │ ├── OdsActionLogConvert.scala │ │ ├── OdsBaseLogConvert.scala │ │ ├── OrderDetailAndCouponMerger.scala │ │ ├── OrderDetailAndDimUserJoin.scala │ │ └── OrderInfoAndDetailMerger.scala │ └── test │ └── java │ ├── JedisSetTest.scala │ └── KafkaConsumerTest.scala ├── pom.xml └── tools ├── pom.xml └── src └── main ├── resources ├── ods_entity.template └── ods_meta.json └── scala └── com └── gmall └── data └── tools └── OdsModelGenerator.scala /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | ### Scala template 3 | *.class 4 | *.log 5 | 6 | .idea 7 | *.iml 8 | 9 | target/ 10 | -------------------------------------------------------------------------------- /README.MD: -------------------------------------------------------------------------------- 1 | # 电商实时数仓项目 2 | 3 | ## 产品需求 4 | 5 | [产品需求](docs/产品需求.md) 6 | 7 | [数据埋点需求](docs/1数据产品需求.md) 8 | 9 | ## 实时数仓架构 10 | 11 | [1.项目简介](docs/0项目简介.md) 12 | 13 | [2.实时数仓架构](docs/2实时数仓架构.md) 14 | 15 | [3.实时数仓分层设计](docs/3实时数仓分层设计.md) 16 | 17 | ## 数据采集 18 | 19 | [1.业务数据采集](docs/4业务数据采集.md) 20 | 21 | [2.用户行为数据采集](docs/5用户行为数据采集.md) 22 | 23 | ## dwd 24 | 25 | [1.业务数据DWD层](docs/6业务数据DWD层.md) 26 | 27 | [2.流量数据DWD层](docs/7流量数据DWD层.md) 28 | 29 | ## dws 30 | 31 | ## 实时数仓的应用场景 32 | 33 | [0.如何从0到1构建指标体系](docs/如何从0到1构建指标体系.md) 34 | 35 | [1.实时指标](docs/8实时指标.md) 36 | 37 | [2.实时OLAP](docs/9实时OLAP.md) 38 | 39 | [3.实时标签]() 40 | 41 | [4.实时特征]() 42 | 43 | **番外** 44 | 45 | [基于Docker部署canal环境](docs/基于docker部署canal环境.md) 46 | -------------------------------------------------------------------------------- /collections/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | realtime-dw 7 | com.gmall.data 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | collections 13 | 14 | 15 | 16 | com.alibaba.otter 17 | canal.client 18 | 1.1.4 19 | 20 | 21 | org.apache.spark 22 | spark-core_2.11 23 | 2.1.1 24 | 25 | 26 | org.apache.spark 27 | spark-sql_2.11 28 | 2.1.1 29 | 30 | 31 | com.fasterxml.jackson.core 32 | jackson-databind 33 | 2.6.6 34 | 35 | 36 | com.fasterxml.jackson.dataformat 37 | jackson-dataformat-csv 38 | 2.6.6 39 | 40 | 41 | 42 | com.alibaba 43 | fastjson 44 | 1.2.73 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | org.apache.maven.plugins 53 | maven-compiler-plugin 54 | 3.1 55 | 56 | 1.8 57 | 1.8 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /collections/src/main/java/com/gmall/data/collection/flow/Csv2JsonDemo02.java: -------------------------------------------------------------------------------- 1 | package com.gmall.data.collection.flow; 2 | 3 | import java.io.File; 4 | import java.util.List; 5 | import java.util.Map; 6 | 7 | import com.fasterxml.jackson.databind.ObjectMapper; 8 | import com.fasterxml.jackson.dataformat.csv.CsvMapper; 9 | import com.fasterxml.jackson.dataformat.csv.CsvSchema; 10 | public class Csv2JsonDemo02 { 11 | 12 | public static void main(String[] args) throws Exception{ 13 | 14 | File input = new File("/Users/wufengchi/Downloads/shence_log0509.csv"); 15 | File output = new File("collections/src/main/resources/data/shence_log0509.json"); 16 | 17 | CsvSchema csvSchema = CsvSchema.builder().setUseHeader(true).build(); 18 | CsvMapper csvMapper = new CsvMapper(); 19 | 20 | // Read data from CSV file 21 | List readAll = csvMapper.readerFor(Map.class).with(csvSchema).readValues(input).readAll(); 22 | 23 | ObjectMapper mapper = new ObjectMapper(); 24 | 25 | // Write JSON formated data to output.json file 26 | mapper.writerWithDefaultPrettyPrinter().writeValue(output, readAll); 27 | 28 | // Write JSON formated data to stdout 29 | // System.out.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(readAll)); 30 | 31 | 32 | } 33 | 34 | 35 | } 36 | -------------------------------------------------------------------------------- /collections/src/main/java/com/gmall/data/collection/flow/GenLogData.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.collection.flow 2 | 3 | import com.alibaba.fastjson.JSON 4 | import org.apache.flink.api.common.functions.FlatMapFunction 5 | import org.apache.flink.streaming.api.scala._ 6 | import org.apache.flink.util.Collector 7 | import scala.collection.JavaConverters._ 8 | 9 | object GenLogData { 10 | 11 | def main(args: Array[String]): Unit = { 12 | 13 | val env = StreamExecutionEnvironment.getExecutionEnvironment 14 | 15 | val stream = env.readTextFile("collections/src/main/resources/data/shence_log0509.json") 16 | 17 | 18 | stream.print() 19 | 20 | env.execute("job") 21 | 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /collections/src/main/java/com/gmall/data/collection/flow/SparkDemo.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.collection.flow 2 | 3 | import com.alibaba.fastjson.{JSON, JSONArray} 4 | import org.apache.spark.{SparkConf, SparkContext} 5 | 6 | object SparkDemo { 7 | 8 | def main(args: Array[String]): Unit = { 9 | 10 | val conf = new SparkConf().setAppName("sss").setMaster("local[1]") 11 | val sc = new SparkContext(conf) 12 | 13 | sc.textFile("collections/src/main/resources/data/shence_log0509.json") 14 | .flatMap(r => { 15 | val array = JSON.parseArray(r) 16 | array.toArray 17 | }) 18 | .saveAsTextFile("collections/src/main/resources/data/aaa") 19 | 20 | 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /collections/src/main/resources/canal/conf/example/instance.properties: -------------------------------------------------------------------------------- 1 | ################################################# 2 | ## mysql serverId , v1.0.26+ will autoGen 3 | canal.instance.mysql.slaveId=3 4 | 5 | # enable gtid use true/false 6 | canal.instance.gtidon=false 7 | 8 | # position info 9 | canal.instance.master.address=localhost:3306 10 | canal.instance.master.journal.name= 11 | canal.instance.master.position= 12 | canal.instance.master.timestamp= 13 | canal.instance.master.gtid= 14 | 15 | # rds oss binlog 16 | canal.instance.rds.accesskey= 17 | canal.instance.rds.secretkey= 18 | canal.instance.rds.instanceId= 19 | 20 | # table meta tsdb info 21 | canal.instance.tsdb.enable=false 22 | #canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb 23 | canal.instance.tsdb.dbUsername=canal 24 | canal.instance.tsdb.dbPassword=canal 25 | 26 | #canal.instance.standby.address = 27 | #canal.instance.standby.journal.name = 28 | #canal.instance.standby.position = 29 | #canal.instance.standby.timestamp = 30 | #canal.instance.standby.gtid= 31 | 32 | # username/password 33 | canal.instance.dbUsername=canal 34 | canal.instance.dbPassword=canal 35 | canal.instance.connectionCharset = UTF-8 36 | # enable druid Decrypt database password 37 | canal.instance.enableDruid=false 38 | #canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ== 39 | 40 | # table regex 41 | canal.instance.filter.regex=gmall2021.*\\..* 42 | # table black regex 43 | canal.instance.filter.black.regex= 44 | # table field filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2) 45 | #canal.instance.filter.field=test1.t_product:id/subject/keywords,test2.t_company:id/name/contact/ch 46 | # table field black filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2) 47 | #canal.instance.filter.black.field=test1.t_product:subject/product_image,test2.t_company:id/name/contact/ch 48 | 49 | # mq config 50 | canal.mq.topic=canal_test 51 | # dynamic topic route by schema or table regex 52 | canal.mq.dynamicTopic=.*\\..* 53 | canal.mq.partition=0 54 | # hash partition config 55 | canal.mq.partitionsNum=1 56 | canal.mq.partitionHash=.*\\..*:$pk$ 57 | ################################################# 58 | -------------------------------------------------------------------------------- /collections/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=ERROR, console 2 | -------------------------------------------------------------------------------- /collections/src/main/resources/sales/application.properties: -------------------------------------------------------------------------------- 1 | logging.level.root=info 2 | 3 | 4 | spring.datasource.driver-class-name=com.mysql.jdbc.Driver 5 | spring.datasource.url=jdbc:mysql://localhost:3306/gmall2021?characterEncoding=utf-8&useSSL=false&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true 6 | spring.datasource.username=root 7 | spring.datasource.password=992318abC 8 | 9 | logging.pattern.console=%m%n 10 | 11 | 12 | mybatis-plus.global-config.db-config.field-strategy=not_null 13 | 14 | 15 | #业务日期 16 | mock.date=2021-05-20 17 | #是否重置 18 | mock.clear=1 19 | #是否重置用户 20 | mock.clear.user=0 21 | 22 | #生成新用户数量 23 | mock.user.count=5000 24 | #男性比例 25 | mock.user.male-rate=20 26 | #用户数据变化概率 27 | mock.user.update-rate:20 28 | 29 | #收藏取消比例 30 | mock.favor.cancel-rate=5 31 | #收藏数量 32 | mock.favor.count=100 33 | 34 | #购物车数量 35 | mock.cart.count=300 36 | #每个商品最多购物个数 37 | mock.cart.sku-maxcount-per-cart=10 38 | #购物车来源 用户查询，商品推广，智能推荐, 促销活动 39 | mock.cart.source-type-rate=60:20:10:10 40 | 41 | #用户下单比例 42 | mock.order.user-rate=95 43 | #用户从购物中购买商品比例 44 | mock.order.sku-rate=70 45 | #是否参加活动 46 | mock.order.join-activity=1 47 | #是否使用购物券 48 | mock.order.use-coupon=1 49 | #购物券领取人数 50 | mock.coupon.user-count=20 51 | 52 | #支付比例 53 | mock.payment.rate=70 54 | #支付方式 支付宝：微信 ：银联 55 | mock.payment.payment-type=30:60:10 56 | 57 | 58 | #评价比例 好：中：差：自动 59 | mock.comment.appraise-rate=30:10:10:50 60 | 61 | #退款原因比例：质量问题 商品描述与实际描述不一致 缺货 号码不合适 拍错 不想买了 其他 62 | mock.refund.reason-rate=30:10:20:5:15:5:5 63 | -------------------------------------------------------------------------------- /collections/src/main/resources/sales/gmall2020-mock-db-2020-11-27.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/collections/src/main/resources/sales/gmall2020-mock-db-2020-11-27.jar -------------------------------------------------------------------------------- /collections/src/main/resources/sales/start_data.sh: -------------------------------------------------------------------------------- 1 | java -jar gmall2020-mock-db-2020-11-27.jar -------------------------------------------------------------------------------- /common/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | realtime-dw 7 | com.gmall.data 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | common 13 | 14 | 15 | 16 | com.alibaba.otter 17 | canal.protocol 18 | 1.1.4 19 | 20 | 21 | jsr305 22 | com.google.code.findbugs 23 | 24 | 25 | slf4j-api 26 | org.slf4j 27 | 28 | 29 | logback-classic 30 | ch.qos.logback 31 | 32 | 33 | logback-core 34 | ch.qos.logback 35 | 36 | 37 | zkclient 38 | com.101tec 39 | 40 | 41 | fastjson 42 | com.alibaba 43 | 44 | 45 | commons-logging 46 | commons-logging 47 | 48 | 49 | protobuf-java 50 | com.google.protobuf 51 | 52 | 53 | 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /common/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | kafka.brokers=localhost:9092 2 | hbase.zookeeper.quorum=localhost:2181 3 | 4 | redis.host=r-uf6fee295232d7e4pd.redis.rds.aliyuncs.com 5 | redis.port=6379 6 | redis.password=992318abC 7 | redis.db=0 -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/config/Config.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.config 2 | 3 | import com.typesafe.config.{Config, ConfigFactory} 4 | 5 | object Config { 6 | 7 | private val load: Config = ConfigFactory.load() 8 | 9 | val kafkaBrokers : String = load.getString("kafka.brokers") 10 | val hbaseZookeeperQuorum: String = load.getString("hbase.zookeeper.quorum") 11 | 12 | 13 | val redisHost: String = load.getString("redis.host") 14 | val redisPort: Int = load.getInt("redis.port") 15 | val redisPassword: String = load.getString("redis.password") 16 | val redisDb: Int = load.getInt("redis.db") 17 | 18 | } 19 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/config/KafkaConfig.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.config 2 | 3 | /** 4 | * kafka配置类 5 | * @param brokers 6 | * @param groupId 7 | * @param mode 8 | * @param timestamp 9 | */ 10 | case class KafkaConfig(brokers: String, groupId: String, mode: String, timestamp: String) extends Serializable { 11 | } 12 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/config/RedisConfig.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.config 2 | 3 | case class RedisConfig(host: String, port: Int, password: String, db: Int) extends Serializable { 4 | } 5 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/dim/DimUserInfo.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.dim 2 | 3 | class DimUserInfo extends Serializable { 4 | 5 | var userBirthday: String = _ 6 | var userGender: String = _ 7 | var userLoginName: String= _ 8 | 9 | override def toString = s"DimUserInfo(user_age=$userBirthday, user_gender=$userGender)" 10 | 11 | } 12 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/Model.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods 2 | 3 | trait Model { 4 | var ts : Long 5 | } 6 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/OdsModel.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods 2 | 3 | import scala.collection.mutable 4 | 5 | abstract class OdsModel extends Model with Serializable { 6 | var database: String 7 | var table : String 8 | var sqlType : SqlType.Value 9 | var old : mutable.Map[String, String] 10 | 11 | override def toString = s"OdsModel($database, $table, $sqlType)" 12 | } 13 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/SqlType.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods 2 | 3 | /** 4 | * mysql binlog类型的枚举类 5 | * INSERT, UPDATE, DELETE对应插入,更新,删除 6 | */ 7 | object SqlType extends Enumeration { 8 | type SqlType = Value 9 | val INSERT, UPDATE, DELETE = Value 10 | } 11 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/flow/DwdBaseLog.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.flow 2 | 3 | /** 4 | * 流量数据dwd的基类，也就是ods层common中的字段 5 | */ 6 | abstract class DwdBaseLog { 7 | 8 | var ts: Long //事件时间 9 | var ar: String //区域 10 | var ba: String // 手机品牌 11 | var ch: String // 渠道号，应用从哪个渠道来的 12 | var is_new: String // 是否新用户 13 | var md: String // 手机型号 14 | var mid: String // 设备唯一标识 15 | var os: String // 系统版本 16 | var uid: String // 用户uid 17 | var vc: String // 程序版本号 18 | 19 | override def toString = s"DwdBaseLog(ts=$ts, ar=$ar, ba=$ba, ch=$ch, is_new=$is_new, md=$md, mid=$mid, os=$os, uid=$uid, vc=$vc)" 20 | } 21 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/flow/DwdDisplayLog.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.flow 2 | 3 | import com.gmall.data.common.utils.GsonUtil 4 | import com.google.gson.{JsonElement, JsonObject} 5 | 6 | /** 7 | * 曝光日志 8 | */ 9 | class DwdDisplayLog extends DwdBaseLog { 10 | override var ts: Long = _ 11 | override var ar: String = _ 12 | override var ba: String = _ 13 | override var ch: String = _ 14 | override var is_new: String = _ 15 | override var md: String = _ 16 | override var mid: String = _ 17 | override var os: String = _ 18 | override var uid: String = _ 19 | override var vc: String = _ 20 | /** 21 | * promotion-商品推广,recommend-算法推荐商品,query-查询结果商品,activity-促销活动 22 | */ 23 | var display_type: String = _ 24 | var item: String = _ 25 | /** 26 | * sku_id-商品skuId,keyword-搜索关键词,sku_ids-多个商品skuId,activity_id-活动id-,coupon_id-购物券id 27 | */ 28 | var item_type: String = _ 29 | var order: Long = _ 30 | var pos_id: String = _ 31 | var page_id: String = _ 32 | 33 | def from(ts: Long): DwdDisplayLog = { 34 | this.ts = ts 35 | this 36 | } 37 | 38 | /** 39 | * 曝光日志属性 40 | * @param display 41 | * @return 42 | */ 43 | def from(display: JsonElement): DwdDisplayLog = { 44 | 45 | this.display_type = GsonUtil.getString(display.getAsJsonObject, "display_type") 46 | this.item = GsonUtil.getString(display.getAsJsonObject, "item") 47 | this.item_type = GsonUtil.getString(display.getAsJsonObject, "item_type") 48 | this.order = GsonUtil.getLong(display.getAsJsonObject, "order") 49 | this.pos_id = GsonUtil.getString(display.getAsJsonObject, "pos_id") 50 | this 51 | } 52 | 53 | /** 54 | * 曝光日志的页面属性 55 | * @param page 56 | * @return 57 | */ 58 | def from(page: JsonObject): DwdDisplayLog = { 59 | this.page_id = GsonUtil.getString(page, "page_id") 60 | this 61 | } 62 | 63 | override def toString = s"DwdDisplayLog(ts=$ts, ar=$ar, ba=$ba, ch=$ch, is_new=$is_new, md=$md, mid=$mid, os=$os, uid=$uid, vc=$vc, display_type=$display_type, item=$item, item_type=$item_type, order=$order, pos_id=$pos_id,page_id=$page_id)" 64 | } 65 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/flow/DwdPageLog.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.flow 2 | 3 | import com.gmall.data.common.utils.GsonUtil 4 | import com.google.gson.JsonObject 5 | 6 | /** 7 | * 页面访问日志 8 | */ 9 | class DwdPageLog extends DwdBaseLog { 10 | override var ts: Long = _ 11 | override var ar: String = _ 12 | override var ba: String = _ 13 | override var ch: String = _ 14 | override var is_new: String = _ 15 | override var md: String = _ 16 | override var mid: String = _ 17 | override var os: String = _ 18 | override var uid: String = _ 19 | override var vc: String = _ 20 | /** 21 | * 页面留存时间 22 | */ 23 | var during_time: Long = _ 24 | /** 25 | * home-首页, category-分类页, discovery-发现页, top_n-热门排行, favor-收藏页, 26 | * search-搜索页, good_list-商品列表页, good_detail-商品详情, good_spec-商品规格, 27 | * comment-评价, comment_done-评价完成, comment_list-评价列表, cart-购物车, 28 | * trade-下单结算, payment-支付页面, payment_done-支付完成, orders_all-全部订单, 29 | * orders_unpaid-订单待支付, orders_undelivered-订单待发货, 30 | * orders_unreceipted-订单待收货, orders_wait_comment-订单待评价, 31 | * mine-我的, activity-活动, login-登录, register-注册; 32 | */ 33 | var page_id: String = _ 34 | var last_page_id: String = _ 35 | var item: String = _ 36 | var item_type: String = _ 37 | var source_type: String = _ 38 | 39 | def from(ts: Long): DwdPageLog = { 40 | this.ts = ts 41 | this 42 | } 43 | 44 | def from(page: JsonObject): DwdPageLog = { 45 | this.during_time = GsonUtil.getLong(page, "during_time") 46 | this.page_id = GsonUtil.getString(page, "page_id") 47 | this.last_page_id = GsonUtil.getString(page, "last_page_id") 48 | this.item = GsonUtil.getString(page, "item") 49 | this.item_type = GsonUtil.getString(page, "item_type") 50 | this.source_type = GsonUtil.getString(page, "source_type") 51 | this 52 | } 53 | 54 | 55 | override def toString = s"DwdPageLog(ts=$ts, ar=$ar, ba=$ba, ch=$ch, is_new=$is_new, md=$md, mid=$mid, os=$os, uid=$uid, vc=$vc, during_time=$during_time, page_id=$page_id, last_page_id=$last_page_id, item=$item, item_type=$item_type, source_type=$source_type)" 56 | } 57 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/flow/DwdStartLog.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.flow 2 | 3 | import com.gmall.data.common.utils.GsonUtil 4 | import com.google.gson.JsonObject 5 | 6 | /** 7 | * 启动日志 8 | */ 9 | class DwdStartLog extends DwdBaseLog { 10 | override var ts: Long = _ 11 | override var ar: String = _ 12 | override var ba: String = _ 13 | override var ch: String = _ 14 | override var is_new: String = _ 15 | override var md: String = _ 16 | override var mid: String = _ 17 | override var os: String = _ 18 | override var uid: String = _ 19 | override var vc: String = _ 20 | var entry: String = _ // 入口: 安装后进入=install，点击图标= icon，点击通知= notice 21 | var loading_time: Long = _ // 加载时长：计算下拉开始到接口返回数据的时间，（开始加载报0，加载成功或加载失败才上报时间） 22 | var open_ad_id: Long = _ // 开屏广告Id 23 | var open_ad_ms: Long = _ // 开屏广告持续时间 24 | var open_ad_skip_ms: Long = _ // 开屏广告点击掉过的时间，未点击为0 25 | 26 | def from(ts: Long): DwdStartLog = { 27 | this.ts = ts 28 | this 29 | } 30 | 31 | /** 32 | * 启动日志属性 33 | * @param start 34 | * @return 35 | */ 36 | def from(start: JsonObject): DwdStartLog = { 37 | this.entry = GsonUtil.getString(start, "entry") 38 | this.loading_time = GsonUtil.getLong(start, "loading_time") 39 | this.open_ad_id = GsonUtil.getLong(start, "open_ad_id") 40 | this.open_ad_ms = GsonUtil.getLong(start, "open_ad_ms") 41 | this.open_ad_skip_ms = GsonUtil.getLong(start, "open_ad_skip_ms") 42 | this 43 | } 44 | 45 | override def toString = s"DwdStartLog(ts=$ts, ar=$ar, ba=$ba, ch=$ch, is_new=$is_new, md=$md, mid=$mid, os=$os, uid=$uid, vc=$vc, entry=$entry, loading_time=$loading_time, open_ad_id=$open_ad_id, open_ad_ms=$open_ad_ms, open_ad_skip_ms=$open_ad_skip_ms)" 46 | } 47 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/flow/DwdUserBaseLog.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.flow 2 | 3 | class DwdUserBaseLog { 4 | 5 | var event: String = _ // 事件 6 | var event_name: String = _ // 时间名称 7 | var user_id: String = _ // 用户ID 8 | var distinct_id: String = _ // 唯一ID 9 | var event_time: Long = _ // 事件时间 10 | var event_time_stamp: String = _ // 事件时间戳 11 | var app_name: String = _ // 应用端 12 | var app_version: String = _ // 应用的版本 13 | var is_login: String = _ // 是否首次登陆 1/0 14 | var is_vip: String = _ // 是否VIP 15 | var wifi: String = _ 16 | var page_title: String = _ // 所在页面 17 | var page_type: String = _ // 页面类型 18 | var platform_type: String = _ // 平台类型 19 | var store_id: String = _ // 门店ID 20 | var store_name: String = _ // 门店名称 21 | var supplier_id: String = _ // 供应商ID 22 | var supplier_name: String = _ // 供应商名称 23 | var room_id: String = _ // 直播间id 24 | var room_name: String = _ // 直播间名称 25 | var vip_level: String = _ // VIP等级 26 | var lib: String = _ // SDK类型，例如python、iOS等 27 | var browser: String = _ // 浏览器名，例如Chrome 28 | var browser_version: String = _ // 浏览器版本，例如Chrome 45 29 | var carrier: String = _ // 运营商名称，例如ChinaNet 30 | var province: String = _ 31 | var city: String = _ 32 | var country: String = _ 33 | var os: String = _ // 操作系统，例如iOS 34 | var os_version: String = _ // 操作系统版本，例如8.1.1 35 | var model: String = _ // 设备型号，例如iphone6 36 | var utm_campaign: String = _ // 广告系列名称 37 | var utm_content: String = _ // 广告系列内容 38 | var utm_matching_type: String = _ // 渠道追踪匹配模式 39 | var utm_medium: String = _ // 广告系列媒介 40 | var utm_source: String = _ // 广告系列来源 41 | var utm_term: String = _ // 广告系列字词 42 | var url: String = _ // url 43 | var referrer: String = _ // 向前地址 44 | var scene: String = _ // 启动场景 45 | var spu_id: String = _ // 商品ID 46 | var spu_name: String = _ // 商品名 47 | var spu_quantity: String = _ // 商品数量 48 | 49 | override def toString = s"DwdUserBaseLog(event=$event, event_name=$event_name, user_id=$user_id, distinct_id=$distinct_id, event_time=$event_time, event_time_stamp=$event_time_stamp, app_name=$app_name, app_version=$app_version, is_login=$is_login, is_vip=$is_vip, wifi=$wifi, page_title=$page_title, page_type=$page_type, platform_type=$platform_type, store_id=$store_id, store_name=$store_name, supplier_id=$supplier_id, supplier_name=$supplier_name, room_id=$room_id, room_name=$room_name, vip_level=$vip_level, lib=$lib, browser=$browser, browser_version=$browser_version, carrier=$carrier, province=$province, city=$city, country=$country, os=$os, os_version=$os_version, model=$model, utm_campaign=$utm_campaign, utm_content=$utm_content, utm_matching_type=$utm_matching_type, utm_medium=$utm_medium, utm_source=$utm_source, utm_term=$utm_term, url=$url, referrer=$referrer, scene=$scene, spu_id=$spu_id, spu_name=$spu_name, spu_quantity=$spu_quantity)" 50 | } 51 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/flow/OdsBaseLog.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.flow 2 | 3 | import com.google.gson.{JsonArray, JsonObject} 4 | 5 | class OdsBaseLog { 6 | 7 | var actions: JsonArray = _ 8 | var common: JsonObject = _ 9 | var start: JsonObject = _ // 启动类型日志 10 | var displays: JsonArray = _ // 曝光类型日志 11 | var page: JsonObject = _ // 页面访问类型日志 12 | var ts: Long = _ 13 | 14 | override def toString = s"OdsBaseLog(actions=$actions, common=$common, start=$start, displays=$displays, page=$page, ts=$ts)" 15 | } 16 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/flow/OdsUserActionLog.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.flow 2 | 3 | class OdsUserActionLog { 4 | 5 | var event: String = _ // 事件 6 | var user_id: String = _ // 用户ID 7 | var distinct_id: String = _ // 唯一ID 8 | var time: String = _ // 事件时间 9 | var app_name: String = _ // 应用端 10 | var commodity_id: String = _ // 商品ID 11 | var commodity_name: String = _ // 商品名 12 | var commodity_quantity: String = _ // 商品数量 13 | var is_login: String = _ // 是否首次登陆 1/0 14 | var is_vip: String = _ // 是否VIP 15 | var page_title: String = _ // 所在页面 16 | var page_type: String = _ // 页面类型 17 | var platform_type: String = _ // 平台类型 18 | var store_id: String = _ // 门店ID 19 | var store_name: String = _ // 门店名称 20 | var supplier_id: String = _ // 供应商ID 21 | var supplier_name: String = _ // 供应商名称 22 | var room_id: String = _ // 直播间id 23 | var room_name: String = _ // 直播间名称 24 | var vip_level: String = _ // VIP等级 25 | var $lib: String = _ // SDK类型，例如python、iOS等 26 | var $app_version: String = _ // 应用的版本 27 | var $browser: String = _ // 浏览器名，例如Chrome 28 | var $browser_version: String = _ // 浏览器版本，例如Chrome 45 29 | var $carrier: String = _ // 运营商名称，例如ChinaNet 30 | var $province: String = _ 31 | var $city: String = _ 32 | var $country: String = _ 33 | var $os: String = _ // 操作系统，例如iOS 34 | var $os_version: String = _ // 操作系统版本，例如8.1.1 35 | var $model: String = _ // 设备型号，例如iphone6 36 | var $wifi: String = _ 37 | var $utm_campaign: String = _ // 广告系列名称 38 | var $utm_content: String = _ // 广告系列内容 39 | var $utm_matching_type: String = _ // 渠道追踪匹配模式 40 | var $utm_medium: String = _ // 广告系列媒介 41 | var $utm_source: String = _ // 广告系列来源 42 | var $utm_term: String = _ // 广告系列字词 43 | var $url: String = _ // url 44 | var $referrer: String = _ // 向前地址 45 | var $scene: String = _ // 启动场景 46 | 47 | 48 | override def toString = s"OdsUserActionLog(event=$event, user_id=$user_id, distinct_id=$distinct_id, time=$time, app_name=$app_name, commodity_id=$commodity_id, commodity_name=$commodity_name, commodity_quantity=$commodity_quantity, is_login=$is_login, is_vip=$is_vip, page_title=$page_title, page_type=$page_type, platform_type=$platform_type, store_id=$store_id, store_name=$store_name, supplier_id=$supplier_id, supplier_name=$supplier_name, vip_level=$vip_level, lib=${$lib}, app_version=${$app_version}, browser=${$browser}, browser_version=${$browser_version}, carrier=${$carrier}, province=${$province}, city=${$city}, country=${$country}, os=${$os}, os_version=${$os_version})" 49 | } 50 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/ActivityInfo.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class ActivityInfo extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var activity_name: String = _ 17 | var activity_type: String = _ 18 | var activity_desc: String = _ 19 | var start_time: String = _ 20 | var end_time: String = _ 21 | var create_time: String = _ 22 | 23 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 24 | this() 25 | this.database = database 26 | this.table = table 27 | this.sqlType = sqlType 28 | this.ts = ts 29 | this.old = old 30 | } 31 | 32 | override def toString = s"ActivityInfo(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, activity_name=${activity_name}, activity_type=${activity_type}, activity_desc=${activity_desc}, start_time=${start_time}, end_time=${end_time}, create_time=${create_time})" 33 | 34 | } 35 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/ActivityRule.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class ActivityRule extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var activity_id: String = _ 17 | var activity_type: String = _ 18 | var condition_amount: String = _ 19 | var condition_num: String = _ 20 | var benefit_amount: String = _ 21 | var benefit_discount: String = _ 22 | var benefit_level: String = _ 23 | 24 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 25 | this() 26 | this.database = database 27 | this.table = table 28 | this.sqlType = sqlType 29 | this.ts = ts 30 | this.old = old 31 | } 32 | 33 | override def toString = s"ActivityRule(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, activity_id=${activity_id}, activity_type=${activity_type}, condition_amount=${condition_amount}, condition_num=${condition_num}, benefit_amount=${benefit_amount}, benefit_discount=${benefit_discount}, benefit_level=${benefit_level})" 34 | 35 | } 36 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/ActivitySku.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class ActivitySku extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var activity_id: String = _ 17 | var sku_id: String = _ 18 | var create_time: String = _ 19 | 20 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 21 | this() 22 | this.database = database 23 | this.table = table 24 | this.sqlType = sqlType 25 | this.ts = ts 26 | this.old = old 27 | } 28 | 29 | override def toString = s"ActivitySku(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, activity_id=${activity_id}, sku_id=${sku_id}, create_time=${create_time})" 30 | 31 | } 32 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/BaseAttrInfo.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class BaseAttrInfo extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var attr_name: String = _ 17 | var category_id: String = _ 18 | var category_level: String = _ 19 | 20 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 21 | this() 22 | this.database = database 23 | this.table = table 24 | this.sqlType = sqlType 25 | this.ts = ts 26 | this.old = old 27 | } 28 | 29 | override def toString = s"BaseAttrInfo(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, attr_name=${attr_name}, category_id=${category_id}, category_level=${category_level})" 30 | 31 | } 32 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/BaseAttrValue.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class BaseAttrValue extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var value_name: String = _ 17 | var attr_id: String = _ 18 | 19 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 20 | this() 21 | this.database = database 22 | this.table = table 23 | this.sqlType = sqlType 24 | this.ts = ts 25 | this.old = old 26 | } 27 | 28 | override def toString = s"BaseAttrValue(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, value_name=${value_name}, attr_id=${attr_id})" 29 | 30 | } 31 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/BaseCategory1.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class BaseCategory1 extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var name: String = _ 17 | 18 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 19 | this() 20 | this.database = database 21 | this.table = table 22 | this.sqlType = sqlType 23 | this.ts = ts 24 | this.old = old 25 | } 26 | 27 | override def toString = s"BaseCategory1(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, name=${name})" 28 | 29 | } 30 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/BaseCategory2.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class BaseCategory2 extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var name: String = _ 17 | var category1_id: String = _ 18 | 19 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 20 | this() 21 | this.database = database 22 | this.table = table 23 | this.sqlType = sqlType 24 | this.ts = ts 25 | this.old = old 26 | } 27 | 28 | override def toString = s"BaseCategory2(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, name=${name}, category1_id=${category1_id})" 29 | 30 | } 31 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/BaseCategory3.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class BaseCategory3 extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var name: String = _ 17 | var category2_id: String = _ 18 | 19 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 20 | this() 21 | this.database = database 22 | this.table = table 23 | this.sqlType = sqlType 24 | this.ts = ts 25 | this.old = old 26 | } 27 | 28 | override def toString = s"BaseCategory3(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, name=${name}, category2_id=${category2_id})" 29 | 30 | } 31 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/BaseCategoryView.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class BaseCategoryView extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var category1_id: String = _ 17 | var category1_name: String = _ 18 | var category2_id: String = _ 19 | var category2_name: String = _ 20 | var category3_id: String = _ 21 | var category3_name: String = _ 22 | 23 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 24 | this() 25 | this.database = database 26 | this.table = table 27 | this.sqlType = sqlType 28 | this.ts = ts 29 | this.old = old 30 | } 31 | 32 | override def toString = s"BaseCategoryView(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, category1_id=${category1_id}, category1_name=${category1_name}, category2_id=${category2_id}, category2_name=${category2_name}, category3_id=${category3_id}, category3_name=${category3_name})" 33 | 34 | } 35 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/BaseDic.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class BaseDic extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var dic_name: String = _ 17 | var parent_code: String = _ 18 | var create_time: String = _ 19 | var operate_time: String = _ 20 | 21 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 22 | this() 23 | this.database = database 24 | this.table = table 25 | this.sqlType = sqlType 26 | this.ts = ts 27 | this.old = old 28 | } 29 | 30 | override def toString = s"BaseDic(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, dic_name=${dic_name}, parent_code=${parent_code}, create_time=${create_time}, operate_time=${operate_time})" 31 | 32 | } 33 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/BaseFrontendParam.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class BaseFrontendParam extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var code: String = _ 17 | var delete_id: String = _ 18 | 19 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 20 | this() 21 | this.database = database 22 | this.table = table 23 | this.sqlType = sqlType 24 | this.ts = ts 25 | this.old = old 26 | } 27 | 28 | override def toString = s"BaseFrontendParam(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, code=${code}, delete_id=${delete_id})" 29 | 30 | } 31 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/BaseProvince.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class BaseProvince extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var name: String = _ 17 | var region_id: String = _ 18 | var area_code: String = _ 19 | var iso_code: String = _ 20 | var iso_3166_2: String = _ 21 | 22 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 23 | this() 24 | this.database = database 25 | this.table = table 26 | this.sqlType = sqlType 27 | this.ts = ts 28 | this.old = old 29 | } 30 | 31 | override def toString = s"BaseProvince(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, name=${name}, region_id=${region_id}, area_code=${area_code}, iso_code=${iso_code}, iso_3166_2=${iso_3166_2})" 32 | 33 | } 34 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/BaseRegion.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class BaseRegion extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var region_name: String = _ 17 | 18 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 19 | this() 20 | this.database = database 21 | this.table = table 22 | this.sqlType = sqlType 23 | this.ts = ts 24 | this.old = old 25 | } 26 | 27 | override def toString = s"BaseRegion(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, region_name=${region_name})" 28 | 29 | } 30 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/BaseSaleAttr.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class BaseSaleAttr extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var name: String = _ 17 | 18 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 19 | this() 20 | this.database = database 21 | this.table = table 22 | this.sqlType = sqlType 23 | this.ts = ts 24 | this.old = old 25 | } 26 | 27 | override def toString = s"BaseSaleAttr(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, name=${name})" 28 | 29 | } 30 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/BaseTrademark.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class BaseTrademark extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var tm_name: String = _ 17 | var logo_url: String = _ 18 | 19 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 20 | this() 21 | this.database = database 22 | this.table = table 23 | this.sqlType = sqlType 24 | this.ts = ts 25 | this.old = old 26 | } 27 | 28 | override def toString = s"BaseTrademark(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, tm_name=${tm_name}, logo_url=${logo_url})" 29 | 30 | } 31 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/CartInfo.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class CartInfo extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var user_id: String = _ 17 | var sku_id: String = _ 18 | var cart_price: String = _ 19 | var sku_num: String = _ 20 | var img_url: String = _ 21 | var sku_name: String = _ 22 | var is_checked: String = _ 23 | var create_time: String = _ 24 | var operate_time: String = _ 25 | var is_ordered: String = _ 26 | var order_time: String = _ 27 | var source_type: String = _ 28 | var source_id: String = _ 29 | 30 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 31 | this() 32 | this.database = database 33 | this.table = table 34 | this.sqlType = sqlType 35 | this.ts = ts 36 | this.old = old 37 | } 38 | 39 | override def toString = s"CartInfo(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, user_id=${user_id}, sku_id=${sku_id}, cart_price=${cart_price}, sku_num=${sku_num}, img_url=${img_url}, sku_name=${sku_name}, is_checked=${is_checked}, create_time=${create_time}, operate_time=${operate_time}, is_ordered=${is_ordered}, order_time=${order_time}, source_type=${source_type}, source_id=${source_id})" 40 | 41 | } 42 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/CmsBanner.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class CmsBanner extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var title: String = _ 17 | var image_url: String = _ 18 | var link_url: String = _ 19 | var sort: String = _ 20 | 21 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 22 | this() 23 | this.database = database 24 | this.table = table 25 | this.sqlType = sqlType 26 | this.ts = ts 27 | this.old = old 28 | } 29 | 30 | override def toString = s"CmsBanner(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, title=${title}, image_url=${image_url}, link_url=${link_url}, sort=${sort})" 31 | 32 | } 33 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/CommentInfo.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class CommentInfo extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var user_id: String = _ 17 | var nick_name: String = _ 18 | var head_img: String = _ 19 | var sku_id: String = _ 20 | var spu_id: String = _ 21 | var order_id: String = _ 22 | var appraise: String = _ 23 | var comment_txt: String = _ 24 | var create_time: String = _ 25 | var operate_time: String = _ 26 | 27 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 28 | this() 29 | this.database = database 30 | this.table = table 31 | this.sqlType = sqlType 32 | this.ts = ts 33 | this.old = old 34 | } 35 | 36 | override def toString = s"CommentInfo(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, user_id=${user_id}, nick_name=${nick_name}, head_img=${head_img}, sku_id=${sku_id}, spu_id=${spu_id}, order_id=${order_id}, appraise=${appraise}, comment_txt=${comment_txt}, create_time=${create_time}, operate_time=${operate_time})" 37 | 38 | } 39 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/CouponInfo.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class CouponInfo extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var coupon_name: String = _ 17 | var coupon_type: String = _ 18 | var condition_amount: String = _ 19 | var condition_num: String = _ 20 | var activity_id: String = _ 21 | var benefit_amount: String = _ 22 | var benefit_discount: String = _ 23 | var create_time: String = _ 24 | var range_type: String = _ 25 | var limit_num: String = _ 26 | var taken_count: String = _ 27 | var start_time: String = _ 28 | var end_time: String = _ 29 | var operate_time: String = _ 30 | var expire_time: String = _ 31 | var range_desc: String = _ 32 | 33 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 34 | this() 35 | this.database = database 36 | this.table = table 37 | this.sqlType = sqlType 38 | this.ts = ts 39 | this.old = old 40 | } 41 | 42 | override def toString = s"CouponInfo(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, coupon_name=${coupon_name}, coupon_type=${coupon_type}, condition_amount=${condition_amount}, condition_num=${condition_num}, activity_id=${activity_id}, benefit_amount=${benefit_amount}, benefit_discount=${benefit_discount}, create_time=${create_time}, range_type=${range_type}, limit_num=${limit_num}, taken_count=${taken_count}, start_time=${start_time}, end_time=${end_time}, operate_time=${operate_time}, expire_time=${expire_time}, range_desc=${range_desc})" 43 | 44 | } 45 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/CouponRange.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class CouponRange extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var coupon_id: String = _ 17 | var range_type: String = _ 18 | var range_id: String = _ 19 | 20 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 21 | this() 22 | this.database = database 23 | this.table = table 24 | this.sqlType = sqlType 25 | this.ts = ts 26 | this.old = old 27 | } 28 | 29 | override def toString = s"CouponRange(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, coupon_id=${coupon_id}, range_type=${range_type}, range_id=${range_id})" 30 | 31 | } 32 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/CouponUse.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class CouponUse extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var coupon_id: String = _ 17 | var user_id: String = _ 18 | var order_id: String = _ 19 | var coupon_status: String = _ 20 | var get_time: String = _ 21 | var using_time: String = _ 22 | var used_time: String = _ 23 | var expire_time: String = _ 24 | 25 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 26 | this() 27 | this.database = database 28 | this.table = table 29 | this.sqlType = sqlType 30 | this.ts = ts 31 | this.old = old 32 | } 33 | 34 | override def toString = s"CouponUse(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, coupon_id=${coupon_id}, user_id=${user_id}, order_id=${order_id}, coupon_status=${coupon_status}, get_time=${get_time}, using_time=${using_time}, used_time=${used_time}, expire_time=${expire_time})" 35 | 36 | } 37 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/FavorInfo.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class FavorInfo extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var user_id: String = _ 17 | var sku_id: String = _ 18 | var spu_id: String = _ 19 | var is_cancel: String = _ 20 | var create_time: String = _ 21 | var cancel_time: String = _ 22 | 23 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 24 | this() 25 | this.database = database 26 | this.table = table 27 | this.sqlType = sqlType 28 | this.ts = ts 29 | this.old = old 30 | } 31 | 32 | override def toString = s"FavorInfo(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, user_id=${user_id}, sku_id=${sku_id}, spu_id=${spu_id}, is_cancel=${is_cancel}, create_time=${create_time}, cancel_time=${cancel_time})" 33 | 34 | } 35 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/FinancialSkuCost.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class FinancialSkuCost extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var sku_id: String = _ 17 | var sku_name: String = _ 18 | var busi_date: String = _ 19 | var is_lastest: String = _ 20 | var sku_cost: String = _ 21 | var create_time: String = _ 22 | 23 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 24 | this() 25 | this.database = database 26 | this.table = table 27 | this.sqlType = sqlType 28 | this.ts = ts 29 | this.old = old 30 | } 31 | 32 | override def toString = s"FinancialSkuCost(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, sku_id=${sku_id}, sku_name=${sku_name}, busi_date=${busi_date}, is_lastest=${is_lastest}, sku_cost=${sku_cost}, create_time=${create_time})" 33 | 34 | } 35 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/OrderDetail.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class OrderDetail extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var order_id: String = _ 17 | var sku_id: String = _ 18 | var sku_name: String = _ 19 | var img_url: String = _ 20 | var order_price: String = _ 21 | var sku_num: String = _ 22 | var create_time: String = _ 23 | var source_type: String = _ 24 | var source_id: String = _ 25 | var split_total_amount: String = _ 26 | var split_activity_amount: String = _ 27 | var split_coupon_amount: String = _ 28 | 29 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 30 | this() 31 | this.database = database 32 | this.table = table 33 | this.sqlType = sqlType 34 | this.ts = ts 35 | this.old = old 36 | } 37 | 38 | override def toString = s"OrderDetail(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, order_id=${order_id}, sku_id=${sku_id}, sku_name=${sku_name}, img_url=${img_url}, order_price=${order_price}, sku_num=${sku_num}, create_time=${create_time}, source_type=${source_type}, source_id=${source_id}, split_total_amount=${split_total_amount}, split_activity_amount=${split_activity_amount}, split_coupon_amount=${split_coupon_amount})" 39 | 40 | } 41 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/OrderDetailActivity.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class OrderDetailActivity extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var order_id: String = _ 17 | var order_detail_id: String = _ 18 | var activity_id: String = _ 19 | var activity_rule_id: String = _ 20 | var sku_id: String = _ 21 | var create_time: String = _ 22 | 23 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 24 | this() 25 | this.database = database 26 | this.table = table 27 | this.sqlType = sqlType 28 | this.ts = ts 29 | this.old = old 30 | } 31 | 32 | override def toString = s"OrderDetailActivity(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, order_id=${order_id}, order_detail_id=${order_detail_id}, activity_id=${activity_id}, activity_rule_id=${activity_rule_id}, sku_id=${sku_id}, create_time=${create_time})" 33 | 34 | } 35 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/OrderDetailCoupon.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class OrderDetailCoupon extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var order_id: String = _ 17 | var order_detail_id: String = _ 18 | var coupon_id: String = _ 19 | var coupon_use_id: String = _ 20 | var sku_id: String = _ 21 | var create_time: String = _ 22 | 23 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 24 | this() 25 | this.database = database 26 | this.table = table 27 | this.sqlType = sqlType 28 | this.ts = ts 29 | this.old = old 30 | } 31 | 32 | override def toString = s"OrderDetailCoupon(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, order_id=${order_id}, order_detail_id=${order_detail_id}, coupon_id=${coupon_id}, coupon_use_id=${coupon_use_id}, sku_id=${sku_id}, create_time=${create_time})" 33 | 34 | } 35 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/OrderInfo.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class OrderInfo extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var consignee: String = _ 17 | var consignee_tel: String = _ 18 | var total_amount: String = _ 19 | var order_status: String = _ 20 | var user_id: String = _ 21 | var payment_way: String = _ 22 | var delivery_address: String = _ 23 | var order_comment: String = _ 24 | var out_trade_no: String = _ 25 | var trade_body: String = _ 26 | var create_time: String = _ 27 | var operate_time: String = _ 28 | var expire_time: String = _ 29 | var process_status: String = _ 30 | var tracking_no: String = _ 31 | var parent_order_id: String = _ 32 | var img_url: String = _ 33 | var province_id: String = _ 34 | var activity_reduce_amount: String = _ 35 | var coupon_reduce_amount: String = _ 36 | var original_total_amount: String = _ 37 | var feight_fee: String = _ 38 | var feight_fee_reduce: String = _ 39 | var refundable_time: String = _ 40 | 41 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 42 | this() 43 | this.database = database 44 | this.table = table 45 | this.sqlType = sqlType 46 | this.ts = ts 47 | this.old = old 48 | } 49 | 50 | override def toString = s"OrderInfo(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, consignee=${consignee}, consignee_tel=${consignee_tel}, total_amount=${total_amount}, order_status=${order_status}, user_id=${user_id}, payment_way=${payment_way}, delivery_address=${delivery_address}, order_comment=${order_comment}, out_trade_no=${out_trade_no}, trade_body=${trade_body}, create_time=${create_time}, operate_time=${operate_time}, expire_time=${expire_time}, process_status=${process_status}, tracking_no=${tracking_no}, parent_order_id=${parent_order_id}, img_url=${img_url}, province_id=${province_id}, activity_reduce_amount=${activity_reduce_amount}, coupon_reduce_amount=${coupon_reduce_amount}, original_total_amount=${original_total_amount}, feight_fee=${feight_fee}, feight_fee_reduce=${feight_fee_reduce}, refundable_time=${refundable_time})" 51 | 52 | } 53 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/OrderRefundInfo.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class OrderRefundInfo extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var user_id: String = _ 17 | var order_id: String = _ 18 | var sku_id: String = _ 19 | var refund_type: String = _ 20 | var refund_num: String = _ 21 | var refund_amount: String = _ 22 | var refund_reason_type: String = _ 23 | var refund_reason_txt: String = _ 24 | var refund_status: String = _ 25 | var create_time: String = _ 26 | 27 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 28 | this() 29 | this.database = database 30 | this.table = table 31 | this.sqlType = sqlType 32 | this.ts = ts 33 | this.old = old 34 | } 35 | 36 | override def toString = s"OrderRefundInfo(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, user_id=${user_id}, order_id=${order_id}, sku_id=${sku_id}, refund_type=${refund_type}, refund_num=${refund_num}, refund_amount=${refund_amount}, refund_reason_type=${refund_reason_type}, refund_reason_txt=${refund_reason_txt}, refund_status=${refund_status}, create_time=${create_time})" 37 | 38 | } 39 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/OrderStatusLog.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class OrderStatusLog extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var order_id: String = _ 17 | var order_status: String = _ 18 | var operate_time: String = _ 19 | 20 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 21 | this() 22 | this.database = database 23 | this.table = table 24 | this.sqlType = sqlType 25 | this.ts = ts 26 | this.old = old 27 | } 28 | 29 | override def toString = s"OrderStatusLog(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, order_id=${order_id}, order_status=${order_status}, operate_time=${operate_time})" 30 | 31 | } 32 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/PaymentInfo.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class PaymentInfo extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var out_trade_no: String = _ 17 | var order_id: String = _ 18 | var user_id: String = _ 19 | var payment_type: String = _ 20 | var trade_no: String = _ 21 | var total_amount: String = _ 22 | var subject: String = _ 23 | var payment_status: String = _ 24 | var create_time: String = _ 25 | var callback_time: String = _ 26 | var callback_content: String = _ 27 | 28 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 29 | this() 30 | this.database = database 31 | this.table = table 32 | this.sqlType = sqlType 33 | this.ts = ts 34 | this.old = old 35 | } 36 | 37 | override def toString = s"PaymentInfo(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, out_trade_no=${out_trade_no}, order_id=${order_id}, user_id=${user_id}, payment_type=${payment_type}, trade_no=${trade_no}, total_amount=${total_amount}, subject=${subject}, payment_status=${payment_status}, create_time=${create_time}, callback_time=${callback_time}, callback_content=${callback_content})" 38 | 39 | } 40 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/RefundPayment.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class RefundPayment extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var out_trade_no: String = _ 17 | var order_id: String = _ 18 | var sku_id: String = _ 19 | var payment_type: String = _ 20 | var trade_no: String = _ 21 | var total_amount: String = _ 22 | var subject: String = _ 23 | var refund_status: String = _ 24 | var create_time: String = _ 25 | var callback_time: String = _ 26 | var callback_content: String = _ 27 | 28 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 29 | this() 30 | this.database = database 31 | this.table = table 32 | this.sqlType = sqlType 33 | this.ts = ts 34 | this.old = old 35 | } 36 | 37 | override def toString = s"RefundPayment(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, out_trade_no=${out_trade_no}, order_id=${order_id}, sku_id=${sku_id}, payment_type=${payment_type}, trade_no=${trade_no}, total_amount=${total_amount}, subject=${subject}, refund_status=${refund_status}, create_time=${create_time}, callback_time=${callback_time}, callback_content=${callback_content})" 38 | 39 | } 40 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/SeckillGoods.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class SeckillGoods extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var spu_id: String = _ 17 | var sku_id: String = _ 18 | var sku_name: String = _ 19 | var sku_default_img: String = _ 20 | var price: String = _ 21 | var cost_price: String = _ 22 | var create_time: String = _ 23 | var check_time: String = _ 24 | var status: String = _ 25 | var start_time: String = _ 26 | var end_time: String = _ 27 | var num: String = _ 28 | var stock_count: String = _ 29 | var sku_desc: String = _ 30 | 31 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 32 | this() 33 | this.database = database 34 | this.table = table 35 | this.sqlType = sqlType 36 | this.ts = ts 37 | this.old = old 38 | } 39 | 40 | override def toString = s"SeckillGoods(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, spu_id=${spu_id}, sku_id=${sku_id}, sku_name=${sku_name}, sku_default_img=${sku_default_img}, price=${price}, cost_price=${cost_price}, create_time=${create_time}, check_time=${check_time}, status=${status}, start_time=${start_time}, end_time=${end_time}, num=${num}, stock_count=${stock_count}, sku_desc=${sku_desc})" 41 | 42 | } 43 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/SkuAttrValue.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class SkuAttrValue extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var attr_id: String = _ 17 | var value_id: String = _ 18 | var sku_id: String = _ 19 | var attr_name: String = _ 20 | var value_name: String = _ 21 | 22 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 23 | this() 24 | this.database = database 25 | this.table = table 26 | this.sqlType = sqlType 27 | this.ts = ts 28 | this.old = old 29 | } 30 | 31 | override def toString = s"SkuAttrValue(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, attr_id=${attr_id}, value_id=${value_id}, sku_id=${sku_id}, attr_name=${attr_name}, value_name=${value_name})" 32 | 33 | } 34 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/SkuImage.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class SkuImage extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var sku_id: String = _ 17 | var img_name: String = _ 18 | var img_url: String = _ 19 | var spu_img_id: String = _ 20 | var is_default: String = _ 21 | 22 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 23 | this() 24 | this.database = database 25 | this.table = table 26 | this.sqlType = sqlType 27 | this.ts = ts 28 | this.old = old 29 | } 30 | 31 | override def toString = s"SkuImage(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, sku_id=${sku_id}, img_name=${img_name}, img_url=${img_url}, spu_img_id=${spu_img_id}, is_default=${is_default})" 32 | 33 | } 34 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/SkuInfo.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class SkuInfo extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var spu_id: String = _ 17 | var price: String = _ 18 | var sku_name: String = _ 19 | var sku_desc: String = _ 20 | var weight: String = _ 21 | var tm_id: String = _ 22 | var category3_id: String = _ 23 | var sku_default_img: String = _ 24 | var is_sale: String = _ 25 | var create_time: String = _ 26 | 27 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 28 | this() 29 | this.database = database 30 | this.table = table 31 | this.sqlType = sqlType 32 | this.ts = ts 33 | this.old = old 34 | } 35 | 36 | override def toString = s"SkuInfo(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, spu_id=${spu_id}, price=${price}, sku_name=${sku_name}, sku_desc=${sku_desc}, weight=${weight}, tm_id=${tm_id}, category3_id=${category3_id}, sku_default_img=${sku_default_img}, is_sale=${is_sale}, create_time=${create_time})" 37 | 38 | } 39 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/SkuSaleAttrValue.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class SkuSaleAttrValue extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var sku_id: String = _ 17 | var spu_id: String = _ 18 | var sale_attr_value_id: String = _ 19 | var sale_attr_id: String = _ 20 | var sale_attr_name: String = _ 21 | var sale_attr_value_name: String = _ 22 | 23 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 24 | this() 25 | this.database = database 26 | this.table = table 27 | this.sqlType = sqlType 28 | this.ts = ts 29 | this.old = old 30 | } 31 | 32 | override def toString = s"SkuSaleAttrValue(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, sku_id=${sku_id}, spu_id=${spu_id}, sale_attr_value_id=${sale_attr_value_id}, sale_attr_id=${sale_attr_id}, sale_attr_name=${sale_attr_name}, sale_attr_value_name=${sale_attr_value_name})" 33 | 34 | } 35 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/SpuImage.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class SpuImage extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var spu_id: String = _ 17 | var img_name: String = _ 18 | var img_url: String = _ 19 | 20 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 21 | this() 22 | this.database = database 23 | this.table = table 24 | this.sqlType = sqlType 25 | this.ts = ts 26 | this.old = old 27 | } 28 | 29 | override def toString = s"SpuImage(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, spu_id=${spu_id}, img_name=${img_name}, img_url=${img_url})" 30 | 31 | } 32 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/SpuInfo.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class SpuInfo extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var spu_name: String = _ 17 | var description: String = _ 18 | var category3_id: String = _ 19 | var tm_id: String = _ 20 | 21 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 22 | this() 23 | this.database = database 24 | this.table = table 25 | this.sqlType = sqlType 26 | this.ts = ts 27 | this.old = old 28 | } 29 | 30 | override def toString = s"SpuInfo(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, spu_name=${spu_name}, description=${description}, category3_id=${category3_id}, tm_id=${tm_id})" 31 | 32 | } 33 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/SpuPoster.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class SpuPoster extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var spu_id: String = _ 17 | var img_name: String = _ 18 | var img_url: String = _ 19 | var create_time: String = _ 20 | var update_time: String = _ 21 | var is_deleted: String = _ 22 | 23 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 24 | this() 25 | this.database = database 26 | this.table = table 27 | this.sqlType = sqlType 28 | this.ts = ts 29 | this.old = old 30 | } 31 | 32 | override def toString = s"SpuPoster(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, spu_id=${spu_id}, img_name=${img_name}, img_url=${img_url}, create_time=${create_time}, update_time=${update_time}, is_deleted=${is_deleted})" 33 | 34 | } 35 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/SpuSaleAttr.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class SpuSaleAttr extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var spu_id: String = _ 17 | var base_sale_attr_id: String = _ 18 | var sale_attr_name: String = _ 19 | 20 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 21 | this() 22 | this.database = database 23 | this.table = table 24 | this.sqlType = sqlType 25 | this.ts = ts 26 | this.old = old 27 | } 28 | 29 | override def toString = s"SpuSaleAttr(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, spu_id=${spu_id}, base_sale_attr_id=${base_sale_attr_id}, sale_attr_name=${sale_attr_name})" 30 | 31 | } 32 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/SpuSaleAttrValue.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class SpuSaleAttrValue extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var spu_id: String = _ 17 | var base_sale_attr_id: String = _ 18 | var sale_attr_value_name: String = _ 19 | var sale_attr_name: String = _ 20 | 21 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 22 | this() 23 | this.database = database 24 | this.table = table 25 | this.sqlType = sqlType 26 | this.ts = ts 27 | this.old = old 28 | } 29 | 30 | override def toString = s"SpuSaleAttrValue(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, spu_id=${spu_id}, base_sale_attr_id=${base_sale_attr_id}, sale_attr_value_name=${sale_attr_value_name}, sale_attr_name=${sale_attr_name})" 31 | 32 | } 33 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/UserAddress.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class UserAddress extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var user_id: String = _ 17 | var province_id: String = _ 18 | var user_address: String = _ 19 | var consignee: String = _ 20 | var phone_num: String = _ 21 | var is_default: String = _ 22 | 23 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 24 | this() 25 | this.database = database 26 | this.table = table 27 | this.sqlType = sqlType 28 | this.ts = ts 29 | this.old = old 30 | } 31 | 32 | override def toString = s"UserAddress(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, user_id=${user_id}, province_id=${province_id}, user_address=${user_address}, consignee=${consignee}, phone_num=${phone_num}, is_default=${is_default})" 33 | 34 | } 35 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/UserInfo.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class UserInfo extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var login_name: String = _ 17 | var nick_name: String = _ 18 | var passwd: String = _ 19 | var name: String = _ 20 | var phone_num: String = _ 21 | var email: String = _ 22 | var head_img: String = _ 23 | var user_level: String = _ 24 | var birthday: String = _ 25 | var gender: String = _ 26 | var create_time: String = _ 27 | var operate_time: String = _ 28 | var status: String = _ 29 | 30 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 31 | this() 32 | this.database = database 33 | this.table = table 34 | this.sqlType = sqlType 35 | this.ts = ts 36 | this.old = old 37 | } 38 | 39 | override def toString = s"UserInfo(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, login_name=${login_name}, nick_name=${nick_name}, passwd=${passwd}, name=${name}, phone_num=${phone_num}, email=${email}, head_img=${head_img}, user_level=${user_level}, birthday=${birthday}, gender=${gender}, create_time=${create_time}, operate_time=${operate_time}, status=${status})" 40 | 41 | } 42 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/WareInfo.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class WareInfo extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var name: String = _ 17 | var address: String = _ 18 | var areacode: String = _ 19 | 20 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 21 | this() 22 | this.database = database 23 | this.table = table 24 | this.sqlType = sqlType 25 | this.ts = ts 26 | this.old = old 27 | } 28 | 29 | override def toString = s"WareInfo(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, name=${name}, address=${address}, areacode=${areacode})" 30 | 31 | } 32 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/WareOrderTask.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class WareOrderTask extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var order_id: String = _ 17 | var consignee: String = _ 18 | var consignee_tel: String = _ 19 | var delivery_address: String = _ 20 | var order_comment: String = _ 21 | var payment_way: String = _ 22 | var task_status: String = _ 23 | var order_body: String = _ 24 | var tracking_no: String = _ 25 | var create_time: String = _ 26 | var ware_id: String = _ 27 | var task_comment: String = _ 28 | 29 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 30 | this() 31 | this.database = database 32 | this.table = table 33 | this.sqlType = sqlType 34 | this.ts = ts 35 | this.old = old 36 | } 37 | 38 | override def toString = s"WareOrderTask(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, order_id=${order_id}, consignee=${consignee}, consignee_tel=${consignee_tel}, delivery_address=${delivery_address}, order_comment=${order_comment}, payment_way=${payment_way}, task_status=${task_status}, order_body=${order_body}, tracking_no=${tracking_no}, create_time=${create_time}, ware_id=${ware_id}, task_comment=${task_comment})" 39 | 40 | } 41 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/WareOrderTaskDetail.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class WareOrderTaskDetail extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var sku_id: String = _ 17 | var sku_name: String = _ 18 | var sku_num: String = _ 19 | var task_id: String = _ 20 | var refund_status: String = _ 21 | 22 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 23 | this() 24 | this.database = database 25 | this.table = table 26 | this.sqlType = sqlType 27 | this.ts = ts 28 | this.old = old 29 | } 30 | 31 | override def toString = s"WareOrderTaskDetail(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, sku_id=${sku_id}, sku_name=${sku_name}, sku_num=${sku_num}, task_id=${task_id}, refund_status=${refund_status})" 32 | 33 | } 34 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/entity/ods/gmall2021/WareSku.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.gmall2021 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class WareSku extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | var id: String = _ 16 | var sku_id: String = _ 17 | var warehouse_id: String = _ 18 | var stock: String = _ 19 | var stock_name: String = _ 20 | var stock_locked: String = _ 21 | 22 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 23 | this() 24 | this.database = database 25 | this.table = table 26 | this.sqlType = sqlType 27 | this.ts = ts 28 | this.old = old 29 | } 30 | 31 | override def toString = s"WareSku(database=$database, table=$table, sqlType=$sqlType, ts=$ts, old=$old, id=${id}, sku_id=${sku_id}, warehouse_id=${warehouse_id}, stock=${stock}, stock_name=${stock_name}, stock_locked=${stock_locked})" 32 | 33 | } 34 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/sink/SinkFactory.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.sink 2 | 3 | import java.util.Properties 4 | 5 | import com.gmall.data.common.config.Config 6 | import com.gmall.data.common.sink.kafka.EventKafkaSerializationSchema 7 | import org.apache.flink.api.common.serialization.SimpleStringSchema 8 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer 9 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer.Semantic 10 | import org.apache.kafka.clients.producer.ProducerConfig 11 | 12 | import scala.reflect.ClassTag 13 | 14 | /** 15 | * 创建一个SinkFunction 16 | */ 17 | object SinkFactory { 18 | 19 | def createKafkaProducer[T](topicName: String)(implicit classTag: ClassTag[T]): FlinkKafkaProducer[T] = { 20 | val props = new Properties 21 | props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, Config.kafkaBrokers) 22 | props.setProperty(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, "600000") // 等待事务状态变更的最长时间10分钟 23 | props.setProperty(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "1") //有序 24 | props.setProperty(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "true") // 幂等性 25 | 26 | new FlinkKafkaProducer[T](topicName, new EventKafkaSerializationSchema[T](topicName), props, Semantic.EXACTLY_ONCE) 27 | } 28 | 29 | def createProducer(topicName: String): FlinkKafkaProducer[String] = { 30 | val props = new Properties 31 | props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, Config.kafkaBrokers) 32 | 33 | new FlinkKafkaProducer(Config.kafkaBrokers, topicName, new SimpleStringSchema()) 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/sink/kafka/EventKafkaSerializationSchema.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.sink.kafka 2 | 3 | import java.lang 4 | 5 | import com.gmall.data.common.utils.GsonUtil 6 | import org.apache.flink.streaming.connectors.kafka.KafkaSerializationSchema 7 | import org.apache.kafka.clients.producer.ProducerRecord 8 | import org.slf4j.LoggerFactory 9 | 10 | import scala.reflect.ClassTag 11 | 12 | /** 13 | * 自定义Kafka序列化器，DataStream[T] -> Json[T] 14 | * @param topic 15 | * @param classTag 16 | * @tparam T 17 | */ 18 | class EventKafkaSerializationSchema[T](topic: String)(implicit classTag: ClassTag[T]) extends KafkaSerializationSchema[T]{ 19 | 20 | private lazy val gson = GsonUtil.gson 21 | lazy val logger = LoggerFactory.getLogger(this.getClass) 22 | 23 | override def serialize(element: T, timestamp: lang.Long): ProducerRecord[Array[Byte], Array[Byte]] = { 24 | var jsonBytes: Array[Byte] = null 25 | 26 | try { 27 | jsonBytes = gson.toJson(element, classTag.runtimeClass).getBytes() 28 | } catch { 29 | case e: Exception => logger.warn(s"failed to serialize element(${element})", e) 30 | } 31 | 32 | new ProducerRecord[Array[Byte], Array[Byte]](topic, jsonBytes) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/transform/Convert.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.transform 2 | 3 | import org.apache.flink.streaming.api.scala._ 4 | 5 | /** 6 | * 流转换 7 | * @tparam T1 8 | * @tparam T2 9 | */ 10 | abstract class Convert[T1, T2] extends Serializable { 11 | 12 | // 请子类取一个有意义的方法名的方法来调用convert方法 13 | def convert(input: DataStream[T1]): DataStream[T2] = { 14 | doConvert(input).uid(s"convert_${getUid}").name(s"convert_${getName}") 15 | } 16 | 17 | def getUid: String 18 | 19 | def getName: String 20 | 21 | // 基于业务,封装在doConvert方法中,日志转换 22 | protected def doConvert(input: DataStream[T1]): DataStream[T2] 23 | 24 | } 25 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/transform/Format.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.transform 2 | 3 | import org.apache.flink.streaming.api.scala._ 4 | 5 | /** 6 | * 流格式化 7 | * @tparam T 8 | */ 9 | abstract class Format[T] { 10 | 11 | // 请子类取一个有意义的方法名的方法来调用format方法 12 | def format(input: DataStream[T]): DataStream[T] = { 13 | doFormat(input).uid(s"format_${getUid}").name(s"format_${getName}") 14 | } 15 | 16 | def getUid: String 17 | 18 | def getName: String 19 | 20 | // 基于业务,封装在check方法中 21 | protected def doFormat(input: DataStream[T]): DataStream[T] 22 | 23 | } 24 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/transform/Merger.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.transform 2 | 3 | import org.apache.flink.streaming.api.scala._ 4 | 5 | /** 6 | * DataStream[T1] and DataStream[T2] to DataStream[T] 7 | * 双流join 8 | * @param source 9 | * @tparam T1 10 | * @tparam T2 11 | * @tparam T 12 | */ 13 | abstract class Merger[T1, T2, T](source: DataStream[T1]) extends Serializable { 14 | 15 | def joinStream(input: DataStream[T2]): DataStream[T] = { 16 | merge(source, input).name(s"merge_${getName}").uid(s"merger_${getUid}") 17 | } 18 | 19 | def getName: String 20 | 21 | def getUid: String 22 | 23 | protected def merge(input1: DataStream[T1], input2: DataStream[T2]): DataStream[T] 24 | 25 | } 26 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/transform/Sink.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.transform 2 | 3 | import org.apache.flink.streaming.api.scala._ 4 | 5 | /** 6 | * sink接口，流sink函数需要继承该抽象类 7 | * 8 | * @tparam T 9 | */ 10 | abstract class Sink[T] { 11 | 12 | // 请子类取一个有意义的方法名的方法来调用sink方法 13 | def sink(input: DataStream[T]): Unit = { 14 | doSink(input) 15 | } 16 | 17 | // 基于业务,封装在doSink方法中 18 | protected def doSink(input: DataStream[T]): Unit 19 | 20 | } 21 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/utils/CacheUtil.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.utils 2 | 3 | import com.google.common.cache.Cache 4 | 5 | /** 6 | * 查询HBase时本地缓存 7 | */ 8 | object CacheUtil { 9 | 10 | var cache: Cache[String, String] = _ 11 | 12 | 13 | } 14 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/utils/GsonUtil.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.utils 2 | 3 | import com.google.gson.{Gson, JsonArray, JsonObject, JsonParser} 4 | 5 | 6 | /** 7 | * gson公共类 8 | */ 9 | object GsonUtil { 10 | 11 | val gson = new Gson() 12 | 13 | val parser = new JsonParser() 14 | 15 | def getLong(jsonObject: JsonObject, name: String): Long = try { 16 | jsonObject.get(name).getAsLong 17 | } catch { 18 | case _: Throwable => 0L 19 | } 20 | 21 | def getString(jsonObject: JsonObject, name: String): String = try { 22 | jsonObject.get(name).getAsString 23 | } catch { 24 | case _: Throwable => null 25 | } 26 | 27 | def getAsString(jsonObject: JsonObject, name: String): String = try { 28 | val obj = jsonObject.get(name) 29 | if (obj.isJsonObject) { 30 | obj.getAsJsonObject.toString 31 | } else { 32 | obj.getAsString 33 | } 34 | } catch { 35 | case _: Throwable => null 36 | } 37 | 38 | def getJsonObject(props: JsonObject, name: String): JsonObject = try { 39 | props.get(name).getAsJsonObject 40 | } catch { 41 | case _: Throwable => null 42 | } 43 | 44 | def getJsonArray(column: String): JsonArray = try { 45 | parser.parse(column).getAsJsonArray 46 | } catch { 47 | case e: Throwable => null 48 | } 49 | 50 | def getAsJsonObject(props: JsonObject, name: String): JsonObject = try { 51 | val eventMore = getString(props, name) 52 | parser.parse(eventMore).getAsJsonObject 53 | } catch { 54 | case _: Throwable => null 55 | } 56 | 57 | 58 | 59 | } 60 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/utils/HBaseUtil.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.utils 2 | 3 | import java.util.concurrent.CompletableFuture 4 | 5 | import com.gmall.data.common.config.Config 6 | import org.apache.hadoop.hbase.{HBaseConfiguration, HConstants} 7 | import org.apache.hadoop.hbase.client.{AsyncConnection, Connection, ConnectionFactory} 8 | import org.slf4j.{Logger, LoggerFactory} 9 | 10 | /** 11 | * HBase工具类 12 | */ 13 | object HBaseUtil { 14 | 15 | private val logger: Logger = LoggerFactory.getLogger(this.getClass) 16 | 17 | var conn: Connection = _ 18 | var asyncConn: CompletableFuture[AsyncConnection] = _ 19 | 20 | /** 21 | * 获取HBase连接单例对象 22 | * 23 | * @return 24 | */ 25 | def getConn(): Connection = { 26 | try { 27 | val conf = HBaseConfiguration.create() 28 | conf.set(HConstants.ZOOKEEPER_QUORUM, Config.hbaseZookeeperQuorum) 29 | conn = ConnectionFactory.createConnection(conf) 30 | } catch { 31 | case e: Exception => LoggerUtil.error(logger, e, 32 | "failed to get HBase connection") 33 | } 34 | conn 35 | } 36 | 37 | /** 38 | * 获取HBase异步客户端 39 | * 40 | * @return 41 | */ 42 | def getAsyncConn(): CompletableFuture[AsyncConnection] = { 43 | try { 44 | val conf = HBaseConfiguration.create() 45 | conf.set(HConstants.ZOOKEEPER_QUORUM, Config.hbaseZookeeperQuorum) 46 | asyncConn = ConnectionFactory.createAsyncConnection(conf) 47 | } catch { 48 | case e: Exception => LoggerUtil.error(logger, e, 49 | "failed to get HBase async connection") 50 | } 51 | asyncConn 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/utils/JedisConnectionPool.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.utils 2 | 3 | import java.util.concurrent.ConcurrentHashMap 4 | 5 | import com.gmall.data.common.config.RedisConfig 6 | import redis.clients.jedis.exceptions.JedisConnectionException 7 | import redis.clients.jedis.{Jedis, JedisPool, JedisPoolConfig} 8 | 9 | import scala.collection.JavaConversions._ 10 | 11 | /** 12 | * Jedis的连接池 13 | */ 14 | object JedisConnectionPool { 15 | @transient private lazy val pools: ConcurrentHashMap[RedisConfig, JedisPool] = 16 | new ConcurrentHashMap[RedisConfig, JedisPool]() 17 | 18 | def connect(implicit re: RedisConfig): Jedis = { 19 | val pool = pools.getOrElseUpdate(re, 20 | { 21 | val poolConfig: JedisPoolConfig = new JedisPoolConfig(); 22 | poolConfig.setMaxTotal(250) 23 | poolConfig.setMaxIdle(32) 24 | poolConfig.setTestOnBorrow(false) 25 | poolConfig.setTestOnReturn(false) 26 | poolConfig.setTestWhileIdle(false) 27 | poolConfig.setMinEvictableIdleTimeMillis(60000) 28 | poolConfig.setTimeBetweenEvictionRunsMillis(30000) 29 | poolConfig.setNumTestsPerEvictionRun(-1) 30 | new JedisPool(poolConfig, re.host, re.port, 10000, re.password, re.db) 31 | } 32 | ) 33 | var sleepTime: Int = 4 34 | var conn: Jedis = null 35 | while (conn == null) { 36 | try { 37 | conn = pool.getResource 38 | } 39 | catch { 40 | case e: JedisConnectionException if e.getCause.toString. 41 | contains("ERR max number of clients reached") => { 42 | if (sleepTime < 500) sleepTime *= 2 43 | Thread.sleep(sleepTime) 44 | } 45 | case e: Exception => throw e 46 | } 47 | } 48 | conn 49 | } 50 | } 51 | 52 | 53 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/utils/JedisWrapper.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.utils 2 | 3 | import com.gmall.data.common.config.RedisConfig 4 | import org.slf4j.LoggerFactory 5 | import redis.clients.jedis.Jedis 6 | 7 | /** 8 | * Redis表操作的封装 9 | */ 10 | object JedisWrapper { 11 | val logger = LoggerFactory.getLogger(this.getClass) 12 | 13 | def wrap(op: Jedis => Unit, errmsg: String)(implicit redisConfig: RedisConfig): Unit = { 14 | var jedisConn: Jedis = null 15 | try { 16 | jedisConn = JedisConnectionPool.connect 17 | op.apply(jedisConn) 18 | } catch { 19 | case e: Exception => logger.error(errmsg, e) 20 | } finally { 21 | if (jedisConn != null) { 22 | jedisConn.close() 23 | } 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/utils/LoggerUtil.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.utils 2 | 3 | import java.util.concurrent.TimeoutException 4 | 5 | import org.slf4j.Logger 6 | 7 | /** 8 | * 日志公共类 9 | */ 10 | object LoggerUtil { 11 | 12 | /** 13 | * 打印错误日志 14 | * 15 | * @param logger Logger 对象 16 | * @param e 错误对象 17 | * @param message 消息体 18 | */ 19 | def error(logger: Logger, e: Exception, message: String) = { 20 | logger.error(message + ":{},{},{}", e.getClass(), e.getMessage, e.getStackTrace) 21 | } 22 | 23 | def error(logger: Logger, e: InterruptedException, message: String) = { 24 | logger.error(message + ":{},{},{}", e.getClass(), e.getMessage, e.getStackTrace) 25 | } 26 | 27 | def error(logger: Logger, e: TimeoutException, message: String) = { 28 | logger.error(message + ":{},{},{}", e.getClass(), e.getMessage, e.getStackTrace) 29 | } 30 | 31 | def error(logger: Logger, message: String): Unit ={ 32 | logger.error(message + ":{}", message) 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/utils/StringUtil.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.utils 2 | 3 | import org.apache.commons.lang3.StringUtils 4 | 5 | /** 6 | * StringUtil 7 | */ 8 | object StringUtil { 9 | 10 | def keyValue(srcStr: String, split1: Char, split2: Char): Map[String, String] = { 11 | srcStr 12 | .split(split1) 13 | .map(kv => if (kv.contains(split2)) { 14 | (kv.substring(0, kv.indexOf(split2)), kv.substring(kv.indexOf(split2) + 1)) 15 | } else null) 16 | .filter(_ != null) 17 | .toMap 18 | } 19 | 20 | def paramMap(params: String): Map[String, String] = { 21 | if (StringUtils.isEmpty(params)) return Map() 22 | keyValue(params, '&', '=') 23 | } 24 | 25 | def isAnyNotEmpty(strs: String*): Boolean = { 26 | 27 | var count = strs.length 28 | for (str <- strs) { 29 | if (str == null || str.isEmpty) 30 | count -= 1 31 | } 32 | 33 | count != 0 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/utils/TimeUtil.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.utils 2 | 3 | import java.time.{Instant, LocalDateTime, ZoneId} 4 | 5 | /** 6 | * 时间工具类 7 | */ 8 | object TimeUtil { 9 | 10 | /** 11 | * 将long类型时间戳转换为字符串格式 12 | * @param milliseconds 13 | * @return 14 | */ 15 | def getDateTimeString(milliseconds: Long): String = { 16 | Constants.DATE_TIME_FORMATTER.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(milliseconds), ZoneId.systemDefault())) 17 | } 18 | 19 | /** 20 | * 将long类型时间戳转换为dt类型 21 | * @param milliseconds 22 | * @return 23 | */ 24 | def getDateString(milliseconds: Long): String = { 25 | Constants.DT_DATE_TIME_FORMATTER.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(milliseconds), ZoneId.systemDefault())) 26 | } 27 | 28 | /** 29 | * 将string类型时间转换为long类型时间戳 30 | * @param time 31 | * @return 32 | */ 33 | def getTimestamp(time: String): Long = try { 34 | LocalDateTime.from(LocalDateTime.parse(time, Constants.DATE_TIME_FORMATTER)).atZone(ZoneId.systemDefault()).toInstant.toEpochMilli 35 | } catch { 36 | case _: Throwable => 0L 37 | } 38 | 39 | /** 40 | * 将业务库中的秒级时间转换为dt类型 41 | * @param datetime 42 | * @return 43 | */ 44 | def formatDt(datetime: String): String = { 45 | Constants.DT_DATE_TIME_FORMATTER.format(LocalDateTime.parse(datetime, Constants.DATE_TIME_MIN_FORMATTER)) 46 | } 47 | 48 | 49 | } 50 | -------------------------------------------------------------------------------- /common/src/main/scala/com/gmall/data/common/utils/Util.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.utils 2 | 3 | object Util { 4 | 5 | def tableClassName(table: String): String = 6 | table.split("_").map(_.capitalize).mkString("") 7 | 8 | def databasePackageName(database: String): String = database 9 | 10 | /** 11 | * string类型转double 12 | * @param item 13 | * @return 14 | */ 15 | def toDouble(item: String): Double = 16 | try { 17 | item.toDouble 18 | } catch { 19 | case _: Throwable => 0 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /common/src/test/java/FlinkRedisConnectorTest.scala: -------------------------------------------------------------------------------- 1 | import org.apache.flink.streaming.api.scala._ 2 | import org.apache.flink.table.api.EnvironmentSettings 3 | import org.apache.flink.table.api.bridge.scala.StreamTableEnvironment 4 | 5 | object FlinkRedisConnectorTest { 6 | 7 | def main(args: Array[String]): Unit = { 8 | val env = StreamExecutionEnvironment.getExecutionEnvironment 9 | env.setParallelism(1) 10 | val settings = EnvironmentSettings.newInstance().useBlinkPlanner().inStreamingMode().build() 11 | val tableEnv = StreamTableEnvironment.create(env, settings) 12 | 13 | tableEnv.executeSql( 14 | """ 15 | |CREATE TABLE `orders` ( 16 | | `order_uid` STRING, 17 | | `product_id` STRING, 18 | | `price` DOUBLE 19 | |) 20 | |COMMENT '' 21 | |WITH ( 22 | | 'connector' = 'datagen' 23 | |) 24 | |""".stripMargin) 25 | 26 | tableEnv.executeSql("SELECT * FROM orders").print() 27 | 28 | 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /common/src/test/java/KafkaConsumerDemo.scala: -------------------------------------------------------------------------------- 1 | import java.util.Properties 2 | 3 | import org.apache.flink.api.common.serialization.SimpleStringSchema 4 | import org.apache.flink.streaming.api.scala._ 5 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer 6 | 7 | object KafkaConsumerDemo { 8 | 9 | def main(args: Array[String]): Unit = { 10 | 11 | val env = StreamExecutionEnvironment.getExecutionEnvironment 12 | env.setParallelism(1) 13 | 14 | val properties = new Properties() 15 | properties.setProperty("bootstrap.servers", "sc-data-03.yit.net:9092") 16 | properties.setProperty("group.id", "kafkaConfig.groupId") 17 | 18 | val kafkaConsumer = new FlinkKafkaConsumer[String]("event_topic", new SimpleStringSchema(), properties) 19 | 20 | // kafkaConsumer.setStartFromEarliest() 21 | 22 | env.addSource(kafkaConsumer).print() 23 | 24 | 25 | env.execute("job") 26 | 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /common/src/test/java/KafkaConsumerTest.scala: -------------------------------------------------------------------------------- 1 | import java.util.Properties 2 | 3 | import com.gmall.data.common.config.{Config, KafkaConfig} 4 | import com.gmall.data.common.entity.ods.gmall2021.OrderInfo 5 | import com.gmall.data.common.source.SourceFactory 6 | import com.gmall.data.common.utils.Constants 7 | import org.apache.flink.api.java.utils.ParameterTool 8 | import org.apache.flink.streaming.api.scala._ 9 | import org.slf4j.LoggerFactory 10 | 11 | object KafkaConsumerTest { 12 | 13 | lazy val logger = LoggerFactory.getLogger(this.getClass) 14 | val OPTION_MODE = "mode" 15 | val OPTION_TIMESTAMP = "timestamp" 16 | 17 | def main(args: Array[String]): Unit = { 18 | 19 | val parameterTool = ParameterTool.fromArgs(args) 20 | 21 | val mode = parameterTool.get(OPTION_MODE, Constants.CONSUMER_MODE_COMMITTED) 22 | val timestamp = parameterTool.get(OPTION_TIMESTAMP, "") 23 | 24 | implicit val env = StreamExecutionEnvironment.getExecutionEnvironment 25 | env.setParallelism(1) 26 | 27 | val kafkaConfig = KafkaConfig(Config.kafkaBrokers, "test", mode, timestamp) 28 | 29 | val value = SourceFactory.createBinlogStream[OrderInfo](kafkaConfig, "gmall2021.order_info") 30 | // SourceFactory.createKafkaStream[FlatMessage](kafkaConfig, "gmall2021.order_info").print() 31 | 32 | value.print() 33 | 34 | env.execute("job") 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /docs/0项目简介.md: -------------------------------------------------------------------------------- 1 | **`写在前面`** 2 | 3 | 随着数据科学的进步以及处理技术的成熟，业务/开发人员早已不在满足于传统的离线数仓报表或者提前设置好维度与指标的OLAP查询。近些年来，实时场景需求的不断增加，比如实时报表、实时指标、实时OLAP、实时监控等，一方面对实时数据提出了更多的要求，另一方面也促进了实时处理框架的发展，从storm、spark streaming、flink，flink作为目前最热门的实时处理框架，我们将从它开始慢慢揭露实时数仓的秘密。 4 | 5 | 该项目以尚硅谷大数据实时数仓项目为基础，结合电商互联网公司实时数据场景，对项目作了一定的扩展，从实时数仓理论基础开始，试着让读者理解实时数仓，并熟悉互联网领域常见的数据需求，学到有用的技术。时间回到2018年，作为转行入门大数据，当时的学习资源还没这么丰富，最开始在网上找视频看(在这里再次鸣谢尚硅谷)，然后和大部分群友一样，来到一线城市，海投简历...我也将把自己的经历分享给大家 6 | 7 | **`项目技术栈`** 8 | 9 | - 开发语言：Java、Scala、Sql 10 | - 数据库：Mysql 11 | - 数据采集：Canal 12 | - 消息队列：Kafka 13 | - 实时计算：Flink 14 | - 实时OLAP：ClickHouse 15 | - 数据存储：Phoenix、HBase、Redis 16 | - 监控：Prometheus、Grafana 17 | - 实时计算平台：基于Flink Sql、SpringBoot 18 | - 实时数据服务：Spring、SpringBoot 19 | 20 | 如果你对以上某些技术栈并不熟悉，没有关系，我们将会一个一个深入学习。 21 | 22 | **`一个通用的实时数仓架构`** 23 | 24 | ![image-20210402102703175](项目简介.assets/image-20210402102703175.png) -------------------------------------------------------------------------------- /docs/1数据产品需求.md: -------------------------------------------------------------------------------- 1 | # 产品需求 2 | 3 | 首先需要了解一个互联网公司有哪些常见的数据需求，在公司发展前期，往往是基于需求来驱动数据平台的发展的。这里我们以一个电商互联网为例，借鉴神策数据平台体验demo，了解数据需求。 4 | 5 | 6 | 7 | # 数据概览 8 | 9 | 也就是通常说的数据大屏，一般来说，大屏包含了一些基础的指标数据。 10 | 11 | ![image-20210405165428593](产品需求.assets/image-20210405165428593.png) 12 | 13 | ## 整体概览 14 | 15 | ![image-20210405165204052](产品需求.assets/image-20210405165204052.png) 16 | 17 | 这里列举了几个基础的指标，包括实时指标与离线指标。 18 | 19 | 20 | 21 | ## 商品概览 22 | 23 | ![image-20210405205643466](产品需求.assets/image-20210405205643466.png) 24 | 25 | ## 运营看板 26 | 27 | ![image-20210405205804475](产品需求.assets/image-20210405205804475.png) 28 | 29 | ## 功能看板 30 | 31 | ![image-20210405205922293](产品需求.assets/image-20210405205922293.png) 32 | 33 | ## 实时统计 34 | 35 | ![image-20210405210015951](产品需求.assets/image-20210405210015951.png) 36 | 37 | 38 | 39 | # 数据分析 40 | 41 | 一般是一些OLAP的分析，能够支持用户自定义的维度以及度量值做灵活查询。 42 | 43 | ## 漏斗分析 44 | 45 | ![image-20210405210345486](产品需求.assets/image-20210405210345486.png) 46 | 47 | 这里就有一个典型的漏斗分析的OLAP需求，是需要实时数仓的支持的。 48 | 49 | ## 留存分析 50 | 51 | ![image-20210405210615660](产品需求.assets/image-20210405210615660.png) 52 | 53 | ## 用户路径 54 | 55 | ![image-20210405210828060](产品需求.assets/image-20210405210828060.png) 56 | 57 | ## 归因分析 58 | 59 | ![image-20210405211124781](产品需求.assets/image-20210405211124781.png) 60 | 61 | 这里就有一个很典型的订单归因分析，比如我们想实时的知道一个用户从打开APP开始到提交订单的转化贡献数据。 62 | 63 | 64 | 65 | # 用户画像 66 | 67 | **TODO** 68 | 69 | # 总结 70 | 71 | - 互联网公司的一些实时数据需求。 -------------------------------------------------------------------------------- /docs/3实时数仓分层设计.assets/224955E00-1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/3实时数仓分层设计.assets/224955E00-1.jpg -------------------------------------------------------------------------------- /docs/3实时数仓分层设计.assets/224955I30-0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/3实时数仓分层设计.assets/224955I30-0.jpg -------------------------------------------------------------------------------- /docs/3实时数仓分层设计.assets/224955O15-2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/3实时数仓分层设计.assets/224955O15-2.jpg -------------------------------------------------------------------------------- /docs/3实时数仓分层设计.assets/640-20210518140232661.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/3实时数仓分层设计.assets/640-20210518140232661.png -------------------------------------------------------------------------------- /docs/3实时数仓分层设计.assets/640-20210518140232689.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/3实时数仓分层设计.assets/640-20210518140232689.png -------------------------------------------------------------------------------- /docs/3实时数仓分层设计.assets/640-20210518140233105.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/3实时数仓分层设计.assets/640-20210518140233105.png -------------------------------------------------------------------------------- /docs/3实时数仓分层设计.assets/640.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/3实时数仓分层设计.assets/640.png -------------------------------------------------------------------------------- /docs/3实时数仓分层设计.assets/p44562.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/3实时数仓分层设计.assets/p44562.png -------------------------------------------------------------------------------- /docs/3实时数仓分层设计.assets/p58877.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/3实时数仓分层设计.assets/p58877.png -------------------------------------------------------------------------------- /docs/3实时数仓分层设计.assets/p59651.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/3实时数仓分层设计.assets/p59651.png -------------------------------------------------------------------------------- /docs/3实时数仓设计.assets/640: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/3实时数仓设计.assets/640 -------------------------------------------------------------------------------- /docs/3实时数仓设计.assets/640-20210515184302228: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/3实时数仓设计.assets/640-20210515184302228 -------------------------------------------------------------------------------- /docs/3实时数仓设计.assets/640-20210515184302234: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/3实时数仓设计.assets/640-20210515184302234 -------------------------------------------------------------------------------- /docs/3实时数仓设计.assets/640-20210515184302235: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/3实时数仓设计.assets/640-20210515184302235 -------------------------------------------------------------------------------- /docs/3实时数仓设计.assets/640-20210515184302239: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/3实时数仓设计.assets/640-20210515184302239 -------------------------------------------------------------------------------- /docs/3实时数仓设计.assets/640-20210515184302246: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/3实时数仓设计.assets/640-20210515184302246 -------------------------------------------------------------------------------- /docs/3实时数仓设计.assets/640-20210515184302246-1075382.: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/3实时数仓设计.assets/640-20210515184302246-1075382. -------------------------------------------------------------------------------- /docs/3实时数仓设计.assets/640-20210515184302251: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/3实时数仓设计.assets/640-20210515184302251 -------------------------------------------------------------------------------- /docs/3实时数仓设计.assets/640-20210515184302255.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/3实时数仓设计.assets/640-20210515184302255.png -------------------------------------------------------------------------------- /docs/3实时数仓设计.assets/640-20210515184302298.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/3实时数仓设计.assets/640-20210515184302298.png -------------------------------------------------------------------------------- /docs/3实时数仓设计.assets/640-20210515184302307.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/3实时数仓设计.assets/640-20210515184302307.png -------------------------------------------------------------------------------- /docs/3实时数仓设计.assets/640-20210515184302375.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/3实时数仓设计.assets/640-20210515184302375.png -------------------------------------------------------------------------------- /docs/3实时数仓设计.assets/Untitled.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/3实时数仓设计.assets/Untitled.md -------------------------------------------------------------------------------- /docs/8实时指标.assets/640-20210516224001799: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/8实时指标.assets/640-20210516224001799 -------------------------------------------------------------------------------- /docs/8实时指标.assets/640-20210516225935573.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/8实时指标.assets/640-20210516225935573.png -------------------------------------------------------------------------------- /docs/8实时指标.assets/640-20210516232619115: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/8实时指标.assets/640-20210516232619115 -------------------------------------------------------------------------------- /docs/8实时指标.assets/640-20210516232619151.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/8实时指标.assets/640-20210516232619151.png -------------------------------------------------------------------------------- /docs/8实时指标.assets/640-20210516232619160.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/8实时指标.assets/640-20210516232619160.png -------------------------------------------------------------------------------- /docs/8实时指标.assets/640-20210516232730698.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/8实时指标.assets/640-20210516232730698.png -------------------------------------------------------------------------------- /docs/8实时指标.assets/640-20210516233805869.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/8实时指标.assets/640-20210516233805869.png -------------------------------------------------------------------------------- /docs/8实时指标.assets/FFBekRjksJE8Q4rtGN28.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/8实时指标.assets/FFBekRjksJE8Q4rtGN28.png -------------------------------------------------------------------------------- /docs/8实时指标.assets/Qktv9k4uWadLTLiXqNOz.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/8实时指标.assets/Qktv9k4uWadLTLiXqNOz.png -------------------------------------------------------------------------------- /docs/8实时指标.assets/U0PhVDw2bYXZmBKdBrjR.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/8实时指标.assets/U0PhVDw2bYXZmBKdBrjR.png -------------------------------------------------------------------------------- /docs/8实时指标.assets/ZoCJhNbfM6TwIqBvP7t4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/8实时指标.assets/ZoCJhNbfM6TwIqBvP7t4.png -------------------------------------------------------------------------------- /docs/8实时指标.assets/append-mode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/8实时指标.assets/append-mode.png -------------------------------------------------------------------------------- /docs/8实时指标.assets/image-20210516235107922.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/8实时指标.assets/image-20210516235107922.png -------------------------------------------------------------------------------- /docs/8实时指标.assets/image-20210518232430888.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/8实时指标.assets/image-20210518232430888.png -------------------------------------------------------------------------------- /docs/8实时指标.assets/image-20210518232504987.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/8实时指标.assets/image-20210518232504987.png -------------------------------------------------------------------------------- /docs/8实时指标.assets/image-20210518233251037.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/8实时指标.assets/image-20210518233251037.png -------------------------------------------------------------------------------- /docs/8实时指标.assets/image-20210519223858298.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/8实时指标.assets/image-20210519223858298.png -------------------------------------------------------------------------------- /docs/8实时指标.assets/image-20210522004340330.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/8实时指标.assets/image-20210522004340330.png -------------------------------------------------------------------------------- /docs/8实时指标.assets/image-20210522004416835.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/8实时指标.assets/image-20210522004416835.png -------------------------------------------------------------------------------- /docs/8实时指标.assets/pbfzmZFLAeBF4O6cFtF2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/8实时指标.assets/pbfzmZFLAeBF4O6cFtF2.png -------------------------------------------------------------------------------- /docs/8实时指标.assets/query-groupBy-cnt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/8实时指标.assets/query-groupBy-cnt.png -------------------------------------------------------------------------------- /docs/8实时指标.assets/query-groupBy-window-cnt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/8实时指标.assets/query-groupBy-window-cnt.png -------------------------------------------------------------------------------- /docs/8实时指标.assets/stream-query-stream.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/8实时指标.assets/stream-query-stream.png -------------------------------------------------------------------------------- /docs/9实时OLAP.assets/2bd1edeecda8a379bde481389ddff425.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/9实时OLAP.assets/2bd1edeecda8a379bde481389ddff425.png -------------------------------------------------------------------------------- /docs/9实时OLAP.assets/640.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/9实时OLAP.assets/640.jpeg -------------------------------------------------------------------------------- /docs/9实时OLAP.assets/640.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/9实时OLAP.assets/640.png -------------------------------------------------------------------------------- /docs/9实时OLAP.assets/Data-Cube.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/9实时OLAP.assets/Data-Cube.png -------------------------------------------------------------------------------- /docs/9实时OLAP.assets/OLAP.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/9实时OLAP.assets/OLAP.png -------------------------------------------------------------------------------- /docs/9实时OLAP.assets/image-20210523193954290.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/9实时OLAP.assets/image-20210523193954290.png -------------------------------------------------------------------------------- /docs/9实时OLAP.assets/image-20210523194032047.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/9实时OLAP.assets/image-20210523194032047.png -------------------------------------------------------------------------------- /docs/9实时OLAP.assets/image-20210524001623967.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/9实时OLAP.assets/image-20210524001623967.png -------------------------------------------------------------------------------- /docs/9实时OLAP.assets/kylin_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/9实时OLAP.assets/kylin_diagram.png -------------------------------------------------------------------------------- /docs/9实时OLAP.assets/v2-6a455b3039e28e98b7592f66910a0793_1440w.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/9实时OLAP.assets/v2-6a455b3039e28e98b7592f66910a0793_1440w.jpg -------------------------------------------------------------------------------- /docs/Apache Kylin与ClickHouse 的对比.assets/640: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/Apache Kylin与ClickHouse 的对比.assets/640 -------------------------------------------------------------------------------- /docs/Apache Kylin与ClickHouse 的对比.assets/640-20210523225447666.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/Apache Kylin与ClickHouse 的对比.assets/640-20210523225447666.png -------------------------------------------------------------------------------- /docs/Apache Kylin与ClickHouse 的对比.assets/640-20210523225447678.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/Apache Kylin与ClickHouse 的对比.assets/640-20210523225447678.png -------------------------------------------------------------------------------- /docs/Apache Kylin与ClickHouse 的对比.assets/640-20210523225447718.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/Apache Kylin与ClickHouse 的对比.assets/640-20210523225447718.png -------------------------------------------------------------------------------- /docs/Apache Kylin与ClickHouse 的对比.assets/640-20210523225447735: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/Apache Kylin与ClickHouse 的对比.assets/640-20210523225447735 -------------------------------------------------------------------------------- /docs/Apache Kylin与ClickHouse 的对比.assets/640-20210523225447783.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/Apache Kylin与ClickHouse 的对比.assets/640-20210523225447783.png -------------------------------------------------------------------------------- /docs/Apache Kylin与ClickHouse 的对比.assets/640-20210523225447860.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/Apache Kylin与ClickHouse 的对比.assets/640-20210523225447860.png -------------------------------------------------------------------------------- /docs/Apache Kylin与ClickHouse 的对比.assets/640-20210523225447981.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/Apache Kylin与ClickHouse 的对比.assets/640-20210523225447981.png -------------------------------------------------------------------------------- /docs/DWD层数据准备.assets/image-20210407224537117.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/DWD层数据准备.assets/image-20210407224537117.png -------------------------------------------------------------------------------- /docs/DWD层数据准备.assets/image-20210409233246633.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/DWD层数据准备.assets/image-20210409233246633.png -------------------------------------------------------------------------------- /docs/DWD层数据准备.assets/image-20210410000728103.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/DWD层数据准备.assets/image-20210410000728103.png -------------------------------------------------------------------------------- /docs/DWD层数据准备.assets/image-20210410000759718.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/DWD层数据准备.assets/image-20210410000759718.png -------------------------------------------------------------------------------- /docs/DWD层数据准备.assets/image-20210413225611694.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/DWD层数据准备.assets/image-20210413225611694.png -------------------------------------------------------------------------------- /docs/DWD层数据准备.assets/image-20210413230248772.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/DWD层数据准备.assets/image-20210413230248772.png -------------------------------------------------------------------------------- /docs/DWD层数据准备.assets/image-20210413230458950.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/DWD层数据准备.assets/image-20210413230458950.png -------------------------------------------------------------------------------- /docs/DWD层数据准备.assets/image-20210413230514762.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/DWD层数据准备.assets/image-20210413230514762.png -------------------------------------------------------------------------------- /docs/DWD层数据准备.assets/image-20210413231328430.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/DWD层数据准备.assets/image-20210413231328430.png -------------------------------------------------------------------------------- /docs/DWD层数据准备.assets/image-20210418152453150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/DWD层数据准备.assets/image-20210418152453150.png -------------------------------------------------------------------------------- /docs/业务数据采集.assets/image-20210329235325068.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/业务数据采集.assets/image-20210329235325068.png -------------------------------------------------------------------------------- /docs/业务数据采集.assets/image-20210329235403290.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/业务数据采集.assets/image-20210329235403290.png -------------------------------------------------------------------------------- /docs/业务数据采集.assets/image-20210401220353400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/业务数据采集.assets/image-20210401220353400.png -------------------------------------------------------------------------------- /docs/业务数据采集.assets/image-20210404232758656.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/业务数据采集.assets/image-20210404232758656.png -------------------------------------------------------------------------------- /docs/业务数据采集.assets/image-20210404233127046.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/业务数据采集.assets/image-20210404233127046.png -------------------------------------------------------------------------------- /docs/业务数据采集.assets/image-20210404233331763.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/业务数据采集.assets/image-20210404233331763.png -------------------------------------------------------------------------------- /docs/产品需求.assets/image-20210405165204052.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/产品需求.assets/image-20210405165204052.png -------------------------------------------------------------------------------- /docs/产品需求.assets/image-20210405165428593.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/产品需求.assets/image-20210405165428593.png -------------------------------------------------------------------------------- /docs/产品需求.assets/image-20210405205643466.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/产品需求.assets/image-20210405205643466.png -------------------------------------------------------------------------------- /docs/产品需求.assets/image-20210405205804475.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/产品需求.assets/image-20210405205804475.png -------------------------------------------------------------------------------- /docs/产品需求.assets/image-20210405205922293.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/产品需求.assets/image-20210405205922293.png -------------------------------------------------------------------------------- /docs/产品需求.assets/image-20210405210015951.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/产品需求.assets/image-20210405210015951.png -------------------------------------------------------------------------------- /docs/产品需求.assets/image-20210405210345486.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/产品需求.assets/image-20210405210345486.png -------------------------------------------------------------------------------- /docs/产品需求.assets/image-20210405210615660.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/产品需求.assets/image-20210405210615660.png -------------------------------------------------------------------------------- /docs/产品需求.assets/image-20210405210828060.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/产品需求.assets/image-20210405210828060.png -------------------------------------------------------------------------------- /docs/产品需求.assets/image-20210405211124781.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/产品需求.assets/image-20210405211124781.png -------------------------------------------------------------------------------- /docs/基于docker部署canal环境.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/基于docker部署canal环境.md -------------------------------------------------------------------------------- /docs/如何从0到1构建指标体系.assets/2sVUM4J4nNKHOdqCLVJi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/如何从0到1构建指标体系.assets/2sVUM4J4nNKHOdqCLVJi.png -------------------------------------------------------------------------------- /docs/如何从0到1构建指标体系.assets/66eVXE58NipBmJO4Zsvu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/如何从0到1构建指标体系.assets/66eVXE58NipBmJO4Zsvu.png -------------------------------------------------------------------------------- /docs/如何从0到1构建指标体系.assets/FFBekRjksJE8Q4rtGN28-20210516233233469.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/如何从0到1构建指标体系.assets/FFBekRjksJE8Q4rtGN28-20210516233233469.png -------------------------------------------------------------------------------- /docs/如何从0到1构建指标体系.assets/FltPuhpqlLUUPeLzk7cJ.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/如何从0到1构建指标体系.assets/FltPuhpqlLUUPeLzk7cJ.png -------------------------------------------------------------------------------- /docs/如何从0到1构建指标体系.assets/IODeeazNNoOlmLJ4S0Lu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/如何从0到1构建指标体系.assets/IODeeazNNoOlmLJ4S0Lu.png -------------------------------------------------------------------------------- /docs/如何从0到1构建指标体系.assets/KkQIpy08fKwfnLJgKZQI.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/如何从0到1构建指标体系.assets/KkQIpy08fKwfnLJgKZQI.png -------------------------------------------------------------------------------- /docs/如何从0到1构建指标体系.assets/Qktv9k4uWadLTLiXqNOz-20210516233233452.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/如何从0到1构建指标体系.assets/Qktv9k4uWadLTLiXqNOz-20210516233233452.png -------------------------------------------------------------------------------- /docs/如何从0到1构建指标体系.assets/Rv2rBKjEaRKnQ8xuHMmW.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/如何从0到1构建指标体系.assets/Rv2rBKjEaRKnQ8xuHMmW.png -------------------------------------------------------------------------------- /docs/如何从0到1构建指标体系.assets/TAOT112YZQUkjsxPBEv1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/如何从0到1构建指标体系.assets/TAOT112YZQUkjsxPBEv1.jpg -------------------------------------------------------------------------------- /docs/如何从0到1构建指标体系.assets/U0PhVDw2bYXZmBKdBrjR-20210516233233322.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/如何从0到1构建指标体系.assets/U0PhVDw2bYXZmBKdBrjR-20210516233233322.png -------------------------------------------------------------------------------- /docs/如何从0到1构建指标体系.assets/WtbMmcHjgmkuFiFLhP8g.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/如何从0到1构建指标体系.assets/WtbMmcHjgmkuFiFLhP8g.png -------------------------------------------------------------------------------- /docs/如何从0到1构建指标体系.assets/ZoCJhNbfM6TwIqBvP7t4-20210516233233415.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/如何从0到1构建指标体系.assets/ZoCJhNbfM6TwIqBvP7t4-20210516233233415.png -------------------------------------------------------------------------------- /docs/如何从0到1构建指标体系.assets/gDwC1ZSk527aLIdCTUbY.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/如何从0到1构建指标体系.assets/gDwC1ZSk527aLIdCTUbY.png -------------------------------------------------------------------------------- /docs/如何从0到1构建指标体系.assets/hpipN9GYAin3xPDvJG3L.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/如何从0到1构建指标体系.assets/hpipN9GYAin3xPDvJG3L.png -------------------------------------------------------------------------------- /docs/如何从0到1构建指标体系.assets/itMgHpcqrkHHogkaO7A1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/如何从0到1构建指标体系.assets/itMgHpcqrkHHogkaO7A1.png -------------------------------------------------------------------------------- /docs/如何从0到1构建指标体系.assets/pbfzmZFLAeBF4O6cFtF2-20210516233233439.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/如何从0到1构建指标体系.assets/pbfzmZFLAeBF4O6cFtF2-20210516233233439.png -------------------------------------------------------------------------------- /docs/如何从0到1构建指标体系.assets/r82Fof1uKLtZa7gtsbe4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/如何从0到1构建指标体系.assets/r82Fof1uKLtZa7gtsbe4.png -------------------------------------------------------------------------------- /docs/如何从0到1构建指标体系.assets/u7LlkgKdFht23fDZvEAO.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/如何从0到1构建指标体系.assets/u7LlkgKdFht23fDZvEAO.png -------------------------------------------------------------------------------- /docs/实时数仓架构.assets/image-20210328225453095.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/实时数仓架构.assets/image-20210328225453095.png -------------------------------------------------------------------------------- /docs/实时数仓架构.assets/image-20210328231618797.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/实时数仓架构.assets/image-20210328231618797.png -------------------------------------------------------------------------------- /docs/实时数仓架构.assets/image-20210328232802457.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/实时数仓架构.assets/image-20210328232802457.png -------------------------------------------------------------------------------- /docs/实时数仓架构.assets/image-20210328233222184.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/实时数仓架构.assets/image-20210328233222184.png -------------------------------------------------------------------------------- /docs/实时数仓架构.assets/image-20210328233335006.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/实时数仓架构.assets/image-20210328233335006.png -------------------------------------------------------------------------------- /docs/实时数仓架构.assets/image-20210328233720452.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/实时数仓架构.assets/image-20210328233720452.png -------------------------------------------------------------------------------- /docs/实时数仓架构.assets/image-20210411153254131.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/实时数仓架构.assets/image-20210411153254131.png -------------------------------------------------------------------------------- /docs/数据埋点需求.assets/link-solid.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/用户行为数据采集.assets/640.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/用户行为数据采集.assets/640.png -------------------------------------------------------------------------------- /docs/项目简介.assets/image-20210402102703175.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fengchi66/realtime-dw/b819109175c2bfabd8a9fbb1806c4a4d3f0646de/docs/项目简介.assets/image-20210402102703175.png -------------------------------------------------------------------------------- /dwd2dws/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | realtime-dw 7 | com.gmall.data 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | dwd2dws 13 | 14 | 15 | 16 | com.gmall.data 17 | common 18 | 1.0-SNAPSHOT 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /dwd2dws/src/main/scala/com/gmall/data/App.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data 2 | 3 | import org.apache.flink.streaming.api.scala._ 4 | import org.apache.flink.table.api.EnvironmentSettings 5 | import org.apache.flink.table.api.bridge.scala.StreamTableEnvironment 6 | import org.apache.flink.types.Row 7 | 8 | object App { 9 | 10 | def main(args: Array[String]): Unit = { 11 | 12 | val env = StreamExecutionEnvironment.getExecutionEnvironment 13 | env.setParallelism(1) 14 | 15 | val bsSettings = EnvironmentSettings.newInstance.useBlinkPlanner.inStreamingMode.build 16 | val tableEnv = StreamTableEnvironment.create(env, bsSettings) 17 | 18 | // tableEnv.executeSql(Sql.sales_kafka_source) 19 | tableEnv.executeSql(Sql.dwd_user_action_log) 20 | tableEnv.executeSql(Sql.dws_core_point) 21 | tableEnv.execute(Sql.dws_flow_app_pv_10min) 22 | 23 | // tableEnv.executeSql( 24 | // """ 25 | // |INSERT INTO dws_core_point 26 | // |SELECT 27 | // | CONCAT_WS(':', 'dws_core_point', DATE_FORMAT(event_time_stamp, 'yyyyMMdd')), 28 | // | 'uv', 29 | // | CAST(COUNT(DISTINCT(user_id)) AS STRING) 30 | // |FROM dwd_user_action_log 31 | // |GROUP BY CONCAT_WS(':', 'dws_core_point', DATE_FORMAT(event_time_stamp, 'yyyyMMdd')) 32 | // | 33 | // |""".stripMargin) 34 | 35 | // tableEnv.toAppendStream[Row](table).print() 36 | 37 | tableEnv.executeSql( 38 | """ 39 | |INSERT INTO dws_flow_app_pv_10min 40 | |SELECT 41 | | app_name, 42 | | TUMBLE_START(event_time_stamp, INTERVAL '10' MINUTE), 43 | | TUMBLE_END(event_time_stamp, INTERVAL '10' MINUTE), 44 | | COUNT(1) 45 | |FROM dwd_user_action_log 46 | |GROUP BY TUMBLE(event_time_stamp, INTERVAL '10' MINUTE), app_name") 47 | | 48 | |""".stripMargin) 49 | 50 | 51 | // env.execute("job") 52 | 53 | 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /ods2dim/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | realtime-dw 7 | com.gmall.data 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | ods2dim 13 | 14 | 15 | 16 | com.gmall.data 17 | common 18 | 1.0-SNAPSHOT 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /ods2dim/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, console 2 | 3 | log4j.logger.org.apache.kafka=WARN 4 | log4j.logger.org.apache.flink=WARN 5 | log4j.logger.com.alibaba.blink=WARN 6 | 7 | log4j.appender.console=org.apache.log4j.ConsoleAppender 8 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 9 | log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-30c %-30t %x - %m%n 10 | -------------------------------------------------------------------------------- /ods2dim/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /ods2dim/src/main/scala/com/gmall/data/dim/App.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.dim 2 | 3 | import java.util.concurrent.TimeUnit 4 | 5 | import com.gmall.data.common.config.{Config, KafkaConfig} 6 | import com.gmall.data.common.utils.Constants 7 | import org.apache.flink.api.common.restartstrategy.RestartStrategies 8 | import org.apache.flink.api.common.time.Time 9 | import org.apache.flink.api.java.utils.ParameterTool 10 | import org.apache.flink.streaming.api.scala._ 11 | 12 | /** 13 | * 启动参数有-mode earliest/latest/committed/timestamp -timestamp "2021-04-10 00:00:00" 14 | * -mode earliest从kafka topic中最早开始消费 15 | * -mode latest从kafka topic中最新开始消费 16 | * -mode committed从flink 上次commit的消息开始消费 17 | * -mode timestamp从指定时间戳开始消费 18 | * -timestamp 只适用于timestamp mode 19 | */ 20 | 21 | object App { 22 | val OPTION_MODE = "mode" 23 | val OPTION_TIMESTAMP = "timestamp" 24 | val GROUP_ID = "ods-dim" 25 | 26 | def main(args: Array[String]): Unit = { 27 | val parameterTool = ParameterTool.fromArgs(args) 28 | 29 | val mode = parameterTool.get(OPTION_MODE, Constants.CONSUMER_MODE_COMMITTED) 30 | val timestamp = parameterTool.get(OPTION_TIMESTAMP, "") 31 | 32 | implicit val env = StreamExecutionEnvironment.getExecutionEnvironment 33 | env.enableCheckpointing(120000) 34 | env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, Time.of(20, TimeUnit.SECONDS))) 35 | env.setParallelism(1) 36 | 37 | val kafkaConfig = KafkaConfig(Config.kafkaBrokers, GROUP_ID, mode, timestamp) 38 | 39 | StreamTopology.build(kafkaConfig) 40 | 41 | env.execute("ods-dim-streaming-job") 42 | 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /ods2dim/src/main/scala/com/gmall/data/dim/Step.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.dim 2 | 3 | import com.gmall.data.common.entity.ods.gmall2021.{BaseCategory3, BaseProvince, BaseTrademark, SkuInfo, SpuInfo, UserInfo} 4 | import com.gmall.data.dim.sink.{Category3InfoSink, ProvinceInfoSink, SkuInfoSink, SpuInfoSink, TradeInfoSink, UserInfoSink} 5 | import org.apache.flink.streaming.api.scala._ 6 | 7 | object Step { 8 | 9 | /** 10 | * user_info维度表写HBase 11 | * @param input 12 | * @return 13 | */ 14 | implicit def userInfoSink(input: DataStream[UserInfo]) = 15 | UserInfoSink(input) 16 | 17 | /** 18 | * 商品: spu维度表写入Hbase 19 | * @param input 20 | * @return 21 | */ 22 | implicit def spuInfoSink(input: DataStream[SpuInfo]) = 23 | SpuInfoSink(input) 24 | 25 | /** 26 | * 商品: sku维度表写入HBase 27 | * @param input 28 | * @return 29 | */ 30 | implicit def skuInfoSink(input: DataStream[SkuInfo]) = 31 | SkuInfoSink(input) 32 | 33 | /** 34 | * 省份信息维度表写Hbase 35 | * @param input 36 | * @return 37 | */ 38 | implicit def provinceInfoSink(input: DataStream[BaseProvince]) = 39 | ProvinceInfoSink(input) 40 | 41 | /** 42 | * 品牌维度表写HBase 43 | * @param input 44 | * @return 45 | */ 46 | implicit def tradeInfoSink(input: DataStream[BaseTrademark]) = 47 | TradeInfoSink(input) 48 | 49 | /** 50 | * 三级类目维度表写HBase 51 | * @param input 52 | * @return 53 | */ 54 | implicit def category3InfoSink(input: DataStream[BaseCategory3]) = 55 | Category3InfoSink(input) 56 | } 57 | -------------------------------------------------------------------------------- /ods2dim/src/main/scala/com/gmall/data/dim/StreamTopology.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.dim 2 | 3 | import com.gmall.data.common.config.KafkaConfig 4 | import com.gmall.data.common.entity.ods.gmall2021.{BaseCategory3, BaseProvince, BaseTrademark, SkuInfo, SpuInfo, UserInfo} 5 | import com.gmall.data.common.source.SourceFactory 6 | import com.gmall.data.common.utils.Constants 7 | import org.apache.flink.streaming.api.scala._ 8 | import com.gmall.data.dim.Step._ 9 | 10 | object StreamTopology { 11 | 12 | def build(kafkaConfig: KafkaConfig)( 13 | implicit env: StreamExecutionEnvironment): Unit = { 14 | 15 | // 用户 16 | val odsUserInfoStream = SourceFactory.createBinlogStream[UserInfo](kafkaConfig, Constants.USER_INFO_TOPIC) 17 | // spu 18 | val odsSpuInfoStream = SourceFactory.createBinlogStream[SpuInfo](kafkaConfig, Constants.SPU_INFO_TOPIC) 19 | // sku 20 | val odsSkuInfoStream = SourceFactory.createBinlogStream[SkuInfo](kafkaConfig, Constants.SKU_INFO_TOPIC) 21 | // 地区 22 | val odsProInfoStream = SourceFactory.createBinlogStream[BaseProvince](kafkaConfig, Constants.PROVINCE_INFO_TOPIC) 23 | // 品牌 24 | val odsTradeInfoStream = SourceFactory.createBinlogStream[BaseTrademark](kafkaConfig, Constants.TRADEMARK_INFO_TOPIC) 25 | // 三级类目 26 | val odsCateInfoStream = SourceFactory.createBinlogStream[BaseCategory3](kafkaConfig, Constants.CATEGORY3_INFO_TOPIC) 27 | 28 | /** 29 | * 维度数据写入HBase 30 | */ 31 | odsUserInfoStream.sink(odsUserInfoStream) 32 | odsSpuInfoStream.sink(odsSpuInfoStream) 33 | odsSkuInfoStream.sink(odsSkuInfoStream) 34 | odsProInfoStream.sink(odsProInfoStream) 35 | odsTradeInfoStream.sink(odsTradeInfoStream) 36 | odsCateInfoStream.sink(odsCateInfoStream) 37 | 38 | 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /ods2dim/src/main/scala/com/gmall/data/dim/sink/Category3InfoSink.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.dim.sink 2 | 3 | import com.gmall.data.common.entity.ods.gmall2021.BaseCategory3 4 | import com.gmall.data.common.transform.Sink 5 | import com.gmall.data.common.utils.{Constants, HBaseUtil, LoggerUtil} 6 | import com.gmall.data.dim.sink.Category3InfoSink.Category3InfoSinkFunc 7 | import org.apache.flink.configuration.Configuration 8 | import org.apache.flink.streaming.api.functions.sink.{RichSinkFunction, SinkFunction} 9 | import org.apache.flink.streaming.api.scala._ 10 | import org.apache.hadoop.hbase.TableName 11 | import org.apache.hadoop.hbase.client.{Connection, Put, Table} 12 | import org.apache.hadoop.hbase.util.Bytes 13 | import org.slf4j.{Logger, LoggerFactory} 14 | 15 | /** 16 | * 三级类目维度表写HBase 17 | * 18 | * @param input 19 | */ 20 | case class Category3InfoSink(input: DataStream[BaseCategory3]) 21 | extends Sink[BaseCategory3] { 22 | override protected def doSink(input: DataStream[BaseCategory3]): Unit = 23 | input.addSink(new Category3InfoSinkFunc) 24 | } 25 | 26 | object Category3InfoSink { 27 | 28 | private val logger: Logger = LoggerFactory.getLogger(this.getClass) 29 | 30 | class Category3InfoSinkFunc extends RichSinkFunction[BaseCategory3] { 31 | 32 | private var conn: Connection = _ 33 | private var table: Table = _ 34 | 35 | override def open(parameters: Configuration): Unit = { 36 | conn = HBaseUtil.getConn() 37 | table = conn.getTable(TableName.valueOf(Constants.DIM_CATEGORY3_INFO)) 38 | } 39 | 40 | override def invoke(value: BaseCategory3, context: SinkFunction.Context[_]): Unit = 41 | 42 | try { 43 | val put = new Put(Bytes.toBytes(value.id)) 44 | 45 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("name"), Bytes.toBytes(value.name)) 46 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("category2_id"), Bytes.toBytes(value.category2_id)) 47 | 48 | table.put(put) 49 | } catch { 50 | case e: Exception => LoggerUtil.error(logger, e, 51 | s"failed to Category3InfoSinkFunc.invoke,input=${value}") 52 | } 53 | 54 | override def close(): Unit = { 55 | if (table != null) table.close() 56 | if (conn != null) conn.close() 57 | } 58 | } 59 | 60 | } -------------------------------------------------------------------------------- /ods2dim/src/main/scala/com/gmall/data/dim/sink/ProvinceInfoSink.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.dim.sink 2 | 3 | import com.gmall.data.common.entity.ods.gmall2021.BaseProvince 4 | import com.gmall.data.common.transform.Sink 5 | import com.gmall.data.common.utils.{Constants, HBaseUtil, LoggerUtil} 6 | import com.gmall.data.dim.sink.ProvinceInfoSink.ProvinceInfoSinkFunc 7 | import org.apache.flink.configuration.Configuration 8 | import org.apache.flink.streaming.api.functions.sink.{RichSinkFunction, SinkFunction} 9 | import org.apache.flink.streaming.api.scala._ 10 | import org.apache.hadoop.hbase.TableName 11 | import org.apache.hadoop.hbase.client.{Connection, Put, Table} 12 | import org.apache.hadoop.hbase.util.Bytes 13 | import org.slf4j.{Logger, LoggerFactory} 14 | 15 | /** 16 | * 省份信息维度表写HBase 17 | * @param input 18 | */ 19 | case class ProvinceInfoSink(input: DataStream[BaseProvince]) 20 | extends Sink[BaseProvince] { 21 | override protected def doSink(input: DataStream[BaseProvince]): Unit = 22 | input.addSink(new ProvinceInfoSinkFunc) 23 | } 24 | 25 | object ProvinceInfoSink { 26 | 27 | private val logger: Logger = LoggerFactory.getLogger(this.getClass) 28 | 29 | class ProvinceInfoSinkFunc extends RichSinkFunction[BaseProvince] { 30 | 31 | private var conn: Connection = _ 32 | private var table: Table = _ 33 | 34 | override def open(parameters: Configuration): Unit = { 35 | conn = HBaseUtil.getConn() 36 | table = conn.getTable(TableName.valueOf(Constants.DIM_PROVINCE_INFO)) 37 | } 38 | 39 | override def invoke(value: BaseProvince, context: SinkFunction.Context[_]): Unit = 40 | try { 41 | val put = new Put(Bytes.toBytes(value.id)) 42 | 43 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("name"), Bytes.toBytes(value.name)) 44 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("region_id"), Bytes.toBytes(value.region_id)) 45 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("area_code"), Bytes.toBytes(value.area_code)) 46 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("iso_code"), Bytes.toBytes(value.iso_code)) 47 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("iso_3166_2"), Bytes.toBytes(value.iso_3166_2)) 48 | 49 | table.put(put) 50 | } catch { 51 | case e: Exception => LoggerUtil.error(logger, e, 52 | s"failed to ProvinceInfoSinkFunc.invoke,input=${value}") 53 | } 54 | 55 | override def close(): Unit = { 56 | if (table != null) table.close() 57 | if (conn != null) conn.close() 58 | } 59 | 60 | } 61 | 62 | } -------------------------------------------------------------------------------- /ods2dim/src/main/scala/com/gmall/data/dim/sink/SkuInfoSink.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.dim.sink 2 | 3 | import com.gmall.data.common.entity.ods.gmall2021.{SkuInfo, SpuInfo} 4 | import com.gmall.data.common.transform.Sink 5 | import com.gmall.data.common.utils.{Constants, HBaseUtil, LoggerUtil} 6 | import com.gmall.data.dim.sink.SkuInfoSink.SKuInfoSinkFunc 7 | import com.gmall.data.dim.sink.SpuInfoSink.logger 8 | import org.apache.flink.configuration.Configuration 9 | import org.apache.flink.streaming.api.functions.sink.{RichSinkFunction, SinkFunction} 10 | import org.apache.flink.streaming.api.scala._ 11 | import org.apache.hadoop.hbase.TableName 12 | import org.apache.hadoop.hbase.client.{Connection, Put, Table} 13 | import org.apache.hadoop.hbase.util.Bytes 14 | import org.slf4j.{Logger, LoggerFactory} 15 | 16 | case class SkuInfoSink(input: DataStream[SkuInfo]) 17 | extends Sink[SkuInfo] { 18 | override protected def doSink(input: DataStream[SkuInfo]): Unit = 19 | input.addSink(new SKuInfoSinkFunc) 20 | } 21 | 22 | object SkuInfoSink { 23 | 24 | private val logger: Logger = LoggerFactory.getLogger(this.getClass) 25 | 26 | class SKuInfoSinkFunc extends RichSinkFunction[SkuInfo] { 27 | 28 | private var conn: Connection = _ 29 | private var table: Table = _ 30 | 31 | override def open(parameters: Configuration): Unit = { 32 | conn = HBaseUtil.getConn() 33 | table = conn.getTable(TableName.valueOf(Constants.DIM_SKU_INFO)) 34 | } 35 | 36 | override def invoke(input: SkuInfo, context: SinkFunction.Context[_]): Unit = 37 | try { 38 | val put = new Put(Bytes.toBytes(input.id)) 39 | 40 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("spu_id"), Bytes.toBytes(input.spu_id)) 41 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("price"), Bytes.toBytes(input.price)) 42 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("sku_name"), Bytes.toBytes(input.sku_name)) 43 | // put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("sku_desc"), Bytes.toBytes(input.sku_desc)) 44 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("weight"), Bytes.toBytes(input.weight)) 45 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("tm_id"), Bytes.toBytes(input.tm_id)) 46 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("category3_id"), Bytes.toBytes(input.category3_id)) 47 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("sku_default_img"), Bytes.toBytes(input.sku_default_img)) 48 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("is_sale"), Bytes.toBytes(input.is_sale)) 49 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("create_time"), Bytes.toBytes(input.create_time)) 50 | 51 | table.put(put) 52 | } catch { 53 | case e: Exception => LoggerUtil.error(logger, e, 54 | s"failed to SkuInfoSinkFunc.invoke,input=${input}") 55 | } 56 | 57 | override def close(): Unit = { 58 | if (table != null) table.close() 59 | if (conn != null) conn.close() 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /ods2dim/src/main/scala/com/gmall/data/dim/sink/SpuInfoSink.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.dim.sink 2 | 3 | import com.gmall.data.common.entity.ods.gmall2021.SpuInfo 4 | import com.gmall.data.common.transform.Sink 5 | import com.gmall.data.common.utils.{Constants, HBaseUtil, LoggerUtil} 6 | import com.gmall.data.dim.sink.SpuInfoSink.SpuInfoSinkFunc 7 | import org.apache.flink.configuration.Configuration 8 | import org.apache.flink.streaming.api.functions.sink.{RichSinkFunction, SinkFunction} 9 | import org.apache.flink.streaming.api.scala.DataStream 10 | import org.apache.hadoop.hbase.TableName 11 | import org.apache.hadoop.hbase.client.{Connection, Put, Table} 12 | import org.apache.hadoop.hbase.util.Bytes 13 | import org.slf4j.{Logger, LoggerFactory} 14 | 15 | /** 16 | * 商品:spu维度表写入HBase 17 | * 18 | * @param input 19 | */ 20 | case class SpuInfoSink(input: DataStream[SpuInfo]) 21 | extends Sink[SpuInfo] { 22 | 23 | override protected def doSink(input: DataStream[SpuInfo]): Unit = 24 | input.addSink(new SpuInfoSinkFunc) 25 | } 26 | 27 | object SpuInfoSink { 28 | 29 | private val logger: Logger = LoggerFactory.getLogger(this.getClass) 30 | 31 | class SpuInfoSinkFunc extends RichSinkFunction[SpuInfo] { 32 | 33 | private var conn: Connection = _ 34 | private var table: Table = _ 35 | 36 | override def open(parameters: Configuration): Unit = { 37 | conn = HBaseUtil.getConn() 38 | table = conn.getTable(TableName.valueOf(Constants.DIM_SPU_INFO)) 39 | } 40 | 41 | override def invoke(input: SpuInfo, context: SinkFunction.Context[_]): Unit = 42 | try { 43 | val put = new Put(Bytes.toBytes(input.id)) 44 | 45 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("spu_name"), Bytes.toBytes(input.spu_name)) 46 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("description"), Bytes.toBytes(input.description)) 47 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("category3_id"), Bytes.toBytes(input.category3_id)) 48 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("tm_id"), Bytes.toBytes(input.tm_id)) 49 | 50 | table.put(put) 51 | } catch { 52 | case e: Exception => LoggerUtil.error(logger, e, 53 | s"failed to SpuInfoSinkFunc.invoke,input=${input}") 54 | } 55 | 56 | override def close(): Unit = { 57 | if (table != null) table.close() 58 | if (conn != null) conn.close() 59 | } 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /ods2dim/src/main/scala/com/gmall/data/dim/sink/TradeInfoSink.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.dim.sink 2 | 3 | import com.gmall.data.common.entity.ods.gmall2021.BaseTrademark 4 | import com.gmall.data.common.transform.Sink 5 | import com.gmall.data.common.utils.{Constants, HBaseUtil, LoggerUtil} 6 | import com.gmall.data.dim.sink.TradeInfoSink.TradeInfoSinkFunc 7 | import org.apache.flink.configuration.Configuration 8 | import org.apache.flink.streaming.api.functions.sink.{RichSinkFunction, SinkFunction} 9 | import org.apache.flink.streaming.api.scala.DataStream 10 | import org.apache.hadoop.hbase.TableName 11 | import org.apache.hadoop.hbase.client.{Connection, Put, Table} 12 | import org.apache.hadoop.hbase.util.Bytes 13 | import org.slf4j.{Logger, LoggerFactory} 14 | 15 | /** 16 | * 品牌维度表写HBase 17 | * 18 | * @param input 19 | */ 20 | case class TradeInfoSink(input: DataStream[BaseTrademark]) 21 | extends Sink[BaseTrademark] { 22 | override protected def doSink(input: DataStream[BaseTrademark]): Unit = 23 | input.addSink(new TradeInfoSinkFunc) 24 | } 25 | 26 | object TradeInfoSink { 27 | 28 | private val logger: Logger = LoggerFactory.getLogger(this.getClass) 29 | 30 | class TradeInfoSinkFunc extends RichSinkFunction[BaseTrademark] { 31 | 32 | private var conn: Connection = _ 33 | private var table: Table = _ 34 | 35 | override def open(parameters: Configuration): Unit = { 36 | conn = HBaseUtil.getConn() 37 | table = conn.getTable(TableName.valueOf(Constants.DIM_TRADEMARK_INFO)) 38 | } 39 | 40 | override def invoke(value: BaseTrademark, context: SinkFunction.Context[_]): Unit = 41 | try { 42 | val put = new Put(Bytes.toBytes(value.id)) 43 | 44 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("tm_name"), Bytes.toBytes(value.tm_name)) 45 | if (value.logo_url != null) 46 | put.addColumn(Bytes.toBytes("f1"), Bytes.toBytes("logo_url"), Bytes.toBytes(value.logo_url)) 47 | 48 | table.put(put) 49 | } catch { 50 | case e: Exception => LoggerUtil.error(logger, e, 51 | s"failed to TradeInfoSinkFunc.invoke,input=${value}") 52 | } 53 | 54 | override def close(): Unit = { 55 | if (table != null) table.close() 56 | if (conn != null) conn.close() 57 | } 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /ods2dwd/src/main/resources/event_mapping.txt: -------------------------------------------------------------------------------- 1 | ProductExposure,商品曝光 2 | distributionMode,分销模式 3 | inRoom,进入直播间 4 | joinGroupFail,拼团失败 5 | joinGroupFinish,拼团成功 6 | leaveRoom,退出直播间 7 | payGroupOrder,拼团支付订单 8 | pullNew,拉新成功 9 | roomaddToCart,直播间-加入购物车 10 | roomanchorShelfGoods,主播上架商品 11 | roombarrage,直播-发弹幕 12 | roombeginLive,开播 13 | roombuyNowClick,直播间-点击立即购买 14 | roomclickRoomGoods,点击直播间商品 15 | roomclickShoppingBag,直播间-点击购物袋 16 | roomfinishLive,结束直播 17 | roomfollowOrCancelFollow,关注主播 18 | roompayOrderDetails,直播-支付订单 19 | roomshareClick,直播-点击分享 20 | roomsubmitOrderDetail,直播-提交订单 21 | roomviewGoodsDetail,直播间-浏览商品详情 22 | $WechatClickMsgmenu,点击微信菜单会话 23 | $WechatClickMenu,点击微信公众号菜单 24 | $WechatUpdateUserLocation,上报微信用户地理位置 25 | $WechatScanQrcodeScene,扫描微信参数二维码 26 | $WechatUnfollow,取关微信公众号 27 | $WechatFollow,关注微信公众号 28 | $WechatReceiveMsgFromUser,微信公众号接收用户消息 29 | $WechatServiceMsgSendDone,微信客服消息终态信息 30 | $AppChannelMatching,访问渠道追踪链接 31 | $ShortUrlRedirect,短链跳转 32 | AppCrashed,App 崩溃 33 | activity_pageview,活动页面浏览 34 | activity_participate,活动参与 35 | banner_exposure,坑位曝光 36 | buyNow,立即购买 37 | SubmitOrderDetail,提交订单详情 38 | SubmitOrder,提交订单 39 | ShareResult,分享结果 40 | ShareMethod,选择分享方式 41 | ShareClick,点击分享 42 | SendCommodity,发货 43 | SelectReceiverAddress,选择地址 44 | SearchResultClick,点击搜索结果 45 | SearchRequest,搜索返回结果 46 | SearchColumClick,点击搜索栏 47 | SearchButtonClick,点击搜索按钮 48 | RegisterResult,注册结果 49 | RegisterButtonClick,点击注册按钮 50 | ReceiveDiscount,领取优惠券 51 | PushReceived,Push 通知送达 52 | PushClick,Push点击1 53 | PayOrderDetail,支付订单详情 54 | PayOrder,支付订单 55 | MPStartGroupBuy,小程序开团 56 | MPQRCodeScan,小程序线下扫码 57 | MPJoinGroupBuw,小程序参与拼团 58 | MPInviteHanggle,小程序邀请好友砍价 59 | MPHelpHanggle,小程序帮助砍价 60 | MPHangleSuccess,小程序砍价成功 61 | MPAuthorization,小程序授权 有数据 62 | LoginResult,登录 有数据 63 | LoginButtonClick,点击登录按钮 64 | GetCodeResult,获取验证码结果 65 | GetCode,点击获取验证码按钮 66 | Contact,联系客服 67 | ConfirmOrder,确认收货 68 | CommodityDetail,浏览商品详情页 69 | Comment,评价商品 70 | Collect,收藏商品 71 | CancelOrderDetail,取消订单详情 72 | CancelOrder,取消订单 73 | BannerClick,运营位点击 74 | ApplyReturn,申请退货 75 | AppOpenNotification,Push 点击 76 | AppInstall,App 激活 77 | AddToShoppingCart,加入购物车 78 | $pageview,Web 浏览页面 79 | $WebStay,Web 视区停留 80 | $WebClick,Web 元素点击 81 | $PlanMsgSendPrepare,消息已准备 82 | $PlanMsgSendDone,消息已发送 83 | $PlanConverted,推送转化 84 | $MPViewScreen,小程序页面浏览 85 | $MPShow,小程序启动 86 | $MPShare,小程序分享 87 | $MPLaunch,小程序首次启动 88 | $MPHide,小程序进入后台 89 | $AppViewScreen,App 浏览页面 90 | $AppStart,启动APP 91 | $AppEnd,退出APP 92 | $AppClick,App 元素点击 -------------------------------------------------------------------------------- /ods2dwd/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, console 2 | 3 | log4j.logger.org.apache.kafka=WARN 4 | log4j.logger.org.apache.flink=WARN 5 | log4j.logger.com.alibaba.blink=WARN 6 | 7 | log4j.appender.console=org.apache.log4j.ConsoleAppender 8 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 9 | log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-30c %-30t %x - %m%n 10 | -------------------------------------------------------------------------------- /ods2dwd/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /ods2dwd/src/main/scala/com/gmall/data/dwd/App.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.dwd 2 | 3 | import java.util.concurrent.TimeUnit 4 | 5 | import com.gmall.data.common.config.{Config, KafkaConfig} 6 | import com.gmall.data.common.utils.Constants 7 | import org.apache.flink.api.common.restartstrategy.RestartStrategies 8 | import org.apache.flink.api.common.time.Time 9 | import org.apache.flink.api.java.utils.ParameterTool 10 | import org.apache.flink.streaming.api.scala._ 11 | 12 | /** 13 | * 启动参数有-mode earliest/latest/committed/timestamp -timestamp "2021-04-10 00:00:00" 14 | * -mode earliest从kafka topic中最早开始消费 15 | * -mode latest从kafka topic中最新开始消费 16 | * -mode committed从flink 上次commit的消息开始消费 17 | * -mode timestamp从指定时间戳开始消费 18 | * -timestamp 只适用于timestamp mode 19 | */ 20 | 21 | object App { 22 | 23 | val OPTION_MODE = "mode" 24 | val OPTION_TIMESTAMP = "timestamp" 25 | val GROUP_ID = "ods-dwd" 26 | 27 | def main(args: Array[String]): Unit = { 28 | 29 | val parameterTool = ParameterTool.fromArgs(args) 30 | 31 | val mode = parameterTool.get(OPTION_MODE, Constants.CONSUMER_MODE_COMMITTED) 32 | val timestamp = parameterTool.get(OPTION_TIMESTAMP, "") 33 | 34 | implicit val env = StreamExecutionEnvironment.getExecutionEnvironment 35 | env.enableCheckpointing(120000) 36 | env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, Time.of(20, TimeUnit.SECONDS))) 37 | // env.setParallelism(1) 38 | 39 | val kafkaConfig = KafkaConfig(Config.kafkaBrokers, GROUP_ID, mode, timestamp) 40 | 41 | StreamTopology.build(kafkaConfig) 42 | // FlowTopology.build(kafkaConfig) 43 | // FlowTopologyV2.build(kafkaConfig) 44 | 45 | env.execute("ods-dwd-streaming-job") 46 | 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /ods2dwd/src/main/scala/com/gmall/data/dwd/FlowTopology.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.dwd 2 | 3 | import com.gmall.data.common.config.KafkaConfig 4 | import com.gmall.data.common.entity.ods.flow.{DwdDisplayLog, DwdPageLog, DwdStartLog, OdsBaseLog} 5 | import com.gmall.data.common.sink.SinkFactory 6 | import com.gmall.data.common.source.SourceFactory 7 | import com.gmall.data.common.utils.{Constants, GsonUtil} 8 | import org.apache.flink.streaming.api.scala._ 9 | import com.gmall.data.dwd.Step._ 10 | import org.apache.flink.api.common.serialization.SimpleStringSchema 11 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer 12 | 13 | object FlowTopology { 14 | 15 | private val dwdStartProducer = SinkFactory.createKafkaProducer[DwdStartLog](Constants.DWD_START_LOG_TOPIC) 16 | private val dwdDisplayProducer = SinkFactory.createKafkaProducer[DwdDisplayLog](Constants.DWD_DISPLAY_LOG_TOPIC) 17 | private val dwdPageProducer = SinkFactory.createKafkaProducer[DwdPageLog](Constants.DWD_PAGE_LOG_TOPIC) 18 | 19 | private val value1 = new FlinkKafkaProducer[String]("localhost:9092", "dwd_start_log", new SimpleStringSchema) 20 | private val value2 = new FlinkKafkaProducer[String]("localhost:9092", "dwd_display_log", new SimpleStringSchema) 21 | private val value3 = new FlinkKafkaProducer[String]("localhost:9092", "dwd_page_log", new SimpleStringSchema) 22 | 23 | 24 | def build(kafkaConfig: KafkaConfig)( 25 | implicit env: StreamExecutionEnvironment): Unit = { 26 | 27 | // 读取kafka中ods层的原始日志 28 | val odsLogStream = SourceFactory.createKafkaStream[OdsBaseLog](kafkaConfig, Constants.ODS_BASE_LOG_TOPIC) 29 | 30 | // ods层的数据格式转换为dwd层，并做分流 31 | val dwdBaseLog = odsLogStream.convert() 32 | 33 | // 将启动日志、曝光日志、页面访问日志分别输出到kafka 34 | val dwdStartLog = dwdBaseLog.filter(_.isInstanceOf[DwdStartLog]) 35 | .map(_.asInstanceOf[DwdStartLog]) 36 | 37 | dwdStartLog.print() 38 | dwdStartLog.map(GsonUtil.gson.toJson(_)).addSink(value1) 39 | 40 | val dwdDisplayLog = dwdBaseLog.filter(_.isInstanceOf[DwdDisplayLog]) 41 | .map(_.asInstanceOf[DwdDisplayLog]) 42 | 43 | dwdDisplayLog.print() 44 | // dwdDisplayLog.addSink(dwdDisplayProducer) 45 | dwdDisplayLog.map(GsonUtil.gson.toJson(_)).addSink(value2) 46 | 47 | val dwdPageLog = dwdBaseLog.filter(_.isInstanceOf[DwdPageLog]) 48 | .map(_.asInstanceOf[DwdPageLog]) 49 | 50 | dwdPageLog.print() 51 | // dwdPageLog.addSink(dwdPageProducer) 52 | dwdPageLog.map(GsonUtil.gson.toJson(_)).addSink(value3) 53 | 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /ods2dwd/src/main/scala/com/gmall/data/dwd/FlowTopologyV2.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.dwd 2 | 3 | import com.gmall.data.common.config.{Config, KafkaConfig, RedisConfig} 4 | import com.gmall.data.common.entity.ods.flow.{DwdUserBaseLog, OdsUserActionLog} 5 | import com.gmall.data.common.sink.SinkFactory 6 | import com.gmall.data.common.source.SourceFactory 7 | import com.gmall.data.common.utils.{Constants, GsonUtil} 8 | import org.apache.flink.streaming.api.scala._ 9 | import com.gmall.data.dwd.Step._ 10 | import com.gmall.data.dwd.transform.EventTypeSplitFunc 11 | 12 | object FlowTopologyV2 { 13 | 14 | implicit private val redisConfig = RedisConfig(Config.redisHost, Config.redisPort, Config.redisPassword, Config.redisDb) 15 | 16 | // 定义侧输出的标签 17 | private val exposeOutPut = OutputTag[DwdUserBaseLog]("exposeOutPut") 18 | private val pageOutPut = OutputTag[DwdUserBaseLog]("pageOutPut") 19 | 20 | // kafka输出 21 | // private val actionLogProducer = SinkFactory.createKafkaProducer[DwdUserBaseLog](Constants.DWD_USER_ACTION_LOG) 22 | // private val exposeLogProducer = SinkFactory.createKafkaProducer[DwdUserBaseLog](Constants.DWD_USER_EXPOSE_LOG) 23 | // private val pageLogProducer = SinkFactory.createKafkaProducer[DwdUserBaseLog](Constants.DWD_PAGE_VIEW_LOG) 24 | 25 | private val actionLogProducer = SinkFactory.createProducer(Constants.DWD_USER_ACTION_LOG) 26 | private val exposeLogProducer = SinkFactory.createProducer(Constants.DWD_USER_EXPOSE_LOG) 27 | private val pageLogProducer = SinkFactory.createProducer(Constants.DWD_PAGE_VIEW_LOG) 28 | 29 | def build(kafkaConfig: KafkaConfig)( 30 | implicit env: StreamExecutionEnvironment): Unit = { 31 | 32 | /** 33 | * 1. 读取Kafka中ODS层用户行为日志 34 | * 2. 数据ETL处理，统一格式 35 | * 3. 关联eventName维度表信息 36 | * 4. 对点击、曝光、页面浏览事件分流 37 | */ 38 | val eventStream = SourceFactory.createKafkaStream[OdsUserActionLog](kafkaConfig, Constants.ODS_USER_ACTION_LOG) 39 | .convertToDwdLog() 40 | .joinDimEvent() 41 | .process(EventTypeSplitFunc(exposeOutPut, pageOutPut)) 42 | 43 | 44 | /** 45 | * 从侧输出的标签取出对应的曝光、页面点击测流 46 | */ 47 | val exposeStream = eventStream.getSideOutput(exposeOutPut) 48 | val pageStream = eventStream.getSideOutput(pageOutPut) 49 | 50 | 51 | /** 52 | * 点击、曝光、页面浏览事件流输出到Kafka，作为实时数仓DWD层 53 | */ 54 | // eventStream.print() 55 | // exposeStream.print() 56 | // pageStream.print() 57 | eventStream.map(GsonUtil.gson.toJson(_)).addSink(actionLogProducer) 58 | exposeStream.map(GsonUtil.gson.toJson(_)).addSink(exposeLogProducer) 59 | pageStream.map(GsonUtil.gson.toJson(_)).addSink(pageLogProducer) 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /ods2dwd/src/main/scala/com/gmall/data/dwd/Step.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.dwd 2 | 3 | import com.gmall.data.common.config.RedisConfig 4 | import com.gmall.data.common.entity.dwd.DwdOrderDetail 5 | import com.gmall.data.common.entity.ods.flow.{DwdUserBaseLog, OdsBaseLog, OdsUserActionLog} 6 | import com.gmall.data.common.entity.ods.gmall2021.OrderInfo 7 | import com.gmall.data.dwd.transform.{DwdLogJoinDimEvent, OdsActionLogConvert, OdsBaseLogConvert, OrderDetailAndCouponMerger, OrderDetailAndDimUserJoin, OrderInfoAndDetailMerger} 8 | import org.apache.flink.streaming.api.scala._ 9 | 10 | /** 11 | * 隐式转换函数抽象 12 | */ 13 | object Step { 14 | 15 | implicit def orderInfoAndDetailMerger(input: DataStream[OrderInfo]) = 16 | OrderInfoAndDetailMerger(input) 17 | 18 | implicit def orderDetailAndCouponMerger(input: DataStream[DwdOrderDetail]) = 19 | OrderDetailAndCouponMerger(input) 20 | 21 | implicit def orderDetailAndDimUserJoin(input: DataStream[DwdOrderDetail]) = 22 | OrderDetailAndDimUserJoin(input) 23 | 24 | implicit def odsBaseLogConvert(input: DataStream[OdsBaseLog]) = 25 | OdsBaseLogConvert(input) 26 | 27 | implicit def odsActionLogConvert(input: DataStream[OdsUserActionLog]) = 28 | OdsActionLogConvert(input) 29 | 30 | implicit def dwdLogJoinDimEvent(input: DataStream[DwdUserBaseLog])(implicit redisConfig: RedisConfig) = 31 | DwdLogJoinDimEvent(input) 32 | 33 | 34 | } 35 | -------------------------------------------------------------------------------- /ods2dwd/src/main/scala/com/gmall/data/dwd/StreamTopology.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.dwd 2 | 3 | import com.gmall.data.common.config.KafkaConfig 4 | import com.gmall.data.common.entity.dwd.DwdOrderDetail 5 | import com.gmall.data.common.entity.ods.SqlType 6 | import com.gmall.data.common.entity.ods.gmall2021.{OrderDetail, OrderDetailCoupon, OrderInfo} 7 | import com.gmall.data.common.sink.SinkFactory 8 | import com.gmall.data.common.source.SourceFactory 9 | import com.gmall.data.common.utils.{Constants, GsonUtil} 10 | import org.apache.flink.streaming.api.scala._ 11 | import com.gmall.data.dwd.Step._ 12 | 13 | object StreamTopology { 14 | 15 | // private val dwdProducer = SinkFactory.createKafkaProducer[DwdOrderDetail](Constants.DWD_ORDER_DETAIL_TOPIC) 16 | private val dwdProducer = SinkFactory.createProducer(Constants.DWD_ORDER_DETAIL_TOPIC) 17 | 18 | def build(kafkaConfig: KafkaConfig)( 19 | implicit env: StreamExecutionEnvironment): Unit = { 20 | 21 | /** 22 | * 创建流并分配watermark 23 | */ 24 | val odsOrderInfoStream = SourceFactory.createBinlogStream[OrderInfo](kafkaConfig, Constants.ORDER_INFO_TOPIC) 25 | .filter(_.sqlType == SqlType.INSERT) // 取订单支付事实 26 | .assignAscendingTimestamps(_.ts) 27 | 28 | val odsOrderDetailStream = SourceFactory.createBinlogStream[OrderDetail](kafkaConfig, Constants.ORDER_DETAIL_TOPIC) 29 | .filter(_.sqlType == SqlType.INSERT) // 取订单支付事实 30 | .assignAscendingTimestamps(_.ts) 31 | 32 | val orderCouponStream = SourceFactory.createBinlogStream[OrderDetailCoupon](kafkaConfig, Constants.ORDER_DETAIL_COUPON_TOPIC) 33 | .filter(_.sqlType == SqlType.INSERT) 34 | .assignAscendingTimestamps(_.ts) 35 | 36 | // 订单流与订单明细流join 37 | val dwdStream = odsOrderInfoStream 38 | .joinStream(odsOrderDetailStream) 39 | .assignAscendingTimestamps(_.ts) 40 | // .joinStream(orderCouponStream) 41 | // .joinDimUser() 42 | 43 | dwdStream.map(GsonUtil.gson.toJson(_)).addSink(dwdProducer) 44 | dwdStream.print() 45 | 46 | 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /ods2dwd/src/main/scala/com/gmall/data/dwd/transform/DwdLogJoinDimEvent.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.dwd.transform 2 | 3 | import com.gmall.data.common.config.RedisConfig 4 | import com.gmall.data.common.entity.ods.flow.DwdUserBaseLog 5 | import com.gmall.data.common.transform.Format 6 | import com.gmall.data.common.utils.JedisWrapper 7 | import com.gmall.data.dwd.transform.DwdLogJoinDimEvent.DwdLogJoinDimEventFunc 8 | import org.apache.flink.streaming.api.functions.ProcessFunction 9 | import org.apache.flink.streaming.api.scala._ 10 | import org.apache.flink.util.Collector 11 | 12 | case class DwdLogJoinDimEvent(input: DataStream[DwdUserBaseLog])(implicit redisConfig: RedisConfig) extends Format[DwdUserBaseLog] { 13 | override def getUid: String = "DwdLogJoinDimEvent" 14 | 15 | override def getName: String = "DwdLogJoinDimEvent" 16 | 17 | def joinDimEvent(): DataStream[DwdUserBaseLog] = super.format(input) 18 | 19 | override protected def doFormat(input: DataStream[DwdUserBaseLog]): DataStream[DwdUserBaseLog] = 20 | 21 | input.process(new DwdLogJoinDimEventFunc) 22 | 23 | } 24 | 25 | object DwdLogJoinDimEvent { 26 | 27 | class DwdLogJoinDimEventFunc(implicit redisConfig: RedisConfig) extends ProcessFunction[DwdUserBaseLog, DwdUserBaseLog] { 28 | override def processElement(i: DwdUserBaseLog, 29 | context: ProcessFunction[DwdUserBaseLog, DwdUserBaseLog]#Context, 30 | collector: Collector[DwdUserBaseLog]): Unit = { 31 | 32 | JedisWrapper.wrap(jedis => { 33 | // 从redis维度表中查询eventName 34 | val eventName = jedis.get("dim:event:" + i.event) 35 | i.event_name = eventName 36 | }, "") 37 | collector.collect(i) 38 | } 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /ods2dwd/src/main/scala/com/gmall/data/dwd/transform/EventTypeSplitFunc.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.dwd.transform 2 | 3 | import com.gmall.data.common.entity.ods.flow.DwdUserBaseLog 4 | import com.gmall.data.common.utils.Constants 5 | import org.apache.flink.streaming.api.functions.ProcessFunction 6 | import org.apache.flink.streaming.api.scala.OutputTag 7 | import org.apache.flink.util.Collector 8 | 9 | case class EventTypeSplitFunc(exposeOutPut: OutputTag[DwdUserBaseLog], pageOutPut: OutputTag[DwdUserBaseLog]) extends ProcessFunction[DwdUserBaseLog, DwdUserBaseLog] { 10 | override def processElement(in: DwdUserBaseLog, context: ProcessFunction[DwdUserBaseLog, DwdUserBaseLog]#Context, 11 | out: Collector[DwdUserBaseLog]): Unit = { 12 | 13 | // 曝光埋点分流：一般会有专门的字段标识这是一个曝光埋点的事件，本demo中基于事件名来分 14 | if (Constants.EXPOSE_EVENT_LIST.contains(in.event)) { 15 | context.output(exposeOutPut, in) 16 | } else if (Constants.PAGE_EVENT_LIST.contains(in.event)) { // 页面浏览事件 17 | context.output(pageOutPut, in) 18 | } else { 19 | out.collect(in) 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /ods2dwd/src/main/scala/com/gmall/data/dwd/transform/OdsActionLogConvert.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.dwd.transform 2 | 3 | import com.gmall.data.common.entity.ods.flow.{DwdUserBaseLog, OdsUserActionLog} 4 | import com.gmall.data.common.transform.Convert 5 | import com.gmall.data.common.utils.TimeUtil 6 | import org.apache.flink.streaming.api.scala._ 7 | 8 | /** 9 | * ods层的用户行为日志转换为dwd 10 | * @param input 11 | */ 12 | case class OdsActionLogConvert(input: DataStream[OdsUserActionLog]) extends Convert[OdsUserActionLog, DwdUserBaseLog] { 13 | override def getUid: String = "OdsActionLogConvert" 14 | 15 | override def getName: String = "OdsActionLogConvert" 16 | 17 | def convertToDwdLog(): DataStream[DwdUserBaseLog] = super.convert(input) 18 | 19 | override protected def doConvert(input: DataStream[OdsUserActionLog]): DataStream[DwdUserBaseLog] = 20 | input.map(r => { 21 | 22 | val log = new DwdUserBaseLog 23 | 24 | log.event = r.event 25 | log.user_id = r.user_id 26 | log.distinct_id = r.distinct_id 27 | log.event_time = TimeUtil.getTimestamp(r.time) 28 | log.event_time_stamp = r.time 29 | log.app_name = r.app_name 30 | log.app_version = r.$app_version 31 | log.is_login = r.is_login 32 | log.is_vip = r.is_vip 33 | log.wifi = r.$wifi 34 | log.page_title = r.page_title 35 | log.page_type = r.page_type 36 | log.platform_type = r.platform_type 37 | log.url = r.$url 38 | log.referrer = r.$referrer 39 | log.vip_level = r.vip_level 40 | log.lib = r.$lib 41 | log.browser = r.$browser 42 | log.browser_version = r.$browser_version 43 | log.carrier = r.$carrier 44 | log.province = r.$province 45 | log.city = r.$city 46 | log.country = r.$country 47 | log.os = r.$os 48 | log.os_version = r.$os_version 49 | log.model = r.$model 50 | log.utm_campaign = r.$utm_campaign 51 | log.utm_content = r.$utm_content 52 | log.utm_medium = r.$utm_medium 53 | log.utm_source = r.$utm_source 54 | log.utm_term = r.$utm_term 55 | log.scene = r.$scene 56 | log.spu_id = r.commodity_id 57 | log.spu_name = r.commodity_name 58 | log.spu_quantity = r.commodity_quantity 59 | log.store_id = r.store_id 60 | log.store_name = r.store_name 61 | log.supplier_id = r.supplier_id 62 | log.supplier_name = r.supplier_name 63 | log.room_id = r.room_id 64 | log.room_name = r.room_name 65 | 66 | log 67 | }) 68 | } 69 | -------------------------------------------------------------------------------- /ods2dwd/src/main/scala/com/gmall/data/dwd/transform/OdsBaseLogConvert.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.dwd.transform 2 | 3 | import com.gmall.data.common.entity.ods.flow.{DwdBaseLog, DwdDisplayLog, DwdPageLog, DwdStartLog, OdsBaseLog} 4 | import com.gmall.data.common.transform.Convert 5 | import com.gmall.data.common.utils.{GsonUtil, LoggerUtil} 6 | import org.apache.flink.streaming.api.functions.ProcessFunction 7 | import org.apache.flink.streaming.api.scala._ 8 | import org.apache.flink.util.Collector 9 | import org.slf4j.{Logger, LoggerFactory} 10 | 11 | import scala.collection.JavaConverters._ 12 | 13 | /** 14 | * 流量数据：ods层原始日志转换为dwd层的启动、曝光、页面访问日志 15 | * 16 | * @param input 17 | */ 18 | case class OdsBaseLogConvert(input: DataStream[OdsBaseLog]) 19 | extends Convert[OdsBaseLog, DwdBaseLog] { 20 | override def getUid: String = "OdsBaseLogConvert" 21 | 22 | override def getName: String = "OdsBaseLogConvert" 23 | 24 | def convert(): DataStream[DwdBaseLog] = super.convert(input) 25 | 26 | override protected def doConvert(input: DataStream[OdsBaseLog]): DataStream[DwdBaseLog] = 27 | input.process(new ProcessFunction[OdsBaseLog, DwdBaseLog] { 28 | override def processElement(input: OdsBaseLog, 29 | context: ProcessFunction[OdsBaseLog, DwdBaseLog]#Context, 30 | out: Collector[DwdBaseLog]): Unit = 31 | try { 32 | if (input.start != null) { // 启动日志 33 | val dwdStartLog = GsonUtil.gson.fromJson(input.common, classOf[DwdStartLog]) 34 | out.collect(dwdStartLog.from(input.start).from(input.ts)) 35 | } else { // 非启动日志,则为页面日志或者曝光日志(携带页面信息) 36 | if (input.displays != null) { //曝光日志，做flatMap 37 | val dwdDisplayLog = GsonUtil.gson.fromJson(input.common, classOf[DwdDisplayLog]) 38 | for (elem <- input.displays.asScala) { 39 | out.collect(dwdDisplayLog.from(input.ts).from(elem).from(input.page)) 40 | } 41 | } else { // 页面访问日志 42 | val dwdPageLog = GsonUtil.gson.fromJson(input.common, classOf[DwdPageLog]) 43 | out.collect(dwdPageLog.from(input.page).from(input.ts)) 44 | } 45 | } 46 | } catch { 47 | case e: Exception => LoggerUtil.error(OdsBaseLogConvert.logger, e, 48 | s"failed to OdsBaseLogConvert.doConvert,input=${input}") 49 | } 50 | }) 51 | } 52 | 53 | object OdsBaseLogConvert { 54 | 55 | private val logger: Logger = LoggerFactory.getLogger(this.getClass) 56 | 57 | } -------------------------------------------------------------------------------- /ods2dwd/src/main/scala/com/gmall/data/dwd/transform/OrderDetailAndCouponMerger.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.dwd.transform 2 | 3 | import java.lang 4 | 5 | import com.gmall.data.common.entity.dwd.DwdOrderDetail 6 | import com.gmall.data.common.entity.ods.gmall2021.OrderDetailCoupon 7 | import com.gmall.data.common.transform.Merger 8 | import com.gmall.data.common.utils.LoggerUtil 9 | import org.apache.flink.api.common.functions.CoGroupFunction 10 | import org.apache.flink.streaming.api.scala._ 11 | import org.apache.flink.streaming.api.windowing.assigners.EventTimeSessionWindows 12 | import org.apache.flink.streaming.api.windowing.time.Time 13 | import org.apache.flink.util.Collector 14 | import org.slf4j.{Logger, LoggerFactory} 15 | 16 | import scala.collection.JavaConverters._ 17 | 18 | /** 19 | * 订单明细与订单coupon基于eventTimeSessionWindow实现left join 20 | * 21 | * @param input 22 | */ 23 | case class OrderDetailAndCouponMerger(input: DataStream[DwdOrderDetail]) 24 | extends Merger[DwdOrderDetail, OrderDetailCoupon, DwdOrderDetail](input) { 25 | override def getName: String = "OrderDetailAndCouponMerger" 26 | 27 | override def getUid: String = "OrderDetailAndCouponMerger" 28 | 29 | override protected def merge(input1: DataStream[DwdOrderDetail], input2: DataStream[OrderDetailCoupon]): DataStream[DwdOrderDetail] = 30 | input1.coGroup(input2) 31 | .where(_.detail_id) 32 | .equalTo(_.order_detail_id) 33 | .window(EventTimeSessionWindows.withGap(Time.seconds(5))) 34 | .apply(new CoGroupFunction[DwdOrderDetail, OrderDetailCoupon, DwdOrderDetail] { 35 | override def coGroup(iterable: lang.Iterable[DwdOrderDetail], 36 | iterable1: lang.Iterable[OrderDetailCoupon], 37 | out: Collector[DwdOrderDetail]): Unit = 38 | try { 39 | val left = iterable.asScala.toSeq 40 | val right = iterable1.asScala.toSeq 41 | 42 | if (left.isEmpty) 43 | OrderDetailAndCouponMerger.logger.warn("DwdOrderDetail is empty") 44 | else { 45 | if (right.isEmpty) // 直接发出结果 46 | out.collect(left.head) 47 | else // 关联coupon信息 48 | out.collect(left.head.from(right.head)) 49 | } 50 | } catch { 51 | case e: Exception => LoggerUtil.error(OrderDetailAndCouponMerger.logger, e, 52 | s"failed to OrderDetailAndCouponMerger.merger,left=${iterable},right=${iterable1}") 53 | } 54 | }) 55 | } 56 | 57 | object OrderDetailAndCouponMerger { 58 | 59 | private val logger: Logger = LoggerFactory.getLogger(this.getClass) 60 | 61 | } -------------------------------------------------------------------------------- /ods2dwd/src/main/scala/com/gmall/data/dwd/transform/OrderInfoAndDetailMerger.scala: -------------------------------------------------------------------------------- 1 | package com.gmall.data.dwd.transform 2 | 3 | import com.gmall.data.common.entity.dwd.DwdOrderDetail 4 | import com.gmall.data.common.entity.ods.gmall2021.{OrderDetail, OrderInfo} 5 | import com.gmall.data.common.transform.Merger 6 | import com.gmall.data.common.utils.LoggerUtil 7 | import org.apache.flink.streaming.api.functions.co.ProcessJoinFunction 8 | import org.apache.flink.streaming.api.scala._ 9 | import org.apache.flink.streaming.api.windowing.time.Time 10 | import org.apache.flink.util.Collector 11 | import org.slf4j.{Logger, LoggerFactory} 12 | 13 | /** 14 | * 订单流与订单明细流基于interval join实现双流inner join 15 | * 16 | * @param input 17 | */ 18 | case class OrderInfoAndDetailMerger(input: DataStream[OrderInfo]) 19 | extends Merger[OrderInfo, OrderDetail, DwdOrderDetail](input) { 20 | override def getName: String = "OrderInfoAndDetailMerger" 21 | 22 | override def getUid: String = "OrderInfoAndDetailMerger" 23 | 24 | override protected def merge(input1: DataStream[OrderInfo], input2: DataStream[OrderDetail]): DataStream[DwdOrderDetail] = 25 | input1.keyBy(_.id) 26 | .intervalJoin(input2.keyBy(_.order_id)) 27 | .between(Time.seconds(-5), Time.seconds(5)) 28 | .process(new ProcessJoinFunction[OrderInfo, OrderDetail, DwdOrderDetail] { 29 | override def processElement(left: OrderInfo, 30 | right: OrderDetail, 31 | context: ProcessJoinFunction[OrderInfo, OrderDetail, DwdOrderDetail]#Context, 32 | out: Collector[DwdOrderDetail]): Unit = 33 | try { 34 | val orderDetail = DwdOrderDetail().from(left).from(right) 35 | out.collect(orderDetail) 36 | } catch { 37 | case e: Exception => LoggerUtil.error(OrderInfoAndDetailMerger.logger, e, 38 | s"failed to OrderInfoAndDetailMerger.merger,left=${left},right=${right}") 39 | } 40 | }) 41 | } 42 | 43 | object OrderInfoAndDetailMerger { 44 | 45 | private val logger: Logger = LoggerFactory.getLogger(this.getClass) 46 | 47 | } 48 | -------------------------------------------------------------------------------- /ods2dwd/src/test/java/JedisSetTest.scala: -------------------------------------------------------------------------------- 1 | import com.gmall.data.common.config.{Config, RedisConfig} 2 | import com.gmall.data.common.utils.JedisWrapper 3 | import org.apache.flink.streaming.api.functions.sink.SinkFunction 4 | import org.apache.flink.streaming.api.scala._ 5 | 6 | object JedisSetTest { 7 | 8 | def main(args: Array[String]): Unit = { 9 | 10 | val env = StreamExecutionEnvironment.getExecutionEnvironment 11 | env.setParallelism(1) 12 | 13 | implicit val redisConfig = RedisConfig(Config.redisHost, Config.redisPort, Config.redisPassword, Config.redisDb) 14 | 15 | env.readTextFile("ods2dwd/src/main/resources/event_mapping.txt") 16 | .addSink(new EventNameSinkFunc) 17 | 18 | env.execute("job") 19 | 20 | } 21 | 22 | class EventNameSinkFunc(implicit redisConfig: RedisConfig) extends SinkFunction[String] { 23 | override def invoke(input: String, context: SinkFunction.Context): Unit = { 24 | 25 | val arr = input.split("\\,", -1) 26 | val event = arr(0).trim 27 | val eventName = arr(1).trim 28 | JedisWrapper.wrap(jedis => { 29 | jedis.set("dim:event:" + event, eventName) 30 | }, "") 31 | } 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /ods2dwd/src/test/java/KafkaConsumerTest.scala: -------------------------------------------------------------------------------- 1 | import java.util.Properties 2 | 3 | import com.gmall.data.common.config.{Config, KafkaConfig} 4 | import com.gmall.data.common.source.SourceFactory 5 | import com.gmall.data.dwd.App.GROUP_ID 6 | import org.apache.flink.api.common.serialization.SimpleStringSchema 7 | import org.apache.flink.streaming.api.scala._ 8 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer 9 | 10 | object KafkaConsumerTest { 11 | 12 | def main(args: Array[String]): Unit = { 13 | 14 | val env = StreamExecutionEnvironment.getExecutionEnvironment 15 | env.setParallelism(1) 16 | 17 | val kafkaConfig = KafkaConfig(Config.kafkaBrokers, GROUP_ID, "mode", "timestamp") 18 | 19 | val properties = new Properties() 20 | properties.setProperty("bootstrap.servers", kafkaConfig.brokers) 21 | properties.setProperty("group.id", kafkaConfig.groupId) 22 | 23 | val kafkaConsumer = new FlinkKafkaConsumer[String]("dwd_order_detail", new SimpleStringSchema(), properties) 24 | kafkaConsumer.setStartFromEarliest() 25 | 26 | env.addSource(kafkaConsumer).print() 27 | 28 | env.execute("job") 29 | 30 | 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /tools/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | realtime-dw 7 | com.gmall.data 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | tools 13 | 14 | 15 | UTF-8 16 | 2.11.12 17 | 2.11 18 | 19 | 20 | 21 | 22 | org.scala-lang 23 | scala-library 24 | ${scala.version} 25 | 26 | 27 | mysql 28 | mysql-connector-java 29 | 8.0.17 30 | 31 | 32 | protobuf-java 33 | com.google.protobuf 34 | 35 | 36 | 37 | 38 | com.gmall.data 39 | common 40 | 1.0-SNAPSHOT 41 | 42 | 43 | com.github.pathikrit 44 | better-files_2.11 45 | 3.8.0 46 | 47 | 48 | org.json4s 49 | json4s-native_2.11 50 | 3.6.7 51 | 52 | 53 | 54 | ${project.artifactId}-${project.version} 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /tools/src/main/resources/ods_entity.template: -------------------------------------------------------------------------------- 1 | package com.gmall.data.common.entity.ods.{ods_model_package} 2 | 3 | import com.gmall.data.common.entity.ods 4 | import com.gmall.data.common.entity.ods.OdsModel 5 | import com.gmall.data.common.entity.ods.SqlType.SqlType 6 | import scala.collection.mutable 7 | 8 | // IMPORTANT!!! Don't do any changes, this is auto-generated by OdsModelGenerator 9 | class {ods_model_class_name} extends OdsModel { 10 | override var database: String = _ 11 | override var table : String = _ 12 | override var ts : Long = _ 13 | override var sqlType : ods.SqlType.Value = _ 14 | override var old : mutable.Map[String, String] = _ 15 | {ods_model_fields} 16 | 17 | def this(database: String, table: String, sqlType: SqlType, ts: Long, old: mutable.Map[String, String]) { 18 | this() 19 | this.database = database 20 | this.table = table 21 | this.sqlType = sqlType 22 | this.ts = ts 23 | this.old = old 24 | } 25 | 26 | override def toString = s"{ods_model_to_string}" 27 | 28 | } 29 | --------------------------------------------------------------------------------