├── .gitattributes ├── .gitignore ├── README.md ├── bdp-collect ├── .gitignore ├── README.md ├── build.bat ├── pom.xml └── src │ └── main │ ├── assembly │ ├── bin-delta.xml │ └── bin.xml │ ├── java │ └── com │ │ └── github │ │ └── bdp │ │ └── collect │ │ ├── Main.java │ │ └── processors │ │ └── DateShiftProcessor.java │ ├── profiles │ ├── cluster.properties │ └── standalone.properties │ └── resources │ ├── bin │ └── bdp-collect.sh │ ├── conf │ ├── bdp-collect.properties │ ├── camel-context.xml │ └── log4j.properties │ └── deploy.bat ├── bdp-dwh ├── .gitignore ├── README.md ├── build.bat ├── pom.xml └── src │ └── main │ ├── assembly │ └── bin.xml │ ├── profiles │ ├── cluster.properties │ └── standalone.properties │ ├── resources │ ├── bin │ │ ├── bdp-dwh.sh │ │ ├── dmt-infra-metric.sh │ │ ├── dmt-master-data.sh │ │ ├── dwh-bdp-master.sh │ │ ├── dwh-bdp-metric.sh │ │ ├── src-bdp-master.sh │ │ ├── src-bdp-metric.sh │ │ └── util.sh │ ├── deploy.bat │ └── lib │ │ ├── dmt │ │ ├── infra-metric │ │ │ ├── action │ │ │ │ ├── build-fact_metric.sql │ │ │ │ ├── build-sum_metric_avg.sql │ │ │ │ └── build-wide_metric_avg.sql │ │ │ ├── bin │ │ │ │ └── spark-actions.sh │ │ │ └── schema │ │ │ │ ├── fact_metric.sql │ │ │ │ ├── sum_metric_avg.sql │ │ │ │ └── wide_metric_avg.sql │ │ └── master-data │ │ │ ├── action │ │ │ ├── build-dim_app.sql │ │ │ ├── build-dim_hour.sql │ │ │ ├── build-dim_metric_index.sql │ │ │ ├── build-dim_metric_threshold.sql │ │ │ └── build-dim_server.sql │ │ │ ├── bin │ │ │ └── spark-actions.sh │ │ │ ├── data │ │ │ └── dim_hour.csv │ │ │ └── schema │ │ │ ├── dim_app.sql │ │ │ ├── dim_hour.sql │ │ │ ├── dim_metric_index.sql │ │ │ ├── dim_metric_threshold.sql │ │ │ └── dim_server.sql │ │ ├── dwh │ │ ├── bdp-master │ │ │ ├── action │ │ │ │ ├── build-app.sql │ │ │ │ ├── build-metric_index.sql │ │ │ │ ├── build-metric_threshold.sql │ │ │ │ └── build-server.sql │ │ │ ├── bin │ │ │ │ └── spark-actions.sh │ │ │ └── schema │ │ │ │ ├── app.sql │ │ │ │ ├── metric_index.sql │ │ │ │ ├── metric_threshold.sql │ │ │ │ └── server.sql │ │ └── bdp-metric │ │ │ ├── action │ │ │ └── build-metric.sql │ │ │ ├── bin │ │ │ └── spark-actions.sh │ │ │ └── schema │ │ │ └── metric.sql │ │ └── src │ │ ├── bdp-master │ │ ├── action │ │ │ ├── build-app.sql │ │ │ ├── build-metric_index.sql │ │ │ ├── build-metric_threshold.sql │ │ │ └── build-server.sql │ │ ├── bin │ │ │ ├── spark-actions.sh │ │ │ └── sqoop-actions.sh │ │ └── schema │ │ │ ├── app.sql │ │ │ ├── metric_index.sql │ │ │ ├── metric_threshold.sql │ │ │ └── server.sql │ │ └── bdp-metric │ │ ├── action │ │ └── build-metric.sql │ │ ├── bin │ │ ├── spark-actions.sh │ │ └── sqoop-actions.sh │ │ └── schema │ │ └── metric.sql │ └── scala │ └── com.github.bdp.dwh.udf │ └── GenRag.scala ├── bdp-import ├── .gitignore ├── README.md ├── build.bat ├── pom.xml └── src │ └── main │ ├── assembly │ └── bin.xml │ ├── profiles │ ├── cluster.properties │ └── standalone.properties │ └── resources │ ├── bin │ ├── bdp-import.sh │ ├── bdp-master-import.sh │ ├── bdp-metric-import.sh │ └── util.sh │ └── deploy.bat ├── bdp-master-client ├── .gitignore ├── README.md ├── pom.xml └── src │ └── main │ ├── profiles │ ├── cluster.properties │ ├── local.properties │ └── standalone.properties │ ├── resources │ ├── bdp-master-client.conf │ └── log4j.properties │ └── scala │ └── com │ └── github │ └── bdp │ └── master │ └── client │ ├── Constants.scala │ ├── Main.scala │ ├── domain │ ├── AlertIndex.scala │ ├── App.scala │ ├── MetricIndex.scala │ ├── MetricThreshold.scala │ ├── SEVERITY.scala │ ├── Server.scala │ └── TSD.scala │ ├── service │ ├── AlertIndexService.scala │ ├── AppService.scala │ ├── MetricIndexService.scala │ └── ServerService.scala │ └── util │ ├── JsonDecoder.scala │ └── RedisClient.scala ├── bdp-master-server ├── .gitignore ├── README.md ├── build.bat ├── pom.xml └── src │ └── main │ ├── assembly │ ├── bin-delta.xml │ └── bin.xml │ ├── java │ └── com │ │ └── github │ │ └── bdp │ │ └── master │ │ └── server │ │ ├── Constants.java │ │ ├── Main.java │ │ ├── RedisConfig.java │ │ ├── controller │ │ ├── AlertIndexController.java │ │ ├── AppController.java │ │ ├── AppStartupListener.java │ │ ├── MetricIndexController.java │ │ └── ServerController.java │ │ ├── domain │ │ ├── AlertIndex.java │ │ ├── App.java │ │ ├── MetricIndex.java │ │ ├── MetricThreshold.java │ │ └── Server.java │ │ ├── repository │ │ ├── AlertIndexJpaRepository.java │ │ ├── AlertIndexRedisRepository.java │ │ ├── AppJpaRepository.java │ │ ├── AppRedisRepository.java │ │ ├── MetricIndexJpaRepository.java │ │ ├── MetricIndexRedisRepository.java │ │ ├── MetricThresholdRepository.java │ │ ├── ServerJpaRepository.java │ │ ├── ServerRedisRepository.java │ │ └── impl │ │ │ ├── AlertIndexRedisRepositoryImpl.java │ │ │ ├── AppRedisRepositoryImpl.java │ │ │ ├── MetricIndexRedisRepositoryImpl.java │ │ │ └── ServerRedisRepositoryImpl.java │ │ └── service │ │ ├── AlertIndexService.java │ │ ├── AppService.java │ │ ├── MetricIndexService.java │ │ ├── ServerService.java │ │ └── impl │ │ ├── AlertIndexServiceImpl.java │ │ ├── AppServiceImpl.java │ │ ├── MetricIndexServiceImpl.java │ │ └── ServerServiceImpl.java │ ├── profiles │ ├── cluster.properties │ └── standalone.properties │ └── resources │ ├── bin │ └── bdp-master-server.sh │ ├── conf │ ├── application.properties │ ├── bdp-master-data-2018-09-01.sql │ ├── bdp-master-data-2018-09-02.sql │ └── logback.xml │ └── deploy.bat ├── bdp-metric ├── .gitignore ├── README.md ├── build.bat ├── pom.xml └── src │ └── main │ ├── assembly │ └── bin.xml │ ├── profiles │ ├── cluster.properties │ └── standalone.properties │ └── resources │ ├── bin │ └── bdp-metric.sh │ ├── deploy.bat │ └── sql │ ├── gen-alert.sql │ ├── gen-cpu-usage.sql │ ├── gen-mem-used.sql │ └── schema.sql ├── bdp-parent ├── .gitignore ├── README.md └── pom.xml ├── bdp-stream ├── .gitignore ├── README.md ├── build.bat ├── pom.xml └── src │ └── main │ ├── assembly │ ├── bin-delta.xml │ └── bin.xml │ ├── profiles │ ├── cluster.properties │ └── standalone.properties │ ├── resources │ ├── bin │ │ └── bdp-stream.sh │ ├── conf │ │ ├── bdp-stream.conf │ │ ├── fairscheduler.xml │ │ ├── hbase-site.xml │ │ ├── log4j-driver.properties │ │ └── log4j-executor.properties │ └── deploy.bat │ └── scala │ └── com │ └── github │ └── bdp │ └── stream │ ├── AlertStream.scala │ ├── Constants.scala │ ├── Main.scala │ ├── MetricStream.scala │ ├── assembler │ ├── AlertAssembler.scala │ ├── MetricAssembler.scala │ └── ServerStateAssembler.scala │ ├── model │ ├── Alert.scala │ ├── AlertRegistry.scala │ ├── Metric.scala │ └── ServerState.scala │ ├── service │ ├── AlertService.scala │ └── MetricService.scala │ └── util │ ├── ForeachWriters.scala │ ├── HBaseClient.scala │ └── JsonDecoder.scala ├── bdp-workflow ├── .gitignore ├── README.md ├── build.bat ├── pom.xml └── src │ └── main │ ├── assembly │ └── bin.xml │ ├── profiles │ ├── cluster.properties │ └── standalone.properties │ └── resources │ ├── bin │ ├── bdp-workflow.sh │ └── util.sh │ ├── deploy.bat │ └── lib │ ├── ds-bdp-master-daily-build │ ├── coordinator.xml │ ├── sub-workflow │ │ ├── app.xml │ │ ├── metric-index.xml │ │ ├── metric-threshold.xml │ │ └── server.xml │ └── workflow.xml │ ├── ds-bdp-metric-daily-build │ ├── coordinator.xml │ ├── sub-workflow │ │ └── metric.xml │ └── workflow.xml │ ├── sj-infra-metric-daily-build │ ├── coordinator.xml │ ├── sub-workflow │ │ ├── fact-metric.xml │ │ ├── sum-metric-avg.xml │ │ └── wide-metric-avg.xml │ └── workflow.xml │ └── sj-master-data-daily-build │ ├── coordinator.xml │ ├── sub-workflow │ ├── app.xml │ ├── metric-index.xml │ ├── metric-threshold.xml │ └── server.xml │ └── workflow.xml └── book.jpg /.gitattributes: -------------------------------------------------------------------------------- 1 | *.sh -crlf linguist-language=java 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | /*.iml 3 | target 4 | -------------------------------------------------------------------------------- /bdp-collect/.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | /*.iml 3 | target 4 | -------------------------------------------------------------------------------- /bdp-collect/README.md: -------------------------------------------------------------------------------- 1 | 关于本子项目的部署、运行与代码细节，请参考《大数据平台架构与原型实现：数据中台建设实战》一书第5章以及第4章4.5节 -------------------------------------------------------------------------------- /bdp-collect/build.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | rem A batch script to build -> deploy -> restart 3 | rem -- Laurence Geng 4 | if [%1]==[] ( 5 | echo. 6 | echo Usage: %0 [-delta] maven-profile-1 maven-profile-2 ... 7 | echo. 8 | echo Option: -delta: only deploy modified part, i.e. project artifact, used for development deploy. 9 | goto end 10 | ) 11 | 12 | set deltaDeploy=0 13 | if "%~1"=="-delta" ( 14 | set deltaDeploy=1 15 | shift 16 | ) 17 | 18 | set profiles=%~1 19 | 20 | :loopProfiles 21 | shift 22 | if "%~1"=="" ( 23 | goto build 24 | ) else ( 25 | set profiles=%profiles%,%~1 26 | goto loopProfiles 27 | ) 28 | 29 | :build 30 | echo. 31 | echo *************************************************************************************** 32 | echo BUILD... 33 | echo *************************************************************************************** 34 | echo. 35 | 36 | if "%profiles%"=="" ( 37 | call mvn clean install -DskipTests=true 38 | ) else ( 39 | call mvn clean install -DskipTests=true -P%profiles% 40 | ) 41 | 42 | if "%errorlevel%"=="1" goto :buildfailed 43 | 44 | if "%deltaDeploy%"=="1" ( 45 | call target\classes\deploy.bat -delta 46 | ) else ( 47 | call target\classes\deploy.bat 48 | ) 49 | 50 | goto buildsuccess 51 | 52 | :buildsuccess 53 | echo. 54 | echo. 55 | echo *************************************************************************************** 56 | echo BUILD SUCCESS!! 57 | echo *************************************************************************************** 58 | goto end 59 | 60 | :buildfailed 61 | echo. 62 | echo. 63 | echo *************************************************************************************** 64 | echo BUILD FAILED!! 65 | echo *************************************************************************************** 66 | goto end 67 | 68 | :end 69 | -------------------------------------------------------------------------------- /bdp-collect/src/main/assembly/bin-delta.xml: -------------------------------------------------------------------------------- 1 | 4 | bin-delta 5 | 6 | zip 7 | 8 | true 9 | 10 | 11 | target/classes/bin 12 | ./bin 13 | 14 | *.sh 15 | 16 | 755 17 | 18 | 19 | target/classes/conf 20 | ./conf 21 | 22 | *.xml 23 | *.conf 24 | *.properties 25 | 26 | 27 | 28 | 29 | 30 | ./lib 31 | 32 | com.github:bdp-collect 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /bdp-collect/src/main/assembly/bin.xml: -------------------------------------------------------------------------------- 1 | 4 | bin 5 | 6 | zip 7 | 8 | true 9 | 10 | 11 | target 12 | ./lib 13 | 14 | *.jar 15 | 16 | 17 | 18 | target/classes/bin 19 | ./bin 20 | 21 | *.sh 22 | 23 | 755 24 | 25 | 26 | target/classes/conf 27 | ./conf 28 | 29 | *.xml 30 | *.conf 31 | *.properties 32 | 33 | 34 | 35 | 36 | 37 | ./lib 38 | 39 | 40 | -------------------------------------------------------------------------------- /bdp-collect/src/main/java/com/github/bdp/collect/Main.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.collect; 2 | 3 | import org.apache.camel.spring.SpringCamelContext; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import org.springframework.context.support.ClassPathXmlApplicationContext; 7 | 8 | public class Main { 9 | private static final Logger logger = LoggerFactory.getLogger(Main.class); 10 | 11 | public static void main(String[] args) throws Exception { 12 | 13 | ClassPathXmlApplicationContext context = new ClassPathXmlApplicationContext("camel-context.xml"); 14 | 15 | SpringCamelContext camelContext = context.getBean(SpringCamelContext.class); 16 | 17 | try { 18 | camelContext.start(); 19 | Runtime.getRuntime().addShutdownHook(new Thread() { 20 | @Override 21 | public void run() { 22 | try { 23 | camelContext.stop(); 24 | context.close(); 25 | } catch (Exception ex) { 26 | logger.error("stop camel context error:" + ex.getMessage()); 27 | } 28 | } 29 | }); 30 | logger.info("camel context started!"); 31 | camelContext.start(); 32 | Thread.sleep(Long.MAX_VALUE); 33 | } catch (Exception e) { 34 | camelContext.stop(); 35 | context.close(); 36 | e.printStackTrace(); 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /bdp-collect/src/main/java/com/github/bdp/collect/processors/DateShiftProcessor.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.collect.processors; 2 | 3 | import org.apache.camel.Exchange; 4 | import org.apache.camel.Message; 5 | import org.apache.camel.Processor; 6 | import org.joda.time.DateTime; 7 | 8 | import java.util.Date; 9 | 10 | public class DateShiftProcessor implements Processor { 11 | 12 | @Override 13 | public void process(Exchange exchange) throws Exception { 14 | Message message = exchange.getIn(); 15 | Integer offset = message.getHeader("offset", Integer.class); 16 | Date firedTime = message.getHeader("firedTime", Date.class); 17 | DateTime dateTime = new DateTime(firedTime); 18 | DateTime shiftedTime = dateTime.minusSeconds(offset); 19 | message.setHeader("shiftedTime", shiftedTime.toDate()); 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /bdp-collect/src/main/profiles/cluster.properties: -------------------------------------------------------------------------------- 1 | # app specific configs 2 | app.name=${project.artifactId} 3 | app.host=gateway1.cluster 4 | app.home=${app.user.home}/${project.build.finalName} 5 | app.user.name=${project.artifactId} 6 | app.user.password=Bdpp1234! 7 | app.user.home=/home/${app.user.name} 8 | app.mainClass=com.github.bdp.collect.Main 9 | app.log.home=${app.home}/log 10 | app.log.level=INFO 11 | 12 | # bdp_metric jdbc configs 13 | bdp.metric.db.host=loadbalancer1.cluster 14 | bdp.metric.jdbc.driverClassName=com.mysql.jdbc.Driver 15 | bdp.metric.jdbc.url=jdbc:mysql://${bdp.metric.db.host}/bdp_metric?useSSL=false&createDatabaseIfNotExist=true&useUnicode=true&characterEncoding=utf-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=GMT 16 | bdp.metric.jdbc.username=bdp_metric 17 | bdp.metric.jdbc.password=Bdpp1234! 18 | 19 | # kafka configs 20 | # Be careful the hostname of broker, you can check it on broker server via CLI: "netstat -ntlp|grep 9092" 21 | kafka.brokers=worker1.cluster:9092,worker2.cluster:9092,worker3.cluster:9092 -------------------------------------------------------------------------------- /bdp-collect/src/main/profiles/standalone.properties: -------------------------------------------------------------------------------- 1 | # app specific configs 2 | app.name=${project.artifactId} 3 | app.host=node1.cluster 4 | app.home=${app.user.home}/${project.build.finalName} 5 | app.user.name=${project.artifactId} 6 | app.user.password=Bdpp1234! 7 | app.user.home=/home/${app.user.name} 8 | app.mainClass=com.github.bdp.collect.Main 9 | app.log.home=${app.home}/log 10 | app.log.level=INFO 11 | 12 | # bdp_metric jdbc configs 13 | bdp.metric.db.host=node1.cluster:3306 14 | bdp.metric.jdbc.driverClassName=com.mysql.jdbc.Driver 15 | bdp.metric.jdbc.url=jdbc:mysql://${bdp.metric.db.host}/bdp_metric?useSSL=false&createDatabaseIfNotExist=true&useUnicode=true&characterEncoding=utf-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=GMT 16 | bdp.metric.jdbc.username=bdp_metric 17 | bdp.metric.jdbc.password=Bdpp1234! 18 | 19 | # kafka configs 20 | # Be careful the hostname of broker, you can check it on broker server via CLI: "netstat -ntlp|grep 9092" 21 | kafka.brokers=node1.cluster:9092 -------------------------------------------------------------------------------- /bdp-collect/src/main/resources/bin/bdp-collect.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export BDP_COLLECT_HOME="$(cd "`dirname $(readlink -nf "$0")`"/..; pwd -P)" 4 | 5 | BDP_COLLECT_LIB_DIR=${BDP_COLLECT_HOME}/lib 6 | BDP_COLLECT_CONF_DIR=${BDP_COLLECT_HOME}/conf 7 | BDP_COLLECT_PID=/tmp/${project.artifactId}.pid 8 | BDP_COLLECT_MAIN_CLASS="${app.mainClass}" 9 | 10 | # ------------------------------------------------ Common Methods ------------------------------------------------ # 11 | 12 | showUsage() { 13 | printHeading "BDP-COLLECT USAGE" 14 | echo "# 启动程序" 15 | echo "$0 start" 16 | echo 17 | echo "# 终止程序" 18 | echo "$0 stop" 19 | echo 20 | echo "# 重新启动程序（先终止先启动）" 21 | echo "$0 restart" 22 | echo 23 | echo "# 监控日志输出" 24 | echo "$0 tail-log" 25 | echo 26 | echo "# 重新启动程序并持续监控日志输出" 27 | echo "$0 restart-with-logging" 28 | echo 29 | } 30 | 31 | printHeading() { 32 | title="$1" 33 | paddingWidth=$((($(tput cols)-${#title})/2-3)) 34 | printf "\n%${paddingWidth}s"|tr ' ' '=' 35 | printf " [ $title ] " 36 | printf "%${paddingWidth}s\n\n"|tr ' ' '=' 37 | } 38 | 39 | getJavaCmd() { 40 | if [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]]; then 41 | echo "$JAVA_HOME/bin/java" 42 | else 43 | echo "java" 44 | fi 45 | } 46 | 47 | # ------------------------------------------------ Major Methods ------------------------------------------------ # 48 | 49 | start() { 50 | java=$(getJavaCmd) 51 | nohup $java -Duser.timezone=Asia/Shanghai -classpath "$BDP_COLLECT_CONF_DIR:$BDP_COLLECT_LIB_DIR/*" $BDP_COLLECT_MAIN_CLASS >/dev/null 2>&1 & 52 | echo $! > $BDP_COLLECT_PID 53 | } 54 | 55 | stop() { 56 | if [ -f $BDP_COLLECT_PID ]; then 57 | # kill -0 == see if the PID exists 58 | if kill -0 `cat $BDP_COLLECT_PID` > /dev/null 2>&1; then 59 | kill -9 `cat $BDP_COLLECT_PID` > /dev/null 2>&1 60 | fi 61 | fi 62 | } 63 | 64 | restart() { 65 | stop 66 | start 67 | } 68 | 69 | tailLog() { 70 | tail -F ${app.log.home}/${project.artifactId}.log 71 | } 72 | 73 | # ----------------------------------------------- Shell Scripts Entry -------------------------------------------- # 74 | 75 | case $1 in 76 | (start) 77 | start 78 | ;; 79 | (stop) 80 | stop 81 | ;; 82 | (restart) 83 | restart 84 | ;; 85 | (tail-log) 86 | tailLog 87 | ;; 88 | (restart-with-logging) 89 | restart 90 | tailLog 91 | ;; 92 | (help) 93 | showUsage 94 | ;; 95 | (*) 96 | showUsage 97 | ;; 98 | esac 99 | -------------------------------------------------------------------------------- /bdp-collect/src/main/resources/conf/bdp-collect.properties: -------------------------------------------------------------------------------- 1 | job.cpu.sql=select id, name, hostname as hostname, value, UNIX_TIMESTAMP(timestamp) as timestamp from metric where name='cpu.usage' and `timestamp` > date_add(:#timestamp, interval -5 second) and `timestamp` <= :#timestamp?parametersCount=2 2 | job.cpu.period=5s 3 | 4 | job.mem.sql=select id, name, hostname, value, UNIX_TIMESTAMP(timestamp) as timestamp from metric where name='mem.used' and `timestamp` > date_add(:#timestamp, interval -5 second) and `timestamp` <= :#timestamp?parametersCount=2 5 | job.mem.wave1.period=5s 6 | job.mem.wave2.offset=60 7 | job.mem.wave2.period=5s 8 | 9 | job.alert.sql=select id, message, hostname, status, UNIX_TIMESTAMP(timestamp) as timestamp from alert where created_time > date_add(:#timestamp, interval -5 second) and created_time <= :#timestamp?parametersCount=2 10 | job.alert.period=5s 11 | 12 | #kafka brokers and topics configuration 13 | kafka.brokers=${kafka.brokers} 14 | kafka.topic.cpuUsage=cpu.usage 15 | kafka.topic.memUsed=mem.used 16 | kafka.topic.alert=alert 17 | 18 | ##add for new model 19 | kafka.prefix.cpu.usage=cu 20 | kafka.prefix.mem.used=mu 21 | kafka.prefix.alert=al -------------------------------------------------------------------------------- /bdp-collect/src/main/resources/conf/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger options 2 | log4j.rootLogger=INFO, ROLLING_BY_SIZE 3 | 4 | # App root logger options 5 | log4j.logger.com.github.bdp = ${app.log.level}, ROLLING_BY_SIZE, ERROR_ROLLING_BY_SIZE 6 | 7 | # Console appender 8 | log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender 9 | log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout 10 | log4j.appender.CONSOLE.layout.ConversionPattern=[%d{yyyy-MM-dd HH:mm:ss}] [%p] [%t] [%c{1}.%M(%L)] -- %m%n 11 | 12 | # Rolling file by size appender for bdp-metric 13 | log4j.appender.ROLLING_BY_SIZE=org.apache.log4j.RollingFileAppender 14 | log4j.appender.ROLLING_BY_SIZE.MaxFileSize=64MB 15 | log4j.appender.ROLLING_BY_SIZE.MaxBackupIndex=16 16 | log4j.appender.ROLLING_BY_SIZE.File=${app.log.home}/${project.artifactId}.log 17 | log4j.appender.ROLLING_BY_SIZE.layout=org.apache.log4j.PatternLayout 18 | log4j.appender.ROLLING_BY_SIZE.layout.ConversionPattern=[%d{yyyy-MM-dd HH:mm:ss}] [%p] [%t] [%c{1}.%M(%L)] -- %m%n 19 | 20 | # Rolling file by size appender for bdp-metric warns & errors 21 | log4j.appender.ERROR_ROLLING_BY_SIZE=org.apache.log4j.RollingFileAppender 22 | log4j.appender.ERROR_ROLLING_BY_SIZE.Threshold = WARN 23 | log4j.appender.ERROR_ROLLING_BY_SIZE.MaxFileSize=64MB 24 | log4j.appender.ERROR_ROLLING_BY_SIZE.MaxBackupIndex=16 25 | log4j.appender.ERROR_ROLLING_BY_SIZE.File=${app.log.home}/${project.artifactId}.error.log 26 | log4j.appender.ERROR_ROLLING_BY_SIZE.layout=org.apache.log4j.PatternLayout 27 | log4j.appender.ERROR_ROLLING_BY_SIZE.layout.ConversionPattern=[%d{yyyy-MM-dd HH:mm:ss}] [%p] [%t] [%c{1}.%M(%L)] -- %m%n 28 | 29 | # -------------------------------------------- Not Used Yet ------------------------------------------------------------ 30 | 31 | # Rolling file by time appender for bdp-metric 32 | # Warning: If you enable the following appender it will fill up your disk if you don't have a cleanup lib! 33 | # This uses the updated rolling file appender from log4j-extras that supports a reliable time-based rolling policy. 34 | # See http://logging.apache.org/log4j/companions/extras/apidocs/org/apache/log4j/rolling/TimeBasedRollingPolicy.html 35 | # Add "ROLLING_BY_TIME" to flume.root.logger above if you want to use this 36 | log4j.appender.ROLLING_BY_TIME=org.apache.log4j.rolling.RollingFileAppender 37 | log4j.appender.ROLLING_BY_TIME.rollingPolicy=org.apache.log4j.rolling.TimeBasedRollingPolicy 38 | log4j.appender.ROLLING_BY_TIME.rollingPolicy.ActiveFileName=${app.log.home}/${project.artifactId}.log 39 | log4j.appender.ROLLING_BY_TIME.rollingPolicy.FileNamePattern=${app.log.home}/${project.artifactId}.log.%d{yyyyMMdd} 40 | log4j.appender.ROLLING_BY_TIME.layout=org.apache.log4j.PatternLayout 41 | log4j.appender.ROLLING_BY_TIME.layout.ConversionPattern=[%d{yyyy-MM-dd HH:mm:ss}] [%p] [%t] [%c{1}.%M(%L)] ── %m%n -------------------------------------------------------------------------------- /bdp-collect/src/main/resources/deploy.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | set host=${app.host} 4 | set user=${app.user.name} 5 | set password=${app.user.password} 6 | set baseDir=${app.user.home} 7 | set home=${app.home} 8 | set buildDir=${project.build.directory} 9 | set binZip=${project.build.finalName}-bin.zip 10 | set deltaBinZip=${project.build.finalName}-bin-delta.zip 11 | set logHome=${app.log.home} 12 | 13 | echo. 14 | echo *************************************************************************************** 15 | echo UPLOAD... 16 | echo *************************************************************************************** 17 | 18 | if "%~1"=="-delta" ( 19 | goto uploadDeltaBinZip 20 | ) else ( 21 | goto uploadBinZip 22 | ) 23 | 24 | :uploadBinZip 25 | @echo on 26 | PSCP -l %user% -pw %password% "%buildDir%\\%binZip%" "%host%:/tmp/" 27 | PLINK -l %user% -pw %password% %host% -t "if [ ! -d '%baseDir%' ];then mkdir %baseDir%;fi" 28 | PLINK -l %user% -pw %password% %host% -t "if [ -d '%home%' ];then rm -rf %home%;fi" 29 | PLINK -l %user% -pw %password% %host% -t "unzip /tmp/%binZip% -d %baseDir%/" 30 | PLINK -l %user% -pw %password% %host% -t "mkdir %logHome%/" 31 | @echo off 32 | goto startup 33 | 34 | :uploadDeltaBinZip 35 | @echo on 36 | PSCP -l %user% -pw %password% "%buildDir%\\%deltaBinZip%" "%host%:/tmp/" 37 | PLINK -l %user% -pw %password% %host% -t "unzip -o /tmp/%deltaBinZip% -d %baseDir%/" 38 | @echo off 39 | goto startup 40 | 41 | :startup 42 | echo. 43 | echo *************************************************************************************** 44 | echo STARTUP... 45 | echo *************************************************************************************** 46 | 47 | @echo on 48 | :: if you want to start program automatically after deploy, uncomment next line. 49 | :: PLINK -l %user% -pw %password% %host% -t "%baseDir%/${project.build.finalName}/bin/${project.artifactId}.sh restart-with-logging" 50 | @echo off 51 | -------------------------------------------------------------------------------- /bdp-dwh/.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | /*.iml 3 | target 4 | -------------------------------------------------------------------------------- /bdp-dwh/README.md: -------------------------------------------------------------------------------- 1 | 关于本子项目的部署、运行与代码细节，请参考《大数据平台架构与原型实现：数据中台建设实战》一书第8章以及第4章4.5节 2 | 3 | 友情提示：如果该项目提交后迟迟进入不到运行状态，请确认你的集群资源是否充足，同时可以考虑将profile文件中spark.num.executors与spark.executor.cores两个参数的数值调低。 4 | 5 | 如无必要，不建议同时启动bdp-stream和bdp-dwh的作业，以免因为资源不足导致作业pending，如果确实需要同时运行，在确保资源充足的前提下，可以通过Yarn的动态资源池为bdp-stream和bdp-dwh两个用户分配隔离资源。 6 | 7 | 在真实的生产环境中，流计算项目一般不于批处理项目在同一集群上运行。 -------------------------------------------------------------------------------- /bdp-dwh/build.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | rem A batch script to build -> deploy -> restart 3 | rem -- Laurence Geng 4 | if [%1]==[] ( 5 | echo. 6 | echo Usage: %0 maven-profile-1 maven-profile-2 ... 7 | echo. 8 | goto end 9 | ) 10 | 11 | set profiles=%~1 12 | 13 | :loopProfiles 14 | shift 15 | if "%~1"=="" ( 16 | goto build 17 | ) else ( 18 | set profiles=%profiles%,%~1 19 | goto loopProfiles 20 | ) 21 | 22 | :build 23 | echo. 24 | echo *************************************************************************************** 25 | echo BUILD... 26 | echo *************************************************************************************** 27 | echo. 28 | 29 | if "%profiles%"=="" ( 30 | call mvn clean install -DskipTests=true 31 | ) else ( 32 | call mvn clean install -DskipTests=true -P%profiles% 33 | ) 34 | if "%errorlevel%"=="1" goto :releasefailed 35 | 36 | call target\classes\deploy.bat 37 | 38 | if "%errorlevel%"=="1" goto :releasefailed 39 | 40 | goto releasesuccess 41 | 42 | :releasesuccess 43 | echo. 44 | echo. 45 | echo *************************************************************************************** 46 | echo RELEASE SUCCESS!! 47 | echo *************************************************************************************** 48 | goto end 49 | 50 | :releasefailed 51 | echo. 52 | echo. 53 | echo *************************************************************************************** 54 | echo RELEASE FAILED!! 55 | echo *************************************************************************************** 56 | goto end 57 | 58 | :end 59 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/assembly/bin.xml: -------------------------------------------------------------------------------- 1 | 4 | bin 5 | 6 | zip 7 | 8 | true 9 | 10 | 11 | target/classes 12 | . 13 | 14 | **/*.sh 15 | 16 | 755 17 | 18 | 19 | target/classes 20 | . 21 | 22 | **/*.sql 23 | **/*.csv 24 | 25 | 26 | 27 | target 28 | ./jar 29 | 30 | *.jar 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/profiles/cluster.properties: -------------------------------------------------------------------------------- 1 | # app specific configs 2 | 3 | cluster.hiverserver=gateway1.cluster 4 | 5 | app.name=${project.artifactId} 6 | app.host=gateway1.cluster 7 | app.home=${app.user.home}/${project.build.finalName} 8 | app.user.name=${project.artifactId} 9 | app.user.password=Bdpp1234! 10 | app.user.home=/home/${app.user.name} 11 | app.hdfs.user.name=bdp-dwh 12 | app.hdfs.user.home=hdfs:///user/${app.hdfs.user.name} 13 | 14 | # spark configs 15 | spark.num.executors=3 16 | spark.executor.cores=3 17 | spark.executor.memory=1024m 18 | 19 | # bdp_metric jdbc configs 20 | # 此处的数据库主机不要使用balancer1.cluster, 因为运行MR作业的节点是3个worker节点,也就是在均衡中配置的三个节点 21 | # 而阿里云的四层负载均衡服务不支持负载均衡后端ECS实例作为客户端直接访问负载均衡，所以如果使用balancer1.cluster作为地址会经常连接数据库失败 22 | bdp.metric.db.host=master1.cluster 23 | bdp.metric.jdbc.url=jdbc:mysql://${bdp.metric.db.host}/bdp_metric?useSSL=false&useUnicode=true&characterEncoding=utf-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=GMT 24 | bdp.metric.jdbc.user=bdp_metric 25 | bdp.metric.jdbc.password=Bdpp1234! 26 | 27 | # bdp_master jdbc configs 28 | # 此处的数据库主机不要使用balancer1.cluster, 因为运行MR作业的节点是3个worker节点,也就是在均衡中配置的三个节点 29 | # 而阿里云的四层负载均衡服务不支持负载均衡后端ECS实例作为客户端直接访问负载均衡，所以如果使用balancer1.cluster作为地址会经常连接数据库失败 30 | bdp.master.db.host=master1.cluster 31 | bdp.master.jdbc.url=jdbc:mysql://${bdp.master.db.host}/bdp_master?useSSL=false&useUnicode=true&characterEncoding=utf-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=GMT 32 | bdp.master.jdbc.user=bdp_master 33 | bdp.master.jdbc.password=Bdpp1234! 34 | 35 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/profiles/standalone.properties: -------------------------------------------------------------------------------- 1 | # app specific configs 2 | 3 | cluster.hiverserver=node1.cluster 4 | 5 | app.name=${project.artifactId} 6 | app.host=node1.cluster 7 | app.home=${app.user.home}/${project.build.finalName} 8 | app.user.name=${project.artifactId} 9 | app.user.password=Bdpp1234! 10 | app.user.home=/home/${app.user.name} 11 | app.hdfs.user.name=bdp-dwh 12 | app.hdfs.user.home=hdfs:///user/${app.hdfs.user.name} 13 | 14 | # spark configs 15 | spark.num.executors=1 16 | spark.executor.cores=2 17 | spark.executor.memory=1024m 18 | 19 | # bdp_metric jdbc configs 20 | bdp.metric.db.host=node1.cluster:3306 21 | bdp.metric.jdbc.url=jdbc:mysql://${bdp.metric.db.host}/bdp_metric?useSSL=false&useUnicode=true&characterEncoding=utf-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=GMT 22 | bdp.metric.jdbc.user=bdp_metric 23 | bdp.metric.jdbc.password=Bdpp1234! 24 | 25 | # bdp_master jdbc configs 26 | bdp.master.db.host=node1.cluster:3306 27 | bdp.master.jdbc.url=jdbc:mysql://${bdp.master.db.host}/bdp_master?useSSL=false&useUnicode=true&characterEncoding=utf-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=GMT 28 | bdp.master.jdbc.user=bdp_master 29 | bdp.master.jdbc.password=Bdpp1234! 30 | 31 | 32 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/bin/dmt-infra-metric.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export BDP_DWH_HOME="$(cd "`dirname $(readlink -nf "$0")`"/..; pwd -P)" 4 | export DMT_INFRA_METRIC_HOME="$BDP_DWH_HOME/lib/dmt/infra-metric" 5 | export SUBJECT="dmt :: infra-metric" 6 | export UNDER_LAYER_SUBJECT="dwh :: bdp-metric" 7 | 8 | source "$BDP_DWH_HOME/bin/util.sh" 9 | source "$DMT_INFRA_METRIC_HOME/bin/spark-actions.sh" 10 | 11 | # ------------------------------------------------ Common Methods ------------------------------------------------ # 12 | 13 | showUsage() 14 | { 15 | printHeading "MODULE: [ $(echo "$SUBJECT" | tr 'a-z' 'A-Z') ] USAGE" 16 | 17 | echo "# 说明：创建metric表的schema" 18 | echo "$0 create-fact-metric" 19 | echo 20 | 21 | echo "# 说明：从dwh导入指定时间范围内的metric数据到dmt, 构建metric事实表" 22 | echo "$0 build-fact-metric START_TIME END_TIME" 23 | echo 24 | 25 | echo "# 示例：从dwh导入2018-09-01的metric数据到dmt" 26 | echo "$0 build-fact-metric '2018-09-01T00:00+0800' '2018-09-02T00:00+0800'" 27 | echo 28 | 29 | echo "# 说明：创建metric-avg表的schema" 30 | echo "$0 create-sum-metric-avg" 31 | echo 32 | 33 | echo "# 说明：基于dmt上指定时间范围内的metric数据构建汇总数据metric-avg" 34 | echo "$0 build-sum-metric-avg START_TIME END_TIME" 35 | echo 36 | 37 | echo "# 示例：基于2018-09-01的metric数据数据构建2018-09-01的汇总数据metric-avg" 38 | echo "$0 build-sum-metric-avg '2018-09-01T00:00+0800' '2018-09-02T00:00+0800'" 39 | echo 40 | 41 | echo "# 说明：创建wide-metric-avg表的schema" 42 | echo "$0 create-wide-metric-avg" 43 | echo 44 | 45 | echo "# 说明：基于dmt上指定时间范围内的metric数据构建宽表数据wide-metric-avg" 46 | echo "$0 build-wide-metric-avg START_TIME END_TIME" 47 | echo 48 | 49 | echo "# 示例：基于2018-09-01的metric数据数据构建2018-09-01的宽表数据metric-avg" 50 | echo "$0 build-wide-metric-avg '2018-09-01T00:00+0800' '2018-09-02T00:00+0800'" 51 | echo 52 | 53 | } 54 | 55 | # ---------------------------------------------- Scripts Entrance ---------------------------------------------- # 56 | 57 | case $1 in 58 | (create-fact-metric) 59 | createFactMetric 60 | ;; 61 | (build-fact-metric) 62 | shift 63 | buildFactMetric "$@" 64 | ;; 65 | (create-sum-metric-avg) 66 | createSumMetricAvg 67 | ;; 68 | (build-sum-metric-avg) 69 | shift 70 | buildSumMetricAvg "$@" 71 | ;; 72 | (create-wide-metric-avg) 73 | createWideMetricAvg 74 | ;; 75 | (build-wide-metric-avg) 76 | shift 77 | buildWideMetricAvg "$@" 78 | ;; 79 | (help) 80 | showUsage 81 | ;; 82 | (*) 83 | showUsage 84 | ;; 85 | esac 86 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/bin/dwh-bdp-metric.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export BDP_DWH_HOME="$(cd "`dirname $(readlink -nf "$0")`"/..; pwd -P)" 4 | export DWH_BDP_METRIC_HOME="$BDP_DWH_HOME/lib/dwh/bdp-metric" 5 | export SUBJECT="dwh :: bdp-metric" 6 | export UNDER_LAYER_SUBJECT="src :: bdp-metric" 7 | 8 | source "$BDP_DWH_HOME/bin/util.sh" 9 | source "$DWH_BDP_METRIC_HOME/bin/spark-actions.sh" 10 | 11 | # ------------------------------------------------ Common Methods ------------------------------------------------ # 12 | 13 | showUsage() 14 | { 15 | printHeading "MODULE: [ $(echo "$SUBJECT" | tr 'a-z' 'A-Z') ] USAGE" 16 | 17 | echo "# 说明：创建metric表的schema" 18 | echo "$0 create-metric" 19 | echo 20 | 21 | echo "# 说明：从src导入指定时间范围内的metric数据到dwh" 22 | echo "$0 build-metric START_TIME END_TIME" 23 | echo 24 | 25 | echo "# 示例：从src导入2018-09-01的metric数据到dwh" 26 | echo "$0 build-metric '2018-09-01T00:00+0800' '2018-09-02T00:00+0800'" 27 | echo 28 | } 29 | 30 | # ---------------------------------------------- Scripts Entrance ---------------------------------------------- # 31 | 32 | case $1 in 33 | (create-metric) 34 | createMetric 35 | ;; 36 | (build-metric) 37 | shift 38 | buildMetric "$@" 39 | ;; 40 | (help) 41 | showUsage 42 | ;; 43 | (*) 44 | showUsage 45 | ;; 46 | esac 47 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/bin/src-bdp-metric.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export BDP_DWH_HOME="$(cd "`dirname $(readlink -nf "$0")`"/..; pwd -P)" 4 | export SRC_BDP_METRIC_HOME="$BDP_DWH_HOME/lib/src/bdp-metric" 5 | export SUBJECT="src :: bdp-metric" 6 | export UNDER_LAYER_SUBJECT="tmp :: bdp-metric" 7 | 8 | source "$BDP_DWH_HOME/bin/util.sh" 9 | source "$SRC_BDP_METRIC_HOME/bin/sqoop-actions.sh" 10 | source "$SRC_BDP_METRIC_HOME/bin/spark-actions.sh" 11 | 12 | # ------------------------------------------------ Common Methods ------------------------------------------------ # 13 | 14 | showUsage() 15 | { 16 | printHeading "MODULE: [ $(echo "$SUBJECT" | tr 'a-z' 'A-Z') ] USAGE" 17 | 18 | echo "# 说明：创建metric表的schema" 19 | echo "$0 create-metric" 20 | echo 21 | 22 | echo "# 说明：从数据源导入指定时间范围内的metric数据到src" 23 | echo "$0 build-metric START_TIME END_TIME" 24 | echo 25 | 26 | echo "# 示例：从数据源导入2018-09-01的metric数据到src" 27 | echo "$0 build-metric '2018-09-01T00:00+0800' '2018-09-02T00:00+0800'" 28 | echo 29 | } 30 | 31 | # ---------------------------------------------- Scripts Entrance ---------------------------------------------- # 32 | 33 | case $1 in 34 | (create-metric) 35 | createMetricToTmp 36 | createMetric 37 | ;; 38 | (build-metric) 39 | shift 40 | buildMetricToTmp "$@" 41 | buildMetric "$@" 42 | ;; 43 | (help) 44 | showUsage 45 | ;; 46 | (*) 47 | showUsage 48 | ;; 49 | esac 50 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/deploy.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | set host=${app.host} 4 | set user=${app.user.name} 5 | set password=${app.user.password} 6 | set baseDir=${app.user.home} 7 | set home=${app.home} 8 | set buildDir=${project.build.directory} 9 | set binZip=${project.build.finalName}-bin.zip 10 | 11 | echo. 12 | echo *************************************************************************************** 13 | echo UPLOAD... 14 | echo *************************************************************************************** 15 | 16 | @echo on 17 | PSCP -l %user% -pw %password% "%buildDir%\\%binZip%" "%host%:/tmp/" 18 | PLINK -l %user% -pw %password% %host% -t "if [ ! -d '%baseDir%' ];then mkdir %baseDir%;fi" 19 | PLINK -l %user% -pw %password% %host% -t "if [ -d '%home%' ];then rm -rf %home%;fi" 20 | PLINK -l %user% -pw %password% %host% -t "unzip /tmp/%binZip% -d %baseDir%/" 21 | @echo off -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dmt/infra-metric/action/build-fact_metric.sql: -------------------------------------------------------------------------------- 1 | -- noinspection SqlNoDataSourceInspectionForFile 2 | -- src.bdp_metric_metric or dwh.bdp_metric_metric ?? 3 | insert overwrite table dmt.fact_metric partition(creation_date) 4 | select m.id, a.dwid as app_dwid, s.dwid as server_dwid, mi.dwid as metric_index_dwid, d.dwid as hour_dwid, m.`timestamp`, m.value, m.creation_date 5 | from dwh.bdp_metric_metric m 6 | join dmt.dim_server s on m.hostname = s.hostname 7 | join dmt.dim_app a on a.id = s.app_id 8 | join dmt.dim_metric_index mi on mi.name = m.name 9 | join dmt.dim_hour d on from_unixtime(unix_timestamp(m.`timestamp`),'yyyy-MM-dd HH:00:00') = d.db_hour 10 | where 11 | m.`timestamp` >= s.valid_from and (m.`timestamp` < s.valid_to or s.valid_to is null) and 12 | m.`timestamp` >= a.valid_from and (m.`timestamp` < a.valid_to or a.valid_to is null) and 13 | m.`timestamp` >= mi.valid_from and (m.`timestamp` < mi.valid_to or mi.valid_to is null) and 14 | m.creation_date >= '@startDate@' and m.creation_date < '@endDate@'; -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dmt/infra-metric/action/build-sum_metric_avg.sql: -------------------------------------------------------------------------------- 1 | -- noinspection SqlNoDataSourceInspectionForFile 2 | create temporary function gen_rag as 'com.github.bdp.dwh.udf.GenRag' using jar '${app.home}/jar/${project.build.finalName}.jar'; 3 | 4 | insert overwrite table dmt.sum_metric_avg partition(creation_date) 5 | select 6 | ma.app_dwid, 7 | ma.server_dwid, 8 | ma.metric_index_dwid, 9 | t.dwid, 10 | ma.hour_dwid, 11 | ma.avg_value, 12 | gen_rag(ma.avg_value, t.amber_threshold, t.red_threshold) as rag, 13 | ma.creation_date 14 | from ( 15 | select 16 | m.app_dwid, 17 | m.server_dwid, 18 | m.metric_index_dwid, 19 | m.hour_dwid, 20 | cast(round(avg(m.`value`)) as int) as avg_value, 21 | m.creation_date 22 | from dmt.fact_metric m 23 | where m.creation_date >= '@startDate@' and m.creation_date < '@endDate@' 24 | group by m.creation_date,m.app_dwid, m.server_dwid, m.metric_index_dwid, m.hour_dwid 25 | ) ma 26 | join dmt.dim_server s on s.dwid = ma.server_dwid 27 | join dmt.dim_metric_index dm on dm.dwid = ma.metric_index_dwid 28 | join dmt.dim_metric_threshold t on t.server_id = s.id and t.metric_name = dm.name 29 | where 30 | cast(ma.creation_date as timestamp) >= s.valid_from and (cast(ma.creation_date as timestamp) < s.valid_to or s.valid_to is null) and 31 | cast(ma.creation_date as timestamp) >= dm.valid_from and (cast(ma.creation_date as timestamp) < dm.valid_to or dm.valid_to is null) and 32 | cast(ma.creation_date as timestamp) >= t.valid_from and (cast(ma.creation_date as timestamp) < t.valid_to or t.valid_to is null); -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dmt/infra-metric/action/build-wide_metric_avg.sql: -------------------------------------------------------------------------------- 1 | -- noinspection SqlNoDataSourceInspectionForFile 2 | insert overwrite table dmt.wide_metric_avg partition(creation_date) 3 | select 4 | a.dwid as app_dwid, 5 | a.name as app_name, 6 | a.description as app_description, 7 | a.version as app_version, 8 | s.dwid as server_dwid, 9 | s.hostname as server_hostname, 10 | s.cpu_cores as server_cpu_cores, 11 | s.memory as server_memory, 12 | m.dwid as metric_index_dwid, 13 | m.name as metric_name, 14 | m.description as metric_description, 15 | m.category as metric_category, 16 | t.dwid as metric_threshold_dwid, 17 | t.amber_threshold as amber_threshold, 18 | t.red_threshold as red_threshold, 19 | h.dwid as hour_dwid, 20 | h.db_date as db_date, 21 | h.db_hour as db_hour, 22 | h.year as year, 23 | h.month as month, 24 | h.day as day, 25 | h.hour as hour, 26 | h.quarter as quarter, 27 | h.week as week, 28 | h.day_name as day_name, 29 | h.month_name as month_name, 30 | h.weekend_flag as weekend_flag, 31 | avg_value, 32 | rag, 33 | creation_date 34 | from dmt.sum_metric_avg ma 35 | join dmt.dim_app a on a.dwid = ma.app_dwid 36 | join dmt.dim_server s on s.dwid = ma.server_dwid 37 | join dmt.dim_metric_index m on m.dwid = ma.metric_index_dwid 38 | join dmt.dim_metric_threshold t on t.server_id = s.id and t.metric_name = m.name 39 | join dmt.dim_hour h on h.dwid = ma.hour_dwid 40 | where ma.creation_date >= '@startDate@' and ma.creation_date < '@endDate@'; -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dmt/infra-metric/bin/spark-actions.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | create() 4 | { 5 | target="$1" 6 | execSql "job name: create schema of [ $target @ $SUBJECT ]" "$DMT_INFRA_METRIC_HOME/schema/$target.sql" 7 | } 8 | 9 | build() 10 | { 11 | target="$1" 12 | validateTime "$2" 13 | validateTime "$3" 14 | 15 | startDate=$(date -d "$2" +"%F") 16 | endDate=$(date -d "$3" +"%F") 17 | template="build-$target.sql" 18 | 19 | sed "s/@startDate@/$startDate/g" "$DMT_INFRA_METRIC_HOME/action/$template" | \ 20 | sed "s/@endDate@/$endDate/g" > "$DMT_INFRA_METRIC_HOME/action/.$template" 21 | 22 | execSql "job name: build [ $target ] data from [ $target @ $UNDER_LAYER_SUBJECT ] to [ $target @ $SUBJECT ]" \ 23 | "$DMT_INFRA_METRIC_HOME/action/.$template" 24 | } 25 | 26 | createFactMetric() 27 | { 28 | create "fact_metric" 29 | } 30 | 31 | buildFactMetric() 32 | { 33 | build "fact_metric" "$1" "$2" 34 | } 35 | 36 | createSumMetricAvg() 37 | { 38 | create "sum_metric_avg" 39 | 40 | } 41 | 42 | buildSumMetricAvg() 43 | { 44 | build "sum_metric_avg" "$1" "$2" 45 | } 46 | 47 | createWideMetricAvg() 48 | { 49 | create "wide_metric_avg" 50 | 51 | } 52 | 53 | buildWideMetricAvg() 54 | { 55 | build "wide_metric_avg" "$1" "$2" 56 | } 57 | 58 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dmt/infra-metric/schema/fact_metric.sql: -------------------------------------------------------------------------------- 1 | -- noinspection sqlnodatasourceinspectionforfile 2 | 3 | drop table if exists dmt.fact_metric; 4 | create table if not exists dmt.fact_metric ( 5 | id bigint, 6 | app_dwid bigint, 7 | server_dwid bigint, 8 | metric_index_dwid bigint, 9 | hour_dwid bigint, 10 | `timestamp` timestamp, 11 | value bigint 12 | ) 13 | partitioned by (creation_date string) 14 | stored as parquet; 15 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dmt/infra-metric/schema/sum_metric_avg.sql: -------------------------------------------------------------------------------- 1 | -- noinspection sqlnodatasourceinspectionforfile 2 | 3 | drop table if exists dmt.sum_metric_avg; 4 | create table if not exists dmt.sum_metric_avg ( 5 | app_dwid bigint, 6 | server_dwid bigint, 7 | metric_index_dwid bigint, 8 | metric_threshold_dwid bigint, 9 | hour_dwid bigint, 10 | avg_value int, 11 | rag string 12 | ) 13 | partitioned by (creation_date string) 14 | stored as parquet; 15 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dmt/infra-metric/schema/wide_metric_avg.sql: -------------------------------------------------------------------------------- 1 | -- noinspection sqlnodatasourceinspectionforfile 2 | 3 | drop table if exists dmt.wide_metric_avg; 4 | create table if not exists dmt.wide_metric_avg ( 5 | app_dwid bigint, 6 | app_name string, 7 | app_description string, 8 | app_version string, 9 | server_dwid bigint, 10 | server_hostname string, 11 | server_cpu_cores int, 12 | server_memory int, 13 | metric_index_dwid bigint, 14 | metric_name string, 15 | metric_description string, 16 | metric_category string, 17 | metric_threshold_dwid bigint, 18 | amber_threshold int, 19 | red_threshold int, 20 | hour_dwid bigint, 21 | db_date string, 22 | db_hour timestamp, 23 | year int, 24 | month int, 25 | day int, 26 | hour int, 27 | quarter int, 28 | week int, 29 | day_name string, 30 | month_name string, 31 | weekend_flag boolean, 32 | avg_value bigint, 33 | rag string 34 | ) 35 | partitioned by (creation_date string) 36 | stored as parquet; -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dmt/master-data/action/build-dim_app.sql: -------------------------------------------------------------------------------- 1 | -- noinspection SqlNoDataSourceInspectionForFile 2 | 3 | set spark.sql.hive.convertMetastoreParquet=false; 4 | set spark.sql.parser.quotedRegexColumnNames=true; 5 | 6 | -- 将新增和变更的数据定义为一个独立数据集，便于后续操作中引用 7 | create or replace temporary view updated_and_added_records as 8 | select 9 | s.`(creation_time|update_time|imported_time)?+.+` 10 | from 11 | src.bdp_master_app s 12 | where 13 | s.update_date >='@startDate@' and s.update_date < '@endDate@'; 14 | 15 | insert overwrite table dmt.dim_app 16 | select 17 | * 18 | from( 19 | -- 针对DMT全量表的操作: 20 | -- 操作1.1: 将DMT全量表中的“更新前的数据”复制到结果集，失效日期取SRC增量表中记录的更新时间，有效标记位置为"false" 21 | -- 操作1.2: 将DMT全量表中的“变更历史记录”复制到结果集，不做任何修改 22 | select 23 | m.`(valid_to|eff_flag)?+.+`, 24 | -- 如果是DMT中的“更新前的记录”，失效日期取增量记录里的更新时间，否则沿用DMT全量表中的原有值 25 | case when m.eff_flag = true and u.id is not null then 26 | u.update_date 27 | else 28 | m.valid_to 29 | end as 30 | valid_to, 31 | -- 如果是DMT中的“更新前的记录”，有效标记位置为"false"，否则沿用DMT全量表中的原有值 32 | case when m.eff_flag = true and u.id is not null then 33 | false 34 | else 35 | m.eff_flag 36 | end as 37 | eff_flag 38 | from 39 | dmt.dim_app m 40 | left join 41 | updated_and_added_records u 42 | on 43 | m.id = u.id 44 | union all 45 | -- 操作2: 针对SRC增量表(新增和变更数据集)的操作: 将增量数据复制到结果集，生效日期取增量记录里的更新时间，有效标记位置为"true" 46 | select 47 | row_number() over(order by 0) + m.max_id as dwid, -- 在最大ID的基础上累加，生成数仓中的代理主键dwid 48 | u.`(update_date)?+.+`, 49 | u.update_date as valid_from, -- 将“更新后的记录”的更新日期作为生效日期 50 | null as valid_to, 51 | true as eff_flag -- 有效标记位置为"true" 52 | from 53 | updated_and_added_records u 54 | cross join 55 | (select coalesce(max(dwid),0) as max_id from dmt.dim_app) m 56 | ); -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dmt/master-data/action/build-dim_hour.sql: -------------------------------------------------------------------------------- 1 | -- noinspection SqlNoDataSourceInspectionForFile 2 | 3 | insert overwrite table dmt.dim_hour 4 | select 5 | dwid, 6 | db_date, 7 | db_hour, 8 | year, 9 | month, 10 | day, 11 | hour, 12 | quarter, 13 | week, 14 | day_name, 15 | month_name, 16 | weekend_flag 17 | from 18 | tmp.dim_hour; -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dmt/master-data/action/build-dim_metric_index.sql: -------------------------------------------------------------------------------- 1 | -- noinspection SqlNoDataSourceInspectionForFile 2 | 3 | set spark.sql.hive.convertMetastoreParquet=false; 4 | set spark.sql.parser.quotedRegexColumnNames=true; 5 | 6 | -- 将新增和变更的数据定义为一个独立数据集，便于后续操作中引用 7 | create or replace temporary view updated_and_added_records as 8 | select 9 | s.`(creation_time|update_time|imported_time)?+.+` 10 | from 11 | src.bdp_master_metric_index s 12 | where 13 | s.update_date >='@startDate@' and s.update_date < '@endDate@'; 14 | 15 | insert overwrite table dmt.dim_metric_index 16 | select 17 | * 18 | from( 19 | -- 针对DMT全量表的操作: 20 | -- 操作1.1: 将DMT全量表中的“更新前的数据”复制到结果集，失效日期取SRC增量表中记录的更新时间，有效标记位置为"false" 21 | -- 操作1.2: 将DMT全量表中的“变更历史记录”复制到结果集，不做任何修改 22 | select 23 | m.`(valid_to|eff_flag)?+.+`, 24 | -- 如果是“更新前的记录”，失效日期取增量记录里的更新时间，否则沿用全量记录中的原有值 25 | case when m.eff_flag = true and u.id is not null then 26 | u.update_date 27 | else 28 | m.valid_to 29 | end as 30 | valid_to, 31 | -- 如果是“更新前的记录”，有效标记位置为"false"，否则沿用全量记录中的原有值 32 | case when m.eff_flag = true and u.id is not null then 33 | false 34 | else 35 | m.eff_flag 36 | end as 37 | eff_flag 38 | from 39 | dmt.dim_metric_index m 40 | left join 41 | updated_and_added_records u 42 | on 43 | m.id = u.id 44 | union all 45 | -- 操作2: 针对SRC增量表(新增和变更数据集)的操作: 将增量数据复制到结果集，生效日期取增量记录里的更新时间，有效标记位置为"true" 46 | select 47 | row_number() over(order by 0) + m.max_id as dwid, -- 在最大ID的基础上累加，生成数仓中的代理主键dwid 48 | u.`(update_date)?+.+`, 49 | u.update_date as valid_from, -- 将“更新后的记录”的更新日期作为生效日期 50 | null as valid_to, 51 | true as eff_flag -- 有效标记位置为"true" 52 | from 53 | updated_and_added_records u 54 | cross join 55 | (select coalesce(max(dwid),0) as max_id from dmt.dim_metric_index) m 56 | ); -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dmt/master-data/action/build-dim_metric_threshold.sql: -------------------------------------------------------------------------------- 1 | -- noinspection SqlNoDataSourceInspectionForFile 2 | 3 | set spark.sql.hive.convertMetastoreParquet=false; 4 | set spark.sql.parser.quotedRegexColumnNames=true; 5 | 6 | -- 将新增和变更的数据定义为一个独立数据集，便于后续操作中引用 7 | create or replace temporary view updated_and_added_records as 8 | select 9 | s.`(creation_time|update_time|imported_time)?+.+` 10 | from 11 | src.bdp_master_metric_threshold s 12 | where 13 | s.update_date >='@startDate@' and s.update_date < '@endDate@'; 14 | 15 | insert overwrite table dmt.dim_metric_threshold 16 | select 17 | * 18 | from( 19 | -- 针对DMT全量表的操作: 20 | -- 操作1.1: 将DMT全量表中的“更新前的数据”复制到结果集，失效日期取SRC增量表中记录的更新时间，有效标记位置为"false" 21 | -- 操作1.2: 将DMT全量表中的“变更历史记录”复制到结果集，不做任何修改 22 | select 23 | m.`(valid_to|eff_flag)?+.+`, 24 | -- 如果是“更新后的记录”，失效日期取增量记录里的更新时间，否则沿用全量记录中的原有值 25 | case when m.eff_flag = true and u.server_id is not null and u.metric_name is not null then 26 | u.update_date 27 | else 28 | m.valid_to 29 | end as 30 | valid_to, 31 | -- 如果是“更新后的记录”，有效标记位置为"false"，否则沿用全量记录中的原有值 32 | case when m.eff_flag = true and u.server_id is not null and u.metric_name is not null then 33 | false 34 | else 35 | m.eff_flag 36 | end as 37 | eff_flag 38 | from 39 | dmt.dim_metric_threshold m 40 | left join 41 | updated_and_added_records u 42 | on 43 | m.server_id = u.server_id and m.metric_name = u.metric_name 44 | union all 45 | -- 操作2: 针对SRC增量表(新增和变更数据集)的操作: 将增量数据复制到结果集，生效日期取增量记录里的更新时间，有效标记位置为"true" 46 | select 47 | row_number() over(order by 0) + m.max_id as dwid, -- 在最大ID的基础上累加，生成数仓中的代理主键dwid 48 | u.`(update_date)?+.+`, 49 | u.update_date as valid_from, -- 将“更新后的记录”的更新日期作为生效日期 50 | null as valid_to, 51 | true as eff_flag -- 有效标记位置为"true" 52 | from 53 | updated_and_added_records u 54 | cross join 55 | (select coalesce(max(dwid),0) as max_id from dmt.dim_metric_threshold) m 56 | ); 57 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dmt/master-data/action/build-dim_server.sql: -------------------------------------------------------------------------------- 1 | -- noinspection SqlNoDataSourceInspectionForFile 2 | 3 | set spark.sql.hive.convertMetastoreParquet=false; 4 | set spark.sql.parser.quotedRegexColumnNames=true; 5 | 6 | -- 将新增和变更的数据定义为一个独立数据集，便于后续操作中引用 7 | create or replace temporary view updated_and_added_records as 8 | select 9 | s.`(creation_time|update_time|imported_time)?+.+` 10 | from 11 | src.bdp_master_server s 12 | where 13 | s.update_date >='@startDate@' and s.update_date < '@endDate@'; 14 | 15 | insert overwrite table dmt.dim_server 16 | select 17 | * 18 | from( 19 | -- 针对DMT全量表的操作: 20 | -- 操作1.1: 将DMT全量表中的“更新前的数据”复制到结果集，失效日期取SRC增量表中记录的更新时间，有效标记位置为"false" 21 | -- 操作1.2: 将DMT全量表中的“变更历史记录”复制到结果集，不做任何修改 22 | select 23 | m.`(valid_to|eff_flag)?+.+`, 24 | -- 如果是“更新后的记录”，失效日期取增量记录里的更新时间，否则沿用全量记录中的原有值 25 | case when m.eff_flag = true and u.id is not null then 26 | u.update_date 27 | else 28 | m.valid_to 29 | end as 30 | valid_to, 31 | -- 如果是“更新后的记录”，有效标记位置为"false"，否则沿用全量记录中的原有值 32 | case when m.eff_flag = true and u.id is not null then 33 | false 34 | else 35 | m.eff_flag 36 | end as 37 | eff_flag 38 | from 39 | dmt.dim_server m 40 | left join 41 | updated_and_added_records u 42 | on 43 | m.id = u.id 44 | union all 45 | -- 操作2: 针对SRC增量表(新增和变更数据集)的操作: 将增量数据复制到结果集，生效日期取增量记录里的更新时间，有效标记位置为"true" 46 | select 47 | row_number() over(order by 0) + m.max_id as dwid, -- 在最大ID的基础上累加，生成数仓中的代理主键dwid 48 | u.`(update_date)?+.+`, 49 | u.update_date as valid_from, -- 将“更新后的记录”的更新日期作为生效日期 50 | null as valid_to, 51 | true as eff_flag -- 有效标记位置为"true" 52 | from 53 | updated_and_added_records u 54 | cross join 55 | (select coalesce(max(dwid),0) as max_id from dmt.dim_server) m 56 | ); -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dmt/master-data/bin/spark-actions.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | create() 4 | { 5 | target="$1" 6 | execSql "job name: create schema of [ $target @ $SUBJECT ]" "$DMT_MASTER_DATA_HOME/schema/$target.sql" 7 | } 8 | 9 | build() 10 | { 11 | target="$1" 12 | validateTime "$2" 13 | validateTime "$3" 14 | 15 | startDate=$(date -d "$2" +"%F") 16 | endDate=$(date -d "$3" +"%F") 17 | template="build-$target.sql" 18 | 19 | sed "s/@startDate@/$startDate/g" "$DMT_MASTER_DATA_HOME/action/$template" | \ 20 | sed "s/@endDate@/$endDate/g" > "$DMT_MASTER_DATA_HOME/action/.$template" 21 | 22 | execSql "job name: build [ $target ] data from [ $target @ $UNDER_LAYER_SUBJECT ] to [ $target @ $SUBJECT ]" \ 23 | "$DMT_MASTER_DATA_HOME/action/.$template" 24 | } 25 | 26 | createApp() 27 | { 28 | create "dim_app" 29 | } 30 | 31 | buildApp() 32 | { 33 | build "dim_app" "$1" "$2" 34 | } 35 | 36 | createServer() 37 | { 38 | create "dim_server" 39 | } 40 | 41 | buildServer() 42 | { 43 | build "dim_server" "$1" "$2" 44 | } 45 | 46 | createMetricIndex() 47 | { 48 | create "dim_metric_index" 49 | } 50 | 51 | buildMetricIndex() 52 | { 53 | build "dim_metric_index" "$1" "$2" 54 | } 55 | 56 | createMetricThreshold() 57 | { 58 | create "dim_metric_threshold" 59 | } 60 | 61 | buildMetricThreshold() 62 | { 63 | build "dim_metric_threshold" "$1" "$2" 64 | } 65 | 66 | createHour() 67 | { 68 | create "dim_hour" 69 | } 70 | 71 | buildHour() 72 | { 73 | # put data dimension data file onto HDFS. 74 | template="build-dim_hour.sql" 75 | dimHourLocalPath="$DMT_MASTER_DATA_HOME/data/dim_hour.csv" 76 | dimHourHdfsDir="/data/tmp/dim_hour" 77 | hdfs dfs -test -d $dimHourHdfsDir && hdfs dfs -rm -r -f -skipTrash $dimHourHdfsDir 78 | hdfs dfs -mkdir -p $dimHourHdfsDir 79 | dimHourHdfsPath="$dimHourHdfsDir/dim_hour.csv" 80 | target="dim_hour" 81 | 82 | hdfs dfs -put -f $dimHourLocalPath $dimHourHdfsPath 83 | execSql "job name: build [ $target ] data for [ $SUBJECT ], data flow: [ $target @ $dimHourLocalPath -> $target @ $SUBJECT ]" "$DMT_MASTER_DATA_HOME/action/$template" 84 | } 85 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dmt/master-data/schema/dim_app.sql: -------------------------------------------------------------------------------- 1 | -- noinspection sqlnodatasourceinspectionforfile 2 | 3 | drop table if exists dmt.dim_app; 4 | create table if not exists dmt.dim_app ( 5 | dwid bigint, 6 | id bigint, 7 | name string, 8 | description string, 9 | version string, 10 | valid_from timestamp, 11 | valid_to timestamp, 12 | eff_flag boolean 13 | ) 14 | stored as parquet; 15 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dmt/master-data/schema/dim_hour.sql: -------------------------------------------------------------------------------- 1 | -- noinspection sqlnodatasourceinspectionforfile 2 | 3 | drop table if exists dmt.dim_hour; 4 | create table if not exists dmt.dim_hour ( 5 | dwid bigint, 6 | db_date string, 7 | db_hour timestamp, 8 | year int, 9 | month int, 10 | day int, 11 | hour int, 12 | quarter int, 13 | week int, 14 | day_name string, 15 | month_name string, 16 | weekend_flag boolean 17 | ) 18 | stored as parquet; 19 | 20 | -- staging tables 21 | 22 | drop table if exists tmp.dim_hour; 23 | create table if not exists tmp.dim_hour ( 24 | dwid bigint, 25 | db_date string, 26 | db_hour timestamp, 27 | year int, 28 | month int, 29 | day int, 30 | hour int, 31 | quarter int, 32 | week int, 33 | day_name string, 34 | month_name string, 35 | weekend_flag boolean 36 | ) 37 | row format delimited 38 | fields terminated by ',' 39 | stored as textfile 40 | location '/data/tmp/dim_hour/'; -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dmt/master-data/schema/dim_metric_index.sql: -------------------------------------------------------------------------------- 1 | -- noinspection sqlnodatasourceinspectionforfile 2 | 3 | drop table if exists dmt.dim_metric_index; 4 | create table if not exists dmt.dim_metric_index ( 5 | dwid bigint, 6 | id bigint, 7 | name string, 8 | description string, 9 | category string, 10 | valid_from timestamp, 11 | valid_to timestamp, 12 | eff_flag boolean 13 | ) 14 | stored as parquet; -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dmt/master-data/schema/dim_metric_threshold.sql: -------------------------------------------------------------------------------- 1 | -- noinspection sqlnodatasourceinspectionforfile 2 | 3 | drop table if exists dmt.dim_metric_threshold; 4 | create table if not exists dmt.dim_metric_threshold ( 5 | dwid bigint, 6 | server_id bigint, 7 | metric_name string, 8 | amber_threshold int, 9 | red_threshold int, 10 | valid_from timestamp, 11 | valid_to timestamp, 12 | eff_flag boolean 13 | ) 14 | stored as parquet; -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dmt/master-data/schema/dim_server.sql: -------------------------------------------------------------------------------- 1 | -- noinspection sqlnodatasourceinspectionforfile 2 | 3 | drop table if exists dmt.dim_server; 4 | create table if not exists dmt.dim_server ( 5 | dwid bigint, 6 | id bigint, 7 | app_id bigint, 8 | hostname string, 9 | cpu_cores int, 10 | memory int, 11 | valid_from timestamp, 12 | valid_to timestamp, 13 | eff_flag boolean 14 | ) 15 | stored as parquet; -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dwh/bdp-master/action/build-app.sql: -------------------------------------------------------------------------------- 1 | -- noinspection SqlNoDataSourceInspectionForFile 2 | -- move consolidated data back to original table. 3 | 4 | set spark.sql.hive.convertMetastoreParquet=false; 5 | set spark.sql.parser.quotedRegexColumnNames=true; 6 | 7 | insert overwrite table dwh.bdp_master_app 8 | select 9 | `(row_num|oc)?+.+` 10 | from ( 11 | select 12 | *, 13 | row_number () over ( 14 | partition by id 15 | order by update_time desc, oc desc 16 | ) as row_num 17 | from ( 18 | select 19 | *, 0 as oc 20 | from 21 | dwh.bdp_master_app 22 | union all 23 | select 24 | `(update_date)?+.+`, 1 as oc 25 | from 26 | src.bdp_master_app 27 | where update_date >= '@startDate@' and update_date < '@endDate@' 28 | )a 29 | ) 30 | where row_num = 1; -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dwh/bdp-master/action/build-metric_index.sql: -------------------------------------------------------------------------------- 1 | -- noinspection SqlNoDataSourceInspectionForFile 2 | -- move consolidated data back to original table. 3 | 4 | set spark.sql.hive.convertMetastoreParquet=false; 5 | set spark.sql.parser.quotedRegexColumnNames=true; 6 | 7 | insert overwrite table dwh.bdp_master_metric_index 8 | select 9 | `(row_num|oc)?+.+` 10 | from ( 11 | select 12 | *, 13 | row_number () over ( 14 | partition by id 15 | order by update_time desc, oc desc 16 | ) as row_num 17 | from ( 18 | select 19 | *, 0 as oc 20 | from 21 | dwh.bdp_master_metric_index 22 | union all 23 | select 24 | `(update_date)?+.+`, 1 as oc 25 | from 26 | src.bdp_master_metric_index 27 | where 28 | update_date >= '@startDate@' and update_date < '@endDate@' 29 | )a 30 | ) 31 | where row_num = 1; -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dwh/bdp-master/action/build-metric_threshold.sql: -------------------------------------------------------------------------------- 1 | -- noinspection SqlNoDataSourceInspectionForFile 2 | -- move consolidated data back to original table. 3 | 4 | set spark.sql.hive.convertMetastoreParquet=false; 5 | set spark.sql.parser.quotedRegexColumnNames=true; 6 | 7 | insert overwrite table dwh.bdp_master_metric_threshold 8 | select 9 | `(row_num|oc)?+.+` 10 | from ( 11 | select 12 | *, 13 | row_number () over ( 14 | partition by server_id, metric_name 15 | order by update_time desc, oc desc 16 | ) as row_num 17 | from ( 18 | select 19 | *, 0 as oc 20 | from 21 | dwh.bdp_master_metric_threshold 22 | union all 23 | select 24 | `(update_date)?+.+`, 1 as oc 25 | from 26 | src.bdp_master_metric_threshold 27 | where 28 | update_date >= '@startDate@' and update_date < '@endDate@' 29 | )a 30 | ) 31 | where row_num = 1; -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dwh/bdp-master/action/build-server.sql: -------------------------------------------------------------------------------- 1 | -- noinspection SqlNoDataSourceInspectionForFile 2 | -- move consolidated data back to original table. 3 | 4 | set spark.sql.hive.convertMetastoreParquet=false; 5 | set spark.sql.parser.quotedRegexColumnNames=true; 6 | 7 | insert overwrite table dwh.bdp_master_server 8 | select 9 | `(row_num|oc)?+.+` 10 | from ( 11 | select 12 | *, 13 | row_number () over ( 14 | partition by id 15 | order by update_time desc, oc desc 16 | ) as row_num 17 | from ( 18 | select 19 | *, 0 as oc 20 | from 21 | dwh.bdp_master_server 22 | union all 23 | select 24 | `(update_date)?+.+`, 1 as oc 25 | from 26 | src.bdp_master_server 27 | where 28 | update_date >= '@startDate@' and update_date < '@endDate@' 29 | )a 30 | ) 31 | where row_num = 1; -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dwh/bdp-master/bin/spark-actions.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | create() 4 | { 5 | target="$1" 6 | execSql "job name: create schema of [ $target @ $SUBJECT ]" "${DWH_BDP_MASTER_HOME}/schema/$target.sql" 7 | } 8 | 9 | build() 10 | { 11 | target="$1" 12 | validateTime "$2" 13 | validateTime "$3" 14 | 15 | startDate=$(date -d "$2" +"%F") 16 | endDate=$(date -d "$3" +"%F") 17 | template="build-$target.sql" 18 | 19 | sed "s/@startDate@/$startDate/g" "$DWH_BDP_MASTER_HOME/action/$template" | \ 20 | sed "s/@endDate@/$endDate/g" > "$DWH_BDP_MASTER_HOME/action/.$template" 21 | 22 | execSql "job name: build [ $target ] data from [ $target @ $UNDER_LAYER_SUBJECT ] to [ $target @ $SUBJECT ]" \ 23 | "$DWH_BDP_MASTER_HOME/action/.$template" 24 | } 25 | 26 | createApp() 27 | { 28 | create "app" 29 | } 30 | 31 | buildApp() 32 | { 33 | build "app" "$1" "$2" 34 | } 35 | 36 | createServer() 37 | { 38 | create "server" 39 | } 40 | 41 | buildServer() 42 | { 43 | build "server" "$1" "$2" 44 | } 45 | 46 | createMetricIndex() 47 | { 48 | create "metric_index" 49 | } 50 | 51 | buildMetricIndex() 52 | { 53 | build "metric_index" "$1" "$2" 54 | } 55 | 56 | createMetricThreshold() 57 | { 58 | create "metric_threshold" 59 | } 60 | 61 | buildMetricThreshold() 62 | { 63 | build "metric_threshold" "$1" "$2" 64 | } -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dwh/bdp-master/schema/app.sql: -------------------------------------------------------------------------------- 1 | -- noinspection sqlnodatasourceinspectionforfile 2 | 3 | drop table if exists dwh.bdp_master_app; 4 | create table if not exists dwh.bdp_master_app ( 5 | id bigint, 6 | name string, 7 | description string, 8 | version string, 9 | creation_time timestamp, 10 | update_time timestamp, 11 | imported_time timestamp 12 | ) 13 | stored as parquet; -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dwh/bdp-master/schema/metric_index.sql: -------------------------------------------------------------------------------- 1 | -- noinspection sqlnodatasourceinspectionforfile 2 | 3 | drop table if exists dwh.bdp_master_metric_index; 4 | create table if not exists dwh.bdp_master_metric_index ( 5 | id bigint, 6 | name string, 7 | description string, 8 | category string, 9 | creation_time timestamp, 10 | update_time timestamp, 11 | imported_time timestamp 12 | ) 13 | stored as parquet; -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dwh/bdp-master/schema/metric_threshold.sql: -------------------------------------------------------------------------------- 1 | -- noinspection sqlnodatasourceinspectionforfile 2 | 3 | drop table if exists dwh.bdp_master_metric_threshold; 4 | create table if not exists dwh.bdp_master_metric_threshold ( 5 | server_id bigint, 6 | metric_name string, 7 | amber_threshold int, 8 | red_threshold int, 9 | creation_time timestamp, 10 | update_time timestamp, 11 | imported_time timestamp 12 | ) 13 | stored as parquet; 14 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dwh/bdp-master/schema/server.sql: -------------------------------------------------------------------------------- 1 | -- noinspection sqlnodatasourceinspectionforfile 2 | 3 | drop table if exists dwh.bdp_master_server; 4 | create table if not exists dwh.bdp_master_server ( 5 | id bigint, 6 | app_id bigint, 7 | hostname string, 8 | cpu_cores int, 9 | memory int, 10 | creation_time timestamp, 11 | update_time timestamp, 12 | imported_time timestamp 13 | ) 14 | stored as parquet; 15 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dwh/bdp-metric/action/build-metric.sql: -------------------------------------------------------------------------------- 1 | -- noinspection SqlNoDataSourceInspectionForFile 2 | 3 | insert overwrite table dwh.bdp_metric_metric partition(creation_date) 4 | select 5 | id, 6 | name, 7 | hostname, 8 | value, 9 | cast(`timestamp` as timestamp), 10 | imported_time, 11 | cast(cast(`timestamp` as date) as string) as creation_date 12 | from 13 | src.bdp_metric_metric 14 | where 15 | creation_date >= '@startDate@' and creation_date < '@endDate@'; -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dwh/bdp-metric/bin/spark-actions.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | create() 4 | { 5 | target="$1" 6 | execSql "job name: create schema of [ $target @ $SUBJECT ]" "$DWH_BDP_METRIC_HOME/schema/$target.sql" 7 | } 8 | 9 | build() 10 | { 11 | target="$1" 12 | validateTime "$2" 13 | validateTime "$3" 14 | 15 | startDate=$(date -d "$2" +"%F") 16 | endDate=$(date -d "$3" +"%F") 17 | template="build-$target.sql" 18 | 19 | sed "s/@startDate@/$startDate/g" "$DWH_BDP_METRIC_HOME/action/$template" | \ 20 | sed "s/@endDate@/$endDate/g" > "$DWH_BDP_METRIC_HOME/action/.$template" 21 | 22 | execSql "job name: build [ $target ] data from [ $target @ $UNDER_LAYER_SUBJECT ] to [ $target @ $SUBJECT ]" \ 23 | "$DWH_BDP_METRIC_HOME/action/.$template" 24 | } 25 | 26 | createMetric() 27 | { 28 | create "metric" 29 | } 30 | 31 | buildMetric() 32 | { 33 | build "metric" "$1" "$2" 34 | } 35 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/dwh/bdp-metric/schema/metric.sql: -------------------------------------------------------------------------------- 1 | -- noinspection sqlnodatasourceinspectionforfile 2 | 3 | drop table if exists dwh.bdp_metric_metric; 4 | create table if not exists dwh.bdp_metric_metric ( 5 | id bigint, 6 | name string, 7 | hostname string, 8 | value bigint, 9 | `timestamp` timestamp, 10 | imported_time timestamp 11 | ) 12 | partitioned by (creation_date string) 13 | stored as parquet; 14 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/src/bdp-master/action/build-app.sql: -------------------------------------------------------------------------------- 1 | -- noinspection SqlNoDataSourceInspectionForFile 2 | 3 | insert overwrite table src.bdp_master_app partition(update_date) 4 | select 5 | id, 6 | name, 7 | description, 8 | version, 9 | cast(creation_time as timestamp) as creation_time, 10 | cast(update_time as timestamp) as update_time, 11 | current_timestamp as imported_time, 12 | cast(cast(`update_time` as date) as string) as update_date 13 | from 14 | tmp.bdp_master_app; 15 | 16 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/src/bdp-master/action/build-metric_index.sql: -------------------------------------------------------------------------------- 1 | -- noinspection SqlNoDataSourceInspectionForFile 2 | 3 | insert overwrite table src.bdp_master_metric_index partition(update_date) 4 | select 5 | id, 6 | name, 7 | description, 8 | category, 9 | cast(creation_time as timestamp) as creation_time, 10 | cast(update_time as timestamp) as update_time, 11 | current_timestamp as imported_time, 12 | cast(cast(update_time as date) as string) as update_date 13 | from 14 | tmp.bdp_master_metric_index; 15 | 16 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/src/bdp-master/action/build-metric_threshold.sql: -------------------------------------------------------------------------------- 1 | -- noinspection SqlNoDataSourceInspectionForFile 2 | 3 | insert overwrite table src.bdp_master_metric_threshold partition(update_date) 4 | select 5 | server_id, 6 | metric_name, 7 | amber_threshold, 8 | red_threshold, 9 | cast(creation_time as timestamp) as creation_time, 10 | cast(update_time as timestamp) as update_time, 11 | current_timestamp as imported_time, 12 | cast(cast(update_time as date) as string) as update_date 13 | from 14 | tmp.bdp_master_metric_threshold; -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/src/bdp-master/action/build-server.sql: -------------------------------------------------------------------------------- 1 | -- noinspection SqlNoDataSourceInspectionForFile 2 | 3 | insert overwrite table src.bdp_master_server partition(update_date) 4 | select 5 | id, 6 | app_id, 7 | hostname, 8 | cpu_cores, 9 | memory, 10 | cast(creation_time as timestamp) as creation_time, 11 | cast(update_time as timestamp) as update_time, 12 | current_timestamp as imported_time, 13 | cast(cast(`update_time` as date) as string) as update_date 14 | from 15 | tmp.bdp_master_server; 16 | 17 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/src/bdp-master/bin/spark-actions.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | create() 4 | { 5 | target="$1" 6 | execSql "job name: create schema of [ $target @ $SUBJECT ]" "${SRC_BDP_MASTER_HOME}/schema/$target.sql" 7 | } 8 | 9 | build() 10 | { 11 | target="$1" 12 | execSql "job name: build [ $target ] data from [ $target @ $UNDER_LAYER_SUBJECT ] to [ $target @ $SUBJECT ]" "${SRC_BDP_MASTER_HOME}/action/build-$target.sql" 13 | } 14 | 15 | createApp() 16 | { 17 | create "app" 18 | } 19 | 20 | buildApp() 21 | { 22 | build "app" 23 | } 24 | 25 | createServer() 26 | { 27 | create "server" 28 | } 29 | 30 | buildServer() 31 | { 32 | build "server" 33 | } 34 | 35 | createMetricIndex() 36 | { 37 | create "metric_index" 38 | } 39 | 40 | buildMetricIndex() 41 | { 42 | build "metric_index" 43 | } 44 | 45 | createMetricThreshold() 46 | { 47 | create "metric_threshold" 48 | } 49 | 50 | buildMetricThreshold() 51 | { 52 | build "metric_threshold" 53 | } -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/src/bdp-master/bin/sqoop-actions.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | createToTmp() 4 | { 5 | srcTable="$1" 6 | sinkTable="$2" 7 | printHeading "job name: create table ${sinkTable}" 8 | 9 | sqoop create-hive-table \ 10 | -D mapred.job.name="job name: create table [$sinkTable]" \ 11 | --connect '${bdp.master.jdbc.url}' \ 12 | --username '${bdp.master.jdbc.user}' \ 13 | --password '${bdp.master.jdbc.password}' \ 14 | --table "$srcTable" \ 15 | --hive-table "$sinkTable" \ 16 | --hive-overwrite 17 | } 18 | 19 | buildToTmp() 20 | { 21 | srcTable="$1" 22 | sinkTable="$2" 23 | splitColumn="$3" 24 | validateTime "$4" 25 | validateTime "$5" 26 | 27 | jobName="subject: $SUBJECT -- build [ $srcTable ] data from data source to tmp layer via sqoop" 28 | 29 | printHeading "${jobName}" 30 | 31 | startTime=$(date -d "$4" +"%F %T") 32 | endTime=$(date -d "$5" +"%F %T") 33 | 34 | sinkTablePath="$TMP_DATA_BASE_DIR/$sinkTable/" 35 | 36 | sqoop import \ 37 | -D mapred.job.name="${jobName}" \ 38 | --connect '${bdp.master.jdbc.url}' \ 39 | --username '${bdp.master.jdbc.user}' \ 40 | --password '${bdp.master.jdbc.password}' \ 41 | --table "$srcTable" \ 42 | --where "update_time between '$startTime' and '$endTime'" \ 43 | --split-by "$splitColumn" \ 44 | --hive-import \ 45 | --hive-overwrite \ 46 | --hive-table "$sinkTable" \ 47 | --target-dir "$sinkTablePath" \ 48 | --outdir "/tmp" \ 49 | --delete-target-dir 50 | } 51 | 52 | createAppToTmp() 53 | { 54 | createToTmp "app" "tmp.bdp_master_app" 55 | } 56 | 57 | buildAppToTmp() 58 | { 59 | buildToTmp "app" "tmp.bdp_master_app" "id" "$1" "$2" 60 | } 61 | 62 | createServerToTmp() 63 | { 64 | createToTmp "server" "tmp.bdp_master_server" 65 | } 66 | 67 | buildServerToTmp() 68 | { 69 | buildToTmp "server" "tmp.bdp_master_server" "id" "$1" "$2" 70 | } 71 | 72 | createMetricIndexToTmp() 73 | { 74 | createToTmp "metric_index" "tmp.bdp_master_metric_index" 75 | } 76 | 77 | buildMetricIndexToTmp() 78 | { 79 | buildToTmp "metric_index" "tmp.bdp_master_metric_index" "id" "$1" "$2" 80 | } 81 | 82 | createMetricThresholdToTmp() 83 | { 84 | createToTmp "metric_threshold" "tmp.bdp_master_metric_threshold" 85 | } 86 | 87 | buildMetricThresholdToTmp() 88 | { 89 | buildToTmp "metric_threshold" "tmp.bdp_master_metric_threshold" "server_id" "$1" "$2" 90 | } 91 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/src/bdp-master/schema/app.sql: -------------------------------------------------------------------------------- 1 | -- noinspection sqlnodatasourceinspectionforfile 2 | 3 | drop table if exists src.bdp_master_app; 4 | create table if not exists src.bdp_master_app ( 5 | id bigint, 6 | name string, 7 | description string, 8 | version string, 9 | creation_time timestamp, 10 | update_time timestamp, 11 | imported_time timestamp 12 | ) 13 | partitioned by (update_date string) 14 | stored as parquet; 15 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/src/bdp-master/schema/metric_index.sql: -------------------------------------------------------------------------------- 1 | -- noinspection sqlnodatasourceinspectionforfile 2 | 3 | drop table if exists src.bdp_master_metric_index; 4 | create table if not exists src.bdp_master_metric_index ( 5 | id bigint, 6 | name string, 7 | description string, 8 | category string, 9 | creation_time timestamp, 10 | update_time timestamp, 11 | imported_time timestamp 12 | ) 13 | partitioned by (update_date string) 14 | stored as parquet; 15 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/src/bdp-master/schema/metric_threshold.sql: -------------------------------------------------------------------------------- 1 | -- noinspection sqlnodatasourceinspectionforfile 2 | 3 | drop table if exists src.bdp_master_metric_threshold; 4 | create table if not exists src.bdp_master_metric_threshold ( 5 | server_id bigint, 6 | metric_name string, 7 | amber_threshold int, 8 | red_threshold int, 9 | creation_time timestamp, 10 | update_time timestamp, 11 | imported_time timestamp 12 | ) 13 | partitioned by (update_date string) 14 | stored as parquet; -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/src/bdp-master/schema/server.sql: -------------------------------------------------------------------------------- 1 | -- noinspection sqlnodatasourceinspectionforfile 2 | 3 | drop table if exists src.bdp_master_server; 4 | create table if not exists src.bdp_master_server ( 5 | id bigint, 6 | app_id bigint, 7 | hostname string, 8 | cpu_cores int, 9 | memory int, 10 | creation_time timestamp, 11 | update_time timestamp, 12 | imported_time timestamp 13 | ) 14 | partitioned by (update_date string) 15 | stored as parquet; -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/src/bdp-metric/action/build-metric.sql: -------------------------------------------------------------------------------- 1 | -- noinspection SqlNoDataSourceInspectionForFile 2 | 3 | insert overwrite table src.bdp_metric_metric partition(creation_date) 4 | select 5 | id, 6 | name, 7 | hostname, 8 | value, 9 | cast(`timestamp` as timestamp) as `timestamp`, 10 | current_timestamp as imported_time, 11 | cast(cast(`timestamp` as date) as string) as creation_date 12 | from 13 | tmp.bdp_metric_metric; -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/src/bdp-metric/bin/spark-actions.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | create() 4 | { 5 | target="$1" 6 | execSql "job name: create schema of [ $target @ $SUBJECT ]" "$SRC_BDP_METRIC_HOME/schema/$target.sql" 7 | } 8 | 9 | build() 10 | { 11 | target="$1" 12 | execSql "job name: build [ $target ] data from [ $target @ $UNDER_LAYER_SUBJECT ] to [ $target @ $SUBJECT ]" "$SRC_BDP_METRIC_HOME/action/build-$target.sql" 13 | } 14 | 15 | createMetric() 16 | { 17 | create "metric" 18 | } 19 | 20 | buildMetric() 21 | { 22 | build "metric" 23 | } 24 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/src/bdp-metric/bin/sqoop-actions.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | createToTmp() 4 | { 5 | srcTable="$1" 6 | sinkTable="$2" 7 | printHeading "job name: create table ${sinkTable}" 8 | 9 | sqoop create-hive-table \ 10 | -D mapred.job.name="job name: create table [$sinkTable]" \ 11 | --connect '${bdp.metric.jdbc.url}' \ 12 | --username '${bdp.metric.jdbc.user}' \ 13 | --password '${bdp.metric.jdbc.password}' \ 14 | --table "$srcTable" \ 15 | --hive-table "$sinkTable" \ 16 | --hive-overwrite 17 | } 18 | 19 | buildToTmp() 20 | { 21 | srcTable="$1" 22 | sinkTable="$2" 23 | splitColumn="$3" 24 | validateTime "$4" 25 | validateTime "$5" 26 | 27 | jobName="subject: $SUBJECT -- build [ $srcTable ] data from data source to tmp layer via sqoop" 28 | 29 | printHeading "${jobName}" 30 | 31 | startTime=$(date -d "$4" +"%F %T") 32 | endTime=$(date -d "$5" +"%F %T") 33 | 34 | sinkTablePath="$TMP_DATA_BASE_DIR/$sinkTable/" 35 | 36 | sqoop import \ 37 | -D mapred.job.name="${jobName}" \ 38 | --connect '${bdp.metric.jdbc.url}' \ 39 | --username '${bdp.metric.jdbc.user}' \ 40 | --password '${bdp.metric.jdbc.password}' \ 41 | --table "$srcTable" \ 42 | --where "timestamp between '$startTime' and '$endTime'" \ 43 | --split-by "$splitColumn" \ 44 | --hive-import \ 45 | --hive-overwrite \ 46 | --hive-table "$sinkTable" \ 47 | --target-dir "$sinkTablePath" \ 48 | --outdir "/tmp" \ 49 | --delete-target-dir 50 | } 51 | 52 | createMetricToTmp() 53 | { 54 | createToTmp "metric" "tmp.bdp_metric_metric" 55 | } 56 | 57 | buildMetricToTmp() 58 | { 59 | buildToTmp "metric" "tmp.bdp_metric_metric" "id" "$1" "$2" 60 | } -------------------------------------------------------------------------------- /bdp-dwh/src/main/resources/lib/src/bdp-metric/schema/metric.sql: -------------------------------------------------------------------------------- 1 | -- noinspection sqlnodatasourceinspectionforfile 2 | 3 | drop table if exists src.bdp_metric_metric; 4 | create table if not exists src.bdp_metric_metric ( 5 | id bigint, 6 | name string, 7 | hostname string, 8 | value bigint, 9 | `timestamp` timestamp, 10 | imported_time timestamp 11 | ) 12 | partitioned by (creation_date string) 13 | stored as parquet; 14 | -------------------------------------------------------------------------------- /bdp-dwh/src/main/scala/com.github.bdp.dwh.udf/GenRag.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.dwh.udf 2 | import org.apache.hadoop.hive.ql.exec.UDF 3 | 4 | class GenRag extends UDF { 5 | def evaluate(avg: Int, amberThreshold: Int, redThreshold: Int): String = { 6 | if (avg < amberThreshold) "GREEN" else if (avg >= redThreshold) "RED" else "AMBER" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /bdp-import/.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | /*.iml 3 | target 4 | -------------------------------------------------------------------------------- /bdp-import/README.md: -------------------------------------------------------------------------------- 1 | 关于本子项目的部署、运行与代码细节，请参考《大数据平台架构与原型实现：数据中台建设实战》一书第5章以及第4章4.5节 -------------------------------------------------------------------------------- /bdp-import/build.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | rem A batch script to build -> deploy -> restart 3 | rem -- Laurence Geng 4 | if [%1]==[] ( 5 | echo. 6 | echo Usage: %0 maven-profile-1 maven-profile-2 ... 7 | echo. 8 | goto end 9 | ) 10 | 11 | set profiles=%~1 12 | 13 | :loopProfiles 14 | shift 15 | if "%~1"=="" ( 16 | goto build 17 | ) else ( 18 | set profiles=%profiles%,%~1 19 | goto loopProfiles 20 | ) 21 | 22 | :build 23 | echo. 24 | echo *************************************************************************************** 25 | echo BUILD... 26 | echo *************************************************************************************** 27 | echo. 28 | 29 | if "%profiles%"=="" ( 30 | call mvn clean install -DskipTests=true 31 | ) else ( 32 | call mvn clean install -DskipTests=true -P%profiles% 33 | ) 34 | if "%errorlevel%"=="1" goto :releasefailed 35 | 36 | call target\classes\deploy.bat 37 | 38 | if "%errorlevel%"=="1" goto :releasefailed 39 | 40 | goto releasesuccess 41 | 42 | :releasesuccess 43 | echo. 44 | echo. 45 | echo *************************************************************************************** 46 | echo RELEASE SUCCESS!! 47 | echo *************************************************************************************** 48 | goto end 49 | 50 | :releasefailed 51 | echo. 52 | echo. 53 | echo *************************************************************************************** 54 | echo RELEASE FAILED!! 55 | echo *************************************************************************************** 56 | goto end 57 | 58 | :end 59 | -------------------------------------------------------------------------------- /bdp-import/src/main/assembly/bin.xml: -------------------------------------------------------------------------------- 1 | 4 | bin 5 | 6 | zip 7 | 8 | true 9 | 10 | 11 | target/classes/bin 12 | ./bin 13 | 14 | *.sh 15 | 16 | 755 17 | 18 | 19 | target/classes/conf 20 | ./conf 21 | 22 | *.xml 23 | *.conf 24 | *.properties 25 | 26 | 27 | 28 | target/classes/sql 29 | ./sql 30 | 31 | **/*.sql 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /bdp-import/src/main/profiles/cluster.properties: -------------------------------------------------------------------------------- 1 | # app specific configs 2 | app.name=${project.artifactId} 3 | app.host=gateway1.cluster 4 | app.home=${app.user.home}/${project.build.finalName} 5 | app.user.name=${app.name} 6 | app.user.password=Bdpp1234! 7 | app.user.home=/home/${app.user.name} 8 | 9 | # bdp_metric jdbc configs 10 | # 此处的数据库主机不要使用balancer1.cluster, 因为运行MR作业的节点是3个worker节点,也就是在均衡中配置的三个节点 11 | # 而阿里云的四层负载均衡服务不支持负载均衡后端ECS实例作为客户端直接访问负载均衡，所以如果使用balancer1.cluster作为地址会经常连接数据库失败 12 | bdp.metric.db.host=master1.cluster 13 | bdp.metric.jdbc.url=jdbc:mysql://${bdp.metric.db.host}/bdp_metric?useSSL=false&autoReconnect=true&allowMultiQueries=true&useUnicode=true&characterEncoding=utf-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=GMT 14 | bdp.metric.jdbc.user=bdp_metric 15 | bdp.metric.jdbc.password=Bdpp1234! 16 | 17 | # bdp_master jdbc configs 18 | # 此处的数据库主机不要使用balancer1.cluster, 因为运行MR作业的节点是3个worker节点,也就是在均衡中配置的三个节点 19 | # 而阿里云的四层负载均衡服务不支持负载均衡后端ECS实例作为客户端直接访问负载均衡，所以如果使用balancer1.cluster作为地址会经常连接数据库失败 20 | bdp.master.db.host=master1.cluster 21 | bdp.master.jdbc.url=jdbc:mysql://${bdp.master.db.host}/bdp_master?useSSL=false&autoReconnect=true&allowMultiQueries=true&useUnicode=true&characterEncoding=utf-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=GMT 22 | bdp.master.jdbc.user=bdp_master 23 | bdp.master.jdbc.password=Bdpp1234! 24 | -------------------------------------------------------------------------------- /bdp-import/src/main/profiles/standalone.properties: -------------------------------------------------------------------------------- 1 | # app specific configs 2 | app.name=${project.artifactId} 3 | app.host=node1.cluster 4 | app.home=${app.user.home}/${project.build.finalName} 5 | app.user.name=${app.name} 6 | app.user.password=Bdpp1234! 7 | app.user.home=/home/${app.user.name} 8 | 9 | # bdp_metric jdbc configs 10 | bdp.metric.db.host=node1.cluster:3306 11 | bdp.metric.jdbc.url=jdbc:mysql://${bdp.metric.db.host}/bdp_metric?useSSL=false&autoReconnect=true&allowMultiQueries=true&useUnicode=true&characterEncoding=utf-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=GMT 12 | bdp.metric.jdbc.user=bdp_metric 13 | bdp.metric.jdbc.password=Bdpp1234! 14 | 15 | # bdp_master jdbc configs 16 | bdp.master.db.host=node1.cluster 17 | bdp.master.jdbc.url=jdbc:mysql://${bdp.master.db.host}/bdp_master?useSSL=false&autoReconnect=true&allowMultiQueries=true&useUnicode=true&characterEncoding=utf-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=GMT 18 | bdp.master.jdbc.user=bdp_master 19 | bdp.master.jdbc.password=Bdpp1234! 20 | -------------------------------------------------------------------------------- /bdp-import/src/main/resources/bin/bdp-import.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export BDP_IMPORT_HOME="$(cd "`dirname $(readlink -nf "$0")`"/..; pwd -P)" 4 | 5 | source "$BDP_IMPORT_HOME/bin/util.sh" 6 | 7 | # ------------------------------------------------ Common Methods ------------------------------------------------ # 8 | 9 | showUsage() 10 | { 11 | showLocalUsage 12 | $BDP_IMPORT_HOME/bin/bdp-master-import.sh help 13 | $BDP_IMPORT_HOME/bin/bdp-metric-import.sh help 14 | } 15 | 16 | showLocalUsage() 17 | { 18 | printHeading "PROJECT [ BDP-IMPORT ] USAGE" 19 | 20 | echo "# 说明：创建所有表在tmp层上的schema，并从数据源导入指定时间范围内的所有表的数据到tmp的对应表" 21 | echo "$0 init-all" 22 | echo 23 | 24 | echo "# 示例：创建所有表在tmp层上的schema，并从数据源导入2018-09-01的所有表的数据到tmp的对应表" 25 | echo "$0 init-all '2018-09-01T00:00+0800' '2018-09-02T00:00+0800'" 26 | echo 27 | 28 | echo "# 说明：创建所有表在tmp层上的schema" 29 | echo "$0 create-all" 30 | echo 31 | 32 | echo "# 说明：从数据源导入指定时间范围内的所有表的数据到tmp的对应表" 33 | echo "$0 import-all START_TIME END_TIME" 34 | echo 35 | 36 | echo "# 示例：从数据源导入2018-09-01的所有表的数据到tmp的对应表" 37 | echo "$0 import-all '2018-09-01T00:00+0800' '2018-09-02T00:00+0800'" 38 | echo 39 | } 40 | 41 | initAll() 42 | { 43 | $BDP_IMPORT_HOME/bin/bdp-master-import.sh init-all "$@" 44 | $BDP_IMPORT_HOME/bin/bdp-metric-import.sh init-all "$@" 45 | } 46 | 47 | createAll() 48 | { 49 | $BDP_IMPORT_HOME/bin/bdp-master-import.sh create-all "$@" 50 | $BDP_IMPORT_HOME/bin/bdp-metric-import.sh create-all "$@" 51 | } 52 | 53 | importAll() 54 | { 55 | $BDP_IMPORT_HOME/bin/bdp-master-import.sh import-all "$@" 56 | $BDP_IMPORT_HOME/bin/bdp-metric-import.sh import-all "$@" 57 | } 58 | 59 | # ---------------------------------------------- Scripts Entrance ---------------------------------------------- # 60 | 61 | case $1 in 62 | (init-all) 63 | shift 64 | initAll "$@" 65 | ;; 66 | (create-all) 67 | createAll 68 | ;; 69 | (import-all) 70 | shift 71 | importAll "$@" 72 | ;; 73 | (help) 74 | showUsage 75 | ;; 76 | (*) 77 | showUsage 78 | ;; 79 | esac 80 | -------------------------------------------------------------------------------- /bdp-import/src/main/resources/bin/util.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export TMP_DATA_BASE_DIR="/data/tmp" 4 | 5 | printHeading() 6 | { 7 | title="$1" 8 | paddingWidth=$((($(tput cols)-${#title})/2-3)) 9 | printf "\n%${paddingWidth}s"|tr ' ' '=' 10 | printf " $title " 11 | printf "%${paddingWidth}s\n\n"|tr ' ' '=' 12 | } 13 | 14 | validateTime() 15 | { 16 | if [ "$1" = "" ] 17 | then 18 | echo "Time is missing!" 19 | exit 1 20 | fi 21 | TIME=$1 22 | date -d "$TIME" >/dev/null 2>&1 23 | if [ "$?" != "0" ] 24 | then 25 | echo "Invalid Time: $TIME" 26 | exit 1 27 | fi 28 | } 29 | -------------------------------------------------------------------------------- /bdp-import/src/main/resources/deploy.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | set host=${app.host} 4 | set user=${app.user.name} 5 | set password=${app.user.password} 6 | set baseDir=${app.user.home} 7 | set home=${app.home} 8 | set buildDir=${project.build.directory} 9 | set binZip=${project.build.finalName}-bin.zip 10 | 11 | echo. 12 | echo *************************************************************************************** 13 | echo UPLOAD... 14 | echo *************************************************************************************** 15 | 16 | @echo on 17 | PSCP -l %user% -pw %password% "%buildDir%\\%binZip%" "%host%:/tmp/" 18 | PLINK -l %user% -pw %password% %host% -t "if [ ! -d '%baseDir%' ];then mkdir %baseDir%;fi" 19 | PLINK -l %user% -pw %password% %host% -t "if [ -d '%home%' ];then rm -rf %home%;fi" 20 | PLINK -l %user% -pw %password% %host% -t "unzip /tmp/%binZip% -d %baseDir%/" 21 | @echo off -------------------------------------------------------------------------------- /bdp-master-client/.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | /*.iml 3 | target 4 | -------------------------------------------------------------------------------- /bdp-master-client/README.md: -------------------------------------------------------------------------------- 1 | 关于本子项目的部署、运行与代码细节，请参考《大数据平台架构与原型实现：数据中台建设实战》一书第7章以及第4章4.5节 -------------------------------------------------------------------------------- /bdp-master-client/src/main/profiles/cluster.properties: -------------------------------------------------------------------------------- 1 | redis.host=gateway1.cluster 2 | redis.port=6379 -------------------------------------------------------------------------------- /bdp-master-client/src/main/profiles/local.properties: -------------------------------------------------------------------------------- 1 | redis.host=localhost 2 | redis.port=6380 -------------------------------------------------------------------------------- /bdp-master-client/src/main/profiles/standalone.properties: -------------------------------------------------------------------------------- 1 | redis.host=node1.cluster 2 | redis.port=6379 -------------------------------------------------------------------------------- /bdp-master-client/src/main/resources/bdp-master-client.conf: -------------------------------------------------------------------------------- 1 | redis.host="${redis.host}" 2 | redis.port=${redis.port} -------------------------------------------------------------------------------- /bdp-master-client/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger options 2 | log4j.rootLogger=INFO, CONSOLE 3 | 4 | # Console appender 5 | log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender 6 | log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout 7 | log4j.appender.CONSOLE.layout.ConversionPattern=[%d{yyyy-MM-dd HH:mm:ss}] [%p] [%t] [%c{1}.%M(%L)] -- %m%n -------------------------------------------------------------------------------- /bdp-master-client/src/main/scala/com/github/bdp/master/client/Constants.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.client 2 | 3 | import com.typesafe.config.{Config, ConfigFactory} 4 | 5 | /** 6 | * All constants in lib must be lazy! 7 | */ 8 | object Constants { 9 | 10 | private val config: Config = ConfigFactory.load("bdp-master-client.conf") 11 | 12 | val APP_SERVICE = "APP_SERVICE" 13 | val SERVER_SERVICE = "SERVER_SERVICE" 14 | val METRIC_THRESHOLD_SERVICE = "METRIC_THRESHOLD_SERVICE" 15 | 16 | val CPU_USAGE = "cpu.usage" 17 | val MEM_USED = "mem.used" 18 | 19 | val APP_KEYSPACE = "bdp-metric" 20 | val SERVER_KEYSPACE = "server" 21 | val ALERT_INDEX_KEYSPACE = "alert_index" 22 | val METRIC_INDEX_KEYSPACE = "metric_index" 23 | 24 | val REDIS_HOST = config.getString("redis.host") 25 | val REDIS_PORT = config.getInt("redis.port") 26 | } -------------------------------------------------------------------------------- /bdp-master-client/src/main/scala/com/github/bdp/master/client/Main.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.client 2 | 3 | import com.github.bdp.master.client.service.{AlertIndexService, AppService, MetricIndexService, ServerService} 4 | import com.typesafe.scalalogging.LazyLogging 5 | 6 | object Main extends App with LazyLogging { 7 | println(AppService.getAppBy(1)) 8 | println(AppService.getAppBy("MyCRM")) 9 | 10 | println(ServerService.getServerBy(1)) 11 | println(ServerService.getServerBy("svr1001")) 12 | 13 | println(AlertIndexService.getAlertIndexBy(1)) 14 | println(AlertIndexService.getAlertIndexBy("free space warning (mb) for host disk")) 15 | 16 | println(MetricIndexService.getMetricIndexBy(1)) 17 | println(MetricIndexService.getMetricIndexBy("cpu.usage")) 18 | } 19 | -------------------------------------------------------------------------------- /bdp-master-client/src/main/scala/com/github/bdp/master/client/domain/AlertIndex.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.client.domain 2 | 3 | import java.sql.Timestamp 4 | 5 | case class AlertIndex(id: Long, name: String, severity: Int, creationTime: Timestamp, updateTime: Timestamp) 6 | -------------------------------------------------------------------------------- /bdp-master-client/src/main/scala/com/github/bdp/master/client/domain/App.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.client.domain 2 | import java.sql.Timestamp 3 | case class App(id: Long, name: String, description: String, version: String, creationTime: Timestamp, updateTime: Timestamp) 4 | -------------------------------------------------------------------------------- /bdp-master-client/src/main/scala/com/github/bdp/master/client/domain/MetricIndex.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.client.domain 2 | 3 | import java.sql.Timestamp 4 | 5 | case class MetricIndex(id: Long, name: String, description: String, category: String, creationTime: Timestamp, updateTime: Timestamp) 6 | -------------------------------------------------------------------------------- /bdp-master-client/src/main/scala/com/github/bdp/master/client/domain/MetricThreshold.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.client.domain 2 | 3 | import java.sql.Timestamp 4 | 5 | case class MetricThreshold(amberThreshold: Int, redThreshold: Int, creationTime: Timestamp, updateTime: Timestamp) 6 | -------------------------------------------------------------------------------- /bdp-master-client/src/main/scala/com/github/bdp/master/client/domain/SEVERITY.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.client.domain 2 | 3 | object SEVERITY extends Enumeration { 4 | type SEVERITY = Value 5 | val GREEN = Value(0, "GREEN") 6 | val AMBER = Value(1, "AMBER") 7 | val RED = Value(2, "RED") 8 | } 9 | -------------------------------------------------------------------------------- /bdp-master-client/src/main/scala/com/github/bdp/master/client/domain/Server.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.client.domain 2 | 3 | import java.sql.Timestamp 4 | 5 | case class Server(id: Long, hostname: String, cpuCores: Int, memory: Int, appId: Long, metricThresholds: Map[String, MetricThreshold], creationTime: Timestamp, updateTime: Timestamp) 6 | -------------------------------------------------------------------------------- /bdp-master-client/src/main/scala/com/github/bdp/master/client/domain/TSD.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.client.domain 2 | 3 | /** 4 | * The case class for OpenTSDB standard metrics format 5 | */ 6 | case class TSD(metric: String, timestamp: Long, value: String, tags: Map[String, String]) -------------------------------------------------------------------------------- /bdp-master-client/src/main/scala/com/github/bdp/master/client/service/AlertIndexService.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.client.service 2 | 3 | import com.github.bdp.master.client.Constants._ 4 | import com.github.bdp.master.client.domain.AlertIndex 5 | import com.github.bdp.master.client.util.{JsonDecoder, RedisClient} 6 | import com.typesafe.scalalogging.LazyLogging 7 | 8 | object AlertIndexService extends LazyLogging { 9 | 10 | def getAlertIndexBy(id: Long): AlertIndex = { 11 | JsonDecoder.decodeAlertIndex(RedisClient.get(s"$ALERT_INDEX_KEYSPACE:$id")) 12 | } 13 | 14 | def getAlertIndexBy(name: String): AlertIndex = { 15 | val key = RedisClient.get(s"i_$ALERT_INDEX_KEYSPACE:$name") 16 | JsonDecoder.decodeAlertIndex(RedisClient.get(key)) 17 | } 18 | } -------------------------------------------------------------------------------- /bdp-master-client/src/main/scala/com/github/bdp/master/client/service/AppService.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.client.service 2 | 3 | import com.github.bdp.master.client.Constants._ 4 | import com.github.bdp.master.client.domain.App 5 | import com.github.bdp.master.client.util.{JsonDecoder, RedisClient} 6 | import com.typesafe.scalalogging.LazyLogging 7 | 8 | object AppService extends LazyLogging { 9 | 10 | def getAppBy(id: Long): App = { 11 | JsonDecoder.decodeApp(RedisClient.get(s"$APP_KEYSPACE:$id")) 12 | } 13 | 14 | def getAppBy(name: String): App = { 15 | val key = RedisClient.get(s"i_$APP_KEYSPACE:$name") 16 | JsonDecoder.decodeApp(RedisClient.get(key)) 17 | } 18 | 19 | } -------------------------------------------------------------------------------- /bdp-master-client/src/main/scala/com/github/bdp/master/client/service/MetricIndexService.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.client.service 2 | 3 | import com.github.bdp.master.client.Constants.METRIC_INDEX_KEYSPACE 4 | import com.github.bdp.master.client.domain.MetricIndex 5 | import com.github.bdp.master.client.util.{JsonDecoder, RedisClient} 6 | import com.typesafe.scalalogging.LazyLogging 7 | 8 | object MetricIndexService extends LazyLogging { 9 | 10 | def getMetricIndexBy(id: Long): MetricIndex = { 11 | JsonDecoder.decodeMetricIndex(RedisClient.get(s"$METRIC_INDEX_KEYSPACE:$id")) 12 | } 13 | 14 | def getMetricIndexBy(name: String): MetricIndex = { 15 | val key = RedisClient.get(s"i_$METRIC_INDEX_KEYSPACE:$name") 16 | JsonDecoder.decodeMetricIndex(RedisClient.get(key)) 17 | } 18 | } -------------------------------------------------------------------------------- /bdp-master-client/src/main/scala/com/github/bdp/master/client/service/ServerService.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.client.service 2 | 3 | import com.github.bdp.master.client.Constants._ 4 | import com.github.bdp.master.client.domain.Server 5 | import com.github.bdp.master.client.util.{JsonDecoder, RedisClient} 6 | import com.typesafe.scalalogging.LazyLogging 7 | 8 | object ServerService extends LazyLogging { 9 | 10 | def getServerBy(id: Long): Server = { 11 | JsonDecoder.decodeServer(RedisClient.get(s"$SERVER_KEYSPACE:$id")) 12 | } 13 | 14 | def getServerBy(hostname: String): Server = { 15 | val key = RedisClient.get(s"i_$SERVER_KEYSPACE:$hostname") 16 | JsonDecoder.decodeServer(RedisClient.get(key)) 17 | } 18 | 19 | } -------------------------------------------------------------------------------- /bdp-master-client/src/main/scala/com/github/bdp/master/client/util/JsonDecoder.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.client.util 2 | 3 | import java.sql.Timestamp 4 | 5 | import com.github.bdp.master.client.domain.{AlertIndex, MetricIndex, MetricThreshold, Server, App} 6 | import io.circe.Decoder.Result 7 | import io.circe.{Decoder, HCursor} 8 | import io.circe.generic.semiauto.deriveDecoder 9 | import io.circe.parser._ 10 | object JsonDecoder { 11 | 12 | implicit private val appDecoder: Decoder[App] = deriveDecoder[App] 13 | implicit private val serverDecoder: Decoder[Server] = deriveDecoder[Server] 14 | implicit private val metricThresholdDecoder: Decoder[MetricThreshold] = deriveDecoder[MetricThreshold] 15 | implicit private val metricIndexDecoder: Decoder[MetricIndex] = deriveDecoder[MetricIndex] 16 | implicit private val alertIndexDecoder: Decoder[AlertIndex] = deriveDecoder[AlertIndex] 17 | implicit private val timestampDecoder = new Decoder[Timestamp] { 18 | override def apply(c: HCursor): Result[Timestamp] = Decoder.decodeLong.map(s => new Timestamp(s)).apply(c) 19 | } 20 | 21 | def decodeApp(json: String): App = { 22 | decode[App](json).right.get 23 | } 24 | 25 | def decodeServer(json: String): Server = { 26 | decode[Server](json).right.get 27 | } 28 | 29 | def decodeMetricIndex(json: String): MetricIndex = { 30 | decode[MetricIndex](json).right.get 31 | } 32 | 33 | def decodeAlertIndex(json: String): AlertIndex = { 34 | decode[AlertIndex](json).right.get 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /bdp-master-client/src/main/scala/com/github/bdp/master/client/util/RedisClient.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.client.util 2 | 3 | import com.github.bdp.master.client.Constants._ 4 | import com.typesafe.scalalogging.LazyLogging 5 | import redis.clients.jedis.{Jedis, JedisPool, JedisPoolConfig} 6 | 7 | object RedisClient extends LazyLogging { 8 | 9 | private val pool = new JedisPool(new JedisPoolConfig(), REDIS_HOST, REDIS_PORT) 10 | 11 | private def withClient[T](f: Jedis => T): T = { 12 | val jedis = pool.getResource 13 | try { 14 | f(jedis) 15 | } catch { 16 | case e: Throwable => 17 | logger.error(s"Redis operation failed! the error message is: ${e.getMessage}") 18 | throw e 19 | } 20 | finally { 21 | jedis.close() 22 | } 23 | } 24 | 25 | def get(key: String): String = { 26 | withClient { 27 | jedis => 28 | jedis.get(key) 29 | } 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /bdp-master-server/.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | /*.iml 3 | target 4 | -------------------------------------------------------------------------------- /bdp-master-server/README.md: -------------------------------------------------------------------------------- 1 | 关于本子项目的部署、运行与代码细节，请参考《大数据平台架构与原型实现：数据中台建设实战》一书第6章以及第4章4.5节 -------------------------------------------------------------------------------- /bdp-master-server/build.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | rem A batch script to build -> deploy -> restart 3 | rem -- Laurence Geng 4 | if [%1]==[] ( 5 | echo. 6 | echo Usage: %0 [-delta] maven-profile-1 maven-profile-2 ... 7 | echo. 8 | echo Option: -delta: only deploy modified part, i.e. project artifact, used for development deploy. 9 | goto end 10 | ) 11 | 12 | set deltaDeploy=0 13 | if "%~1"=="-delta" ( 14 | set deltaDeploy=1 15 | shift 16 | ) 17 | 18 | set profiles=%~1 19 | 20 | :loopProfiles 21 | shift 22 | if "%~1"=="" ( 23 | goto build 24 | ) else ( 25 | set profiles=%profiles%,%~1 26 | goto loopProfiles 27 | ) 28 | 29 | :build 30 | echo. 31 | echo *************************************************************************************** 32 | echo BUILD... 33 | echo *************************************************************************************** 34 | echo. 35 | 36 | if "%profiles%"=="" ( 37 | call mvn clean install -DskipTests=true 38 | ) else ( 39 | call mvn clean install -DskipTests=true -P%profiles% 40 | ) 41 | 42 | if "%errorlevel%"=="1" goto :buildfailed 43 | 44 | rem for 'prd' env, skip deploy! 'prd' is always deployed manually! 45 | if "%profiles%"=="prd" goto "buildsuccess" 46 | 47 | if "%deltaDeploy%"=="1" ( 48 | call target\deploy.bat -delta 49 | ) else ( 50 | call target\deploy.bat 51 | ) 52 | 53 | goto buildsuccess 54 | 55 | :buildsuccess 56 | echo. 57 | echo. 58 | echo *************************************************************************************** 59 | echo BUILD SUCCESS!! 60 | echo *************************************************************************************** 61 | goto end 62 | 63 | :buildfailed 64 | echo. 65 | echo. 66 | echo *************************************************************************************** 67 | echo BUILD FAILED!! 68 | echo *************************************************************************************** 69 | goto end 70 | 71 | :end 72 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/assembly/bin-delta.xml: -------------------------------------------------------------------------------- 1 | 4 | bin-delta 5 | 6 | zip 7 | 8 | true 9 | 10 | 11 | target/classes/bin 12 | ./bin 13 | 14 | *.sh 15 | 16 | 755 17 | 18 | 19 | target/classes/conf 20 | ./conf 21 | 22 | *.xml 23 | *.sql 24 | *.conf 25 | *.properties 26 | 27 | 28 | 29 | 30 | 31 | ./lib 32 | 33 | com.github:bdp-master-server 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/assembly/bin.xml: -------------------------------------------------------------------------------- 1 | 4 | bin 5 | 6 | zip 7 | 8 | true 9 | 10 | 11 | target 12 | ./lib 13 | 14 | *.jar 15 | 16 | 17 | 18 | target/classes/bin 19 | ./bin 20 | 21 | *.sh 22 | 23 | 755 24 | 25 | 26 | target/classes/conf 27 | ./conf 28 | 29 | *.xml 30 | *.sql 31 | *.conf 32 | *.properties 33 | 34 | 35 | 36 | 37 | 38 | ./lib 39 | 40 | 41 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/Constants.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server; 2 | 3 | public interface Constants { 4 | String APP_KEYSPACE = "bdp-metric"; 5 | String APP_KEY_PATTERN = "bdp-metric:*"; 6 | String SERVER_KEYSPACE = "server"; 7 | String SERVER_KEY_PATTERN = "server:*"; 8 | String METRIC_INDEX_KEYSPACE = "metric_index"; 9 | String METRIC_INDEX_KEY_PATTERN = "metric_index:*"; 10 | String ALERT_INDEX_KEYSPACE = "alert_index"; 11 | String ALERT_INDEX_KEY_PATTERN = "alert_index:*"; 12 | String INDEX_PREFIX = "i_"; 13 | String JOIN_PREFIX = "x_"; 14 | } 15 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/Main.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.context.ConfigurableApplicationContext; 6 | 7 | @SpringBootApplication 8 | public class Main { 9 | 10 | public static void main(String[] args) { 11 | SpringApplication.run(Main.class, args); 12 | } 13 | 14 | public static ConfigurableApplicationContext getSpringBeanContext() { 15 | return SpringApplication.run(Main.class, new String[] {}); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/RedisConfig.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.beans.factory.annotation.Value; 5 | import org.springframework.boot.autoconfigure.EnableAutoConfiguration; 6 | import org.springframework.cache.annotation.CachingConfigurerSupport; 7 | import org.springframework.context.annotation.Bean; 8 | import org.springframework.context.annotation.Configuration; 9 | import org.springframework.data.redis.connection.jedis.JedisConnectionFactory; 10 | import org.springframework.data.redis.core.StringRedisTemplate; 11 | import org.springframework.transaction.annotation.EnableTransactionManagement; 12 | 13 | @Configuration 14 | @EnableAutoConfiguration 15 | @EnableTransactionManagement 16 | public class RedisConfig extends CachingConfigurerSupport { 17 | 18 | @Value("${spring.redis.host}") 19 | private String host; 20 | @Value("${spring.redis.port}") 21 | private int port; 22 | @Value("${spring.redis.timeout}") 23 | private int timeout; 24 | @Value("${spring.redis.database}") 25 | private int database; 26 | @Value("${spring.redis.password}") 27 | private String password; 28 | 29 | @Bean 30 | public JedisConnectionFactory jedisConnectionFactory() { 31 | JedisConnectionFactory factory = new JedisConnectionFactory(); 32 | factory.setHostName(host); 33 | factory.setPort(port); 34 | factory.setTimeout(timeout); 35 | factory.setPassword(password); 36 | factory.setDatabase(database); 37 | return factory; 38 | } 39 | 40 | @Bean 41 | @Autowired 42 | public StringRedisTemplate stringRedisTemplate(JedisConnectionFactory jedisConnectionFactory) { 43 | StringRedisTemplate stringRedisTemplate = new StringRedisTemplate(); 44 | stringRedisTemplate.setEnableTransactionSupport(true); 45 | stringRedisTemplate.setConnectionFactory(jedisConnectionFactory); 46 | stringRedisTemplate.afterPropertiesSet(); 47 | return stringRedisTemplate; 48 | } 49 | 50 | } 51 | 52 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/controller/AlertIndexController.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.controller; 2 | 3 | import com.github.bdp.master.server.domain.AlertIndex; 4 | import com.github.bdp.master.server.service.AlertIndexService; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.web.bind.annotation.*; 9 | 10 | import javax.servlet.http.HttpServletRequest; 11 | import java.util.List; 12 | 13 | import static org.springframework.web.bind.annotation.RequestMethod.*; 14 | 15 | @RestController 16 | public class AlertIndexController { 17 | 18 | public static Logger logger = LoggerFactory.getLogger(AlertIndexController.class); 19 | 20 | @Autowired 21 | private AlertIndexService alertIndexService; 22 | 23 | @RequestMapping(method = GET, path = "/alertIndexes") 24 | public List findAll(HttpServletRequest request) { 25 | return alertIndexService.findAll(); 26 | } 27 | 28 | @RequestMapping(method = GET, path = "/alertIndex/{id}") 29 | public AlertIndex find(@PathVariable Long id) { 30 | return alertIndexService.findOne(id); 31 | } 32 | 33 | @RequestMapping(method = GET, path = "/alertIndex") 34 | public AlertIndex find(@RequestParam("name") String name) { 35 | return alertIndexService.findOne(name); 36 | } 37 | 38 | @RequestMapping(method = POST, path = "/alertIndex") 39 | public void save(@RequestBody AlertIndex alertIndex) { 40 | alertIndexService.save(alertIndex); 41 | } 42 | 43 | @RequestMapping(method = DELETE, path = "/alertIndex/{id}") 44 | public void delete(@PathVariable Long id) { 45 | alertIndexService.delete(id); 46 | } 47 | 48 | 49 | } 50 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/controller/AppController.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.controller; 2 | 3 | import com.github.bdp.master.server.domain.App; 4 | import com.github.bdp.master.server.service.AppService; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.web.bind.annotation.*; 9 | 10 | import javax.servlet.http.HttpServletRequest; 11 | import java.util.List; 12 | 13 | @RestController 14 | public class AppController { 15 | 16 | public static Logger logger = LoggerFactory.getLogger(AppController.class); 17 | 18 | @Autowired 19 | private AppService appService; 20 | 21 | @RequestMapping(method = RequestMethod.GET, path = "/apps") 22 | public List findAll(HttpServletRequest request) { 23 | return appService.findAll(); 24 | } 25 | 26 | @RequestMapping(method = RequestMethod.GET, path = "/bdp-metric/{id}") 27 | public App find(@PathVariable Long id) { 28 | return appService.findOne(id); 29 | } 30 | 31 | @RequestMapping(method = RequestMethod.GET, path = "/bdp-metric") 32 | public App find(@RequestParam("appName") String appName) { 33 | return appService.findOne(appName); 34 | } 35 | 36 | @RequestMapping(method = RequestMethod.POST, path = "/bdp-metric") 37 | public void save(@RequestBody App app) { 38 | appService.save(app); 39 | } 40 | 41 | @RequestMapping(method = RequestMethod.DELETE, path = "/bdp-metric/{id}") 42 | public void delete(@PathVariable Long id) { 43 | appService.delete(id); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/controller/AppStartupListener.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.controller; 2 | 3 | import com.github.bdp.master.server.service.AlertIndexService; 4 | import com.github.bdp.master.server.service.AppService; 5 | import com.github.bdp.master.server.service.MetricIndexService; 6 | import com.github.bdp.master.server.service.ServerService; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import org.springframework.beans.factory.annotation.Autowired; 10 | import org.springframework.context.event.ContextRefreshedEvent; 11 | import org.springframework.context.event.EventListener; 12 | import org.springframework.stereotype.Component; 13 | 14 | @Component 15 | public class AppStartupListener { 16 | 17 | public static Logger logger = LoggerFactory.getLogger(AppStartupListener.class); 18 | 19 | @Autowired 20 | private AppService appService; 21 | 22 | @Autowired 23 | private ServerService serverService; 24 | 25 | @Autowired 26 | private MetricIndexService metricIndexService; 27 | 28 | @Autowired 29 | private AlertIndexService alertIndexService; 30 | 31 | @EventListener 32 | public void onApplicationEvent(ContextRefreshedEvent event) { 33 | logger.info("Start to load all data into redis...."); 34 | appService.loadAll(); 35 | serverService.loadAll(); 36 | metricIndexService.loadAll(); 37 | alertIndexService.loadAll(); 38 | logger.info("loading all data into redis is done!"); 39 | } 40 | } -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/controller/MetricIndexController.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.controller; 2 | 3 | import com.github.bdp.master.server.domain.MetricIndex; 4 | import com.github.bdp.master.server.service.MetricIndexService; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.web.bind.annotation.*; 7 | 8 | import javax.servlet.http.HttpServletRequest; 9 | import java.util.List; 10 | 11 | @RestController 12 | public class MetricIndexController { 13 | 14 | @Autowired 15 | private MetricIndexService metricIndexService; 16 | 17 | @RequestMapping(method = RequestMethod.GET, path = "/metricIndexes") 18 | public List findAll(HttpServletRequest request) { 19 | return metricIndexService.findAll(); 20 | } 21 | 22 | @RequestMapping(method = RequestMethod.GET, path = "/metricIndex/{id}") 23 | public MetricIndex find(@PathVariable Long id) { 24 | return metricIndexService.findOne(id); 25 | } 26 | 27 | @RequestMapping(method = RequestMethod.GET, path = "/metricIndex") 28 | public MetricIndex find(@RequestParam("name") String name) { 29 | return metricIndexService.findOne(name); 30 | } 31 | 32 | @RequestMapping(method = RequestMethod.POST, path = "/metricIndex") 33 | public void save(@RequestBody MetricIndex metricIndex) { 34 | metricIndexService.save(metricIndex); 35 | } 36 | 37 | @RequestMapping(method = RequestMethod.DELETE, path = "/metricIndex/{id}") 38 | public void delete(@PathVariable Long id) { 39 | metricIndexService.delete(id); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/controller/ServerController.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.controller; 2 | 3 | import com.github.bdp.master.server.domain.Server; 4 | import com.github.bdp.master.server.service.ServerService; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.web.bind.annotation.*; 9 | 10 | import javax.servlet.http.HttpServletRequest; 11 | import java.util.List; 12 | 13 | @RestController 14 | public class ServerController { 15 | 16 | public static Logger logger = LoggerFactory.getLogger(ServerController.class); 17 | 18 | @Autowired 19 | private ServerService serverService; 20 | 21 | @RequestMapping(method = RequestMethod.GET, path = "/servers") 22 | public List findAll(HttpServletRequest request) { 23 | return serverService.findAll(); 24 | } 25 | 26 | @RequestMapping(method = RequestMethod.GET, path = "/server/{id}") 27 | public Server find(@PathVariable Long id) { 28 | return serverService.findOne(id); 29 | } 30 | 31 | @RequestMapping(method = RequestMethod.GET, path = "/server") 32 | public Server find(@RequestParam("hostname") String hostname) { 33 | return serverService.findOne(hostname); 34 | } 35 | 36 | @RequestMapping(method = RequestMethod.POST, path = "/server") 37 | public void save(@RequestBody Server server) { 38 | serverService.save(server); 39 | } 40 | 41 | @RequestMapping(method = RequestMethod.DELETE, path = "/server/{id}") 42 | public void delete(@PathVariable Long id) { 43 | serverService.delete(id); 44 | } 45 | 46 | 47 | } 48 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/domain/App.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.domain; 2 | 3 | import javax.persistence.*; 4 | import java.io.Serializable; 5 | import java.util.Date; 6 | import java.util.Set; 7 | 8 | @Entity 9 | @Table(name="app") 10 | public class App implements Serializable { 11 | 12 | private static final long serialVersionUID = 1L; 13 | 14 | @Id 15 | @Column(name="id") 16 | @GeneratedValue(strategy = GenerationType.IDENTITY) 17 | private Long id; 18 | 19 | @Column(name = "name", nullable = false) 20 | private String name; 21 | 22 | @Column(name = "description", nullable = false) 23 | private String description; 24 | 25 | @Column(name = "version", nullable = false) 26 | private String version; 27 | 28 | @Temporal(TemporalType.TIMESTAMP) 29 | @Column(name="creation_time", insertable = false, updatable = false, columnDefinition="TIMESTAMP default CURRENT_TIMESTAMP") 30 | private Date creationTime; 31 | 32 | @Temporal(TemporalType.TIMESTAMP) 33 | @Column(name="update_time", insertable = false, columnDefinition="TIMESTAMP default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP") 34 | private Date updateTime; 35 | 36 | 37 | /*-------------------------------------------- Getters/Setters ---------------------------------------------*/ 38 | 39 | public Long getId() { 40 | return id; 41 | } 42 | 43 | public void setId(Long id) { 44 | this.id = id; 45 | } 46 | 47 | public String getName() { 48 | return name; 49 | } 50 | 51 | public void setName(String name) { 52 | this.name = name; 53 | } 54 | 55 | public String getDescription() { 56 | return description; 57 | } 58 | 59 | public void setDescription(String description) { 60 | this.description = description; 61 | } 62 | 63 | public String getVersion() { 64 | return version; 65 | } 66 | 67 | public void setVersion(String version) { 68 | this.version = version; 69 | } 70 | 71 | public Date getCreationTime() { 72 | return creationTime; 73 | } 74 | 75 | public void setCreationTime(Date creationTime) { 76 | this.creationTime = creationTime; 77 | } 78 | 79 | public Date getUpdateTime() { 80 | return updateTime; 81 | } 82 | 83 | public void setUpdateTime(Date updateTime) { 84 | this.updateTime = updateTime; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/domain/MetricIndex.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.domain; 2 | 3 | import javax.persistence.*; 4 | import java.io.Serializable; 5 | import java.util.Date; 6 | 7 | @Entity 8 | @Table(name="metric_index") 9 | public class MetricIndex implements Serializable { 10 | 11 | private static final long serialVersionUID = 1L; 12 | 13 | @Id 14 | @Column(name="id") 15 | @GeneratedValue(strategy = GenerationType.IDENTITY) 16 | private Long id; 17 | 18 | @Column(name = "name", nullable = false) 19 | private String name; 20 | 21 | @Column(name = "description", nullable = false) 22 | private String description; 23 | 24 | @Column(name = "category", nullable = false) 25 | private String category; 26 | 27 | @Temporal(TemporalType.TIMESTAMP) 28 | @Column(name="creation_time", insertable = false, updatable = false, columnDefinition="TIMESTAMP default CURRENT_TIMESTAMP") 29 | private Date creationTime; 30 | 31 | @Temporal(TemporalType.TIMESTAMP) 32 | @Column(name="update_time", insertable = false, columnDefinition="TIMESTAMP default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP") 33 | private Date updateTime; 34 | 35 | /*-------------------------------------------- Getters/Setters ---------------------------------------------*/ 36 | 37 | public MetricIndex() { 38 | 39 | } 40 | 41 | public MetricIndex(Long id, String key, String name) { 42 | super(); 43 | this.id = id; 44 | this.name = name; 45 | } 46 | 47 | public Long getId() { 48 | return id; 49 | } 50 | 51 | public void setId(Long id) { 52 | this.id = id; 53 | } 54 | 55 | public String getName() { 56 | return name; 57 | } 58 | 59 | public void setName(String name) { 60 | this.name = name; 61 | } 62 | 63 | public String getDescription() { 64 | return description; 65 | } 66 | 67 | public void setDescription(String description) { 68 | this.description = description; 69 | } 70 | 71 | public String getCategory() { 72 | return category; 73 | } 74 | 75 | public void setCategory(String category) { 76 | this.category = category; 77 | } 78 | 79 | public Date getCreationTime() { 80 | return creationTime; 81 | } 82 | 83 | public void setCreationTime(Date creationTime) { 84 | this.creationTime = creationTime; 85 | } 86 | 87 | public Date getUpdateTime() { 88 | return updateTime; 89 | } 90 | 91 | public void setUpdateTime(Date updateTime) { 92 | this.updateTime = updateTime; 93 | } 94 | } -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/domain/MetricThreshold.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.domain; 2 | 3 | import javax.persistence.*; 4 | import java.util.Date; 5 | 6 | @Embeddable 7 | //@Table(name="metric_threshold") 8 | public class MetricThreshold { 9 | 10 | private static final long serialVersionUID = 1L; 11 | 12 | @Column(name = "red_threshold") 13 | private Integer redThreshold; 14 | 15 | @Column(name = "amber_threshold") 16 | private Integer amberThreshold; 17 | 18 | @Temporal(TemporalType.TIMESTAMP) 19 | @Column(name="creation_time", insertable = false, updatable = false, columnDefinition="TIMESTAMP default CURRENT_TIMESTAMP") 20 | private Date creationTime; 21 | 22 | @Temporal(TemporalType.TIMESTAMP) 23 | @Column(name="update_time", insertable = false, columnDefinition="TIMESTAMP default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP") 24 | private Date updateTime; 25 | 26 | 27 | /*-------------------------------------------- Getters/Setters ---------------------------------------------*/ 28 | 29 | public Integer getRedThreshold() { 30 | return redThreshold; 31 | } 32 | 33 | public void setRedThreshold(Integer redThreshold) { 34 | this.redThreshold = redThreshold; 35 | } 36 | 37 | public Integer getAmberThreshold() { 38 | return amberThreshold; 39 | } 40 | 41 | public void setAmberThreshold(Integer amberThreshold) { 42 | this.amberThreshold = amberThreshold; 43 | } 44 | 45 | public Date getCreationTime() { 46 | return creationTime; 47 | } 48 | 49 | public void setCreationTime(Date creationTime) { 50 | this.creationTime = creationTime; 51 | } 52 | 53 | public Date getUpdateTime() { 54 | return updateTime; 55 | } 56 | 57 | public void setUpdateTime(Date updateTime) { 58 | this.updateTime = updateTime; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/repository/AlertIndexJpaRepository.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.repository; 2 | 3 | import com.github.bdp.master.server.domain.AlertIndex; 4 | import org.springframework.data.repository.PagingAndSortingRepository; 5 | 6 | public interface AlertIndexJpaRepository extends PagingAndSortingRepository {} 7 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/repository/AlertIndexRedisRepository.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.repository; 2 | 3 | import com.github.bdp.master.server.domain.AlertIndex; 4 | 5 | import java.util.List; 6 | 7 | public interface AlertIndexRedisRepository { 8 | void save(AlertIndex alertIndex); 9 | AlertIndex findOne(Long id); 10 | AlertIndex findOne(String name); 11 | List findAll(); 12 | void delete(AlertIndex alertIndex); 13 | } 14 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/repository/AppJpaRepository.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.repository; 2 | 3 | import com.github.bdp.master.server.domain.App; 4 | import org.springframework.data.repository.PagingAndSortingRepository; 5 | 6 | public interface AppJpaRepository extends PagingAndSortingRepository {} 7 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/repository/AppRedisRepository.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.repository; 2 | 3 | import com.github.bdp.master.server.domain.App; 4 | 5 | import java.util.List; 6 | 7 | public interface AppRedisRepository { 8 | void save(App app); 9 | App findOne(Long id); 10 | App findOne(String name); 11 | List findAll(); 12 | void delete(App app); 13 | } 14 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/repository/MetricIndexJpaRepository.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.repository; 2 | 3 | import com.github.bdp.master.server.domain.MetricIndex; 4 | import org.springframework.data.repository.PagingAndSortingRepository; 5 | 6 | public interface MetricIndexJpaRepository extends PagingAndSortingRepository {} 7 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/repository/MetricIndexRedisRepository.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.repository; 2 | 3 | import com.github.bdp.master.server.domain.MetricIndex; 4 | 5 | import java.util.List; 6 | 7 | public interface MetricIndexRedisRepository { 8 | void save(MetricIndex metricIndex); 9 | MetricIndex findOne(Long id); 10 | MetricIndex findOne(String name); 11 | List findAll(); 12 | void delete(MetricIndex metricIndex); 13 | } 14 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/repository/MetricThresholdRepository.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.repository; 2 | 3 | import com.github.bdp.master.server.domain.MetricThreshold; 4 | import org.springframework.data.repository.PagingAndSortingRepository; 5 | 6 | public interface MetricThresholdRepository extends PagingAndSortingRepository {} 7 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/repository/ServerJpaRepository.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.repository; 2 | 3 | import com.github.bdp.master.server.domain.Server; 4 | import org.springframework.data.repository.PagingAndSortingRepository; 5 | 6 | public interface ServerJpaRepository extends PagingAndSortingRepository {} 7 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/repository/ServerRedisRepository.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.repository; 2 | 3 | import com.github.bdp.master.server.domain.Server; 4 | 5 | import java.util.List; 6 | 7 | public interface ServerRedisRepository { 8 | 9 | void save(Server server); 10 | 11 | Server findOne(Long id); 12 | 13 | Server findOne(String name); 14 | 15 | List findAll(); 16 | 17 | void delete(Server server); 18 | } 19 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/service/AlertIndexService.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.service; 2 | 3 | 4 | import com.github.bdp.master.server.domain.AlertIndex; 5 | 6 | import java.util.List; 7 | 8 | public interface AlertIndexService { 9 | 10 | AlertIndex findOne(Long id); 11 | 12 | AlertIndex findOne(String alertName); 13 | 14 | void save(AlertIndex alertIndex); 15 | 16 | void delete(Long id); 17 | 18 | List findAll(); 19 | 20 | void loadAll(); 21 | } 22 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/service/AppService.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.service; 2 | 3 | 4 | import com.github.bdp.master.server.domain.App; 5 | 6 | import java.util.List; 7 | 8 | public interface AppService { 9 | 10 | App findOne(Long id); 11 | 12 | App findOne(String appName); 13 | 14 | void save(App app); 15 | 16 | void delete(Long id); 17 | 18 | List findAll(); 19 | 20 | void loadAll(); 21 | } 22 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/service/MetricIndexService.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.service; 2 | 3 | 4 | import com.github.bdp.master.server.domain.MetricIndex; 5 | 6 | import java.util.List; 7 | 8 | public interface MetricIndexService { 9 | 10 | MetricIndex findOne(Long id); 11 | 12 | MetricIndex findOne(String metricName); 13 | 14 | void save(MetricIndex metricIndex); 15 | 16 | void delete(Long id); 17 | 18 | List findAll(); 19 | 20 | void loadAll(); 21 | } 22 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/service/ServerService.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.service; 2 | 3 | import java.util.List; 4 | import com.github.bdp.master.server.domain.Server; 5 | 6 | public interface ServerService { 7 | 8 | Server findOne(Long id); 9 | 10 | Server findOne(String serverName); 11 | 12 | void save(Server server); 13 | 14 | void delete(Long id); 15 | 16 | List findAll(); 17 | 18 | void loadAll(); 19 | } 20 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/service/impl/AlertIndexServiceImpl.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.service.impl; 2 | 3 | 4 | import com.github.bdp.master.server.domain.AlertIndex; 5 | import com.github.bdp.master.server.repository.AlertIndexJpaRepository; 6 | import com.github.bdp.master.server.repository.AlertIndexRedisRepository; 7 | import com.github.bdp.master.server.service.AlertIndexService; 8 | import org.springframework.stereotype.Component; 9 | import org.springframework.transaction.annotation.Transactional; 10 | 11 | import java.util.List; 12 | 13 | @Component("alertService") 14 | @Transactional 15 | public class AlertIndexServiceImpl implements AlertIndexService { 16 | 17 | 18 | private final AlertIndexJpaRepository alertIndexJpaRepository; 19 | 20 | private final AlertIndexRedisRepository alertIndexRedisRepository; 21 | 22 | public AlertIndexServiceImpl(AlertIndexJpaRepository alertIndexJpaRepository, 23 | AlertIndexRedisRepository alertIndexRedisRepository) { 24 | this.alertIndexJpaRepository = alertIndexJpaRepository; 25 | this.alertIndexRedisRepository = alertIndexRedisRepository; 26 | } 27 | 28 | @Override 29 | public void save(AlertIndex alertIndex) { 30 | AlertIndex savedAlertIndex = alertIndexJpaRepository.save(alertIndex); 31 | alertIndexRedisRepository.save(savedAlertIndex); 32 | } 33 | 34 | @Override 35 | public AlertIndex findOne(Long id) { 36 | return alertIndexRedisRepository.findOne(id); 37 | } 38 | 39 | @Override 40 | public AlertIndex findOne(String name) { 41 | return alertIndexRedisRepository.findOne(name); 42 | } 43 | 44 | @Override 45 | public void delete(Long id) { 46 | AlertIndex alertIndex = findOne(id); 47 | alertIndexJpaRepository.delete(alertIndex.getId()); 48 | alertIndexRedisRepository.delete(alertIndex); 49 | } 50 | 51 | @Override 52 | public List findAll() { 53 | return alertIndexRedisRepository.findAll(); 54 | } 55 | 56 | @Override 57 | public void loadAll() { 58 | alertIndexJpaRepository.findAll().forEach( 59 | alert -> alertIndexRedisRepository.save(alert) 60 | ); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/service/impl/AppServiceImpl.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.service.impl; 2 | 3 | 4 | import com.github.bdp.master.server.domain.App; 5 | import com.github.bdp.master.server.repository.AppJpaRepository; 6 | import com.github.bdp.master.server.repository.AppRedisRepository; 7 | import com.github.bdp.master.server.service.AppService; 8 | import org.springframework.stereotype.Component; 9 | import org.springframework.transaction.annotation.Transactional; 10 | 11 | import java.util.List; 12 | 13 | @Component 14 | @Transactional 15 | public class AppServiceImpl implements AppService { 16 | 17 | private final AppJpaRepository appJpaRepository; 18 | 19 | private final AppRedisRepository appRedisRepository; 20 | 21 | public AppServiceImpl(AppJpaRepository appJpaRepository, 22 | AppRedisRepository appRedisRepository) { 23 | this.appJpaRepository = appJpaRepository; 24 | this.appRedisRepository = appRedisRepository; 25 | } 26 | 27 | @Override 28 | public void save(App app) { 29 | App savedApp = appJpaRepository.save(app); 30 | appRedisRepository.save(savedApp); 31 | } 32 | 33 | @Override 34 | public App findOne(Long id) { 35 | return appRedisRepository.findOne(id); 36 | } 37 | 38 | @Override 39 | public App findOne(String appName) { 40 | return appRedisRepository.findOne(appName); 41 | } 42 | 43 | @Override 44 | public void delete(Long id) { 45 | App app = findOne(id); 46 | appJpaRepository.delete(app.getId()); 47 | appRedisRepository.delete(app); 48 | } 49 | 50 | @Override 51 | public List findAll() { 52 | return appRedisRepository.findAll(); 53 | } 54 | 55 | @Override 56 | public void loadAll() { 57 | appJpaRepository.findAll().forEach( 58 | app -> appRedisRepository.save(app) 59 | ); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/service/impl/MetricIndexServiceImpl.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.service.impl; 2 | 3 | 4 | import com.github.bdp.master.server.domain.MetricIndex; 5 | import com.github.bdp.master.server.repository.MetricIndexJpaRepository; 6 | import com.github.bdp.master.server.repository.MetricIndexRedisRepository; 7 | import com.github.bdp.master.server.service.MetricIndexService; 8 | import org.springframework.stereotype.Component; 9 | import org.springframework.transaction.annotation.Transactional; 10 | 11 | import java.util.List; 12 | 13 | @Component("metricService") 14 | @Transactional 15 | public class MetricIndexServiceImpl implements MetricIndexService { 16 | 17 | private final MetricIndexJpaRepository metricIndexJpaRepository; 18 | 19 | private final MetricIndexRedisRepository metricIndexRedisRepository; 20 | 21 | public MetricIndexServiceImpl(MetricIndexJpaRepository metricIndexJpaRepository, 22 | MetricIndexRedisRepository metricIndexRedisRepository) { 23 | this.metricIndexJpaRepository = metricIndexJpaRepository; 24 | this.metricIndexRedisRepository = metricIndexRedisRepository; 25 | } 26 | 27 | @Override 28 | public void save(MetricIndex metricIndex) { 29 | MetricIndex savedMetricIndex = metricIndexJpaRepository.save(metricIndex); 30 | metricIndexRedisRepository.save(savedMetricIndex); 31 | } 32 | 33 | @Override 34 | public MetricIndex findOne(Long id) { 35 | return metricIndexRedisRepository.findOne(id); 36 | } 37 | 38 | @Override 39 | public MetricIndex findOne(String metricName) { 40 | return metricIndexRedisRepository.findOne(metricName); 41 | } 42 | 43 | @Override 44 | public void delete(Long id) { 45 | MetricIndex metricIndex = findOne(id); 46 | metricIndexJpaRepository.delete(metricIndex.getId()); 47 | metricIndexRedisRepository.delete(metricIndex); 48 | } 49 | 50 | @Override 51 | public List findAll() { 52 | return metricIndexRedisRepository.findAll(); 53 | } 54 | 55 | @Override 56 | public void loadAll() { 57 | metricIndexJpaRepository.findAll().forEach( 58 | metric -> metricIndexRedisRepository.save(metric) 59 | ); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/java/com/github/bdp/master/server/service/impl/ServerServiceImpl.java: -------------------------------------------------------------------------------- 1 | package com.github.bdp.master.server.service.impl; 2 | 3 | 4 | import com.github.bdp.master.server.domain.Server; 5 | import com.github.bdp.master.server.repository.ServerJpaRepository; 6 | import com.github.bdp.master.server.repository.ServerRedisRepository; 7 | import com.github.bdp.master.server.service.ServerService; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | import org.springframework.stereotype.Service; 11 | import org.springframework.transaction.annotation.Transactional; 12 | 13 | import java.util.List; 14 | 15 | @Service 16 | @Transactional 17 | public class ServerServiceImpl implements ServerService { 18 | 19 | public static Logger logger = LoggerFactory.getLogger(ServerServiceImpl.class); 20 | 21 | private final ServerJpaRepository serverJpaRepository; 22 | 23 | private final ServerRedisRepository serverRedisRepository; 24 | 25 | @SuppressWarnings("unused") 26 | public ServerServiceImpl(ServerJpaRepository serverJpaRepository, 27 | ServerRedisRepository serverRedisRepository) { 28 | this.serverJpaRepository = serverJpaRepository; 29 | this.serverRedisRepository = serverRedisRepository; 30 | } 31 | 32 | @Override 33 | public void loadAll() { 34 | serverJpaRepository.findAll().forEach( 35 | server -> serverRedisRepository.save(server) 36 | ); 37 | } 38 | 39 | @Override 40 | public List findAll() { 41 | return serverRedisRepository.findAll(); 42 | } 43 | 44 | @Override 45 | public void save(Server server) { 46 | logger.debug(server.getHostname()); 47 | Server savedServer = serverJpaRepository.save(server); 48 | serverRedisRepository.save(savedServer); 49 | } 50 | 51 | @Override 52 | public Server findOne(Long id) { 53 | return serverRedisRepository.findOne(id); 54 | } 55 | 56 | @Override 57 | public Server findOne(String hostname) { 58 | return serverRedisRepository.findOne(hostname); 59 | } 60 | 61 | @Override 62 | public void delete(Long id) { 63 | Server app = findOne(id); 64 | serverJpaRepository.delete(app.getId()); 65 | serverRedisRepository.delete(app); 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/profiles/cluster.properties: -------------------------------------------------------------------------------- 1 | # app specific configs 2 | app.name=${project.artifactId} 3 | app.host=gateway1.cluster 4 | app.home=${app.user.home}/${project.build.finalName} 5 | app.user.name=${app.name} 6 | app.user.password=Bdpp1234! 7 | app.user.home=/home/${app.user.name} 8 | app.mainClass=com.github.bdp.master.server.Main 9 | app.log.home=${app.home}/log 10 | app.log.level=DEBUG 11 | 12 | # bdp_master jdbc configs 13 | bdp.master.db.host=loadbalancer1 14 | bdp.master.jdbc.url=jdbc:mysql://${bdp.master.db.host}/bdp_master?useSSL=false&useUnicode=true&characterEncoding=utf-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=GMT 15 | bdp.master.jdbc.user=bdp_master 16 | bdp.master.jdbc.password=Bdpp1234! 17 | 18 | bdp.master.redis.host=gateway1.cluster -------------------------------------------------------------------------------- /bdp-master-server/src/main/profiles/standalone.properties: -------------------------------------------------------------------------------- 1 | # app specific configs 2 | app.name=${project.artifactId} 3 | app.host=node1.cluster 4 | app.home=${app.user.home}/${project.build.finalName} 5 | app.user.name=${app.name} 6 | app.user.password=Bdpp1234! 7 | app.user.home=/home/${app.user.name} 8 | app.mainClass=com.github.bdp.master.server.Main 9 | app.log.home=${app.home}/log 10 | app.log.level=DEBUG 11 | 12 | # bdp_master jdbc configs 13 | bdp.master.db.host=node1.cluster 14 | bdp.master.jdbc.url=jdbc:mysql://${bdp.master.db.host}/bdp_master?useSSL=false&useUnicode=true&characterEncoding=utf-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=GMT 15 | bdp.master.jdbc.user=bdp_master 16 | bdp.master.jdbc.password=Bdpp1234! 17 | 18 | bdp.master.redis.host=node1.cluster -------------------------------------------------------------------------------- /bdp-master-server/src/main/resources/conf/application.properties: -------------------------------------------------------------------------------- 1 | server.port=9090 2 | 3 | spring.datasource.driver-class-name=com.mysql.jdbc.Driver 4 | spring.datasource.url=${bdp.master.jdbc.url} 5 | spring.datasource.username=${bdp.master.jdbc.user} 6 | spring.datasource.password=${bdp.master.jdbc.password} 7 | 8 | # To enable create db when bdp-metric start, uncomment following 2 lines, 9 | # and edit pom.xml, comment out *.sql in resources. 10 | #spring.jpa.hibernate.ddl-auto=none 11 | spring.jpa.hibernate.ddl-auto=create 12 | spring.datasource.data=classpath:/bdp-master-data-2018-09-01.sql 13 | spring.jpa.hibernate.naming.physical-strategy=org.hibernate.boot.model.naming.PhysicalNamingStrategyStandardImpl 14 | spring.jpa.show-sql=true 15 | spring.mvc.dispatch-options-request=true 16 | 17 | # redis configs 18 | 19 | # Redis数据库索引（默认为0）,如果设置为1，那么存入的key-value都存放在select 1中 20 | spring.redis.database=0 21 | # Redis服务器地址 22 | spring.redis.host=${bdp.master.redis.host} 23 | # Redis服务器连接端口 24 | spring.redis.port=6379 25 | # Redis服务器连接密码（默认为空） 26 | spring.redis.password= 27 | #连接池最大连接数（使用负值表示没有限制） 28 | spring.redis.pool.max-active=8 29 | # 连接池最大阻塞等待时间（使用负值表示没有限制） 30 | spring.redis.pool.max-wait=600 31 | # 连接池中的最大空闲连接 32 | spring.redis.pool.max-idle=8 33 | # 连接池中的最小空闲连接 34 | spring.redis.pool.min-idle=0 35 | # 连接超时时间（毫秒） 36 | spring.redis.timeout=0 37 | 38 | redis.keyspace.server=server 39 | redis.keyspace.serverIndex=idx_server 40 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/resources/conf/bdp-master-data-2018-09-02.sql: -------------------------------------------------------------------------------- 1 | update bdp_master.app set version='7.1', update_time='2018-09-02 00:00:00' where id=1; 2 | update bdp_master.server set memory=128000, update_time='2018-09-02 00:00:00' where id=1; 3 | update bdp_master.metric_index set description='The instantaneous usage percent of cpu', update_time='2018-09-02 00:00:00' where id=1; 4 | update bdp_master.metric_threshold set amber_threshold=85, update_time='2018-09-02 00:00:00' where server_id=1 and metric_name='cpu.usage'; -------------------------------------------------------------------------------- /bdp-master-server/src/main/resources/conf/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | [%d{yyyy-MM-dd HH:mm:ss}] [%level] [%thread] [%logger] [%line] -- %msg%n 8 | 9 | 10 | 11 | 12 | 13 | ${app.log.home}/${project.artifactId}.log 14 | 15 | [%d{yyyy-MM-dd HH:mm:ss}] [%level] [%thread] [%logger] [%line] -- %msg%n 16 | 17 | 18 | ${app.log.home}/${project.artifactId}.%d{yyyyMMdd}.%i.log 19 | 128MB 20 | 30 21 | 5GB 22 | 23 | 24 | 25 | 26 | 27 | ERROR 28 | 29 | ${app.log.home}/${project.artifactId}.error.log 30 | 31 | [%d{yyyy-MM-dd HH:mm:ss}] [%level] [%thread] [%logger] [%line] -- %msg%n 32 | 33 | 34 | ${app.log.home}/${project.artifactId}.error.%d{yyyyMMdd}.%i.log 35 | 128MB 36 | 30 37 | 5GB 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /bdp-master-server/src/main/resources/deploy.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | set host=${app.host} 4 | set user=${app.user.name} 5 | set password=${app.user.password} 6 | set baseDir=${app.user.home} 7 | set home=${app.home} 8 | set buildDir=${project.build.directory} 9 | set binZip=${project.build.finalName}-bin.zip 10 | set deltaBinZip=${project.build.finalName}-bin-delta.zip 11 | set logHome=${app.log.home} 12 | 13 | echo. 14 | echo *************************************************************************************** 15 | echo UPLOAD... 16 | echo *************************************************************************************** 17 | 18 | if "%~1"=="-delta" ( 19 | goto uploadDeltaBinZip 20 | ) else ( 21 | goto uploadBinZip 22 | ) 23 | 24 | :uploadBinZip 25 | @echo on 26 | PSCP -l %user% -pw %password% "%buildDir%\\%binZip%" "%host%:/tmp/" 27 | PLINK -l %user% -pw %password% %host% -t "if [ ! -d '%baseDir%' ];then mkdir %baseDir%;fi" 28 | PLINK -l %user% -pw %password% %host% -t "if [ -d '%home%' ];then rm -rf %home%;fi" 29 | PLINK -l %user% -pw %password% %host% -t "unzip /tmp/%binZip% -d %baseDir%/" 30 | PLINK -l %user% -pw %password% %host% -t "mkdir %logHome%/" 31 | @echo off 32 | goto startup 33 | 34 | :uploadDeltaBinZip 35 | @echo on 36 | PSCP -l %user% -pw %password% "%buildDir%\\%deltaBinZip%" "%host%:/tmp/" 37 | PLINK -l %user% -pw %password% %host% -t "unzip -o /tmp/%deltaBinZip% -d %baseDir%/" 38 | @echo off 39 | goto startup 40 | 41 | :startup 42 | echo. 43 | echo *************************************************************************************** 44 | echo STARTUP... 45 | echo *************************************************************************************** 46 | 47 | @echo on 48 | :: if you want to start program automatically after deploy, uncomment next line. 49 | :: PLINK -l %user% -pw %password% %host% -t "%baseDir%/${project.build.finalName}/bin/${project.artifactId}.sh restart-with-logging" 50 | @echo off -------------------------------------------------------------------------------- /bdp-metric/.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | /*.iml 3 | target 4 | -------------------------------------------------------------------------------- /bdp-metric/README.md: -------------------------------------------------------------------------------- 1 | 关于本子项目的部署、运行与代码细节，请参考《大数据平台架构与原型实现：数据中台建设实战》一书第5章以及第4章4.5节 -------------------------------------------------------------------------------- /bdp-metric/build.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | rem A batch script to build -> deploy -> restart 3 | rem -- Laurence Geng 4 | if [%1]==[] ( 5 | echo. 6 | echo Usage: %0 maven-profile-1 maven-profile-2 ... 7 | echo. 8 | goto end 9 | ) 10 | 11 | set profiles=%~1 12 | 13 | :loopProfiles 14 | shift 15 | if "%~1"=="" ( 16 | goto build 17 | ) else ( 18 | set profiles=%profiles%,%~1 19 | goto loopProfiles 20 | ) 21 | 22 | :build 23 | echo. 24 | echo *************************************************************************************** 25 | echo BUILD... 26 | echo *************************************************************************************** 27 | echo. 28 | 29 | if "%profiles%"=="" ( 30 | call mvn clean install -DskipTests=true 31 | ) else ( 32 | call mvn clean install -DskipTests=true -P%profiles% 33 | ) 34 | if "%errorlevel%"=="1" goto :releasefailed 35 | 36 | call target\classes\deploy.bat 37 | 38 | if "%errorlevel%"=="1" goto :releasefailed 39 | 40 | goto releasesuccess 41 | 42 | :releasesuccess 43 | echo. 44 | echo. 45 | echo *************************************************************************************** 46 | echo RELEASE SUCCESS!! 47 | echo *************************************************************************************** 48 | goto end 49 | 50 | :releasefailed 51 | echo. 52 | echo. 53 | echo *************************************************************************************** 54 | echo RELEASE FAILED!! 55 | echo *************************************************************************************** 56 | goto end 57 | 58 | :end 59 | -------------------------------------------------------------------------------- /bdp-metric/src/main/assembly/bin.xml: -------------------------------------------------------------------------------- 1 | 4 | bin 5 | 6 | zip 7 | 8 | true 9 | 10 | 11 | target/classes/bin 12 | ./bin 13 | 14 | *.sh 15 | 16 | 755 17 | 18 | 19 | target/classes/sql 20 | ./sql 21 | 22 | *.sql 23 | 24 | 25 | 26 | target/classes/log 27 | ./log 28 | 29 | 30 | -------------------------------------------------------------------------------- /bdp-metric/src/main/profiles/cluster.properties: -------------------------------------------------------------------------------- 1 | app.name=${project.artifactId} 2 | app.host=gateway1.cluster 3 | app.home=${app.user.home}/${project.build.finalName} 4 | app.pidDir=/tmp 5 | 6 | app.user.name=${app.name} 7 | app.user.password=Bdpp1234! 8 | app.user.home=/home/${app.user.name} 9 | 10 | db.host=loadbalancer1.cluster 11 | db.user=bdp_metric 12 | db.password=Bdpp1234! -------------------------------------------------------------------------------- /bdp-metric/src/main/profiles/standalone.properties: -------------------------------------------------------------------------------- 1 | app.name=${project.artifactId} 2 | app.host=node1.cluster 3 | app.home=${app.user.home}/${project.build.finalName} 4 | app.pidDir=/tmp 5 | 6 | app.user.name=${app.name} 7 | app.user.password=Bdpp1234! 8 | app.user.home=/home/${app.user.name} 9 | 10 | db.host=node1.cluster 11 | db.user=bdp_metric 12 | db.password=Bdpp1234! -------------------------------------------------------------------------------- /bdp-metric/src/main/resources/deploy.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | set host=${app.host} 4 | set user=${app.user.name} 5 | set password=${app.user.password} 6 | set baseDir=${app.user.home} 7 | set home=${app.home} 8 | set buildDir=${project.build.directory} 9 | set binZip=${project.build.finalName}-bin.zip 10 | 11 | echo. 12 | echo *************************************************************************************** 13 | echo UPLOAD... 14 | echo *************************************************************************************** 15 | 16 | @echo on 17 | PSCP -l %user% -pw %password% "%buildDir%\\%binZip%" "%host%:/tmp/" 18 | PLINK -l %user% -pw %password% %host% -t "if [ ! -d '%baseDir%' ];then mkdir %baseDir%;fi" 19 | PLINK -l %user% -pw %password% %host% -t "if [ -d '%home%' ];then rm -rf %home%;fi" 20 | PLINK -l %user% -pw %password% %host% -t "unzip /tmp/%binZip% -d %baseDir%/" 21 | @echo off 22 | 23 | echo. 24 | echo *************************************************************************************** 25 | echo STARTUP... 26 | echo *************************************************************************************** 27 | 28 | @echo on 29 | :: if you want to start program automatically after deploy, uncomment next line. 30 | :: PLINK -l %user% -pw %password% %host% "%baseDir%/${project.build.finalName}/bin/${project.artifactId}.sh restart" 31 | @echo off 32 | -------------------------------------------------------------------------------- /bdp-metric/src/main/resources/sql/gen-alert.sql: -------------------------------------------------------------------------------- 1 | insert into `bdp_metric`.`alert` (`message`, `hostname`, `status`, `timestamp`) values ('free space warning (mb) for host disk', 'svr1001', '@status@', '@timestamp@'); -------------------------------------------------------------------------------- /bdp-metric/src/main/resources/sql/gen-cpu-usage.sql: -------------------------------------------------------------------------------- 1 | insert into `bdp_metric`.`metric` (`name`, `hostname`, `value`, `timestamp`) values ('cpu.usage', 'svr1001', @value1@, '@timestamp@'); 2 | insert into `bdp_metric`.`metric` (`name`, `hostname`, `value`, `timestamp`) values ('cpu.usage', 'svr1002', @value2@, '@timestamp@'); 3 | -------------------------------------------------------------------------------- /bdp-metric/src/main/resources/sql/gen-mem-used.sql: -------------------------------------------------------------------------------- 1 | insert into `bdp_metric`.`metric` (`name`, `hostname`, `value`, `timestamp`) values ('mem.used', 'svr1001', @value1@, '@timestamp@'); 2 | insert into `bdp_metric`.`metric` (`name`, `hostname`, `value`, `timestamp`) values ('mem.used', 'svr1002', @value2@, '@timestamp@'); 3 | -------------------------------------------------------------------------------- /bdp-metric/src/main/resources/sql/schema.sql: -------------------------------------------------------------------------------- 1 | drop database if exists bdp_metric; 2 | create database if not exists bdp_metric; 3 | 4 | drop user if exists 'bdp_metric'@'node1.cluster'; 5 | create user if not exists 'bdp_metric'@'node1.cluster' identified by 'Bdpp1234!'; 6 | grant all privileges on bdp_metric.* to 'bdp_metric'@'node1.cluster' with grant option; 7 | flush privileges; 8 | 9 | use bdp_metric; 10 | 11 | drop table if exists metric; 12 | 13 | create table if not exists metric ( 14 | `id` bigint(20) not null auto_increment, 15 | `name` varchar(255) not null, 16 | `hostname` varchar(255) not null, 17 | `value` bigint(20) not null, 18 | `timestamp` datetime not null, 19 | primary key (`id`) 20 | ) engine=innodb auto_increment=1 default charset=utf8; 21 | 22 | drop table if exists alert; 23 | 24 | create table if not exists alert ( 25 | `id` bigint(20) not null auto_increment, 26 | `message` varchar(255) not null, 27 | `hostname` varchar(255) not null, 28 | `status` varchar(6) not null, 29 | `timestamp` datetime not null, 30 | `created_time` timestamp not null default current_timestamp, 31 | primary key (`id`) 32 | ) engine=innodb auto_increment=1 default charset=utf8; -------------------------------------------------------------------------------- /bdp-parent/.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | /*.iml 3 | target 4 | -------------------------------------------------------------------------------- /bdp-parent/README.md: -------------------------------------------------------------------------------- 1 | # 大数据平台工程原型（Big Data Platform Project Prototype） 2 | 3 | 4 | 2008年Hadoop成为Apache的顶级项目，以此为开端，大数据技术迎来了十多年的持续发展，其间随着Spark的异军突起，整个大数据生态圈又经历了一次“装备升级”，变得更加完善和强大。今天，很多企业已经完成了早期对大数据技术的尝试和探索转而进入到应用阶段，但不得不说的是，大数据平台的架构体系非常庞大，技术堆栈特别深，从事大数据开发的同学对此应该都深有体会。 5 | 6 | 而在很多细分领域（例如实时计算、作业调度）也没有像样的工程模板, 这一现状与Java社区使用Spring Boot信手拈来地搭建Web工程原型形成了鲜明了对比。这导致很多团队在启动大数据平台建设时往往感到无所侍从，也使得希望深入学习大数据技术的开发者由于缺少工程级的示例参考而感到迷茫。 7 | 8 | 该原型项目就是以此为命题创建的，它基于过往项目的最佳实践提炼而来，可以帮助团队快速启动开发，上手就写业务代码。 9 | 10 | 本项目同时是《[大数据平台架构与原型实现：数据中台建设实战](https://item.jd.com/12677623.html)》一书的配套示例代码。该书已由知名IT图书品牌电子工业出版社博文视点出版发行，在京东和当当有售。 11 | 关于如何部署和运行该项目，书中做了非常细致的讲解。 12 | 13 | 京东购书链接：[https://item.jd.com/12677623.html](https://item.jd.com/12677623.html) 14 | 15 | 当当购书链接：[http://product.dangdang.com/28974965.html](http://product.dangdang.com/28974965.html) 16 | 17 | 18 | 点击[《重磅推荐：建大数据平台太难了！给我发个工程原型吧！》](https://laurence.blog.csdn.net/article/details/106851739)了解图书详情，扫码进入京东手机购书页面！ 19 | 20 | ![在这里插入图片描述](https://img-blog.csdnimg.cn/2020061912530549.jpg?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2JsdWlzaGdsYw==,size_16,color_FFFFFF,t_70#pic_center) 21 | 22 | 23 | -------------------------------------------------------------------------------- /bdp-stream/.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | /*.iml 3 | target 4 | -------------------------------------------------------------------------------- /bdp-stream/README.md: -------------------------------------------------------------------------------- 1 | 关于本子项目的部署、运行与代码细节，请参考《大数据平台架构与原型实现：数据中台建设实战》一书第7章以及第4章4.5节 2 | 3 | 友情提示：如果该项目提交后迟迟进入不到运行状态，请确认你的集群资源是否充足，同时可以考虑将profile文件中spark.num.executors与spark.executor.cores两个参数的数值调低。 4 | 5 | 如无必要，不建议同时启动bdp-stream和bdp-dwh的作业，以免因为资源不足导致作业pending，如果确实需要同时运行，在确保资源充足的前提下，可以通过Yarn的动态资源池为bdp-stream和bdp-dwh两个用户分配隔离资源。 6 | 7 | 在真实的生产环境中，流计算项目一般不于批处理项目在同一集群上运行。 -------------------------------------------------------------------------------- /bdp-stream/build.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | rem A batch script to build -> deploy -> restart 3 | rem -- Laurence Geng 4 | if [%1]==[] ( 5 | echo. 6 | echo Usage: %0 [-delta] maven-profile-1 maven-profile-2 ... 7 | echo. 8 | echo Option: -delta: only deploy modified part, i.e. project artifact, used for development deploy. 9 | goto end 10 | ) 11 | 12 | set deltaDeploy=0 13 | if "%~1"=="-delta" ( 14 | set deltaDeploy=1 15 | shift 16 | ) 17 | 18 | set profiles=%~1 19 | 20 | :loopProfiles 21 | shift 22 | if "%~1"=="" ( 23 | goto build 24 | ) else ( 25 | set profiles=%profiles%,%~1 26 | goto loopProfiles 27 | ) 28 | 29 | :build 30 | echo. 31 | echo *************************************************************************************** 32 | echo BUILD... 33 | echo *************************************************************************************** 34 | echo. 35 | 36 | rem for 'prd' env, skip deploy! 'prd' is always deployed manually! 37 | if "%profiles%"=="" ( 38 | cd ..\bdp-master-client 39 | call mvn clean install -DskipTests=true 40 | cd ..\bdp-stream 41 | call mvn clean install -DskipTests=true 42 | ) else ( 43 | cd ..\bdp-master-client 44 | call mvn clean install -DskipTests=true -P%profiles% 45 | cd ..\bdp-stream 46 | call mvn clean install -DskipTests=true -P%profiles% 47 | ) 48 | 49 | if "%errorlevel%"=="1" goto :buildfailed 50 | 51 | rem for 'prd' env, skip deploy! 'prd' is always deployed manually! 52 | if "%profiles%"=="prd" goto "buildsuccess" 53 | 54 | if "%deltaDeploy%"=="1" ( 55 | call target\classes\deploy.bat -delta 56 | ) else ( 57 | call target\classes\deploy.bat 58 | ) 59 | 60 | goto buildsuccess 61 | 62 | :buildsuccess 63 | echo. 64 | echo. 65 | echo *************************************************************************************** 66 | echo BUILD SUCCESS!! 67 | echo *************************************************************************************** 68 | goto end 69 | 70 | :buildfailed 71 | echo. 72 | echo. 73 | echo *************************************************************************************** 74 | echo BUILD FAILED!! 75 | echo *************************************************************************************** 76 | goto end 77 | 78 | :end 79 | -------------------------------------------------------------------------------- /bdp-stream/src/main/assembly/bin-delta.xml: -------------------------------------------------------------------------------- 1 | 4 | bin-delta 5 | 6 | zip 7 | 8 | true 9 | 10 | 11 | target/classes/bin 12 | ./bin 13 | 14 | *.sh 15 | 16 | 755 17 | 18 | 19 | target/classes/conf 20 | ./conf 21 | 22 | *.xml 23 | *.sql 24 | *.conf 25 | *.properties 26 | 27 | 28 | 29 | 30 | 31 | lib 32 | 33 | com.github:bdp-stream 34 | com.github:bdp-master-client 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /bdp-stream/src/main/assembly/bin.xml: -------------------------------------------------------------------------------- 1 | 4 | bin 5 | 6 | zip 7 | 8 | true 9 | 10 | 11 | target/classes/bin 12 | ./bin 13 | 14 | *.sh 15 | 16 | 755 17 | 18 | 19 | target/classes/conf 20 | ./conf 21 | 22 | *.xml 23 | *.sql 24 | *.conf 25 | *.properties 26 | 27 | 28 | 29 | target 30 | ./lib 31 | 32 | *.jar 33 | 34 | 35 | 36 | 37 | 38 | ./lib 39 | 40 | 41 | -------------------------------------------------------------------------------- /bdp-stream/src/main/profiles/cluster.properties: -------------------------------------------------------------------------------- 1 | app.name=${project.artifactId} 2 | app.host=gateway1.cluster 3 | app.home=${app.user.home}/${project.build.finalName} 4 | 5 | app.user.name=${app.name} 6 | app.user.password=Bdpp1234! 7 | app.user.home=/home/${app.user.name} 8 | 9 | app.log.home=/var/log/${app.name} 10 | app.log.level=DEBUG 11 | 12 | app.mainClass=com.github.bdp.stream.Main 13 | 14 | app.cluster.nodes=(gateway1.cluster worker1.cluster worker2.cluster worker3.cluster) 15 | 16 | spark.num.executors=3 17 | spark.executor.cores=3 18 | spark.streaming.concurrentJobs=1 19 | spark.scheduler.mode=FAIR 20 | spark.sql.shuffle.partitions=12 21 | spark.checkpoint=/user/${app.user.name}/checkpoint 22 | 23 | hbase.zkQuorum=master1.cluster:2181,master1.cluster:2181,utility1.cluster:2181 24 | hbase.zkPort=2181 25 | 26 | kafka.brokerList=worker1.cluster:9092,worker2.cluster:9092,worker3.cluster:9092 -------------------------------------------------------------------------------- /bdp-stream/src/main/profiles/standalone.properties: -------------------------------------------------------------------------------- 1 | app.name=${project.artifactId} 2 | app.host=node1.cluster 3 | app.home=${app.user.home}/${project.build.finalName} 4 | 5 | app.user.name=${app.name} 6 | app.user.password=Bdpp1234! 7 | app.user.home=/home/${app.user.name} 8 | 9 | app.log.home=/var/log/${app.name} 10 | app.log.level=DEBUG 11 | 12 | app.mainClass=com.github.bdp.stream.Main 13 | 14 | app.cluster.nodes=(node1.cluster) 15 | 16 | spark.num.executors=1 17 | spark.executor.cores=2 18 | spark.streaming.concurrentJobs=2 19 | spark.scheduler.mode=FAIR 20 | spark.sql.shuffle.partitions=2 21 | spark.checkpoint=/user/${app.user.name}/checkpoint 22 | 23 | hbase.zkQuorum=node1.cluster:2181 24 | hbase.zkPort=2181 25 | 26 | kafka.brokerList=node1.cluster:9092 -------------------------------------------------------------------------------- /bdp-stream/src/main/resources/conf/bdp-stream.conf: -------------------------------------------------------------------------------- 1 | spark { 2 | slide="5 seconds" 3 | window="60 seconds" 4 | checkpoint="${spark.checkpoint}" 5 | } 6 | 7 | hbase { 8 | zkQuorum="${hbase.zkQuorum}" 9 | zkPort="${hbase.zkPort}" 10 | schema { 11 | metric { 12 | table = "metric" 13 | columnFamily = "f" 14 | qualifier { 15 | name = "name" 16 | hostname = "hostname" 17 | timestamp = "timestamp" 18 | value = "value" 19 | } 20 | } 21 | alert { 22 | table="alert" 23 | columnFamily="f" 24 | qualifier { 25 | message="message" 26 | severity="severity" 27 | hostname="hostname" 28 | timestamp="timestamp" 29 | status="status" 30 | } 31 | } 32 | serverState { 33 | table = "server_state" 34 | columnFamily = "f" 35 | qualifier { 36 | serverId = "getServerId" 37 | srcType = "srcType" 38 | timestamp = "timestamp" 39 | severity = "severity" 40 | } 41 | } 42 | } 43 | } 44 | 45 | kafka { 46 | brokerList="${kafka.brokerList}" 47 | } 48 | 49 | akka { 50 | loglevel = ${app.log.level} 51 | loggers = ["akka.event.slf4j.Slf4jLogger"] 52 | } 53 | 54 | stream { 55 | metric { 56 | cpuUsageEnabled = true 57 | cpuUsageKeyPrefix = "cu" 58 | memUsedEnabled = true 59 | memUsedKeyPrefix = "mu" 60 | watermark = "60 seconds" 61 | } 62 | alert { 63 | enabled = true 64 | keyPrefix = "al" 65 | watermark = "86400 seconds" 66 | timeToLive = 60 67 | } 68 | } 69 | 70 | cors.allowed-origin = "*" 71 | -------------------------------------------------------------------------------- /bdp-stream/src/main/resources/conf/fairscheduler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | FAIR 5 | 1 6 | 0 7 | 8 | 9 | FAIR 10 | 1 11 | 0 12 | 13 | 14 | FAIR 15 | 1 16 | 0 17 | 18 | 19 | FAIR 20 | 1 21 | 0 22 | 23 | 24 | FAIR 25 | 1 26 | 0 27 | 28 | 29 | FAIR 30 | 1 31 | 0 32 | 33 | -------------------------------------------------------------------------------- /bdp-stream/src/main/resources/deploy.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | set host=${app.host} 4 | set port=${app.local.port} 5 | set user=${app.user.name} 6 | set password=${app.user.password} 7 | set baseDir=${app.user.home} 8 | set home=${app.home} 9 | set buildDir=${project.build.directory} 10 | set binZip=${project.build.finalName}-bin.zip 11 | set deltaBinZip=${project.build.finalName}-bin-delta.zip 12 | 13 | echo. 14 | echo *************************************************************************************** 15 | echo UPLOAD... 16 | echo *************************************************************************************** 17 | 18 | if "%~1"=="-delta" ( 19 | goto uploadDeltaBinZip 20 | ) else ( 21 | goto uploadBinZip 22 | ) 23 | 24 | :uploadBinZip 25 | @echo on 26 | PSCP -l %user% -pw %password% "%buildDir%\\%binZip%" "%host%:/tmp/" 27 | PLINK -l %user% -pw %password% %host% -t "if [ ! -d '%baseDir%' ];then mkdir %baseDir%;fi" 28 | PLINK -l %user% -pw %password% %host% -t "if [ -d '%home%' ];then rm -rf %home%;fi" 29 | PLINK -l %user% -pw %password% %host% -t "unzip -o /tmp/%binZip% -d %baseDir%/" 30 | @echo off 31 | goto startup 32 | 33 | :uploadDeltaBinZip 34 | @echo on 35 | PSCP -l %user% -pw %password% "%buildDir%\\%deltaBinZip%" "%host%:/tmp/" 36 | PLINK -l %user% -pw %password% %host% -t "unzip -o /tmp/%deltaBinZip% -d %baseDir%/" 37 | @echo off 38 | goto startup 39 | 40 | :startup 41 | echo. 42 | echo *************************************************************************************** 43 | echo STARTUP... 44 | echo *************************************************************************************** 45 | 46 | @echo on 47 | :: if you want to start program automatically after deploy, uncomment next line. 48 | :: PLINK -l %user% -pw %password% %host% -t "%baseDir%/${project.build.finalName}/bin/${project.artifactId}.sh restart-with-logging" 49 | @echo off -------------------------------------------------------------------------------- /bdp-stream/src/main/scala/com/github/bdp/stream/AlertStream.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.stream 2 | 3 | import com.github.bdp.stream.Constants._ 4 | import com.github.bdp.stream.model.{Alert, AlertRegistry} 5 | import com.github.bdp.stream.service.AlertService._ 6 | import com.github.bdp.stream.util.{AlertWriter, ServerStateWriter} 7 | import com.typesafe.scalalogging.LazyLogging 8 | import org.apache.spark.sql.SparkSession 9 | import org.apache.spark.sql.streaming.GroupStateTimeout 10 | 11 | object AlertStream extends LazyLogging { 12 | 13 | def restream(implicit sparkSession: SparkSession): Unit = { 14 | persist 15 | evaluate 16 | } 17 | 18 | /** 19 | * we assume no matter open or closed alert message, it can and only can push one time! 20 | * so here, once there is new alert, no matter what, persist it first! and there won't be duplicated messages. 21 | */ 22 | def persist(implicit sparkSession: SparkSession): Unit = { 23 | import sparkSession.implicits._ 24 | sparkSession.sparkContext.setLocalProperty("spark.scheduler.pool", s"pool_persist_alert") 25 | sparkSession 26 | .sql(s"select * from alert").as[Alert] 27 | .writeStream 28 | .outputMode("update") 29 | .foreach(AlertWriter()) 30 | .queryName(s"persist_alert") 31 | .start 32 | } 33 | 34 | def evaluate(implicit sparkSession: SparkSession): Unit = { 35 | import sparkSession.implicits._ 36 | implicit val stateEncoder = org.apache.spark.sql.Encoders.kryo[AlertRegistry] 37 | sparkSession.sparkContext.setLocalProperty("spark.scheduler.pool", s"pool_evaluate_alert") 38 | sparkSession 39 | .sql(s"select * from alert").as[Alert] 40 | .withWatermark("timestamp", ALERT_WATERMARK) 41 | .groupByKey(alert => getServerId(alert.hostname)) 42 | .mapGroupsWithState(GroupStateTimeout.NoTimeout)(updateAlertGroupState) 43 | .writeStream 44 | .outputMode("update") 45 | .foreach(ServerStateWriter()) 46 | .queryName(s"evaluate_alert") 47 | .start 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /bdp-stream/src/main/scala/com/github/bdp/stream/Main.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.stream 2 | 3 | import com.github.bdp.stream.Constants._ 4 | import com.github.bdp.stream.service.AlertService 5 | import com.github.bdp.stream.service.MetricService.transform 6 | import com.typesafe.scalalogging.LazyLogging 7 | import org.apache.spark.sql.SparkSession 8 | 9 | /** 10 | * CAUTION!! 11 | * You should always use services in , never expose 12 | * any master entity in , because once it's exposed, 13 | * it will be se/de from driver to executor, this will cause 14 | * unnecessary resource cost! 15 | * 16 | * @author lichao.geng 17 | */ 18 | object Main extends LazyLogging { 19 | 20 | def main(args: Array[String]): Unit = { 21 | try { 22 | implicit val sparkSession = SparkSession 23 | .builder 24 | .appName("bdp-stream") 25 | .config("spark.cleaner.referenceTracking.cleanCheckpoints", "true") 26 | .config("spark.streaming.stopGracefullyOnShutdown", "true") 27 | .getOrCreate() 28 | 29 | import sparkSession.implicits._ 30 | 31 | sparkSession 32 | .readStream 33 | .format("kafka") 34 | .option("kafka.bootstrap.servers", KAFKA_BROKER_LIST) 35 | .option("subscribe", s"$TOPIC_CPU_USAGE, $TOPIC_MEM_USED, $TOPIC_ALERT") 36 | .option("startingOffsets", "latest") 37 | .load() 38 | .selectExpr("CAST(key AS STRING)", "CAST(value AS STRING)") 39 | .createTempView("trunk") 40 | 41 | if (CPU_USAGE_STREAM_ENABLED) { 42 | logger.info("[ CPU USAGE ] streaming is enabled!") 43 | sparkSession 44 | .sql(s"select value from trunk where key like '$CPU_USAGE_MSG_KEY_PREFIX%'").as[String] 45 | .map(transform(_)) 46 | .createTempView(CPU_USAGE) 47 | MetricStream.restream(CPU_USAGE) 48 | } 49 | 50 | if (MEM_USED_STREAM_ENABLED) { 51 | logger.info("[ MEM USED ] streaming is enabled!") 52 | sparkSession 53 | .sql(s"select value from trunk where key like '$MEM_USED_MSG_KEY_PREFIX%'").as[String] 54 | .map(transform(_)) 55 | .createTempView(MEM_USED) 56 | MetricStream.restream(MEM_USED) 57 | } 58 | 59 | if (ALERT_STREAM_ENABLED) { 60 | logger.info("[ ALERT ] streaming is enabled!") 61 | sparkSession 62 | .sql(s"select value from trunk where key like '$ALERT_MSG_KEY_PREFIX%'").as[String] 63 | .map(AlertService.transform(_)) 64 | .createTempView(ALERT) 65 | AlertStream.restream 66 | } 67 | sparkSession.streams.awaitAnyTermination() 68 | // sparkSession.streams.awaitAnyTermination(600000) 69 | } catch { 70 | case e: Throwable => e.printStackTrace() 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /bdp-stream/src/main/scala/com/github/bdp/stream/MetricStream.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.stream 2 | 3 | import com.github.bdp.stream.Constants._ 4 | import com.github.bdp.stream.model.Metric 5 | import com.github.bdp.stream.service.MetricService 6 | import com.github.bdp.stream.util.{MetricWriter, ServerStateWriter} 7 | import com.typesafe.scalalogging.LazyLogging 8 | import org.apache.spark.sql.SparkSession 9 | import org.apache.spark.sql.functions._ 10 | 11 | object MetricStream extends LazyLogging { 12 | 13 | def restream(metric: String)(implicit sparkSession: SparkSession): Unit = { 14 | persist(metric) 15 | evaluate(metric) 16 | } 17 | 18 | def persist(metric: String)(implicit sparkSession: SparkSession): Unit = { 19 | import sparkSession.implicits._ 20 | sparkSession.sparkContext.setLocalProperty("spark.scheduler.pool", s"pool_persist_$metric") 21 | sparkSession 22 | .sql(s"select * from $metric").as[Metric] 23 | .writeStream 24 | .outputMode("update") 25 | .foreach(MetricWriter()) 26 | .queryName(s"persist_$metric") 27 | .start 28 | } 29 | 30 | def evaluate(metric: String)(implicit sparkSession: SparkSession): Unit = { 31 | import sparkSession.implicits._ 32 | sparkSession.sparkContext.setLocalProperty("spark.scheduler.pool", s"pool_evaluate_$metric") 33 | sparkSession 34 | .sql(s"select * from $metric").as[Metric] 35 | .withWatermark("timestamp", METRIC_WATERMARK) 36 | .dropDuplicates("id", "timestamp") 37 | .groupBy($"hostname", window($"timestamp", WINDOW, SLIDE)) 38 | .agg(avg($"value") as "avg") 39 | .select($"hostname", (unix_timestamp($"window.end") cast "bigint") as "timestamp", $"avg") 40 | .as[(String, Long, Double)] 41 | .map(MetricService.evaluate(metric, _)) 42 | .writeStream 43 | .outputMode("update") 44 | .foreach(ServerStateWriter()) 45 | .queryName(s"evaluate_${metric}") 46 | .start 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /bdp-stream/src/main/scala/com/github/bdp/stream/assembler/AlertAssembler.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.stream.assembler 2 | 3 | import com.github.bdp.stream.Constants._ 4 | import com.github.bdp.stream.model.Alert 5 | import org.apache.hadoop.hbase.client.Put 6 | import org.apache.hadoop.hbase.util.Bytes 7 | 8 | object AlertAssembler { 9 | def assemble(alert: Alert): Put = { 10 | val put = new Put(Bytes.toBytes(alert.id)) // row key 11 | // column, qualifier, value 12 | put.addColumn(ALERT_COL_FAMILY, ALERT_Q_MESSAGE, Bytes.toBytes(alert.message)) 13 | put.addColumn(ALERT_COL_FAMILY, ALERT_Q_HOSTNAME, Bytes.toBytes(alert.hostname)) 14 | put.addColumn(ALERT_COL_FAMILY, ALERT_Q_TIMESTAMP, Bytes.toBytes(alert.timestamp.getTime)) 15 | put.addColumn(ALERT_COL_FAMILY, ALERT_Q_STATUS, Bytes.toBytes(alert.status)) 16 | put 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /bdp-stream/src/main/scala/com/github/bdp/stream/assembler/MetricAssembler.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.stream.assembler 2 | 3 | import com.github.bdp.stream.Constants._ 4 | import com.github.bdp.stream.model.Metric 5 | import org.apache.hadoop.hbase.client.Put 6 | import org.apache.hadoop.hbase.util.Bytes 7 | 8 | object MetricAssembler { 9 | def assemble(metric: Metric): Put = { 10 | // The row key format: [hostname][metric][timestamp] 11 | // we assume hostname is fixed length, for metric, we use fixed length abbreviation, 12 | // i.e. cpu.usage is cu, memory free is mu and so on. 13 | val put = new Put(Bytes.toBytes(metric.id)) // row key 14 | // column, qualifier, value 15 | put.addColumn(METRIC_COL_FAMILY, METRIC_Q_NAME, Bytes.toBytes(metric.name)) 16 | put.addColumn(METRIC_COL_FAMILY, METRIC_Q_HOSTNAME, Bytes.toBytes(metric.hostname)) 17 | put.addColumn(METRIC_COL_FAMILY, METRIC_Q_TIMESTAMP, Bytes.toBytes(metric.timestamp.getTime)) 18 | put.addColumn(METRIC_COL_FAMILY, METRIC_Q_VALUE, Bytes.toBytes(metric.value)) 19 | put 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /bdp-stream/src/main/scala/com/github/bdp/stream/assembler/ServerStateAssembler.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.stream.assembler 2 | 3 | import com.github.bdp.stream.Constants._ 4 | import com.github.bdp.stream.model.ServerState 5 | import org.apache.hadoop.hbase.client.Put 6 | import org.apache.hadoop.hbase.util.Bytes 7 | 8 | object ServerStateAssembler { 9 | def assemble(serverState: ServerState): Put = { 10 | // The row key format: [hostname][serverState][timestamp] 11 | // we assume hostname is fixed length, for serverState, we use fixed length abbreviation, 12 | // i.e. cpu.usage is cu, memory free is mu and so on. 13 | val put = new Put(Bytes.toBytes(serverState.serverId) ++ Bytes.toBytes(serverState.timestamp)) // row key 14 | // column, qualifier, value 15 | put.addColumn(SERVER_STATE_COL_FAMILY, SERVER_STATE_Q_SRC_TYPE, Bytes.toBytes(serverState.srcType)) 16 | put.addColumn(SERVER_STATE_COL_FAMILY, SERVER_STATE_Q_SEVERITY, Bytes.toBytes(serverState.severity)) 17 | put 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /bdp-stream/src/main/scala/com/github/bdp/stream/model/Alert.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.stream.model 2 | 3 | import java.sql.Timestamp 4 | 5 | case class Alert(id: Long, message: String, hostname: String, status: String, timestamp: Timestamp) -------------------------------------------------------------------------------- /bdp-stream/src/main/scala/com/github/bdp/stream/model/AlertRegistry.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.stream.model 2 | 3 | import com.github.bdp.master.client.service.AlertIndexService 4 | import com.github.bdp.stream.Constants._ 5 | import com.typesafe.scalalogging.LazyLogging 6 | 7 | import scala.collection.mutable 8 | import scala.math._ 9 | 10 | case class AlertRegistry() extends LazyLogging { 11 | 12 | private var registry = mutable.Map[(Long, Long), (Boolean, Boolean)]() 13 | 14 | def updateWith(alerts: Iterator[Alert]): Unit = { 15 | alerts.foreach { 16 | alert => 17 | val id = AlertIndexService.getAlertIndexBy(alert.message).id 18 | val timestamp = alert.timestamp.getTime 19 | val status = alert.status 20 | val key = (id,timestamp) 21 | val oldValue = registry.getOrElse(key, (false,false)) 22 | val newValue = status match { 23 | case "OPEN" => (true, oldValue._2) 24 | case "CLOSED" => (oldValue._1, true) 25 | } 26 | registry.update(key, newValue) 27 | } 28 | } 29 | 30 | /** 31 | * If there is un-closed flag on any timestamp, return given severity, 32 | * because it means, for this server, there is at least an open alert on it. 33 | */ 34 | def evaluate():Int = { 35 | // need fold and go though all elements so as to get highest severity! 36 | registry.foldLeft(0){ 37 | (severity,entry) => 38 | val ((id,_),(open,closed)) = entry 39 | if (open && !closed) { 40 | max(severity,AlertIndexService.getAlertIndexBy(id).severity) 41 | } else { 42 | severity 43 | } 44 | } 45 | } 46 | 47 | /** 48 | * To avoid unbounded state... 49 | * @param now 50 | */ 51 | def cleanUp(now: Long): Unit = { 52 | registry = registry.filter{ 53 | case ((id,timestamp),_) => 54 | logger.debug(s"(CURRENT_TIME-ALERT_TIME)-ALERT_TIME_TO_LIVE=" + 55 | s"($now-$timestamp)-$ALERT_TIME_TO_LIVE = ${(now-timestamp)-ALERT_TIME_TO_LIVE}") 56 | if (now - timestamp < ALERT_TIME_TO_LIVE) { 57 | logger.debug(s"($id, $timestamp) is kept in session because it is LIVE.") 58 | true 59 | } else { 60 | logger.debug(s"($id, $timestamp) is removed from session because it is EXPIRED.") 61 | false 62 | } 63 | } 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /bdp-stream/src/main/scala/com/github/bdp/stream/model/Metric.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.stream.model 2 | 3 | import java.sql.Timestamp 4 | 5 | case class Metric(id: Long, name: String, hostname: String, value: Long, timestamp: Timestamp) -------------------------------------------------------------------------------- /bdp-stream/src/main/scala/com/github/bdp/stream/model/ServerState.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.stream.model 2 | 3 | case class ServerState(serverId: Long, timestamp: Long, srcType: String, severity: Int) 4 | -------------------------------------------------------------------------------- /bdp-stream/src/main/scala/com/github/bdp/stream/service/AlertService.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.stream.service 2 | 3 | import com.github.bdp.master.client.service.ServerService._ 4 | import com.github.bdp.stream.Constants._ 5 | import com.github.bdp.stream.model._ 6 | import com.github.bdp.stream.util.JsonDecoder._ 7 | import com.typesafe.scalalogging.LazyLogging 8 | import org.apache.spark.sql.streaming.GroupState 9 | 10 | 11 | object AlertService extends LazyLogging { 12 | 13 | def transform(alertMsg: String): Alert = { 14 | try { 15 | decodeAlert(alertMsg) 16 | } catch { 17 | case ex: Exception => { 18 | logger.error("decode kafka message error: " + ex.getMessage) 19 | null 20 | } 21 | } 22 | } 23 | 24 | def getServerId(hostname: String): Long = { 25 | getServerBy(hostname).id 26 | } 27 | 28 | def updateAlertGroupState(serverId: Long, alerts: Iterator[Alert], state: GroupState[AlertRegistry]): ServerState = { 29 | val alertRegistry = state.getOption.getOrElse(AlertRegistry()) 30 | val now = System.currentTimeMillis()/1000 31 | alertRegistry.cleanUp(now) 32 | alertRegistry.updateWith(alerts) 33 | state.update(alertRegistry) 34 | val severity = alertRegistry.evaluate() 35 | val timestamp = (now+5)/5*5000 36 | ServerState(serverId, timestamp, ALERT, severity) 37 | } 38 | 39 | } -------------------------------------------------------------------------------- /bdp-stream/src/main/scala/com/github/bdp/stream/service/MetricService.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.stream.service 2 | 3 | import com.github.bdp.master.client.domain.SEVERITY.{AMBER, GREEN, RED} 4 | import com.github.bdp.master.client.service.ServerService._ 5 | import com.github.bdp.stream.model.{Metric, ServerState} 6 | import com.github.bdp.stream.util.JsonDecoder._ 7 | import com.typesafe.scalalogging.LazyLogging 8 | 9 | object MetricService extends LazyLogging { 10 | 11 | def transform(metricMsg: String): Metric = { 12 | try { 13 | decodeMetric(metricMsg) 14 | } catch { 15 | case ex: Exception => { 16 | logger.error("decode kafka message error: " + ex.getMessage) 17 | null 18 | } 19 | } 20 | } 21 | 22 | def evaluate(metric:String, row:(String, Long, Double)): ServerState = { 23 | val (hostname, timestamp, avg) = row 24 | val server = getServerBy(hostname) 25 | val serverId = server.id 26 | val amberThreshold = server.metricThresholds(metric.replace('_','.')).amberThreshold 27 | val redThreshold = server.metricThresholds(metric.replace('_','.')).redThreshold 28 | val severity = avg match { 29 | case avg if avg < amberThreshold => GREEN 30 | case avg if avg >= redThreshold => RED 31 | case _ => AMBER 32 | } 33 | ServerState(serverId, timestamp, metric, severity.id) 34 | } 35 | 36 | } -------------------------------------------------------------------------------- /bdp-stream/src/main/scala/com/github/bdp/stream/util/HBaseClient.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.stream.util 2 | 3 | import com.typesafe.scalalogging.LazyLogging 4 | import org.apache.hadoop.hbase.client.{Connection, ConnectionFactory} 5 | import org.apache.hadoop.hbase.{HBaseConfiguration, TableName} 6 | import org.apache.hadoop.hbase.client.BufferedMutator 7 | import org.apache.hadoop.hbase.client.BufferedMutatorParams 8 | import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException 9 | import scala.collection.JavaConverters._ 10 | import com.github.bdp.stream.Constants._ 11 | 12 | /** 13 | * Typically, a single connection per client application is instantiated and every thread will obtain 14 | * its own Table instance. Caching or pooling of Table and Admin is not recommended. 15 | */ 16 | object HBaseClient extends LazyLogging { 17 | 18 | private val connection = createConnection() 19 | 20 | private val mutatorParams = createMutatorParams() 21 | 22 | private def createMutatorParams(): Map[String, BufferedMutatorParams] = { 23 | Map[String, BufferedMutatorParams]( 24 | METRIC_TABLE_NAME -> createMutatorParams(METRIC_TABLE_NAME), 25 | ALERT_TABLE_NAME -> createMutatorParams(ALERT_TABLE_NAME), 26 | SERVER_STATE_TABLE_NAME -> createMutatorParams(SERVER_STATE_TABLE_NAME) 27 | ) 28 | } 29 | 30 | private def createMutatorParams(tableName: String): BufferedMutatorParams = { 31 | // a callback invoked when an asynchronous write fails. 32 | val listener = new BufferedMutator.ExceptionListener() { 33 | override def onException(e: RetriesExhaustedWithDetailsException, mutator: BufferedMutator): Unit = { 34 | for (cause: Throwable <- e.getCauses.asScala) { 35 | logger.error(s"HBase put operation failed! the error message is: ${cause.getMessage}") 36 | cause.printStackTrace() 37 | } 38 | throw e 39 | } 40 | } 41 | new BufferedMutatorParams(TableName.valueOf(tableName)).listener(listener) 42 | } 43 | 44 | private def createConnection(): Connection = { 45 | try { 46 | val conf = HBaseConfiguration.create() 47 | conf.addResource("hbase-site.xml") 48 | ConnectionFactory.createConnection(conf) 49 | } catch { 50 | case e: Throwable => 51 | logger.error(s"HBase create connection operation failed! the error message is: ${e.getMessage}") 52 | throw e 53 | } 54 | } 55 | 56 | def mutator(tableName: String): BufferedMutator = { 57 | try { 58 | connection.getBufferedMutator(mutatorParams(tableName)) 59 | } catch { 60 | case e: Exception => 61 | logger.error(s"HBase get mutator operation failed! the error message is: ${e.getMessage}") 62 | throw e 63 | } 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /bdp-stream/src/main/scala/com/github/bdp/stream/util/JsonDecoder.scala: -------------------------------------------------------------------------------- 1 | package com.github.bdp.stream.util 2 | 3 | import java.sql.Timestamp 4 | 5 | import com.github.bdp.stream.model.{Alert, Metric} 6 | import io.circe.Decoder.Result 7 | import io.circe.generic.semiauto.deriveDecoder 8 | import io.circe.parser._ 9 | import io.circe.{Decoder, HCursor} 10 | 11 | object JsonDecoder { 12 | 13 | implicit private val metricDecoder: Decoder[Metric] = deriveDecoder 14 | implicit private val alertDecoder: Decoder[Alert] = deriveDecoder 15 | implicit private val timestampDecoder = new Decoder[Timestamp] { 16 | override def apply(c: HCursor): Result[Timestamp] = Decoder.decodeLong.map(s => new Timestamp(s)).apply(c) 17 | } 18 | 19 | def decodeMetric(json: String): Metric = { 20 | decode[Metric](json).right.get 21 | } 22 | 23 | def decodeAlert(json: String): Alert = { 24 | decode[Alert](json).right.get 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /bdp-workflow/.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | /*.iml 3 | target 4 | -------------------------------------------------------------------------------- /bdp-workflow/README.md: -------------------------------------------------------------------------------- 1 | 关于本子项目的部署、运行与代码细节，请参考《大数据平台架构与原型实现：数据中台建设实战》一书第10章以及第4章4.5节 -------------------------------------------------------------------------------- /bdp-workflow/build.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | rem A batch script to build -> deploy -> restart 3 | rem -- Laurence Geng 4 | if [%1]==[] ( 5 | echo. 6 | echo Usage: %0 maven-profile-1 maven-profile-2 ... 7 | echo. 8 | goto end 9 | ) 10 | 11 | set profiles=%~1 12 | 13 | :loopProfiles 14 | shift 15 | if "%~1"=="" ( 16 | goto build 17 | ) else ( 18 | set profiles=%profiles%,%~1 19 | goto loopProfiles 20 | ) 21 | 22 | :build 23 | echo. 24 | echo *************************************************************************************** 25 | echo BUILD... 26 | echo *************************************************************************************** 27 | echo. 28 | 29 | if "%profiles%"=="" ( 30 | call mvn clean install -DskipTests=true 31 | ) else ( 32 | call mvn clean install -DskipTests=true -P%profiles% 33 | ) 34 | if "%errorlevel%"=="1" goto :releasefailed 35 | 36 | call target\classes\deploy.bat 37 | 38 | if "%errorlevel%"=="1" goto :releasefailed 39 | 40 | goto releasesuccess 41 | 42 | :releasesuccess 43 | echo. 44 | echo. 45 | echo *************************************************************************************** 46 | echo RELEASE SUCCESS!! 47 | echo *************************************************************************************** 48 | goto end 49 | 50 | :releasefailed 51 | echo. 52 | echo. 53 | echo *************************************************************************************** 54 | echo RELEASE FAILED!! 55 | echo *************************************************************************************** 56 | goto end 57 | 58 | :end -------------------------------------------------------------------------------- /bdp-workflow/src/main/assembly/bin.xml: -------------------------------------------------------------------------------- 1 | 4 | bin 5 | 6 | zip 7 | 8 | true 9 | 10 | 11 | target/classes/lib 12 | ./lib 13 | 14 | **/*.xml 15 | 16 | 17 | 18 | target/classes/bin 19 | ./bin 20 | 21 | *.sh 22 | 23 | 755 24 | 25 | 26 | -------------------------------------------------------------------------------- /bdp-workflow/src/main/profiles/cluster.properties: -------------------------------------------------------------------------------- 1 | cluster.namenode=nameservice1 2 | cluster.resourcemanager=master1.cluster 3 | cluster.oozie.host=utility1.cluster 4 | cluster.oozie.url=http://${cluster.oozie.host}:11000/oozie 5 | 6 | app.user.name=bdp-workflow 7 | app.user.password=Bdpp1234! 8 | app.user.home=/home/${app.user.name} 9 | 10 | app.host=gateway1.cluster 11 | app.home=${app.user.home}/${project.build.finalName} 12 | 13 | app.hdfs.user.home=/user/${app.user.name} 14 | app.hdfs.home=hdfs://${app.hdfs.user.home}/${project.build.finalName} 15 | 16 | bdp-dwh.user.name=bdp-dwh 17 | bdp-dwh.app.host=gateway1.cluster 18 | bdp-dwh.ssh.host=${bdp-dwh.user.name}@${bdp-dwh.app.host} 19 | bdp-dwh.app.home=/home/bdp-dwh/bdp-dwh-1.0 20 | bdp-dwh.app.bin.home=${bdp-dwh.app.home}/bin 21 | 22 | -------------------------------------------------------------------------------- /bdp-workflow/src/main/profiles/standalone.properties: -------------------------------------------------------------------------------- 1 | cluster.namenode=node1.cluster 2 | cluster.resourcemanager=node1.cluster 3 | cluster.oozie.host=node1.cluster 4 | cluster.oozie.url=http://${cluster.oozie.host}:11000/oozie 5 | 6 | app.user.name=bdp-workflow 7 | app.user.password=Bdpp1234! 8 | app.user.home=/home/${app.user.name} 9 | 10 | app.host=node1.cluster 11 | app.home=${app.user.home}/${project.build.finalName} 12 | 13 | app.hdfs.user.home=/user/${app.user.name} 14 | app.hdfs.home=hdfs://${app.hdfs.user.home}/${project.build.finalName} 15 | 16 | bdp-dwh.user.name=bdp-dwh 17 | bdp-dwh.app.host=node1.cluster 18 | bdp-dwh.ssh.host=${bdp-dwh.user.name}@${bdp-dwh.app.host} 19 | bdp-dwh.app.home=/home/bdp-dwh/bdp-dwh-1.0 20 | bdp-dwh.app.bin.home=${bdp-dwh.app.home}/bin 21 | 22 | -------------------------------------------------------------------------------- /bdp-workflow/src/main/resources/bin/util.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | printHeading() 4 | { 5 | title="$1" 6 | paddingWidth=$((($(tput cols)-${#title})/2-3)) 7 | printf "\n%${paddingWidth}s"|tr ' ' '=' 8 | printf " $title " 9 | printf "%${paddingWidth}s\n\n"|tr ' ' '=' 10 | } 11 | 12 | showTime() 13 | { 14 | date '%FT%H:%M%z' 15 | } 16 | -------------------------------------------------------------------------------- /bdp-workflow/src/main/resources/deploy.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | set host=${app.host} 4 | set port=${app.ssh.port} 5 | set user=${app.user.name} 6 | set password=${app.user.password} 7 | set baseDir=${app.user.home} 8 | set home=${app.home} 9 | set buildDir=${project.build.directory} 10 | set binZip=${project.build.finalName}-bin.zip 11 | set artifactId=${project.artifactId} 12 | 13 | echo. 14 | echo *************************************************************************************** 15 | echo UPLOAD... 16 | echo *************************************************************************************** 17 | 18 | @echo on 19 | PSCP -l %user% -pw %password% "%buildDir%\\%binZip%" "%host%:/tmp/" 20 | PLINK -l %user% -pw %password% %host% -t "if [ ! -d '%baseDir%' ];then mkdir %baseDir%;fi" 21 | PLINK -l %user% -pw %password% %host% -t "if [ -d '%home%' ];then rm -rf %home%;fi" 22 | PLINK -l %user% -pw %password% %host% -t "unzip /tmp/%binZip% -d %baseDir%/" 23 | @echo off 24 | 25 | echo. 26 | echo *************************************************************************************** 27 | echo INITIALIZE... 28 | echo *************************************************************************************** 29 | 30 | @echo on 31 | PLINK -l %user% -pw %password% %host% -t "%home%/bin/%artifactId%.sh init" 32 | @echo off 33 | -------------------------------------------------------------------------------- /bdp-workflow/src/main/resources/lib/ds-bdp-master-daily-build/coordinator.xml: -------------------------------------------------------------------------------- 1 | 4 | 5 | 6 | ${app.hdfs.home}/lib/ds-bdp-master-daily-build/workflow.xml 7 | 8 | 9 | START_TIME 10 | ${coord:dateOffset(coord:nominalTime(), -1, 'DAY')} 11 | 12 | 13 | END_TIME 14 | ${coord:dateOffset(coord:nominalTime(), 0, 'DAY')} 15 | 16 | 17 | DATE_FLAG 18 | ${coord:formatTime(coord:dateOffset(coord:nominalTime(), -1, 'DAY'), "yyyy-MM-dd")} 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /bdp-workflow/src/main/resources/lib/ds-bdp-master-daily-build/sub-workflow/app.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | ${bdp-dwh.ssh.host} 8 | ${bdp-dwh.app.bin.home}/src-bdp-master.sh 9 | build-app 10 | ${START_TIME} 11 | ${END_TIME} 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | ${bdp-dwh.ssh.host} 21 | ${bdp-dwh.app.bin.home}/dwh-bdp-master.sh 22 | build-app 23 | ${START_TIME} 24 | ${END_TIME} 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] 41 | 42 | 43 | 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /bdp-workflow/src/main/resources/lib/ds-bdp-master-daily-build/sub-workflow/metric-index.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | ${bdp-dwh.ssh.host} 8 | ${bdp-dwh.app.bin.home}/src-bdp-master.sh 9 | build-metric-index 10 | ${START_TIME} 11 | ${END_TIME} 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | ${bdp-dwh.ssh.host} 21 | ${bdp-dwh.app.bin.home}/dwh-bdp-master.sh 22 | build-metric-index 23 | ${START_TIME} 24 | ${END_TIME} 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] 41 | 42 | 43 | 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /bdp-workflow/src/main/resources/lib/ds-bdp-master-daily-build/sub-workflow/metric-threshold.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | ${bdp-dwh.ssh.host} 8 | ${bdp-dwh.app.bin.home}/src-bdp-master.sh 9 | build-metric-threshold 10 | ${START_TIME} 11 | ${END_TIME} 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | ${bdp-dwh.ssh.host} 21 | ${bdp-dwh.app.bin.home}/dwh-bdp-master.sh 22 | build-metric-threshold 23 | ${START_TIME} 24 | ${END_TIME} 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] 41 | 42 | 43 | 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /bdp-workflow/src/main/resources/lib/ds-bdp-master-daily-build/sub-workflow/server.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | ${bdp-dwh.ssh.host} 8 | ${bdp-dwh.app.bin.home}/src-bdp-master.sh 9 | build-server 10 | ${START_TIME} 11 | ${END_TIME} 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | ${bdp-dwh.ssh.host} 21 | ${bdp-dwh.app.bin.home}/dwh-bdp-master.sh 22 | build-server 23 | ${START_TIME} 24 | ${END_TIME} 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] 41 | 42 | 43 | 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /bdp-workflow/src/main/resources/lib/ds-bdp-master-daily-build/workflow.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ${cluster.resourcemanager} 5 | hdfs://${cluster.namenode} 6 | 7 | 8 | 9 | 10 | 11 | 12 | ${app.hdfs.home}/lib/ds-bdp-master-daily-build/sub-workflow/app.xml 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | ${app.hdfs.home}/lib/ds-bdp-master-daily-build/sub-workflow/server.xml 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | ${app.hdfs.home}/lib/ds-bdp-master-daily-build/sub-workflow/metric-index.xml 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | ${app.hdfs.home}/lib/ds-bdp-master-daily-build/sub-workflow/metric-threshold.xml 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] 48 | 49 | 50 | 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /bdp-workflow/src/main/resources/lib/ds-bdp-metric-daily-build/coordinator.xml: -------------------------------------------------------------------------------- 1 | 4 | 5 | 6 | ${app.hdfs.home}/lib/ds-bdp-metric-daily-build/workflow.xml 7 | 8 | 9 | START_TIME 10 | ${coord:dateOffset(coord:nominalTime(), -1, 'DAY')} 11 | 12 | 13 | END_TIME 14 | ${coord:dateOffset(coord:nominalTime(), 0, 'DAY')} 15 | 16 | 17 | DATE_FLAG 18 | ${coord:formatTime(coord:dateOffset(coord:nominalTime(), -1, 'DAY'), "yyyy-MM-dd")} 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /bdp-workflow/src/main/resources/lib/ds-bdp-metric-daily-build/sub-workflow/metric.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | ${bdp-dwh.ssh.host} 8 | ${bdp-dwh.app.bin.home}/src-bdp-metric.sh 9 | build-metric 10 | ${START_TIME} 11 | ${END_TIME} 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | ${bdp-dwh.ssh.host} 21 | ${bdp-dwh.app.bin.home}/dwh-bdp-metric.sh 22 | build-metric 23 | ${START_TIME} 24 | ${END_TIME} 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] 41 | 42 | 43 | 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /bdp-workflow/src/main/resources/lib/ds-bdp-metric-daily-build/workflow.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ${cluster.resourcemanager} 5 | hdfs://${cluster.namenode} 6 | 7 | 8 | 9 | 10 | 11 | 12 | ${app.hdfs.home}/lib/ds-bdp-metric-daily-build/sub-workflow/metric.xml 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /bdp-workflow/src/main/resources/lib/sj-infra-metric-daily-build/sub-workflow/fact-metric.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | ${bdp-dwh.ssh.host} 8 | ${bdp-dwh.app.bin.home}/dmt-infra-metric.sh 9 | build-fact-metric 10 | ${START_TIME} 11 | ${END_TIME} 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /bdp-workflow/src/main/resources/lib/sj-infra-metric-daily-build/sub-workflow/sum-metric-avg.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | ${bdp-dwh.ssh.host} 8 | ${bdp-dwh.app.bin.home}/dmt-infra-metric.sh 9 | build-sum-metric-avg 10 | ${START_TIME} 11 | ${END_TIME} 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /bdp-workflow/src/main/resources/lib/sj-infra-metric-daily-build/sub-workflow/wide-metric-avg.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | ${bdp-dwh.ssh.host} 8 | ${bdp-dwh.app.bin.home}/dmt-infra-metric.sh 9 | build-wide-metric-avg 10 | ${START_TIME} 11 | ${END_TIME} 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /bdp-workflow/src/main/resources/lib/sj-infra-metric-daily-build/workflow.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ${cluster.resourcemanager} 5 | hdfs://${cluster.namenode} 6 | 7 | 8 | 9 | 10 | 11 | 12 | ${app.hdfs.home}/lib/sj-infra-metric-daily-build/sub-workflow/fact-metric.xml 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | ${app.hdfs.home}/lib/sj-infra-metric-daily-build/sub-workflow/sum-metric-avg.xml 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | ${app.hdfs.home}/lib/sj-infra-metric-daily-build/sub-workflow/wide-metric-avg.xml 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] 39 | 40 | 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /bdp-workflow/src/main/resources/lib/sj-master-data-daily-build/sub-workflow/app.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | ${bdp-dwh.ssh.host} 8 | ${bdp-dwh.app.bin.home}/dmt-master-data.sh 9 | build-app 10 | ${START_TIME} 11 | ${END_TIME} 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /bdp-workflow/src/main/resources/lib/sj-master-data-daily-build/sub-workflow/metric-index.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | ${bdp-dwh.ssh.host} 8 | ${bdp-dwh.app.bin.home}/dmt-master-data.sh 9 | build-metric-index 10 | ${START_TIME} 11 | ${END_TIME} 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /bdp-workflow/src/main/resources/lib/sj-master-data-daily-build/sub-workflow/metric-threshold.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | ${bdp-dwh.ssh.host} 8 | ${bdp-dwh.app.bin.home}/dmt-master-data.sh 9 | build-metric-threshold 10 | ${START_TIME} 11 | ${END_TIME} 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /bdp-workflow/src/main/resources/lib/sj-master-data-daily-build/sub-workflow/server.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | ${bdp-dwh.ssh.host} 8 | ${bdp-dwh.app.bin.home}/dmt-master-data.sh 9 | build-server 10 | ${START_TIME} 11 | ${END_TIME} 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /bdp-workflow/src/main/resources/lib/sj-master-data-daily-build/workflow.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ${cluster.resourcemanager} 5 | hdfs://${cluster.namenode} 6 | 7 | 8 | 9 | 10 | 11 | 12 | ${app.hdfs.home}/lib/sj-master-data-daily-build/sub-workflow/app.xml 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | ${app.hdfs.home}/lib/sj-master-data-daily-build/sub-workflow/server.xml 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | ${app.hdfs.home}/lib/sj-master-data-daily-build/sub-workflow/metric-index.xml 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | ${app.hdfs.home}/lib/sj-master-data-daily-build/sub-workflow/metric-threshold.xml 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] 48 | 49 | 50 | 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /book.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bluishglc/bdp/3e31b59b52d2c7c297f8cb5852713e791c702a5e/book.jpg --------------------------------------------------------------------------------