├── .gitignore ├── .travis.yml ├── README.md ├── _config.yml ├── alarm-common ├── pom.xml └── src │ └── main │ └── scala │ └── com │ └── netease │ └── spark │ └── alarm │ ├── AlarmConstants.scala │ ├── Alarmist.scala │ ├── AlertMessage.scala │ ├── AlertResp.scala │ ├── AlertType.scala │ └── Components.scala ├── alarm-email ├── pom.xml └── src │ ├── main │ └── scala │ │ └── com │ │ └── netease │ │ └── spark │ │ └── alarm │ │ └── email │ │ ├── EmailAlarmist.scala │ │ └── package.scala │ └── test │ └── scala │ └── com │ └── netease │ └── spark │ └── alarm │ └── email │ └── EmailAlarmistTest.scala ├── alarm-sentry ├── pom.xml └── src │ ├── main │ └── scala │ │ └── com │ │ └── netease │ │ └── spark │ │ └── alarm │ │ └── sentry │ │ ├── SentryAlarmist.scala │ │ ├── SentryStatus.scala │ │ ├── SentryType.scala │ │ └── package.scala │ └── test │ └── scala │ └── com │ └── netease │ └── spark │ └── alarm │ └── sentry │ └── SentryAlarmistTest.scala ├── alarm-smilodon ├── pom.xml └── src │ ├── main │ └── scala │ │ └── com │ │ └── netease │ │ └── spark │ │ └── alarm │ │ └── smilodon │ │ ├── SmilodonAlarmist.scala │ │ ├── SmilodonAlertLevel.scala │ │ ├── SmilodonAlertMessage.scala │ │ ├── channel │ │ ├── ChannelType.scala │ │ └── SmilodonChannel.scala │ │ ├── event │ │ └── SmilodonEvent.scala │ │ └── package.scala │ └── test │ └── scala │ └── com │ └── netease │ └── spark │ └── alarm │ └── smilodon │ └── SmilodonAlarmistTest.scala ├── pom.xml └── streaming-alarmer ├── pom.xml └── src ├── main └── scala │ └── com │ └── netease │ └── spark │ └── alarm │ └── alarmer │ └── streaming │ ├── EmailStreamingAlarmListener.scala │ ├── SentryStreamingAlarmListener.scala │ ├── SmilodonStreamingAlarmListener.scala │ ├── StreamingAlarmListener.scala │ └── package.scala └── test └── scala ├── com └── netease │ └── spark │ └── alarm │ └── alarmer │ └── streaming │ ├── EmailStreamingAlarmListenerTest.scala │ ├── SmilodonStreamingAlarmListenerTest.scala │ └── StreamingAlarmListenerTest.scala └── org └── apache └── spark └── streaming └── AlarmTestInputStream.scala /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | /alarm-common/alarm-common.iml 3 | /spark-alarm.iml 4 | /alarm-email/alarm-email.iml 5 | /alarm-sentry/alarm-sentry.iml 6 | /alarm-smilodon/alarm-smilodon.iml 7 | /.idea/ 8 | .idea/dictionaries/ 9 | .idea/encodings.xml 10 | .idea/misc.xml 11 | .idea/sbt.xml 12 | .idea/thriftCompiler.xml 13 | .idea/vcs.xml 14 | .idea/workspace.xml 15 | alarm-common/target/ 16 | alarm-email/target/ 17 | alarm-sentry/target/ 18 | alarm-smilodon/target/ 19 | /streaming-alarmer/streaming-alarmer.iml 20 | /streaming-alarmer/dependency-reduced-pom.xml 21 | /streaming-alarmer/target/ 22 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | scala: 3 | - 2.11.8 4 | 5 | cache: 6 | directories: 7 | - $HOME/.m2 8 | - ./build 9 | 10 | deploy: 11 | - provider: pages 12 | skip_cleanup: true 13 | github_token: $GITHUB_TOKEN 14 | email: yaooqinn@hotmail.com 15 | name: Kent Yao 16 | on: 17 | branch: master 18 | 19 | jobs: 20 | include: 21 | - stage: spark2.4 22 | language: scala 23 | script: mvn clean install -Pspark-2.4 -Dmaven.javadoc.skip=true -B -V 24 | - stage: spark2.3 25 | language: scala 26 | script: mvn clean install -Pspark-2.3 -Dmaven.javadoc.skip=true -B -V 27 | - stage: spark2.2 28 | language: scala 29 | script: mvn clean install -Pspark-2.2 -Dmaven.javadoc.skip=true -B -V 30 | - stage: spark2.1 31 | language: scala 32 | script: mvn clean install -Pspark-2.1 -Dmaven.javadoc.skip=true -B -V 33 | 34 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Spark-Alarm [![Build Status](https://travis-ci.com/yaooqinn/spark-alarm.svg?branch=master)](https://travis-ci.com/yaooqinn/spark-alarm)[![HitCount](http://hits.dwyl.io/yaooqinn/spark-alarm.svg)](http://hits.dwyl.io/yaooqinn/spark-alarm) 2 | 3 | 4 | ## 项目简介 5 | 6 | 提供一些基本的报警手段，并可以通过SparkListener实现对Spark内部执行逻辑进行监控报警 7 | 8 | | 报警模式 | 使用限制 | 简介 | 9 | | ------ | ------ | ------ | 10 | | 邮件 | 通用，无限制 | 通过 SMTP 协议发送告警 | 11 | | 哨兵 | 网易内部使用 | 通过 HTTP 协议发送告警| 12 | | Smilodon | 网易内部使用 | 通过 HTTP 协议发送告警 | 13 | 14 | ## 使用方法 15 | 16 | ### 编译 17 | 18 | ```bash 19 | # 克隆本项目 20 | git https://github.com/yaooqinn/spark-alarm.git 21 | # cd spark-alarm 22 | # mvn clean package 23 | ``` 24 | 25 | 可以得到内置示例项目jar包：`streaming-alarmer/target/streaming-alarmer-1.0-SNAPSHOT.jar`，该构件实现了对Streaming程序"异常退出"和""任务堆积"等相关关键事件进行简单的告警服务 26 | 27 | ## 配置 28 | 29 | 配置工作分为三个过程： 30 | - 配置报警方式相关参数 31 | - 配置报警规则参数 32 | - 配置spark.extraListeners 33 | 所有的参数都完全等价Spark内置的各项参数，可以通过代码/--conf/spark-default.conf等方式加入 34 | 35 | ### 配置邮件报警 36 | 37 | | 配置 | 默认值 | 说明 | 38 | | ------ | ------ | ------ | 39 | | spark.alarm.email.smtp.host | <空值> | 发信邮件服务器的地址, 比如 smtp-mail.outlook.com | 40 | | spark.alarm.email.smtp.port | 465(ssl)/25(no ssl) | 发信邮件服务器的端口， 587| 41 | | spark.alarm.email.username | <空值> | 发信邮箱的用户名 | 42 | | spark.alarm.email.password | <空值> | 发信邮箱的密码 | 43 | | spark.alarm.email.ssl.on.connect | false | 是否ssl加密连接 | 44 | | spark.alarm.email.from | <空值> | 发信邮箱 | 45 | | spark.alarm.email.to | <空值> | 收信邮箱列表，用逗号分隔 | 46 | 47 | 参见: https://github.com/yaooqinn/spark-alarm/blob/master/alarm-email/src/test/scala/com/netease/spark/alarm/email/EmailAlarmistTest.scala 48 | 49 | ### 配置哨兵报警 50 | 51 | | 配置 | 默认值 | 说明 | 52 | | ------ | ------ | ------ | 53 | | spark.alarm.sentry.url | <空值> | 哨兵服务地址 | 54 | | spark.alarm.sentry.api.type | Stone | 哨兵服务通知到的终端类型 /** 泡泡 */ POPO, /** 易信 */YiXin, /** 短信 */ SMS, /** 语音 */ Voice, Stone, /** 邮件 */ Email| 55 | | spark.alarm.sentry.app.name | <空值> | 哨兵服务的应用名称，若没有可以去哨兵官网添加 | 56 | | spark.alarm.sentry.app.secret | <空值> | 哨兵服务应用对应的密钥 | 57 | | spark.alarm.sentry.to | <空值> | 收信邮箱、或者ID、电话列表，用逗号分隔，根据具体的终端类型决定 | 58 | 59 | 参见: alarm-sentry/src/test/scala/com/netease/spark/alarm/sentry/SentryAlarmistTest.scala 60 | 61 | ### 配置Smilodon报警 62 | 63 | | 配置 | 默认值 | 说明 | 64 | | ------ | ------ | ------ | 65 | | spark.alarm.smilodon.url | http://10.165.139.76:8765/v1/alerts | Smilodon服务地址 | 66 | | spark.alarm.smilodon.channels | POPO,STONE | 通知中心终端类型，目前只支持| 67 | | spark.alarm.smilodon.users | <空值> | 收信者的邮箱账号列表，逗号分隔 | 68 | | spark.alarm.smilodon.groups | <空值> | 暂时不用管 | 69 | 70 | 参见: alarm-smilodon/src/test/scala/com/netease/spark/alarm/smilodon/SmilodonAlarmistTest.scala 71 | 72 | 注意：Smilodon服务有白名单控制，所以需要用户Spark作业的Driver进程所在的服务器在该列表中，对于Cluster模式则需要所有的节点（如NodeManager）在该列表中 73 | 74 | ### 配置报警规则参数 75 | 76 | 该参数列表适用于本项目内置示例项目streaming-alarmer的报警规则，用户可以自己定义相关的SparkListener来定制报警器 77 | 78 | | 配置 | 默认值 | 说明 | 79 | | ------ | ------ | ------ | 80 | | spark.alarm.streaming.application.force.exit | false | 出现Application级别错误的时候，是否直接exit | 81 | | spark.alarm.streaming.batch.error.enable | true | 是否对Batch出错的信息进行告警 | 82 | | spark.alarm.streaming.job.error.enable | false | 是否对job出错的信息进行告警 | 83 | | spark.alarm.streaming.batch.delay.ratio | 1 | Schedule delay时间 / Processing 时间的比值，大于该值，则作为判定为需要告警的必要条件之一 | 84 | | spark.alarm.streaming.batch.process.time.threshold | 1s | 处理时间的阈值，实际处理时间大于该值的批次，才进行报警 | 85 | 86 | 参见：streaming-alarmer/src/test/scala/com/netease/spark/alarm/alarmer/streaming/EmailStreamingAlarmListenerTest.scala 87 | 88 | 89 | ### 配置spark.extraListeners 90 | 91 | 开启邮件报警，也可以设置其他实现，或者多个逗号分隔的报警实现，以及自己拓展的实现 92 | ```scala 93 | spark.extraListeners=com.netease.spark.alarm.alarmer.streaming.EmailStreamingAlarmListener 94 | ``` 95 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-leap-day 2 | -------------------------------------------------------------------------------- /alarm-common/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | spark-alarm 7 | com.netease.spark 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | alarm-common 13 | Spark Alarm Common 14 | jar 15 | 16 | 17 | org.scala-lang 18 | scala-library 19 | 20 | 21 | 22 | 23 | org.scalatest 24 | scalatest_${scala.bin.ver} 25 | 26 | 27 | 28 | 29 | target/scala-${scala.bin.ver}/classes 30 | target/scala-${scala.bin.ver}/test-classes 31 | 32 | 33 | ${project.basedir}/src/main/resources 34 | 35 | 36 | 37 | 38 | org.apache.maven.plugins 39 | maven-compiler-plugin 40 | 41 | 42 | 43 | net.alchim31.maven 44 | scala-maven-plugin 45 | 46 | 47 | 48 | 49 | org.apache.maven.plugins 50 | maven-surefire-plugin 51 | 52 | 53 | 54 | org.scalatest 55 | scalatest-maven-plugin 56 | 1.0 57 | 58 | ${project.build.directory}/surefire-reports 59 | . 60 | TestSuite.txt 61 | 62 | ${project.build.testOutputDirectory}/kyuubi-server-${project.version}.jar 63 | 64 | 65 | ${project.build.testOutputDirectory}/spark-warehouse 66 | 67 | 68 | 69 | 70 | test 71 | 72 | test 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /alarm-common/src/main/scala/com/netease/spark/alarm/AlarmConstants.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm 2 | 3 | object AlarmConstants { 4 | 5 | import Components._ 6 | import AlertType._ 7 | val SERVICE: String = "Spark" 8 | 9 | val STREAMING_APPLICATION_ERROR: String = 10 | "服务[" + SERVICE + "]-组件["+ STREAMING + "]-级别[" + Application + "] 异常告警" 11 | 12 | val STREAMING_BATCH_ERROR: String = 13 | "服务[" + SERVICE + "]-组件["+ STREAMING + "]-级别[" + Batch + "] 异常告警" 14 | 15 | val STREAMING_JOB_ERROR: String = 16 | "服务[" + SERVICE + "]-组件["+ STREAMING + "]-级别[" + Job + "] 异常告警" 17 | } 18 | 19 | 20 | -------------------------------------------------------------------------------- /alarm-common/src/main/scala/com/netease/spark/alarm/Alarmist.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm 2 | 3 | trait Alarmist { 4 | 5 | /** 6 | * Send the alert message to possible external SMS, EMAIL, Phone system. 7 | * 8 | * @param msg the alert message to send 9 | * @return a [[AlertResp]] with status and an optional message 10 | */ 11 | def alarm(msg: AlertMessage): AlertResp 12 | } 13 | -------------------------------------------------------------------------------- /alarm-common/src/main/scala/com/netease/spark/alarm/AlertMessage.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm 2 | 3 | /** 4 | * Message used to be sent by [[Alarmist]] 5 | * @param title message title 6 | * @param content message body 7 | */ 8 | case class AlertMessage(title: String, content: String) 9 | -------------------------------------------------------------------------------- /alarm-common/src/main/scala/com/netease/spark/alarm/AlertResp.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm 2 | 3 | case class AlertResp(status: Boolean, ret: String) 4 | 5 | object AlertResp { 6 | def success(ret: String) = apply(status = true, ret) 7 | def failure(ret: String) = apply(status = false, ret) 8 | } 9 | 10 | -------------------------------------------------------------------------------- /alarm-common/src/main/scala/com/netease/spark/alarm/AlertType.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm 2 | 3 | /** 4 | * 5 | */ 6 | object AlertType extends Enumeration { 7 | type AlertType = Value 8 | val Application, Batch, Job, Stage, Task = Value 9 | } 10 | -------------------------------------------------------------------------------- /alarm-common/src/main/scala/com/netease/spark/alarm/Components.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm 2 | 3 | object Components extends Enumeration { 4 | type Components = Value 5 | val CORE, SQL, STREAMING, MLLIB, GRAPHX = Value 6 | } 7 | -------------------------------------------------------------------------------- /alarm-email/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | spark-alarm 7 | com.netease.spark 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | alarm-email 13 | Spark Alarm Email 14 | jar 15 | 16 | 17 | 18 | com.netease.spark 19 | alarm-common 20 | ${project.version} 21 | 22 | 23 | 24 | org.apache.commons 25 | commons-email 26 | 27 | 28 | 29 | org.apache.spark 30 | spark-core_${scala.bin.ver} 31 | 32 | 33 | 34 | commons-logging 35 | commons-logging 36 | 1.2 37 | 38 | 39 | 40 | 41 | 42 | org.scalatest 43 | scalatest_${scala.bin.ver} 44 | 45 | 46 | 47 | 48 | target/scala-${scala.bin.ver}/classes 49 | target/scala-${scala.bin.ver}/test-classes 50 | 51 | 52 | ${project.basedir}/src/main/resources 53 | 54 | 55 | 56 | 57 | org.apache.maven.plugins 58 | maven-compiler-plugin 59 | 60 | 61 | 62 | net.alchim31.maven 63 | scala-maven-plugin 64 | 65 | 66 | 67 | 68 | org.apache.maven.plugins 69 | maven-surefire-plugin 70 | 71 | 72 | 73 | org.scalatest 74 | scalatest-maven-plugin 75 | 1.0 76 | 77 | ${project.build.directory}/surefire-reports 78 | . 79 | TestSuite.txt 80 | 81 | ${project.build.testOutputDirectory}/kyuubi-server-${project.version}.jar 82 | 83 | 84 | ${project.build.testOutputDirectory}/spark-warehouse 85 | 86 | 87 | 88 | 89 | test 90 | 91 | test 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | -------------------------------------------------------------------------------- /alarm-email/src/main/scala/com/netease/spark/alarm/email/EmailAlarmist.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm.email 2 | 3 | import com.netease.spark.alarm.{Alarmist, AlertMessage, AlertResp} 4 | import org.apache.commons.logging.LogFactory 5 | import org.apache.commons.mail.{DefaultAuthenticator, EmailConstants, SimpleEmail} 6 | import org.apache.spark.SparkConf 7 | 8 | class EmailAlarmist(conf: SparkConf) extends Alarmist { 9 | 10 | private val LOG = LogFactory.getFactory.getInstance(classOf[EmailAlarmist]) 11 | private val hostName = conf.get(HOSTNAME, "") 12 | private val useSSL = conf.getBoolean(SSL_ON_CONNECT, defaultValue = false) 13 | private val port = conf.getOption(SMTP_PORT) match { 14 | case Some(v) => v.toInt 15 | case _ => if (useSSL) 465 else 25 16 | } 17 | private val user = conf.get(USERNAME, "") 18 | private val password = conf.get(PASSWORD, "") 19 | private val from = conf.get(FROM, "") 20 | private val toList = conf.get(TO, from).split(",") 21 | 22 | override def alarm(msg: AlertMessage): AlertResp = { 23 | try { 24 | val email = new SimpleEmail() 25 | email.setHostName(hostName) 26 | email.setSmtpPort(port) 27 | email.setCharset(EmailConstants.UTF_8) 28 | val authn = new DefaultAuthenticator(user, password) 29 | email.setAuthenticator(authn) 30 | email.setSSLOnConnect(useSSL) 31 | email.setFrom(from) 32 | email.setSubject(msg.title) 33 | email.setMsg(msg.content) 34 | email.addTo(toList: _*) 35 | val ret = email.send() 36 | AlertResp.success(ret) 37 | } catch { 38 | case e: Exception => 39 | LOG.error(s"User $user failed to send email from $from to $toList", e) 40 | AlertResp.failure(e.getMessage) 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /alarm-email/src/main/scala/com/netease/spark/alarm/email/package.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm 2 | 3 | package object email { 4 | 5 | private val PREFIX = "spark.alarm.email." 6 | val HOSTNAME = PREFIX + "smtp.host" 7 | val SMTP_PORT = PREFIX + "smtp.port" 8 | val USERNAME = PREFIX + "username" 9 | val PASSWORD = PREFIX + "password" 10 | val SSL_ON_CONNECT = PREFIX + "ssl.on.connect" 11 | val FROM = PREFIX + "from" 12 | val TO = PREFIX + "to" 13 | } 14 | -------------------------------------------------------------------------------- /alarm-email/src/test/scala/com/netease/spark/alarm/email/EmailAlarmistTest.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm.email 2 | 3 | import com.netease.spark.alarm.AlertMessage 4 | import org.apache.spark.SparkConf 5 | import org.scalatest.FunSuite 6 | 7 | class EmailAlarmistTest extends FunSuite { 8 | private val conf = new SparkConf(true) 9 | private val message = AlertMessage("title", "content") 10 | ignore("email alarm") { 11 | conf.set(HOSTNAME, "smtp-mail.outlook.com") 12 | .set(FROM, "yaooqinn@hotmail.com") 13 | .set(SSL_ON_CONNECT, "true") 14 | .set(USERNAME, "yaooqinn") 15 | .set(PASSWORD, "*****") 16 | .set(TO, "yaooqinn@hotmail.com") 17 | val alarmist = new EmailAlarmist(conf) 18 | val response = alarmist.alarm(message) 19 | assert(response.status) 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /alarm-sentry/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | spark-alarm 7 | com.netease.spark 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | alarm-sentry 13 | Spark Alarm Sentry 14 | jar 15 | 16 | 17 | 18 | com.netease.spark 19 | alarm-common 20 | ${project.version} 21 | 22 | 23 | 24 | org.apache.spark 25 | spark-core_${scala.bin.ver} 26 | 27 | 28 | 29 | org.apache.httpcomponents 30 | httpclient 31 | 4.5.13 32 | 33 | 34 | 35 | com.google.code.gson 36 | gson 37 | 38 | 39 | 40 | 41 | 42 | org.scalatest 43 | scalatest_${scala.bin.ver} 44 | 45 | 46 | 47 | 48 | target/scala-${scala.bin.ver}/classes 49 | target/scala-${scala.bin.ver}/test-classes 50 | 51 | 52 | ${project.basedir}/src/main/resources 53 | 54 | 55 | 56 | 57 | org.apache.maven.plugins 58 | maven-compiler-plugin 59 | 60 | 61 | 62 | net.alchim31.maven 63 | scala-maven-plugin 64 | 65 | 66 | 67 | 68 | org.apache.maven.plugins 69 | maven-surefire-plugin 70 | 71 | 72 | 73 | org.scalatest 74 | scalatest-maven-plugin 75 | 1.0 76 | 77 | ${project.build.directory}/surefire-reports 78 | . 79 | TestSuite.txt 80 | 81 | ${project.build.testOutputDirectory}/kyuubi-server-${project.version}.jar 82 | 83 | 84 | ${project.build.testOutputDirectory}/spark-warehouse 85 | 86 | 87 | 88 | 89 | test 90 | 91 | test 92 | 93 | 94 | 95 | 96 | 97 | 98 | -------------------------------------------------------------------------------- /alarm-sentry/src/main/scala/com/netease/spark/alarm/sentry/SentryAlarmist.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm.sentry 2 | 3 | import java.util 4 | 5 | import com.google.gson.Gson 6 | import com.netease.spark.alarm.{Alarmist, AlertMessage, AlertResp} 7 | import org.apache.commons.codec.digest.DigestUtils 8 | import org.apache.commons.codec.Charsets 9 | import org.apache.commons.logging.LogFactory 10 | import org.apache.http.client.entity.UrlEncodedFormEntity 11 | import org.apache.http.client.methods.HttpPost 12 | import org.apache.http.impl.client.HttpClients 13 | import org.apache.http.message.BasicNameValuePair 14 | import org.apache.http.util.EntityUtils 15 | import org.apache.spark.SparkConf 16 | 17 | import scala.collection.JavaConverters._ 18 | import scala.util.control.NonFatal 19 | 20 | class SentryAlarmist(conf: SparkConf) extends Alarmist { 21 | private val LOG = LogFactory.getFactory.getInstance(classOf[SentryAlarmist]) 22 | 23 | private val client = HttpClients.createDefault() 24 | private val url = conf.get(SENTRY_URL, "") 25 | private val typ = conf.get(SENTRY_API_TYPE, SentryType.Stone.toString) 26 | private val api = url + typ 27 | private val to = conf.get(SENTRY_TO_LIST, "") 28 | private val secret = conf.get(SENTRY_APP_SECRET, "") 29 | private val appName = conf.get(SENTRY_APP_NAME, "") 30 | 31 | override def alarm(msg: AlertMessage): AlertResp = { 32 | 33 | val httpPost = new HttpPost(api) 34 | val now = System.currentTimeMillis() 35 | val sig = DigestUtils.md5Hex(secret + now) 36 | val params = List(new BasicNameValuePair("to", to), 37 | new BasicNameValuePair("title", msg.title), 38 | new BasicNameValuePair("content", msg.content), 39 | new BasicNameValuePair("isSync", "true"), 40 | new BasicNameValuePair("ac_appName", appName), 41 | new BasicNameValuePair("ac_timestamp", now.toString), 42 | new BasicNameValuePair("ac_signature", sig)) 43 | httpPost.setEntity(new UrlEncodedFormEntity(params.asJava, Charsets.UTF_8)) 44 | 45 | try { 46 | val response = client.execute(httpPost) 47 | val status = response.getStatusLine.getStatusCode 48 | val resp = EntityUtils.toString(response.getEntity) 49 | if (status == 200) { 50 | val map = new Gson().fromJson(resp, classOf[util.HashMap[String, Any]]) 51 | SentryStatus.handleResponse(map) 52 | } else { 53 | LOG.error(s"Failed to send message to $to using API $api, status: $status" ) 54 | AlertResp.failure(resp) 55 | } 56 | } catch { 57 | case NonFatal(e) => 58 | LOG.error(s"Failed to send message to $to using API $api", e) 59 | val message = e.getMessage 60 | AlertResp.failure(message) 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /alarm-sentry/src/main/scala/com/netease/spark/alarm/sentry/SentryStatus.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm.sentry 2 | 3 | import com.netease.spark.alarm.AlertResp 4 | import org.apache.commons.logging.LogFactory 5 | 6 | object SentryStatus extends Enumeration { 7 | private val LOG = LogFactory.getFactory.getInstance(this.getClass) 8 | 9 | type SentryStatus = Value 10 | 11 | val SUCCESS: SentryStatus = Value(200, "SUCCESS") 12 | val PARTIAL_SUCCESS: SentryStatus = Value(201, "PARTIAL_SUCCESS") 13 | val SYSTEM_ERROR: SentryStatus = Value(500, "SYSTEM_ERROR") 14 | val AUTHENTICATION_ERROR: SentryStatus = Value(501, "AUTHENTICATION_ERROR") 15 | val ILLEGAL_ARGUMENTS: SentryStatus = Value(502, "ILLEGAL_ARGUMENTS") 16 | val QUOTA_EXCEED: SentryStatus = Value(520, "QUOTA_EXCEED") 17 | 18 | def handleResponse(resp: java.util.HashMap[String, Any]): AlertResp = { 19 | val msg = resp.get("message") + "" 20 | resp.get("code").asInstanceOf[Double].toInt match { 21 | case 200 => 22 | LOG.info("Succeed to send alert to Sentry") 23 | AlertResp.success(msg) 24 | case 201 => 25 | LOG.warn("Succeed to send alert to Sentry, but some receiver may not receive the alert") 26 | AlertResp.success(msg) 27 | case 500 => 28 | LOG.error(s"${SYSTEM_ERROR.toString}: failed to send message to Sentry due to $msg") 29 | AlertResp.failure(msg) 30 | case 501 => 31 | LOG.error(s"${AUTHENTICATION_ERROR.toString}: failed to send message to Sentry due to $msg") 32 | AlertResp.failure(msg) 33 | case 502 => 34 | LOG.error(s"${ILLEGAL_ARGUMENTS.toString}: failed to send message to Sentry due to $msg") 35 | AlertResp.failure(msg) 36 | case 520 => 37 | LOG.error(s"${QUOTA_EXCEED.toString}: failed to send message to Sentry due to $msg") 38 | AlertResp.failure(msg) 39 | case _ => 40 | LOG.error(s"UNKNOWN_ERROR: failed to send message to Sentry due to $msg") 41 | AlertResp.failure(msg) 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /alarm-sentry/src/main/scala/com/netease/spark/alarm/sentry/SentryType.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm.sentry 2 | 3 | object SentryType extends Enumeration { 4 | 5 | type SentryType = Value 6 | 7 | val /** 泡泡 */ POPO, /** 易信 */YiXin, /** 短信 */ SMS, /** 语音 */ Voice, Stone, /** 邮件 */ Email = Value 8 | } 9 | -------------------------------------------------------------------------------- /alarm-sentry/src/main/scala/com/netease/spark/alarm/sentry/package.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm 2 | 3 | package object sentry { 4 | private val PREFIX = "spark.alarm.sentry." 5 | val SENTRY_URL: String = PREFIX + "url" 6 | val SENTRY_API_TYPE: String = PREFIX + "api.type" 7 | val SENTRY_TO_LIST: String = PREFIX + "to" 8 | val SENTRY_APP_SECRET: String = PREFIX + "app.secret" 9 | val SENTRY_APP_NAME: String = PREFIX + "app.name" 10 | } 11 | -------------------------------------------------------------------------------- /alarm-sentry/src/test/scala/com/netease/spark/alarm/sentry/SentryAlarmistTest.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm.sentry 2 | 3 | import com.netease.spark.alarm.AlertMessage 4 | import org.apache.spark.SparkConf 5 | import org.scalatest.FunSuite 6 | 7 | class SentryAlarmistTest extends FunSuite { 8 | 9 | private val conf = new SparkConf(true) 10 | private val message = AlertMessage("title", "content") 11 | 12 | ignore("sentry alarm") { 13 | conf.set(SENTRY_API_TYPE, "Stone") 14 | .set(SENTRY_APP_SECRET, "da1ce204-52aa-4d6a-835b-ea659dd2d0d3") 15 | .set(SENTRY_TO_LIST, "yaooqinn@hotmail.com") 16 | .set(SENTRY_APP_NAME, "NeSparkLogAnalyzer") 17 | val alarmist = new SentryAlarmist(conf) 18 | val response = alarmist.alarm(message) 19 | assert(!response.status) 20 | } 21 | 22 | ignore("smilodon alarm") { 23 | conf.set(SENTRY_API_TYPE, "Stone") 24 | .set(SENTRY_APP_SECRET, "4204243f-30bb-4bb9-9bdc-50f034778671") 25 | .set(SENTRY_TO_LIST, "hzyaoqin") 26 | .set(SENTRY_APP_NAME, "smilodon") 27 | val alarmist = new SentryAlarmist(conf) 28 | val response = alarmist.alarm(message) 29 | assert(response.status) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /alarm-smilodon/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | spark-alarm 7 | com.netease.spark 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | alarm-smilodon 13 | Spark Alarm Smilodon 14 | jar 15 | 16 | 17 | 18 | com.netease.spark 19 | alarm-common 20 | ${project.version} 21 | 22 | 23 | 24 | org.apache.spark 25 | spark-core_${scala.bin.ver} 26 | 27 | 28 | 29 | org.apache.httpcomponents 30 | httpclient 31 | 32 | 33 | 34 | com.google.code.gson 35 | gson 36 | 37 | 38 | 39 | 40 | 41 | org.scalatest 42 | scalatest_${scala.bin.ver} 43 | 44 | 45 | 46 | 47 | target/scala-${scala.bin.ver}/classes 48 | target/scala-${scala.bin.ver}/test-classes 49 | 50 | 51 | ${project.basedir}/src/main/resources 52 | 53 | 54 | 55 | 56 | org.apache.maven.plugins 57 | maven-compiler-plugin 58 | 59 | 60 | 61 | net.alchim31.maven 62 | scala-maven-plugin 63 | 64 | 65 | 66 | 67 | org.apache.maven.plugins 68 | maven-surefire-plugin 69 | 70 | 71 | 72 | org.scalatest 73 | scalatest-maven-plugin 74 | 1.0 75 | 76 | ${project.build.directory}/surefire-reports 77 | . 78 | TestSuite.txt 79 | 80 | ${project.build.testOutputDirectory}/kyuubi-server-${project.version}.jar 81 | 82 | 83 | ${project.build.testOutputDirectory}/spark-warehouse 84 | 85 | 86 | 87 | 88 | test 89 | 90 | test 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | -------------------------------------------------------------------------------- /alarm-smilodon/src/main/scala/com/netease/spark/alarm/smilodon/SmilodonAlarmist.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm.smilodon 2 | 3 | import com.google.gson.Gson 4 | import com.netease.spark.alarm.{Alarmist, AlertMessage, AlertResp} 5 | import org.apache.commons.logging.LogFactory 6 | import org.apache.http.client.methods.HttpPost 7 | import org.apache.http.entity.{ContentType, StringEntity} 8 | import org.apache.http.impl.client.HttpClients 9 | import org.apache.http.util.EntityUtils 10 | import org.apache.spark.SparkConf 11 | 12 | import scala.util.control.NonFatal 13 | 14 | class SmilodonAlarmist(conf: SparkConf) extends Alarmist { 15 | private val LOG = LogFactory.getFactory.getInstance(classOf[SmilodonAlarmist]) 16 | 17 | private val client = HttpClients.createDefault() 18 | private val url = conf.get(SMILODON_URL, "http://10.165.139.76:8765/v1/alerts") 19 | override def alarm(msg: AlertMessage): AlertResp = { 20 | val httpPost = new HttpPost(url) 21 | val entity = new StringEntity(msg.content, ContentType.APPLICATION_JSON) 22 | // httpPost.setHeader("Content-Type", "application/json") 23 | httpPost.setEntity(entity) 24 | try { 25 | val response = client.execute(httpPost) 26 | val code = response.getStatusLine.getStatusCode 27 | val resp = EntityUtils.toString(response.getEntity) 28 | 29 | if (code == 200) { 30 | LOG.info("Succeed to send alert to Smilondon") 31 | AlertResp.success(resp) 32 | } else { 33 | LOG.error(s"Failed to send alert to Smilondon, status: $code, msg: $resp") 34 | AlertResp.failure(resp) 35 | } 36 | } catch { 37 | case NonFatal(e) => 38 | LOG.error("Failed to send alert to Smilondon " + msg.content , e) 39 | AlertResp.failure(e.getMessage) 40 | } 41 | } 42 | 43 | def alarm(msg: SmilodonAlertMessage): AlertResp = { 44 | val jsonMsg = new Gson().toJson(msg) 45 | alarm(AlertMessage("", jsonMsg)) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /alarm-smilodon/src/main/scala/com/netease/spark/alarm/smilodon/SmilodonAlertLevel.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm.smilodon 2 | 3 | object SmilodonAlertLevel extends Enumeration { 4 | 5 | type SmilodonAlertLevel = Value 6 | 7 | val GENERAL: SmilodonAlertLevel = Value(1, "general") 8 | val NORMAL: SmilodonAlertLevel = Value(2, "normal") 9 | val WARN: SmilodonAlertLevel = Value(3, "warn") 10 | val ERROR: SmilodonAlertLevel = Value(4, "error") 11 | val FATAL: SmilodonAlertLevel = Value(5, "fatal") 12 | 13 | def getIdByLevel(level: SmilodonAlertLevel): Int = { 14 | level match { 15 | case NORMAL => 2 16 | case WARN => 3 17 | case ERROR => 4 18 | case FATAL => 5 19 | case _ => 1 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /alarm-smilodon/src/main/scala/com/netease/spark/alarm/smilodon/SmilodonAlertMessage.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm.smilodon 2 | 3 | import java.util 4 | 5 | import com.netease.spark.alarm.smilodon.SmilodonAlertLevel.SmilodonAlertLevel 6 | import com.netease.spark.alarm.smilodon.channel.{ChannelType, SmilodonChannel} 7 | import com.netease.spark.alarm.smilodon.event.SmilodonEvent 8 | import org.apache.spark.SparkConf 9 | 10 | import scala.collection.JavaConverters._ 11 | 12 | /** 13 | * 14 | * @param level_id 不填时默认值为1 15 | * @param event 16 | * @param default_title 默认报警信息标题，如发送邮件的主题；不为空时，若channel无title字段，将使用本字段值，若多channel的title相同，建议设置此字段; 17 | * 可为空，使用系统默认title 18 | * @param default_content 默认报警信息；不为空时，若channel无content字段，将使用本字段值，若多channel的content相同，建议设置此字段；可为空，使用 19 | * 系统默认content 20 | * @param channels 报警通道列表，可定制该通道的报警标题和信息 21 | * @param users 接收人email列表 22 | * @param groups 报警组id 列表 23 | */ 24 | case class SmilodonAlertMessage private( 25 | level_id: Int, 26 | event: SmilodonEvent, 27 | default_title: String, 28 | default_content: String, 29 | channels: java.util.List[SmilodonChannel], 30 | users: java.util.List[String], 31 | groups: java.util.List[String]) 32 | 33 | object SmilodonAlertMessage { 34 | def apply( 35 | level: SmilodonAlertLevel, 36 | event: SmilodonEvent, 37 | default_title: String, 38 | default_content: String, 39 | channels: util.List[SmilodonChannel], 40 | users: util.List[String], 41 | groups: util.List[String]): SmilodonAlertMessage = { 42 | val id = SmilodonAlertLevel.getIdByLevel(level) 43 | new SmilodonAlertMessage(id, event, default_title, default_content, channels, users, groups) 44 | } 45 | 46 | def apply( 47 | level: SmilodonAlertLevel, 48 | event: SmilodonEvent, 49 | default_title: String, 50 | default_content: String, 51 | channels: util.List[SmilodonChannel], 52 | users: util.List[String]): SmilodonAlertMessage = { 53 | val id = SmilodonAlertLevel.getIdByLevel(level) 54 | new SmilodonAlertMessage(id, event, default_title, default_content, channels, users, Seq.empty[String].asJava) 55 | } 56 | 57 | def apply( 58 | event: SmilodonEvent, 59 | default_title: String, 60 | default_content: String, 61 | channels: util.List[SmilodonChannel], 62 | users: util.List[String]): SmilodonAlertMessage = { 63 | new SmilodonAlertMessage(1, event, default_title, default_content, channels, users, Seq.empty[String].asJava) 64 | } 65 | 66 | def apply( 67 | level: SmilodonAlertLevel, 68 | event: SmilodonEvent, 69 | default_title: String, 70 | default_content: String, 71 | conf: SparkConf): SmilodonAlertMessage = { 72 | val id = SmilodonAlertLevel.getIdByLevel(level) 73 | val channels = conf.get(SMILODON_ALERT_CHANNELS, "POPO,STONE") 74 | .split(",").map(_.trim) 75 | .map(ChannelType.getIdByName) 76 | .map(SmilodonChannel(_, "", "")).toList.asJava 77 | val users = conf.get(SMILODON_ALERT_USERS, "").split(",").map(_.trim).toList.asJava 78 | val groups = conf.getOption(SMILODON_ALERT_GROUPS) match { 79 | case Some(li) if li.trim.nonEmpty => li.split(",").map(_.trim).toList.asJava 80 | case _ => Seq.empty[String].asJava 81 | } 82 | new SmilodonAlertMessage(id, event, default_title, default_content, channels, users, groups) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /alarm-smilodon/src/main/scala/com/netease/spark/alarm/smilodon/channel/ChannelType.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm.smilodon.channel 2 | 3 | object ChannelType extends Enumeration { 4 | 5 | type ChannelType = Value 6 | 7 | val UNSUPPORTED = Value(1, "EMAIL, SMS, VOICE, YIXIN, etc.") 8 | val POPO = Value(4, "POPO") 9 | val STONE = Value(5, "Stone") 10 | 11 | def getIdByType(typ: ChannelType): Int = typ match { 12 | case POPO => 4 13 | case STONE => 5 14 | case _ => 1 15 | } 16 | 17 | def getIdByName(name: String): Int = name.toUpperCase() match { 18 | case "POPO" => 4 19 | case "STONE" => 5 20 | case _ => 1 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /alarm-smilodon/src/main/scala/com/netease/spark/alarm/smilodon/channel/SmilodonChannel.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm.smilodon.channel 2 | 3 | import com.netease.spark.alarm.smilodon.channel.ChannelType.ChannelType 4 | 5 | case class SmilodonChannel private (channel_id: Int, title: String, content: String) 6 | 7 | object SmilodonChannel { 8 | def apply(typ: ChannelType, title: String, content: String): SmilodonChannel = { 9 | val id = ChannelType.getIdByType(typ) 10 | new SmilodonChannel(id, title, content) 11 | } 12 | 13 | /** 14 | * Send alert message to Smilodon, then it will be alarm by POPO 15 | */ 16 | def popo(title: String, content: String): SmilodonChannel = { 17 | SmilodonChannel(4, title, content) 18 | } 19 | 20 | /** 21 | * Send alert message to Smilodon, then it will be alarm by POPO 22 | */ 23 | def popo(): SmilodonChannel = { 24 | SmilodonChannel(4, "", "") 25 | } 26 | 27 | /** 28 | * Send alert message to Smilodon, then it will be alarm by Stone 29 | */ 30 | def stone(title: String, content: String): SmilodonChannel = { 31 | SmilodonChannel(5, title, content) 32 | } 33 | 34 | /** 35 | * Send alert message to Smilodon, then it will be alarm by Stone 36 | */ 37 | def stone(): SmilodonChannel = { 38 | SmilodonChannel(5, "", "") 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /alarm-smilodon/src/main/scala/com/netease/spark/alarm/smilodon/event/SmilodonEvent.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm.smilodon.event 2 | 3 | import com.netease.spark.alarm.{AlarmConstants, AlertType, Components} 4 | 5 | /** 6 | * 7 | * @param service 报警是由哪个服务产生的 8 | * @param component 报警是由服务的哪个组件产生的 9 | * @param `type` 具体的事件类型，比如任务失败，服务挂掉等 10 | */ 11 | case class SmilodonEvent(service: String, component: String, `type`: String) 12 | 13 | object SmilodonEvent { 14 | def streamingAppEvent(): SmilodonEvent = 15 | SmilodonEvent(AlarmConstants.SERVICE, Components.STREAMING.toString, AlertType.Application.toString) 16 | 17 | def streamingBatchEvent(): SmilodonEvent = 18 | SmilodonEvent(AlarmConstants.SERVICE, Components.STREAMING.toString, AlertType.Batch.toString) 19 | 20 | def streamingJobEvent(): SmilodonEvent = 21 | SmilodonEvent(AlarmConstants.SERVICE, Components.STREAMING.toString, AlertType.Job.toString) 22 | } 23 | -------------------------------------------------------------------------------- /alarm-smilodon/src/main/scala/com/netease/spark/alarm/smilodon/package.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm 2 | 3 | package object smilodon { 4 | 5 | private val PREFIX = "spark.alarm.smilodon." 6 | val SMILODON_URL: String = PREFIX + "url" 7 | val SMILODON_ALERT_CHANNELS: String = PREFIX + "channels" 8 | val SMILODON_ALERT_USERS: String = PREFIX + "users" 9 | val SMILODON_ALERT_GROUPS: String = PREFIX + "groups" 10 | } 11 | -------------------------------------------------------------------------------- /alarm-smilodon/src/test/scala/com/netease/spark/alarm/smilodon/SmilodonAlarmistTest.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm.smilodon 2 | 3 | import com.netease.spark.alarm.smilodon.channel.{ChannelType, SmilodonChannel} 4 | import com.netease.spark.alarm.smilodon.event.SmilodonEvent 5 | import org.apache.spark.SparkConf 6 | import org.scalatest.FunSuite 7 | 8 | import scala.collection.JavaConverters._ 9 | 10 | class SmilodonAlarmistTest extends FunSuite { 11 | private val conf = new SparkConf(true) 12 | private val event = SmilodonEvent("TEST", "ALERT", "TASK_FAILED") 13 | 14 | ignore("smilodon alarm") { 15 | val stone = SmilodonChannel.stone() 16 | val popo = SmilodonChannel(ChannelType.POPO, "title", "spark 服务有异常吗？") 17 | val msg = SmilodonAlertMessage( 18 | SmilodonAlertLevel.WARN, 19 | event, 20 | "Spark 服务异常", 21 | "Spark 服务没什么异常", 22 | Seq(stone, popo).asJava, 23 | List("yaooqinn@hotmail.com").asJava) 24 | 25 | val alarmist = new SmilodonAlarmist(conf) 26 | val response = alarmist.alarm(msg) 27 | assert(response.status) 28 | } 29 | 30 | ignore("smilodon alarm using spark conf") { 31 | conf.set("spark.alarm.smilodon.users", "yaooqinn@hotmail.com") 32 | .set("spark.alarm.smilodon.channels", "POPO,STONE") 33 | 34 | val msg = SmilodonAlertMessage( 35 | SmilodonAlertLevel.WARN, 36 | event, 37 | "Spark 服务异常", 38 | "Spark 服务没什么异常", 39 | conf) 40 | 41 | val alarmist = new SmilodonAlarmist(conf) 42 | val response = alarmist.alarm(msg) 43 | assert(response.status) 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.netease.spark 8 | spark-alarm 9 | Spark Alarm Parent 10 | pom 11 | 1.0-SNAPSHOT 12 | 13 | alarm-common 14 | alarm-email 15 | alarm-sentry 16 | alarm-smilodon 17 | streaming-alarmer 18 | 19 | 20 | 21 | 1.5 22 | 1.2.49 23 | 2.11 24 | 2.11.8 25 | 2.4.7 26 | 27 | 28 | 29 | 30 | 31 | org.scala-lang 32 | scala-library 33 | ${scala.ver} 34 | provided 35 | 36 | 37 | 38 | org.apache.commons 39 | commons-email 40 | ${common.email.ver} 41 | compile 42 | 43 | 44 | 45 | org.apache.spark 46 | spark-core_${scala.bin.ver} 47 | ${spark.ver} 48 | provided 49 | 50 | 51 | 52 | org.apache.spark 53 | spark-streaming_${scala.bin.ver} 54 | ${spark.ver} 55 | provided 56 | 57 | 58 | 59 | org.apache.httpcomponents 60 | httpclient 61 | 4.5.6 62 | provided 63 | 64 | 65 | 66 | com.google.code.gson 67 | gson 68 | 2.2.4 69 | provided 70 | 71 | 72 | 73 | commons-logging 74 | commons-logging 75 | 1.1.3 76 | provided 77 | 78 | 79 | 80 | 81 | org.scalatest 82 | scalatest_${scala.bin.ver} 83 | 2.2.6 84 | test 85 | 86 | 87 | 88 | 89 | 90 | 91 | target/scala-${scala.bin.ver}/classes 92 | target/scala-${scala.bin.ver}/test-classes 93 | 94 | 95 | 96 | org.apache.maven.plugins 97 | maven-compiler-plugin 98 | 3.5.1 99 | 100 | ${java.version} 101 | ${java.version} 102 | UTF-8 103 | 1024m 104 | true 105 | 106 | -Xlint:all,-serial,-path 107 | 108 | 109 | 110 | 111 | 112 | net.alchim31.maven 113 | scala-maven-plugin 114 | 3.3.1 115 | 116 | 117 | eclipse-add-source 118 | 119 | add-source 120 | 121 | 122 | 123 | scala-compile-first 124 | 125 | compile 126 | 127 | 128 | 129 | scala-test-compile-first 130 | 131 | testCompile 132 | 133 | 134 | 135 | 136 | ${scala.ver} 137 | incremental 138 | true 139 | 140 | -unchecked 141 | -deprecation 142 | -feature 143 | -explaintypes 144 | -Yno-adapted-args 145 | 146 | 147 | -Xms1024m 148 | -Xmx1024m 149 | -XX:ReservedCodeCacheSize=512M 150 | 151 | 152 | -source 153 | ${java.version} 154 | -target 155 | ${java.version} 156 | -Xlint:all,-serial,-path,-try 157 | 158 | 159 | 160 | 161 | 162 | 163 | org.apache.maven.plugins 164 | maven-surefire-plugin 165 | 2.12.4 166 | 167 | true 168 | 169 | 170 | 171 | 172 | org.scalatest 173 | scalatest-maven-plugin 174 | 1.0 175 | 176 | ${project.build.directory}/surefire-reports 177 | . 178 | TestSuite.txt 179 | 180 | 181 | 182 | test 183 | 184 | test 185 | 186 | 187 | 188 | 189 | 190 | 191 | org.jacoco 192 | jacoco-maven-plugin 193 | 0.8.0 194 | 195 | 196 | 197 | org.apache.maven.plugins 198 | maven-shade-plugin 199 | 3.2.1 200 | 201 | 202 | 203 | org.apache.maven.plugins 204 | maven-jar-plugin 205 | 3.0.2 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | spark-2.1 214 | 215 | 2.1.3 216 | 217 | 218 | 219 | 220 | spark-2.2 221 | 222 | 2.2.2 223 | 224 | 225 | 226 | 227 | spark-2.3 228 | 229 | 2.3.2 230 | 3.0.3 231 | 232 | 233 | 234 | 235 | spark-2.4 236 | 237 | 2.4.0 238 | 3.0.3 239 | 240 | 241 | 242 | 243 | -------------------------------------------------------------------------------- /streaming-alarmer/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | spark-alarm 7 | com.netease.spark 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | streaming-alarmer 13 | Spark Streaming Alarmer 14 | jar 15 | 16 | 17 | 18 | org.scala-lang 19 | scala-library 20 | 21 | 22 | 23 | com.netease.spark 24 | alarm-common 25 | ${project.version} 26 | 27 | 28 | 29 | com.netease.spark 30 | alarm-email 31 | ${project.version} 32 | 33 | 34 | 35 | com.netease.spark 36 | alarm-sentry 37 | ${project.version} 38 | 39 | 40 | 41 | com.netease.spark 42 | alarm-smilodon 43 | ${project.version} 44 | 45 | 46 | 47 | org.apache.spark 48 | spark-core_${scala.bin.ver} 49 | 50 | 51 | 52 | org.apache.spark 53 | spark-streaming_${scala.bin.ver} 54 | provided 55 | 56 | 57 | 58 | org.apache.httpcomponents 59 | httpclient 60 | 61 | 62 | 63 | com.google.code.gson 64 | gson 65 | 66 | 67 | 68 | 69 | org.scalatest 70 | scalatest_${scala.bin.ver} 71 | 72 | 73 | 74 | 75 | target/scala-${scala.bin.ver}/classes 76 | target/scala-${scala.bin.ver}/test-classes 77 | 78 | 79 | ${project.basedir}/src/main/resources 80 | 81 | 82 | 83 | 84 | org.apache.maven.plugins 85 | maven-compiler-plugin 86 | 87 | 88 | 89 | net.alchim31.maven 90 | scala-maven-plugin 91 | 92 | 93 | 94 | maven-shade-plugin 95 | 96 | false 97 | 98 | 99 | com.netease.spark:alarm-common 100 | com.netease.spark:alarm-email 101 | com.netease.spark:alarm-sentry 102 | com.netease.spark:alarm-smilodon 103 | org.apache.commons:commons-email 104 | com.sun.mail:javax.mail 105 | 106 | 107 | 108 | 109 | 110 | package 111 | 112 | shade 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | org.apache.maven.plugins 122 | maven-surefire-plugin 123 | 124 | 125 | 126 | org.scalatest 127 | scalatest-maven-plugin 128 | 1.0 129 | 130 | ${project.build.directory}/surefire-reports 131 | . 132 | TestSuite.txt 133 | 134 | ${project.build.testOutputDirectory}/kyuubi-server-${project.version}.jar 135 | 136 | 137 | ${project.build.testOutputDirectory}/spark-warehouse 138 | 139 | 140 | 141 | 142 | test 143 | 144 | test 145 | 146 | 147 | 148 | 149 | 150 | 151 | -------------------------------------------------------------------------------- /streaming-alarmer/src/main/scala/com/netease/spark/alarm/alarmer/streaming/EmailStreamingAlarmListener.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm.alarmer.streaming 2 | import com.netease.spark.alarm.Alarmist 3 | import com.netease.spark.alarm.email.EmailAlarmist 4 | import org.apache.spark.SparkConf 5 | 6 | class EmailStreamingAlarmListener(override val conf: SparkConf) extends StreamingAlarmListener { 7 | override val alarmist: Alarmist = new EmailAlarmist(conf) 8 | } 9 | -------------------------------------------------------------------------------- /streaming-alarmer/src/main/scala/com/netease/spark/alarm/alarmer/streaming/SentryStreamingAlarmListener.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm.alarmer.streaming 2 | import com.netease.spark.alarm.Alarmist 3 | import com.netease.spark.alarm.sentry.SentryAlarmist 4 | import org.apache.spark.SparkConf 5 | 6 | class SentryStreamingAlarmListener(override val conf: SparkConf) extends StreamingAlarmListener { 7 | override val alarmist: Alarmist = new SentryAlarmist(conf) 8 | 9 | } 10 | -------------------------------------------------------------------------------- /streaming-alarmer/src/main/scala/com/netease/spark/alarm/alarmer/streaming/SmilodonStreamingAlarmListener.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm.alarmer.streaming 2 | import java.text.SimpleDateFormat 3 | import java.util.Date 4 | 5 | import com.netease.spark.alarm.smilodon.{SmilodonAlarmist, SmilodonAlertLevel, SmilodonAlertMessage} 6 | import com.netease.spark.alarm.smilodon.event.SmilodonEvent 7 | import com.netease.spark.alarm.AlarmConstants._ 8 | import org.apache.commons.logging.LogFactory 9 | import org.apache.spark.SparkConf 10 | import org.apache.spark.scheduler.SparkListenerApplicationEnd 11 | import org.apache.spark.streaming.StreamingContext 12 | import org.apache.spark.streaming.scheduler.{StreamingListenerBatchCompleted, StreamingListenerBatchStarted, StreamingListenerOutputOperationCompleted} 13 | 14 | import scala.collection.JavaConverters._ 15 | 16 | class SmilodonStreamingAlarmListener(override val conf: SparkConf) extends StreamingAlarmListener { 17 | private val LOG = LogFactory.getFactory.getInstance(classOf[SmilodonStreamingAlarmListener]) 18 | override val alarmist: SmilodonAlarmist = new SmilodonAlarmist(conf) 19 | 20 | override def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd): Unit = { 21 | if (LOG.isDebugEnabled) LOG.debug(s"onApplicationEnd called: $applicationEnd") 22 | 23 | val maybeContext = StreamingContext.getActive() 24 | maybeContext.foreach { ssc => 25 | val context = ssc.sparkContext 26 | val dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") 27 | val startTime = dateFormat.format(new Date(context.startTime)) 28 | val endTime = dateFormat.format(new Date(applicationEnd.time)) 29 | val content = appFailContent(startTime, endTime) 30 | LOG.error(content) 31 | val event = SmilodonEvent.streamingAppEvent() 32 | val alertMessage = SmilodonAlertMessage(SmilodonAlertLevel.FATAL, event, STREAMING_APPLICATION_ERROR, content, conf) 33 | alarmist.alarm(alertMessage) 34 | // Spark 内部机制会保障Streaming 无法在ListenerBus中被调用 35 | // ssc.stop(stopSparkContext = false, stopGracefully = false) 36 | } 37 | 38 | if (conf.getBoolean(FORCE_EXIT, defaultValue = false)) { 39 | System.exit(-1) 40 | } 41 | } 42 | 43 | override def onBatchStarted(batchStarted: StreamingListenerBatchStarted): Unit = { 44 | if (LOG.isDebugEnabled) LOG.debug(s"onBatchStarted called: $batchStarted") 45 | 46 | if (batchNoticeEnable) { 47 | val currentTime = System.currentTimeMillis() 48 | batchSet.entrySet().asScala.foreach { kv => 49 | val start = kv.getValue 50 | if (currentTime - start > batchProcessThreshold) { 51 | val content = batchTimeoutContent(currentTime - start) 52 | val event = SmilodonEvent.streamingBatchEvent() 53 | val alertMessage = SmilodonAlertMessage(SmilodonAlertLevel.FATAL, event, STREAMING_BATCH_ERROR, content, conf) 54 | alarmist.alarm(alertMessage) 55 | } 56 | } 57 | val batchInfo = batchStarted.batchInfo 58 | batchSet.put(batchInfo.batchTime, batchInfo.processingStartTime.getOrElse(System.currentTimeMillis())) 59 | } 60 | } 61 | 62 | override def onBatchCompleted(batchCompleted: StreamingListenerBatchCompleted): Unit = { 63 | if (LOG.isDebugEnabled) LOG.debug(s"onBatchCompleted called: $batchCompleted") 64 | 65 | val info = batchCompleted.batchInfo 66 | 67 | if (batchNoticeEnable) { 68 | batchSet.remove(info.batchTime) 69 | val batchFailureReasons = info.outputOperationInfos.values.map(_.failureReason).filter(_.nonEmpty).mkString("\n") 70 | if (batchFailureReasons.nonEmpty) { 71 | val content = batchErrorContent(batchFailureReasons) 72 | val event = SmilodonEvent.streamingBatchEvent() 73 | val alertMessage = SmilodonAlertMessage(SmilodonAlertLevel.FATAL, event, STREAMING_BATCH_ERROR, content, conf) 74 | alarmist.alarm(alertMessage) 75 | } 76 | } 77 | 78 | val schedulingDelay = info.schedulingDelay.getOrElse(0L) 79 | val processingDelay = info.processingDelay.getOrElse(Long.MaxValue) 80 | if (processingDelay > batchProcessThreshold && schedulingDelay / processingDelay > batchDelayRatio) { 81 | val content = batchDelayContent(schedulingDelay, processingDelay) 82 | val event = SmilodonEvent.streamingBatchEvent() 83 | val alertMessage = SmilodonAlertMessage(SmilodonAlertLevel.FATAL, event, STREAMING_BATCH_ERROR, content, conf) 84 | alarmist.alarm(alertMessage) 85 | } 86 | } 87 | 88 | override def onOutputOperationCompleted(outputOperationCompleted: StreamingListenerOutputOperationCompleted): Unit = { 89 | if (LOG.isDebugEnabled) LOG.debug(s"onOutputOperationCompleted called: $outputOperationCompleted") 90 | 91 | if (jobNoticeEnable) { 92 | val jobFailureReason = outputOperationCompleted.outputOperationInfo.failureReason 93 | jobFailureReason.foreach { reason => 94 | val content = jobErrorContent(reason) 95 | val event = SmilodonEvent.streamingJobEvent() 96 | val alertMessage = SmilodonAlertMessage(SmilodonAlertLevel.FATAL, event, STREAMING_JOB_ERROR, content, conf) 97 | alarmist.alarm(alertMessage) 98 | } 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /streaming-alarmer/src/main/scala/com/netease/spark/alarm/alarmer/streaming/StreamingAlarmListener.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm.alarmer.streaming 2 | 3 | import java.text.SimpleDateFormat 4 | import java.util.Date 5 | import java.util.concurrent.ConcurrentHashMap 6 | 7 | import com.netease.spark.alarm.{Alarmist, AlertMessage} 8 | import com.netease.spark.alarm.AlarmConstants._ 9 | import org.apache.commons.logging.LogFactory 10 | import org.apache.spark.scheduler.{SparkListener, SparkListenerApplicationEnd, SparkListenerApplicationStart, SparkListenerEvent} 11 | import org.apache.spark.streaming.scheduler.{StreamingListener, StreamingListenerBatchCompleted, StreamingListenerBatchStarted, StreamingListenerOutputOperationCompleted} 12 | import org.apache.spark.SparkConf 13 | import org.apache.spark.streaming.{StreamingContext, Time} 14 | 15 | import scala.util.{Success, Try} 16 | import scala.collection.JavaConverters._ 17 | 18 | trait StreamingAlarmListener extends SparkListener with StreamingListener { 19 | 20 | import StreamingAlarmListener._ 21 | 22 | private val LOG = LogFactory.getFactory.getInstance(classOf[StreamingAlarmListener]) 23 | 24 | protected def alarmist: Alarmist 25 | 26 | protected def conf: SparkConf 27 | 28 | protected var applicationId: String = conf.getAppId 29 | protected var appName: String = conf.get("spark.app.name") 30 | 31 | protected val batchNoticeEnable: Boolean = conf.getBoolean(BATCH_NOTICE_ENABLE, defaultValue = true) 32 | protected val batchDelayRatio: Double = conf.getDouble(BATCH_DELAY_RATIO, 1) 33 | protected val batchProcessThreshold: Long = conf.getTimeAsMs(BATCH_PROCESS_TIME, "1s") 34 | protected lazy val batchSet: java.util.Map[Time, Long] = new ConcurrentHashMap[Time, Long] 35 | 36 | protected val jobNoticeEnable: Boolean = conf.getBoolean(JOB_NOTICE_ENABLE, defaultValue = false) 37 | 38 | override def onApplicationStart(applicationStart: SparkListenerApplicationStart): Unit = { 39 | if (LOG.isDebugEnabled) LOG.debug(s"onApplicationStart called: $applicationStart") 40 | 41 | StreamingContext.getActive().foreach { ssc => 42 | val context = ssc.sparkContext 43 | applicationId = context.applicationId 44 | appName = context.appName 45 | } 46 | } 47 | /** 48 | * 用于捕获SparkContext变量stop, 但流式作业未正常结束 49 | * 场景举例: 50 | * ApplicationMaster failover失败，当Driver端的Application Monitor检测到AM跪了， 51 | * 会触发调用SparkContext.stop 52 | */ 53 | override def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd): Unit = { 54 | if (LOG.isDebugEnabled) LOG.debug(s"onApplicationEnd called: $applicationEnd") 55 | 56 | val maybeContext = StreamingContext.getActive() 57 | maybeContext.foreach { ssc => 58 | val context = ssc.sparkContext 59 | val dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") 60 | val startTime = dateFormat.format(new Date(context.startTime)) 61 | val endTime = dateFormat.format(new Date(applicationEnd.time)) 62 | val content = appFailContent(startTime, endTime) 63 | alarmist.alarm(AlertMessage(STREAMING_APPLICATION_ERROR, content)) 64 | // ssc.stop(stopSparkContext = false, stopGracefully = false) 65 | } 66 | 67 | if (conf.getBoolean(FORCE_EXIT, defaultValue = false)) { 68 | System.exit(-1) 69 | } 70 | } 71 | 72 | /** 73 | * In [[SparkListener]]s, wrapped streaming events first come here 74 | */ 75 | override def onOtherEvent(event: SparkListenerEvent): Unit = { 76 | if (LOG.isDebugEnabled) LOG.debug(s"onOtherEvent called: $event") 77 | 78 | Try { event.getClass.getDeclaredField("streamingListenerEvent") } match { 79 | case Success(field) => 80 | field.setAccessible(true) 81 | field.get(event) match { 82 | case as: SparkListenerApplicationStart => onApplicationStart(as) 83 | case ae: SparkListenerApplicationEnd => onApplicationEnd(ae) 84 | case bs: StreamingListenerBatchStarted => onBatchStarted(bs) 85 | case bc: StreamingListenerBatchCompleted => onBatchCompleted(bc) 86 | case oc: StreamingListenerOutputOperationCompleted => onOutputOperationCompleted(oc) 87 | case _ => 88 | } 89 | case _ => 90 | } 91 | } 92 | 93 | override def onBatchStarted(batchStarted: StreamingListenerBatchStarted): Unit = { 94 | if (LOG.isDebugEnabled) LOG.debug(s"onBatchStarted called: $batchStarted") 95 | 96 | if (batchNoticeEnable) { 97 | val currentTime = System.currentTimeMillis() 98 | batchSet.entrySet().asScala.foreach { kv => 99 | val start = kv.getValue 100 | if (currentTime - start > batchProcessThreshold) { 101 | alarmist.alarm(AlertMessage(STREAMING_BATCH_ERROR, 102 | batchTimeoutContent(currentTime - start))) 103 | } 104 | } 105 | val batchInfo = batchStarted.batchInfo 106 | batchSet.put(batchInfo.batchTime, batchInfo.processingStartTime.getOrElse(System.currentTimeMillis())) 107 | } 108 | } 109 | 110 | override def onBatchCompleted(batchCompleted: StreamingListenerBatchCompleted): Unit = { 111 | if (LOG.isDebugEnabled) LOG.debug(s"onBatchCompleted called: $batchCompleted") 112 | 113 | val info = batchCompleted.batchInfo 114 | 115 | if (batchNoticeEnable) { 116 | batchSet.remove(info.batchTime) 117 | val batchFailureReasons = info.outputOperationInfos.values.map(_.failureReason).filter(_.nonEmpty).mkString("\n") 118 | if (batchFailureReasons.nonEmpty) { 119 | val content = batchErrorContent(batchFailureReasons) 120 | alarmist.alarm(AlertMessage(STREAMING_BATCH_ERROR, content)) 121 | } 122 | } 123 | 124 | val schedulingDelay = info.schedulingDelay.getOrElse(0L) 125 | val processingDelay = info.processingDelay.getOrElse(Long.MaxValue) 126 | if (processingDelay > batchProcessThreshold && schedulingDelay / processingDelay > batchDelayRatio) { 127 | val content = batchDelayContent(schedulingDelay, processingDelay) 128 | alarmist.alarm(AlertMessage(STREAMING_BATCH_ERROR, content)) 129 | } 130 | } 131 | 132 | override def onOutputOperationCompleted(outputOperationCompleted: StreamingListenerOutputOperationCompleted): Unit = { 133 | if (LOG.isDebugEnabled) LOG.debug(s"onOutputOperationCompleted called: $outputOperationCompleted") 134 | 135 | if (jobNoticeEnable) { 136 | val jobFailureReason = outputOperationCompleted.outputOperationInfo.failureReason 137 | jobFailureReason.foreach { reason => 138 | alarmist.alarm(AlertMessage(STREAMING_JOB_ERROR, jobErrorContent(reason))) 139 | } 140 | } 141 | } 142 | 143 | protected def batchDelayContent(schedulingDelay: Long, processingDelay: Long): String = { 144 | s""" 145 | |$STREAMING_BATCH_ERROR 146 | | 147 | |作业名: $appName 148 | |作业ID: $applicationId 149 | |诊断信息: 150 | | 批次调度延时过高(Scheduling Delay: $schedulingDelay ms / Processing Time: $processingDelay ms > $batchDelayRatio ) 151 | | 单独关闭该类型警报可设置 $BATCH_DELAY_RATIO 的值 152 | """.stripMargin 153 | } 154 | 155 | protected def batchErrorContent(batchFailureReasons: String): String = { 156 | s""" 157 | |$STREAMING_BATCH_ERROR 158 | | 159 | |作业名: $appName 160 | |作业ID: $applicationId 161 | |诊断信息: 162 | | (长度限制: $failureReasonLimit) 163 | | ${getLimitedFailureReason(batchFailureReasons)} 164 | | 单独关闭该类型警报可设置$BATCH_NOTICE_ENABLE=false 165 | """.stripMargin 166 | } 167 | 168 | protected def batchTimeoutContent(processDuration: Long): String = { 169 | s""" 170 | |$STREAMING_BATCH_ERROR 171 | | 172 | |作业名: $appName 173 | |作业ID: $applicationId 174 | |诊断信息: 175 | | 批次调度运行时长已超阈值(Process Duration: $processDuration ms > Processing Threshold: $batchProcessThreshold ms) 176 | | 单独关闭该类型警报可设置 $BATCH_DELAY_RATIO 的值 177 | """.stripMargin 178 | } 179 | 180 | protected def jobErrorContent(jobErrorReason: String): String = { 181 | s""" 182 | |$STREAMING_JOB_ERROR 183 | | 184 | |作业名: $appName 185 | |作业ID: $applicationId 186 | |诊断信息: 187 | | (长度限制: $failureReasonLimit) 188 | | ${getLimitedFailureReason(jobErrorReason)} 189 | | 单独关闭该类型警报可设置$JOB_NOTICE_ENABLE=false 190 | """.stripMargin 191 | } 192 | 193 | protected def appFailContent(startTime: String, endTime: String): String = { 194 | s""" 195 | |$STREAMING_APPLICATION_ERROR 196 | | 197 | |作业名: $appName 198 | |作业ID: $applicationId 199 | |启动时间: $startTime 200 | |停止时间: $endTime 201 | | 202 | |诊断信息: 203 | | [SparkContext]终止与[StreamingContext]之前，尝试终止StreamingContext 204 | | 如进程无法正常退出，可尝试设置$FORCE_EXIT，强制退出（不推荐） 205 | """.stripMargin 206 | } 207 | } 208 | 209 | object StreamingAlarmListener { 210 | // The length limit for failureReason to sent 211 | val failureReasonLimit: Int = 200 212 | val maxLengthFactor: Double = 1.5 213 | 214 | def getLimitedFailureReason(failureReason: String): String = { 215 | val nextLineIndex = failureReason.indexOf("\n", failureReasonLimit) 216 | if (nextLineIndex == -1) { 217 | failureReason.substring(0, math.min(failureReason.length, (failureReasonLimit * maxLengthFactor).toInt)) 218 | } else { 219 | failureReason.substring(0, math.min(nextLineIndex, (failureReasonLimit * maxLengthFactor).toInt)) 220 | } 221 | } 222 | } 223 | -------------------------------------------------------------------------------- /streaming-alarmer/src/main/scala/com/netease/spark/alarm/alarmer/streaming/package.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm.alarmer 2 | 3 | package object streaming { 4 | private val PREFIX = "spark.alarm.streaming." 5 | 6 | val FORCE_EXIT: String = PREFIX + "application.force.exit" 7 | val BATCH_NOTICE_ENABLE: String = PREFIX + "batch.error.enable" 8 | val BATCH_DELAY_RATIO: String = PREFIX + "batch.delay.ratio" 9 | val BATCH_PROCESS_TIME: String = PREFIX + "batch.process.time.threshold" 10 | val JOB_NOTICE_ENABLE: String = PREFIX + "job.error.enable" 11 | 12 | } 13 | -------------------------------------------------------------------------------- /streaming-alarmer/src/test/scala/com/netease/spark/alarm/alarmer/streaming/EmailStreamingAlarmListenerTest.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm.alarmer.streaming 2 | 3 | import com.netease.spark.alarm.email._ 4 | import org.apache.spark.{SparkConf, SparkContext} 5 | import org.apache.spark.streaming._ 6 | import org.scalatest.FunSuite 7 | 8 | class EmailStreamingAlarmListenerTest extends FunSuite { 9 | private val conf = new SparkConf(true) 10 | .setMaster("local[2]") 11 | .setAppName("smilodon") 12 | .set(HOSTNAME, "smtp-mail.outlook.com") 13 | .set(SMTP_PORT, "587") 14 | .set(FROM, "yaooqinn@hotmail.com") 15 | .set(SSL_ON_CONNECT, "true") 16 | .set(USERNAME, "hzyaoqin") 17 | .set(PASSWORD, "***") 18 | .set(TO, "yaooqinn@hotmail.com") 19 | 20 | 21 | ignore("on application end") { 22 | val input = Seq(Seq(1), Seq(2), Seq(3), Seq(4)) 23 | val sc = new SparkContext(conf) 24 | val ssc = new StreamingContext(sc, Durations.seconds(1)) 25 | val listener = new EmailStreamingAlarmListener(sc.getConf) 26 | sc.addSparkListener(listener) 27 | val stream = new AlarmTestInputStream[Int](ssc, input, 2) 28 | stream.foreachRDD(_ => {}) // Dummy output stream 29 | ssc.start() 30 | sc.stop() 31 | assert(ssc.getState() === StreamingContextState.ACTIVE) 32 | } 33 | 34 | ignore("on application end with extra listener") { 35 | conf.set("spark.extraListeners", classOf[EmailStreamingAlarmListener].getName) 36 | val input = Seq(Seq(1), Seq(2), Seq(3), Seq(4)) 37 | val sc = new SparkContext(conf) 38 | val ssc = new StreamingContext(sc, Durations.seconds(1)) 39 | val stream = new AlarmTestInputStream[Int](ssc, input, 2) 40 | stream.foreachRDD(_ => {}) // Dummy output stream 41 | ssc.start() 42 | sc.stop() 43 | assert(ssc.getState() === StreamingContextState.ACTIVE) 44 | } 45 | 46 | ignore("on batch completed") { 47 | conf.set("spark.extraListeners", classOf[EmailStreamingAlarmListener].getName) 48 | val input = Seq(Seq(1), Seq(2), Seq(3), Seq(4)) 49 | val sc = new SparkContext(conf) 50 | val ssc = new StreamingContext(sc, Seconds(1)) 51 | val stream = new AlarmTestInputStream[Int](ssc, input, 2) 52 | stream.foreachRDD(_ => {}) // Dummy output stream 53 | ssc.start() 54 | 55 | Thread.sleep(2000) 56 | ssc.stop(stopSparkContext = true, stopGracefully = true) 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /streaming-alarmer/src/test/scala/com/netease/spark/alarm/alarmer/streaming/SmilodonStreamingAlarmListenerTest.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm.alarmer.streaming 2 | 3 | import org.apache.spark.{SparkConf, SparkContext} 4 | import org.apache.spark.streaming._ 5 | import org.scalatest.FunSuite 6 | 7 | class SmilodonStreamingAlarmListenerTest extends FunSuite { 8 | 9 | val conf: SparkConf = new SparkConf(loadDefaults = true) 10 | .setMaster("local[2]") 11 | .setAppName("smilodon") 12 | .set("spark.alarm.smilodon.users", "yaooqin@hotmail.com") 13 | .set("spark.alarm.smilodon.channels", "POPO,STONE") 14 | // .set("spark.alarm.alarm.force.exit.on.stop", "true") 15 | 16 | ignore("on application end") { 17 | val input = Seq(Seq(1), Seq(2), Seq(3), Seq(4)) 18 | val sc = new SparkContext(conf) 19 | val ssc = new StreamingContext(sc, Durations.seconds(1)) 20 | val listener = new SmilodonStreamingAlarmListener(sc.getConf) 21 | sc.addSparkListener(listener) 22 | val stream = new AlarmTestInputStream[Int](ssc, input, 2) 23 | stream.foreachRDD(_ => {}) // Dummy output stream 24 | ssc.start() 25 | sc.stop() 26 | assert(ssc.getState() === StreamingContextState.ACTIVE) 27 | } 28 | 29 | 30 | ignore("on application end with extra listener") { 31 | conf.set("spark.extraListeners", classOf[SmilodonStreamingAlarmListener].getName) 32 | val input = Seq(Seq(1), Seq(2), Seq(3), Seq(4)) 33 | val sc = new SparkContext(conf) 34 | val ssc = new StreamingContext(sc, Durations.seconds(1)) 35 | val stream = new AlarmTestInputStream[Int](ssc, input, 2) 36 | stream.foreachRDD(_ => {}) // Dummy output stream 37 | ssc.start() 38 | sc.stop() 39 | assert(ssc.getState() === StreamingContextState.ACTIVE) 40 | } 41 | 42 | ignore("on batch completed") { 43 | conf.set("spark.extraListeners", classOf[SmilodonStreamingAlarmListener].getName) 44 | .set("spark.alarm.streaming.batch.process.time.threshold", "10ms") 45 | val input = Seq(Seq(1), Seq(2), Seq(3), Seq(4)) 46 | val sc = new SparkContext(conf) 47 | val ssc = new StreamingContext(sc, Seconds(1)) 48 | val stream = new AlarmTestInputStream[Int](ssc, input, 2) 49 | stream.foreachRDD(_ => {}) // Dummy output stream 50 | ssc.start() 51 | 52 | Thread.sleep(2000) 53 | ssc.stop(stopSparkContext = true, stopGracefully = true) 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /streaming-alarmer/src/test/scala/com/netease/spark/alarm/alarmer/streaming/StreamingAlarmListenerTest.scala: -------------------------------------------------------------------------------- 1 | package com.netease.spark.alarm.alarmer.streaming 2 | 3 | import org.scalatest.FunSuite 4 | 5 | import scala.reflect.runtime.{universe => ru} 6 | 7 | class StreamingAlarmListenerTest extends FunSuite { 8 | import StreamingAlarmListener._ 9 | 10 | test("test get limit failure reason") { 11 | val className = "com.netease.spark.alarm.alarmer.streaming.StreamingAlarmListener" 12 | val limitFieldName = "failureReasonLimit" 13 | val factorFieldName = "maxLengthFactor" 14 | val lengthLimit = 10 15 | setStaticFieldValue(className, limitFieldName, lengthLimit) 16 | // length = 0 17 | val reason0 = "" 18 | // length < limit 19 | 20 | val reason1 = "Failed" 21 | // length > limit 22 | val reason2 = "Task Failed with Exception" 23 | // length > limit with multi lines not than lengthLimit * maxLengthFactor 24 | val reason3 = "Task Failed with Exception\n\tline2" 25 | 26 | setStaticFieldValue(className, factorFieldName, 10.0) 27 | assert(getLimitedFailureReason(reason0) === reason0) 28 | assert(getLimitedFailureReason(reason1) === reason1) 29 | assert(getLimitedFailureReason(reason2) === reason2) 30 | assert(getLimitedFailureReason(reason3) === reason3.split("\n")(0)) 31 | 32 | setStaticFieldValue(className, factorFieldName, 1.1) 33 | assert(getLimitedFailureReason(reason2) === reason2.substring(0, (maxLengthFactor * failureReasonLimit).toInt)) 34 | } 35 | 36 | def setStaticFieldValue(className: String, fieldName: String, value: Any): Unit = { 37 | try { 38 | val mirror = ru.runtimeMirror(getClass.getClassLoader) 39 | val moduleSymbol = mirror.staticModule(className) 40 | val moduleMirror = mirror.reflectModule(moduleSymbol) 41 | val instanceMirror = mirror.reflect(moduleMirror.instance) 42 | val limitField = moduleSymbol.typeSignature.decl(ru.TermName(fieldName)) 43 | val limit = instanceMirror.reflectField(limitField.asTerm) 44 | limit.set(value) 45 | } catch { 46 | case e: Exception => e.printStackTrace() 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /streaming-alarmer/src/test/scala/org/apache/spark/streaming/AlarmTestInputStream.scala: -------------------------------------------------------------------------------- 1 | package org.apache.spark.streaming 2 | 3 | import org.apache.spark.rdd.RDD 4 | import org.apache.spark.streaming.dstream.InputDStream 5 | import org.apache.spark.streaming.scheduler.StreamInputInfo 6 | 7 | import scala.reflect.ClassTag 8 | 9 | /** 10 | * This is a input stream just for the testsuites. This is equivalent to a checkpointable, 11 | * replayable, reliable message queue like Kafka. It requires a sequence as input, and 12 | * returns the i_th element at the i_th batch under manual clock. 13 | */ 14 | class AlarmTestInputStream[T: ClassTag](_ssc: StreamingContext, input: Seq[Seq[T]], numPartitions: Int) 15 | extends InputDStream[T](_ssc) { 16 | 17 | def start() {} 18 | 19 | def stop() {} 20 | 21 | def compute(validTime: Time): Option[RDD[T]] = { 22 | logInfo("Computing RDD for time " + validTime) 23 | val index = ((validTime - zeroTime) / slideDuration - 1).toInt 24 | val selectedInput = if (index < input.size) input(index) else Seq[T]() 25 | 26 | // lets us test cases where RDDs are not created 27 | if (selectedInput == null) { 28 | return None 29 | } 30 | 31 | // Report the input data's information to InputInfoTracker for testing 32 | val inputInfo = StreamInputInfo(id, selectedInput.length.toLong) 33 | ssc.scheduler.inputInfoTracker.reportInfo(validTime, inputInfo) 34 | 35 | val rdd = ssc.sc.makeRDD(selectedInput, numPartitions) 36 | logInfo("Created RDD " + rdd.id + " with " + selectedInput) 37 | Some(rdd) 38 | } 39 | } 40 | --------------------------------------------------------------------------------