Error: {error.message}
48 | )
49 | } else if (!isLoaded) {
50 | return (
51 | Loading...
52 | )
53 | } else {
54 | return (
55 |
56 | {renderState(groups, this.props.endpoint, 'grid-item')}
57 |
58 | )
59 | }
60 | }
61 | }
62 |
--------------------------------------------------------------------------------
/src/test/scala/models/EnvVarSpec.scala:
--------------------------------------------------------------------------------
1 | package me.amanj.greenish.models
2 |
3 | import org.scalatest.matchers.should.Matchers
4 | import org.scalatest.wordspec.AnyWordSpecLike
5 |
6 | class EnvVarSpec() extends Matchers with AnyWordSpecLike {
7 | "EnvVar.apply" must {
8 | "create PlainEnvVar when secure flag is not provided" in {
9 | val (name, value) = ("username", "Homa")
10 | val expected = PlainEnvVar(name, value)
11 | val actual = EnvVar(name, value)
12 | actual shouldBe expected
13 | }
14 |
15 | "create SecureEnvVar when secure flag is provided" in {
16 | val (name, value) = ("username", "secure(Homa)")
17 | val expected = SecureEnvVar(name, "Homa".toSeq)
18 | val actual = EnvVar(name, value)
19 | actual shouldBe expected
20 | }
21 |
22 | "create SecureEnvVar when secure flag is provided but value is empty" in {
23 | val (name, value) = ("username", "secure()")
24 | val expected = SecureEnvVar(name, "".toSeq)
25 | val actual = EnvVar(name, value)
26 | actual shouldBe expected
27 | }
28 | }
29 |
30 | "EnvVar.tupled" must {
31 | "work for secure variables" in {
32 | val (name, value) = ("username", "secure(Homa)")
33 | val origin = EnvVar(name, value)
34 | val expected = (name, "Homa")
35 | val actual = origin.tupled
36 | actual shouldBe expected
37 | }
38 |
39 | "work for plain variables" in {
40 | val (name, value) = ("username", "Homa")
41 | val origin = EnvVar(name, value)
42 | val expected = (name, value)
43 | val actual = origin.tupled
44 | actual shouldBe expected
45 | }
46 | }
47 | }
48 |
49 |
--------------------------------------------------------------------------------
/src/test/scala/models/GroupStatusSpec.scala:
--------------------------------------------------------------------------------
1 | package me.amanj.greenish.models
2 |
3 | import org.scalatest.matchers.should.Matchers
4 | import org.scalatest.wordspec.AnyWordSpecLike
5 | import java.time.ZoneId
6 |
7 | class GroupStatusSpec() extends Matchers
8 | with AnyWordSpecLike {
9 |
10 | val job1 = Job(1, "job1", None, "p1", "foo",
11 | "yyyy-MM-dd-HH", Hourly, 1, ZoneId.of("UTC"),
12 | 4, 0, AlertLevels(0, 1, 2, 3), None, Seq(EnvVar("a", "b"))
13 | )
14 |
15 | val job2 = Job(2, "job2", None, "p2", "bar",
16 | "yyyy-MM-dd-HH", Hourly, 1, ZoneId.of("UTC"),
17 | 4, 0, AlertLevels(0, 1, 2, 3), None, Seq(EnvVar("a", "secure(b)"))
18 | )
19 |
20 | val group1 = Group(0, "group1", Seq(job1))
21 | val group2 = Group(1, "group2", Seq(job2))
22 |
23 | val gs1 = GroupStatus(group1, Array(JobStatus(job1, -1, Seq.empty)))
24 | val gs1Copy = GroupStatus(group1, Array(JobStatus(job1, -1, Seq.empty)))
25 | val gs2 = GroupStatus(group2, Array(JobStatus(job2, -1, Seq.empty)))
26 |
27 | "equals" must {
28 | "work if that is null" in {
29 | val actual = gs1 == null
30 | actual shouldBe false
31 | }
32 |
33 | "work if that is this" in {
34 | val actual = gs1 == gs1
35 | actual shouldBe true
36 | }
37 |
38 | "work if that is a clone of this" in {
39 | val actual = gs1 == gs1Copy
40 | actual shouldBe true
41 | }
42 |
43 | "not be equal to non-GroupStatus objects" in {
44 | val actual = gs1 == job1
45 | actual shouldBe false
46 | }
47 | }
48 |
49 | "hashCode" must {
50 | "be predictive" in {
51 | val actual = gs1.## == gs1.##
52 | actual shouldBe true
53 | }
54 |
55 | "produce the same value for equivalent objects" in {
56 | val actual = gs1.## == gs1Copy.##
57 | actual shouldBe true
58 | }
59 |
60 | "produce differe values for different objects" in {
61 | val actual = gs1.## == gs2.##
62 | actual shouldBe false
63 | }
64 | }
65 | }
66 |
67 |
--------------------------------------------------------------------------------
/src/main/resources/dashboard/common_lib.jsx:
--------------------------------------------------------------------------------
1 | const fetchInterval = 5000
2 |
3 | function renderState(groups, subClassName, keyPrefix) {
4 | return (
5 | groups.map(groupStatus => renderGroup(groupStatus, subClassName, keyPrefix, ""))
6 | );
7 | }
8 |
9 | function encloseInTable(trs, gid, keyPrefix) {
10 | return (
11 | | Job name | 15 |Data set period | 16 |Last updated | 17 |
|---|
31 |
);
34 | }
35 |
36 | function renderJob(jobStatus, gid, keyPrefix) {
37 | if(jobStatus["period_health"] != undefined) {
38 | const job = jobStatus.job;
39 | const jid = job.job_id;
40 | const date = new Date(jobStatus.updated_at).toUTCString();
41 | return(jobStatus.period_health.map((ph, i) => (
42 | {group.name}{sub}
32 | {encloseInTable(jobs, keyPrefix, gid)} 33 |Error: {error.message}
55 | )
56 | } else if (!isLoaded) {
57 | return (
58 | Loading...
59 | )
60 | } else {
61 | const sub = (
62 |
63 | `${loc.pathname}?page=main`}
64 | className="link" onClick={this.handleBack}>
65 | See main dashboard
66 |
67 |
68 | )
69 | return (
70 |
71 | {renderGroup(group, 'group-view', 'grid-item', sub)}
72 |
73 | )
74 | }
75 | }
76 | }
77 |
--------------------------------------------------------------------------------
/src/main/scala/App.scala:
--------------------------------------------------------------------------------
1 | package me.amanj.greenish
2 |
3 | import akka.actor.{ActorSystem, Props}
4 | import akka.http.scaladsl.Http
5 | import scala.language.postfixOps
6 | import akka.stream.ActorMaterializer
7 | import scala.concurrent.duration._
8 | import java.time.ZonedDateTime
9 | import checker.{StatusChecker, Refresh}
10 | import endpoints.Routes
11 | import akka.event.{Logging, LogSource}
12 |
13 | object App {
14 |
15 | private[this] implicit val system = ActorSystem("greenish-system")
16 | private[this] implicit val executionContext = system.dispatcher
17 | private[this] val schedulerActor = system.actorOf(Props.empty)
18 |
19 | implicit val logSource: LogSource[AnyRef] = new LogSource[AnyRef] {
20 | def genString(o: AnyRef): String = o.getClass.getName
21 | override def getClazz(o: AnyRef): Class[_] = o.getClass
22 | }
23 |
24 | private[this] val logger = Logging(system, this)
25 |
26 | def main(args: Array[String]): Unit = {
27 |
28 | val appConfig = AppConfig()
29 |
30 | val statsActor = system.actorOf(
31 | Props(new stats.StatsCollector(getPrometheusIds(appConfig))))
32 |
33 | val statusChecker = system.actorOf(
34 | Props(new StatusChecker(appConfig.groups, statsActor,
35 | appConfig.refreshInSeconds * 3, appConfig.scratchDir)))
36 |
37 | system.scheduler.scheduleWithFixedDelay(
38 | 0 seconds,
39 | appConfig.refreshInSeconds seconds,
40 | statusChecker, Refresh(() => ZonedDateTime.now()))
41 |
42 | val bindingFuture = Http()
43 | .bindAndHandle(
44 | new Routes(appConfig.namespace, appConfig.scratchDir, statusChecker,
45 | statsActor,
46 | // At least there should be one good run in the last 5 refresh sets
47 | appConfig.refreshInSeconds * 1000 * 5).routes,
48 | appConfig.address, appConfig.port)
49 |
50 | println(s"Server online at http://${appConfig.address}:${appConfig.port}...")
51 | }
52 |
53 | def getPrometheusIds(appConfig: AppConfig): Set[String] = {
54 | val prometheusIds = appConfig.groups.flatMap ( g =>
55 | g.jobs.map(j => j.prometheusId))
56 |
57 | val prometheusIdsSet = prometheusIds.toSet
58 | if(prometheusIdsSet.size < prometheusIds.size) {
59 | logger.warning(
60 | "prometheus-id is best to be unique per the entire configuration")
61 | }
62 |
63 | prometheusIdsSet
64 | }
65 |
66 |
67 | }
68 |
--------------------------------------------------------------------------------
/src/test/scala/checker/CheckerSpec.scala:
--------------------------------------------------------------------------------
1 | package me.amanj.greenish.checker
2 |
3 | import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
4 | import org.scalatest.matchers.should.Matchers
5 | import org.scalatest.wordspec.AnyWordSpecLike
6 | import me.amanj.greenish.models.PeriodHealth
7 |
8 | class CheckerSpec() extends AnyWordSpecLike with Matchers {
9 |
10 | "computeOldest" must {
11 | "work for empty period health lists" in {
12 | val periods = Seq.empty[PeriodHealth]
13 | val actual = computeOldest(periods)
14 | val expected = 0
15 | actual shouldBe expected
16 | }
17 |
18 | "work when the first period is missing" in {
19 | val periods = Seq(
20 | PeriodHealth("kaka", false),
21 | PeriodHealth("kaka", true),
22 | PeriodHealth("kaka", true),
23 | PeriodHealth("kaka", true),
24 | )
25 | val actual = computeOldest(periods)
26 | val expected = 4
27 | actual shouldBe expected
28 | }
29 |
30 | "work when a middle period is missing" in {
31 | val periods = Seq(
32 | PeriodHealth("kaka", true),
33 | PeriodHealth("kaka", false),
34 | PeriodHealth("kaka", true),
35 | PeriodHealth("kaka", true),
36 | )
37 | val actual = computeOldest(periods)
38 | val expected = 3
39 | actual shouldBe expected
40 | }
41 |
42 | "work when the last period is missing" in {
43 | val periods = Seq(
44 | PeriodHealth("kaka", true),
45 | PeriodHealth("kaka", true),
46 | PeriodHealth("kaka", true),
47 | PeriodHealth("kaka", false),
48 | )
49 | val actual = computeOldest(periods)
50 | val expected = 1
51 | actual shouldBe expected
52 | }
53 |
54 | "work when more than a period is missing" in {
55 | val periods = Seq(
56 | PeriodHealth("kaka", true),
57 | PeriodHealth("kaka", false),
58 | PeriodHealth("kaka", true),
59 | PeriodHealth("kaka", false),
60 | )
61 | val actual = computeOldest(periods)
62 | val expected = 3
63 | actual shouldBe expected
64 | }
65 |
66 | "work when no period is missing" in {
67 | val periods = Seq(
68 | PeriodHealth("kaka", true),
69 | PeriodHealth("kaka", true),
70 | PeriodHealth("kaka", true),
71 | PeriodHealth("kaka", true),
72 | )
73 | val actual = computeOldest(periods)
74 | val expected = 0
75 | actual shouldBe expected
76 | }
77 | }
78 | }
79 |
80 |
81 |
--------------------------------------------------------------------------------
/src/main/scala/models/CheckFrequency.scala:
--------------------------------------------------------------------------------
1 | package me.amanj.greenish.models
2 |
3 | import java.time.ZonedDateTime
4 | import com.cronutils.model.time.ExecutionTime
5 | import com.cronutils.model.CronType.UNIX
6 | import com.cronutils.parser.CronParser
7 | import com.cronutils.model.definition.CronDefinitionBuilder
8 | import io.circe.{Printer, Decoder, Encoder, HCursor, Json}
9 | import io.circe.syntax.EncoderOps
10 | import io.circe.generic.extras.semiauto.{
11 | deriveEnumerationCodec, deriveConfiguredDecoder, deriveConfiguredEncoder}
12 |
13 | sealed trait CheckFrequency {
14 | def prev(date: ZonedDateTime): ZonedDateTime
15 | }
16 |
17 | object CheckFrequency {
18 | implicit val freqDecoder: Decoder[CheckFrequency] = new Decoder[CheckFrequency] {
19 | final def apply(obj: HCursor): Decoder.Result[CheckFrequency] = {
20 | obj.as[String].map {
21 | case "hourly" => Right(Hourly)
22 | case "daily" => Right(Daily)
23 | case "monthly" => Right(Monthly)
24 | case "annually" => Right(Annually)
25 | }.getOrElse(obj.as[Cron])
26 | }
27 | }
28 |
29 | implicit val freqEncoder: Encoder[CheckFrequency] = Encoder.instance {
30 | case Hourly => "hourly".asJson
31 | case Daily => "daily".asJson
32 | case Monthly => "monthly".asJson
33 | case Annually => "annually".asJson
34 | case other: Cron => other.asJson
35 | }
36 | }
37 |
38 | case class Cron(pattern: String) extends CheckFrequency {
39 | private[this] val parser = new CronParser(
40 | CronDefinitionBuilder.instanceDefinitionFor(UNIX))
41 | private[this] val executionTime = ExecutionTime.forCron(
42 | parser.parse(pattern))
43 |
44 | def prev(date: ZonedDateTime): ZonedDateTime =
45 | executionTime.lastExecution(date).get()
46 | }
47 | object Cron {
48 | implicit val cronDecoder: Decoder[Cron] = deriveConfiguredDecoder
49 | implicit val checkGroupEncoder: Encoder[Cron] = deriveConfiguredEncoder
50 | }
51 |
52 | case object Hourly extends CheckFrequency {
53 | def prev(date: ZonedDateTime): ZonedDateTime =
54 | date
55 | .minusHours(1L)
56 | }
57 |
58 | case object Daily extends CheckFrequency {
59 | def prev(date: ZonedDateTime): ZonedDateTime =
60 | date
61 | .minusDays(1L)
62 | }
63 |
64 | case object Monthly extends CheckFrequency {
65 | def prev(date: ZonedDateTime): ZonedDateTime =
66 | date
67 | .minusMonths(1L)
68 | }
69 |
70 | case object Annually extends CheckFrequency {
71 | def prev(date: ZonedDateTime): ZonedDateTime =
72 | date
73 | .minusYears(1L)
74 | }
75 |
--------------------------------------------------------------------------------
/src/main/resources/dashboard/job_container.jsx:
--------------------------------------------------------------------------------
1 | const Link = ReactRouterDOM.Link;
2 |
3 | class JobContainer extends React.Component {
4 | intervalID
5 | constructor(props) {
6 | super(props);
7 | this.state = {
8 | error: null,
9 | isLoaded: false,
10 | job: null
11 | };
12 | this.handleBack = this.handleBack.bind(this);
13 | }
14 |
15 | componentDidMount() {
16 | this.fetchData()
17 | }
18 |
19 | componentWillUnmount() {
20 | clearTimeout(this.intervalID);
21 | }
22 |
23 | fetchData = () => {
24 | fetch(`/group/${this.props.group}/job/${this.props.job}`)
25 | .then(res => res.json())
26 | .then(
27 | (job) => {
28 | this.setState({
29 | isLoaded: true,
30 | job: job
31 | });
32 | },
33 | // Note: it's important to handle errors here
34 | // instead of a catch() block so that we don't swallow
35 | // exceptions from actual bugs in components.
36 | (error) => {
37 | this.setState({
38 | isLoaded: true,
39 | error
40 | });
41 | }
42 | )
43 | this.intervalID = setTimeout(this.fetchData, fetchInterval);
44 | }
45 |
46 | handleBack() {
47 | this.props.handler("main", null, null);
48 | }
49 |
50 | render() {
51 | const { error, isLoaded, job } = this.state;
52 | if (error) {
53 | return (
54 | Error: {error.message}
55 | )
56 | } else if (!isLoaded) {
57 | return (
58 | Loading...
59 | )
60 | } else {
61 | const jobs = renderJob(job, this.props.group, 'job-view')
62 | return (
63 |
64 |
82 | )
83 | }
84 | }
85 | }
86 |
--------------------------------------------------------------------------------
/src/main/resources/dashboard/main.css:
--------------------------------------------------------------------------------
1 | body {
2 | background-color: #233502;
3 | color: darkolivegreen;
4 | padding: 0px;
5 | margin: 0px;
6 | font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
7 | }
8 |
9 | td {
10 | margin: 5px;
11 | padding: 5px;
12 | }
13 |
14 | th {
15 | margin: 5px;
16 | padding: 5px;
17 | }
18 |
19 | table {
20 | border-collapse: collapse;
21 | width: 100%;
22 | text-align: left;
23 | }
24 |
25 | .grid-container {
26 | display: grid;
27 | grid-gap: 50px 100px;
28 | grid-template-columns: repeat(3, 1fr);
29 | justify-content: center;
30 | }
31 |
32 | .grid-item {
33 | vertical-align: top;
34 | }
35 |
36 | .grid-container-detail {
37 | display: grid;
38 | grid-gap: 50px 100px;
39 | grid-template-columns: repeat(2, 1fr);
40 | justify-content: center;
41 | }
42 |
43 | .detail-div {
44 | padding: 10px;
45 | background-color: #f3f2f2
46 | }
47 |
48 | .summary-div {
49 | padding: 10px;
50 | background-color: #e8ece9;
51 | }
52 |
53 | .great {
54 | background-color: #c9daa7;
55 | color: #345f34;
56 | }
57 |
58 | .normal {
59 | background-color: lightblue;
60 | color: steelblue;
61 | }
62 |
63 | .warn {
64 | background-color: palegoldenrod;
65 | color: sienna;
66 | }
67 |
68 | .critical {
69 | background-color: salmon;
70 | color: maroon;
71 | }
72 |
73 | .link {
74 | color: steelblue;
75 | text-decoration: none;
76 | }
77 |
78 | .link:hover {
79 | cursor: pointer
80 | }
81 |
82 | a {
83 | color: steelblue;
84 | text-decoration: none;
85 | }
86 |
87 | a:hover {
88 | cursor: pointer
89 | }
90 |
91 | .detail-box {
92 | background: whitesmoke;
93 | padding: 10px;
94 | box-shadow: 5px 5px lightgray;
95 | }
96 |
97 | .summary-box {
98 | background: #f6fdfc;
99 | padding: 10px;
100 | box-shadow: 5px 5px lightgray;
101 | }
102 |
103 | .header-div {
104 | width: 100%;
105 | margin-right: 8px;
106 | height: 100px;
107 | }
108 |
109 | .header-left {
110 | float: left;
111 | }
112 |
113 | .header-right {
114 | float: right;
115 | margin-top: 65px;
116 | }
117 |
118 | .namespace-span {
119 | margin-left: 16px;
120 | line-height: 100px;
121 | vertical-align: top;
122 | }
123 |
124 | .owner-div {
125 | margin-bottom: 8px;
126 | }
127 |
128 | .info-div {
129 | margin-bottom: 8px;
130 | }
131 |
132 | .stdout-div {
133 | margin-bottom: 20px;
134 | }
135 |
136 | .greenish-header {
137 | margin: 8px;
138 | color: snow;
139 | }
140 |
141 | .version-div {
142 | text-align: right;
143 | font-size: x-small;
144 | margin-right:4px;
145 | margin-top:4px;
146 | margin-bottom:4px;
147 | color: snow;
148 | }
149 |
150 | .time-div {
151 | text-align: right;
152 | margin-right:4px;
153 | color: snow;
154 | }
155 |
--------------------------------------------------------------------------------
/src/main/scala/models/EnvVar.scala:
--------------------------------------------------------------------------------
1 | package me.amanj.greenish.models
2 |
3 | import io.circe.syntax.EncoderOps
4 | import io.circe.{Encoder, Decoder, HCursor, Json}
5 |
6 | sealed trait EnvVar {
7 | type T <: AnyRef
8 | def name: String
9 | def value: T
10 | def tupled: (String, String)
11 | }
12 | object EnvVar {
13 | private[this] val pattern = """secure\((.*)\)""".r
14 | def apply(key: String, value: String): EnvVar = {
15 | value match {
16 | case pattern(v) => SecureEnvVar(key, v.toSeq)
17 | case _ => PlainEnvVar(key, value)
18 | }
19 | }
20 |
21 | implicit val envVarDecoer: Decoder[EnvVar] = new Decoder[EnvVar] {
22 | final def apply(obj: HCursor): Decoder.Result[EnvVar] = {
23 | obj.downField("type").as[String].flatMap {
24 | case "secure" => obj.as[SecureEnvVar]
25 | case "plain" => obj.as[PlainEnvVar]
26 | }
27 | }
28 | }
29 |
30 | implicit val envVarEncoder: Encoder[EnvVar] = Encoder.instance {
31 | case sec: SecureEnvVar => sec.asJson
32 | case plain: PlainEnvVar => plain.asJson
33 | }
34 | }
35 |
36 | private[models] case class SecureEnvVar(name: String, value: Seq[Char]) extends EnvVar {
37 | type T = Seq[Char]
38 | def tupled: (String, String) = (name, value.mkString(""))
39 | }
40 |
41 | private[models] object SecureEnvVar {
42 | val HIDDEN_PASSWORD = "****"
43 | implicit val secureEnvVarEncoder: Encoder[SecureEnvVar] =
44 | new Encoder[SecureEnvVar] {
45 | final def apply(v: SecureEnvVar): Json = Json.obj(
46 | ("type", Json.fromString("secure")),
47 | ("name", Json.fromString(v.name)),
48 | ("value", Json.fromString(HIDDEN_PASSWORD)),
49 | )
50 | }
51 |
52 | implicit val secureEnvVarDecoder: Decoder[SecureEnvVar] = new Decoder[SecureEnvVar] {
53 | final def apply(c: HCursor): Decoder.Result[SecureEnvVar] =
54 | c.downField("type").as[String].flatMap {
55 | case "secure" =>
56 | for {
57 | name <- c.downField("name").as[String]
58 | value <- c.downField("value").as[String].map(_.toSeq)
59 | } yield SecureEnvVar(name, value)
60 | }
61 | }
62 | }
63 |
64 | private[models] case class PlainEnvVar(name: String, value: String) extends EnvVar {
65 | type T = String
66 | def tupled: (String, String) = (name, value)
67 | }
68 | private[models] object PlainEnvVar {
69 | implicit val plainEnvVarEncoder: Encoder[PlainEnvVar] =
70 | new Encoder[PlainEnvVar] {
71 | final def apply(v: PlainEnvVar): Json = Json.obj(
72 | ("type", Json.fromString("plain")),
73 | ("name", Json.fromString(v.name)),
74 | ("value", Json.fromString(v.value)),
75 | )
76 | }
77 |
78 | implicit val secureEnvVarDecoder: Decoder[PlainEnvVar] = new Decoder[PlainEnvVar] {
79 | final def apply(c: HCursor): Decoder.Result[PlainEnvVar] =
80 | c.downField("type").as[String].flatMap {
81 | case "plain" =>
82 | for {
83 | name <- c.downField("name").as[String]
84 | value <- c.downField("value").as[String]
85 | } yield PlainEnvVar(name, value)
86 | }
87 | }
88 | }
89 |
--------------------------------------------------------------------------------
/src/main/resources/dashboard/main_container.jsx:
--------------------------------------------------------------------------------
1 | const e = React.createElement;
2 | const Link = ReactRouterDOM.Link;
3 |
4 | class MainContainer extends React.Component {
5 | constructor(props) {
6 | super(props);
7 | this.state = {
8 | page: props.page,
9 | gid: props.group,
10 | jid: props.job
11 | }
12 | this.handler = this.handler.bind(this);
13 | this.renderMain = this.renderMain.bind(this);
14 | }
15 |
16 | renderMain(page, gid, jid, handler) {
17 | if (page == 'state') {
18 | return (
19 | 65 | {job.job.name} 66 | 67 | `${loc.pathname}?page=main`} 68 | onClick={this.handleBack} 69 | className="link"> 70 | See main dashboard 71 | 72 | 73 |
74 | {"owner" in job.job?:} 75 | {"info" in job.job?:} 76 |
77 | stdout
79 |
80 | {encloseInTable(jobs, 'job-view', this.props.group)}
81 |
20 |
30 | )
31 | } else if(page == 'group'){
32 | return(
33 | All data sets 21 | 22 | `${loc.pathname}?page=main`} 23 | onClick={() => this.setState({page:"main"})} className="link"> 24 | See main dashboard 25 | 26 | 27 |
28 |
34 |
35 |
36 | )
37 | } else if(page == 'job'){
38 | return(
39 |
40 |
41 |
42 | )
43 | } else { // page == 'main'
44 | return(
45 |
46 |
63 | )
64 | }
65 | }
66 |
67 | handler(page, gid, jid) {
68 |
69 | this.setState({
70 | page: page,
71 | gid: gid,
72 | jid: jid,
73 | })
74 | }
75 |
76 | render() {
77 | return (
78 |
47 |
49 |
50 |
51 | Summary
48 |50 |
52 |
62 | Detailed missing periods 53 | 54 | `${loc.pathname}?page=state`} 55 | onClick={() => this.setState({page:"state"})} className="link"> 56 | See all periods 57 | 58 | 59 |
60 |
79 |
95 | )
96 | }
97 | }
98 |
99 | const domContainer = document.querySelector('#main_container');
100 | const BrowserRouter = ReactRouterDOM.BrowserRouter;
101 | const Route = ReactRouterDOM.Route;
102 | const useLocation = ReactRouterDOM.useLocation;
103 |
104 | function useQuery() {
105 | return new URLSearchParams(useLocation().search);
106 | }
107 |
108 | function ShowPage() {
109 | let query = useQuery();
110 | let page = query.get("page");
111 | let gid = query.get("gid");
112 | let jid = query.get("jid");
113 | return (
80 |
91 |
81 |
82 |
85 |
86 |
82 | 
83 |
85 |
87 |
90 |
92 | {this.renderMain(this.state.page, this.state.gid, this.state.jid)}
93 |
94 | Error: {error.message}
60 | )
61 | } else if (!isLoaded) {
62 | return (
63 | Loading...
64 | )
65 | } else {
66 | return (
67 |
68 | {
69 | items.map(group => {
70 | const gid = group.group_id;
71 | return (
72 |
109 | )
110 | }
111 | }
112 | }
113 |
--------------------------------------------------------------------------------
/src/test/scala/AppSpec.scala:
--------------------------------------------------------------------------------
1 | package me.amanj.greenish
2 |
3 | import org.scalatest.matchers.should.Matchers
4 | import org.scalatest.wordspec.AnyWordSpecLike
5 | import models._
6 | import java.time.ZoneId
7 | import java.io.File
8 |
9 | class AppSpec() extends Matchers
10 | with AnyWordSpecLike {
11 | "getPrometheusIds" must {
12 | "work when there are duplicate IDs" in {
13 | val config = new AppConfig(
14 | Seq(
15 | Group(0, "Group1", Seq(
16 | Job(0, "Job1", None, "job_1", "/tmp/first_script",
17 | "yyyy-MM-dd-HH", Hourly, 3,
18 | ZoneId.of("UTC"), 24, 0,
19 | AlertLevels(0, 1, 2, 3),
20 | None,
21 | Seq(EnvVar("VAR1", "baz"), EnvVar("VAR2", "secure(bar)")),
22 | ),
23 | Job(1, "Job2", None, "job_1", "/tmp/second_script job2",
24 | "yyyy-MM-dd-HH", Daily, 2,
25 | ZoneId.of("UTC"), 24, 0,
26 | AlertLevels(0, 1, 2, 3),
27 | None,
28 | Seq(EnvVar("VAR1", "baz"), EnvVar("VAR2", "secure(bar)")),
29 | ),
30 | )),
31 | Group(1, "Group2", Seq(
32 | Job(0, "Job3", None, "job_2", "/tmp/third_script",
33 | "yyyy-MM-dd", Monthly, 1,
34 | ZoneId.of("UTC"), 3, 0,
35 | AlertLevels(0, 1, 2, 3),
36 | None,
37 | Seq(EnvVar("VAR1", "baz"), EnvVar("VAR2", "secure(bar)")),
38 | ),
39 | Job(1, "Job4", None, "job_2", "/tmp/fourth_script",
40 | "yyyy-01-01", Annually, 1,
41 | ZoneId.of("UTC"), 3, 0,
42 | AlertLevels(0, 1, 2, 3),
43 | None,
44 | Seq(EnvVar("VAR1", "baz"), EnvVar("VAR2", "secure(bar)")),
45 | ),
46 | )),
47 | ),
48 | None,
49 | new File("/tmp/greenish/stdout"),
50 | 30,
51 | "127.0.0.1",
52 | 8080,
53 | )
54 |
55 | val expected = Set("job_1", "job_2")
56 |
57 | val actual = App.getPrometheusIds(config)
58 |
59 | actual shouldBe expected
60 | }
61 |
62 | "work when there are no duplicate IDs" in {
63 | val config = new AppConfig(
64 | Seq(
65 | Group(0, "Group1", Seq(
66 | Job(0, "Job1", None, "job_1", "/tmp/first_script",
67 | "yyyy-MM-dd-HH", Hourly, 3,
68 | ZoneId.of("UTC"), 24, 0,
69 | AlertLevels(0, 1, 2, 3),
70 | None,
71 | Seq(EnvVar("VAR1", "baz"), EnvVar("VAR2", "secure(bar)")),
72 | ),
73 | Job(1, "Job2", None, "job_2", "/tmp/second_script job2",
74 | "yyyy-MM-dd-HH", Daily, 2,
75 | ZoneId.of("UTC"), 24, 0,
76 | AlertLevels(0, 1, 2, 3),
77 | None,
78 | Seq(EnvVar("VAR1", "baz"), EnvVar("VAR2", "secure(bar)")),
79 | ),
80 | )),
81 | Group(1, "Group2", Seq(
82 | Job(0, "Job3", None, "job_3", "/tmp/third_script",
83 | "yyyy-MM-dd", Monthly, 1,
84 | ZoneId.of("UTC"), 3, 0,
85 | AlertLevels(0, 1, 2, 3),
86 | None,
87 | Seq(EnvVar("VAR1", "baz"), EnvVar("VAR2", "secure(bar)")),
88 | ),
89 | Job(1, "Job4", None, "job_4", "/tmp/fourth_script",
90 | "yyyy-01-01", Annually, 1,
91 | ZoneId.of("UTC"), 3, 0,
92 | AlertLevels(0, 1, 2, 3),
93 | None,
94 | Seq(EnvVar("VAR1", "baz"), EnvVar("VAR2", "secure(bar)")),
95 | ),
96 | )),
97 | ),
98 | None,
99 | new File("/tmp/greenish/stdout"),
100 | 30,
101 | "127.0.0.1",
102 | 8080,
103 | )
104 |
105 | val expected = Set("job_1", "job_2", "job_3", "job_4")
106 |
107 | val actual = App.getPrometheusIds(config)
108 |
109 | actual shouldBe expected
110 | }
111 | }
112 | }
113 |
--------------------------------------------------------------------------------
/src/main/scala/checker/CommandRunner.scala:
--------------------------------------------------------------------------------
1 | package me.amanj.greenish.checker
2 |
3 | import me.amanj.greenish.stats._
4 | import me.amanj.greenish.models._
5 | import java.time.ZonedDateTime
6 | import java.io.{File, PrintWriter}
7 | import java.nio.file.{Files, StandardCopyOption}
8 | import scala.sys.process.Process
9 | import scala.util.control.NonFatal
10 | import akka.actor.{Actor, ActorRef, ActorLogging}
11 |
12 | class CommandRunner(statsActor: ActorRef,
13 | scratchDir: File) extends Actor with ActorLogging {
14 | override def receive: Receive = {
15 | case BatchRun(cmd, periods, env, group, job,
16 | prometheusId, clockCounter, expireAt) =>
17 | val startTimeLong = System.currentTimeMillis
18 | if(startTimeLong <= expireAt) {
19 | statsActor ! IncRefresh(prometheusId)
20 | val startTime = startTimeLong.toDouble
21 | try {
22 | run(cmd, periods, env, group, job, prometheusId, clockCounter)
23 | } catch {
24 | case NonFatal(exp) =>
25 | log.error(exp.getMessage())
26 | statsActor ! IncBadRefresh(prometheusId)
27 | } finally {
28 | statsActor ! DecRefresh(prometheusId)
29 | val endTime = System.currentTimeMillis.toDouble
30 | statsActor ! RefreshTime(prometheusId,
31 | (endTime - startTime) / 1000)
32 | }
33 | } else {
34 | statsActor ! IncExpiredRefresh(prometheusId)
35 | }
36 | }
37 |
38 | private[this] def run(
39 | cmd: String,
40 | periods: Seq[String],
41 | env: Seq[(String, String)],
42 | group: Int,
43 | job: Int,
44 | prometheusId: String,
45 | clockCounter: Long): Unit = {
46 | val exec = Seq("bash", "-c", CommandRunner.toBashCommand(cmd, periods))
47 | val output = Process(exec, None, env:_*).lazyLines
48 | CommandRunner.write(debugFile(scratchDir, group, job), output)
49 | val capturedOutput = CommandRunner.parseOutput(output, periods.toSet)
50 | val distinctReturnedPeriods = capturedOutput.map(_._1).distinct
51 | if(capturedOutput.length < periods.size) {
52 | log.error(s"""|Some periods weren't returned for:
53 | |Group ID: $group, Job ID: $job
54 | |$cmd $periods
55 | |state update aborted""".stripMargin)
56 | statsActor ! IncBadRefresh(prometheusId)
57 | } else if(distinctReturnedPeriods.length != capturedOutput.size) {
58 | log.error(s"""|Some periods were returned more than once for:
59 | |$cmd $periods
60 | |Group ID: $group, Job ID: $job
61 | |$cmd $periods
62 | |state update aborted""".stripMargin)
63 | statsActor ! IncBadRefresh(prometheusId)
64 | } else {
65 | val mapped = capturedOutput.toMap
66 | val periodHealths = periods.map {
67 | period => PeriodHealth(period, mapped(period)) }
68 | context.sender ! RunResult(periodHealths, group, job, clockCounter)
69 | statsActor ! MissingPeriods(prometheusId, periodHealths.count(!_.ok))
70 | val oldestMissingPeriod = computeOldest(periodHealths)
71 | statsActor ! OldestMissingPeriod(prometheusId, oldestMissingPeriod)
72 | }
73 | }
74 | }
75 |
76 | object CommandRunner {
77 | private[this] val Matcher = "^greenish-period\t(.*)\t(1|0)$".r
78 |
79 | protected[checker] def write(file: String,
80 | lines: LazyList[String]): Unit = {
81 | val tmp = new File(s"$file.tmp")
82 | val pw = new PrintWriter(tmp)
83 | lines.foreach(pw.println)
84 | pw.close
85 | // FIXME: There is a slight chance of race, but do we care?
86 | Files.move(tmp.toPath, new File(file).toPath,
87 | StandardCopyOption.ATOMIC_MOVE)
88 | }
89 |
90 | protected[checker] def parseOutput(lines: LazyList[String],
91 | periods: Set[String]): Seq[(String, Boolean)] =
92 | lines.map { line =>
93 | line match {
94 | case Matcher(period, "1") => Some((period, true))
95 | case Matcher(period, "0") => Some((period, false))
96 | case _ => None
97 | }
98 | }.collect { case Some(periodStatus) => periodStatus }
99 | .filter { case (period, _) => periods.contains(period) }
100 | .toList
101 |
102 | protected[checker] def toBashCommand(command: String, periods: Seq[String]): String =
103 | s"$command ${periods.map(p => s"'$p'").mkString(" ")}"
104 | }
105 |
--------------------------------------------------------------------------------
/src/main/scala/stats/StatsCollector.scala:
--------------------------------------------------------------------------------
1 | package me.amanj.greenish.stats
2 |
3 | import akka.actor.Actor
4 | import akka.actor.ActorLogging
5 | import io.prometheus.client.{Counter, Gauge, Histogram, CollectorRegistry}
6 | import io.prometheus.client.Collector.MetricFamilySamples
7 | import io.prometheus.client.exporter.common.TextFormat
8 | import java.util.Enumeration
9 | import java.io.{StringWriter, Writer}
10 | import akka.http.scaladsl.model.{MediaType, HttpCharsets, HttpEntity}
11 | import akka.http.scaladsl.marshalling.{ToEntityMarshaller, Marshaller}
12 |
13 | class StatsCollector(jobIDs: Set[String],
14 | registry: CollectorRegistry = new CollectorRegistry()) extends Actor with ActorLogging {
15 |
16 | // Job related metrics
17 | private[this] val refreshGauge = Gauge.build()
18 | .name("greenish_active_refresh_tasks")
19 | .help("Current number active state refresh tasks")
20 | .labelNames("job_id")
21 | .register(registry)
22 |
23 | private[this] val refreshTime = Histogram.build()
24 | .name("greenish_state_refresh_time_seconds")
25 | .help("Job state refreshing time")
26 | .labelNames("job_id")
27 | .buckets(StatsCollector.HistogramTimeBuckets:_*)
28 | .register(registry)
29 |
30 | private[this] val refreshCounter = Counter.build()
31 | .name("greenish_state_refresh_total")
32 | .help("Total number of job state refresh instances")
33 | .labelNames("job_id")
34 | .register(registry)
35 |
36 | private[this] val badRefreshCounter = Counter.build()
37 | .name("greenish_state_refresh_failed_total")
38 | .help("Total number of failed job state refresh instances")
39 | .labelNames("job_id")
40 | .register(registry)
41 |
42 | private[this] val expiredRefreshCounter = Counter.build()
43 | .name("greenish_state_refresh_expired_total")
44 | .help("Total number of expired job state refresh instances")
45 | .labelNames("job_id")
46 | .register(registry)
47 |
48 | private[this] val missingPeriods = Gauge.build()
49 | .name("greenish_missing_periods_total")
50 | .help("Current number of missing dataset periods")
51 | .labelNames("job_id")
52 | .register(registry)
53 |
54 | private[this] val oldestMissingPeriod = Gauge.build()
55 | .name("greenish_oldest_missing_period")
56 | .help("The oldest missing period")
57 | .labelNames("job_id")
58 | .register(registry)
59 |
60 | init()
61 |
62 | private[this] def init(): Unit = {
63 | jobIDs.foreach { jobId =>
64 | refreshGauge.labels(jobId)
65 | refreshTime.labels(jobId)
66 | refreshCounter.labels(jobId)
67 | expiredRefreshCounter.labels(jobId)
68 | badRefreshCounter.labels(jobId)
69 | missingPeriods.labels(jobId)
70 | oldestMissingPeriod.labels(jobId)
71 | }
72 | }
73 |
74 | override def receive: Receive = {
75 | case RefreshTime(jobId, time) =>
76 | refreshTime.labels(jobId).observe(time)
77 | case IncRefresh(jobId) =>
78 | refreshCounter.labels(jobId).inc()
79 | refreshGauge.labels(jobId).inc()
80 | case DecRefresh(jobId) =>
81 | refreshGauge.labels(jobId).dec()
82 | case IncBadRefresh(jobId) =>
83 | badRefreshCounter.labels(jobId).inc()
84 | case MissingPeriods(jobId, count) =>
85 | missingPeriods.labels(jobId).set(count)
86 | case OldestMissingPeriod(jobId, count) =>
87 | oldestMissingPeriod.labels(jobId).set(count)
88 | case IncExpiredRefresh(jobId) =>
89 | refreshCounter.labels(jobId).inc()
90 | expiredRefreshCounter.labels(jobId).inc()
91 | case GetPrometheus =>
92 | import StatsCollector.{fromRegistry, toPrometheusTextFormat}
93 | val metrics = fromRegistry(registry)
94 | context.sender ! metrics
95 | }
96 | }
97 |
98 | object StatsCollector {
99 | case class MetricsEntity(samples: Enumeration[MetricFamilySamples])
100 |
101 | private [StatsCollector] val HistogramTimeBuckets =
102 | Seq(
103 | 0.1, 0.3, 0.5, 0.8, 1, 1.3, 1.5, 1.8, 2, 2.5, 3, 3.5, 4, 4.5)
104 |
105 | private[this] val mediaTypeParams = Map("version" -> "0.0.4")
106 | private[this] val mediaType = MediaType.customWithFixedCharset(
107 | "text", "plain", HttpCharsets.`UTF-8`, params = mediaTypeParams)
108 |
109 | private[stats] def fromRegistry(
110 | collectorRegistry: CollectorRegistry): MetricsEntity = {
111 | MetricsEntity(collectorRegistry.metricFamilySamples())
112 | }
113 |
114 | private[stats] def toPrometheusTextFormat(e: MetricsEntity): String = {
115 | val writer: Writer = new StringWriter()
116 | TextFormat.write004(writer, e.samples)
117 |
118 | writer.toString
119 | }
120 |
121 | implicit val metricsMarshaller: ToEntityMarshaller[MetricsEntity] = {
122 | Marshaller.withFixedContentType(mediaType) { s =>
123 | HttpEntity(mediaType, toPrometheusTextFormat(s))
124 | }
125 | }
126 | }
127 |
--------------------------------------------------------------------------------
/src/main/scala/checker/StatusChecker.scala:
--------------------------------------------------------------------------------
1 | package me.amanj.greenish.checker
2 |
3 | import me.amanj.greenish.stats.StatsCollector
4 | import me.amanj.greenish.models._
5 | import java.time.ZonedDateTime
6 | import java.io.File
7 | import akka.actor.{Actor, Props, ActorRef, ActorLogging}
8 | import scala.sys.process.Process
9 | import scala.concurrent.{Future}
10 | import scala.util.{Success, Failure}
11 | import akka.routing.{ActorRefRoutee, RoundRobinRoutingLogic, Router}
12 | import akka.pattern.pipe
13 | import scala.annotation.tailrec
14 |
15 | trait StatusCheckerApi {
16 | protected[this] var state: IndexedSeq[GroupStatus]
17 |
18 | protected[checker] def getMissing(): IndexedSeq[GroupStatus] = {
19 | state
20 | .map { group =>
21 | val newJobs: Array[JobStatus] = group.status.map { job =>
22 | job.copy(periodHealth = job.periodHealth.filterNot(_.ok))
23 | }.filterNot(_.periodHealth.isEmpty)
24 | .toArray
25 |
26 | group.copy(status = newJobs)
27 | }.filterNot(_.status.isEmpty)
28 | }
29 |
30 | protected[checker] def maxLag(): Lag = {
31 | if(state.isEmpty) Lag(0)
32 | else {
33 | val lag = state.map { group =>
34 | group.status.map(_.countMissing).max
35 | }.max
36 | Lag(lag)
37 | }
38 | }
39 |
40 | protected[checker] def allEntries(): IndexedSeq[GroupStatus] = state
41 |
42 | protected[checker] def summary(): Seq[GroupStatusSummary] =
43 | state.map { group =>
44 | val status = group.status.map { status =>
45 | val missing = status.countMissing
46 | val alertLevel: AlertLevel =
47 | if(missing <= status.job.alertLevels.great) Great
48 | else if(missing <= status.job.alertLevels.normal) Normal
49 | else if(missing <= status.job.alertLevels.warn) Warn
50 | else Critical
51 |
52 | val oldestMissingPeriod = computeOldest(status.periodHealth)
53 | JobStatusSummary(status.job.jobId, status.job.name, missing, oldestMissingPeriod, alertLevel)
54 | }.toSeq
55 | GroupStatusSummary(group.group.groupId, group.group.name, status)
56 | }
57 |
58 | protected[checker] def getGroupStatus(groupId: Int): Option[GroupStatus] =
59 | state.lift(groupId)
60 |
61 | protected[checker] def getJobStatus(groupId: Int, jobId: Int): Option[JobStatus] =
62 | for {
63 | group <- state.lift(groupId)
64 | job <- group.status.lift(jobId)
65 | } yield job
66 | }
67 |
68 | class StatusChecker(groups: Seq[Group],
69 | statsActor: ActorRef,
70 | refreshValidityInSeconds: Long,
71 | scratchDir: File,
72 | clockCounter: () => Long = () => System.currentTimeMillis())
73 | extends Actor with ActorLogging with StatusCheckerApi {
74 | override protected[this] var state = StatusChecker.initState(groups)
75 |
76 | import context.dispatcher
77 |
78 | private[this] val parallelism: Int = groups.map(_.jobs.length).sum
79 |
80 | private[this] val router = {
81 | val routees = (0 until parallelism) map { _ =>
82 | val runner = context.actorOf(
83 | Props(new CommandRunner(statsActor, scratchDir))
84 | .withDispatcher("akka.refresh-dispatcher"))
85 | context watch runner
86 | ActorRefRoutee(runner)
87 | }
88 |
89 | Router(RoundRobinRoutingLogic(), routees)
90 | }
91 |
92 | private[this] def refresh(now: ZonedDateTime, group: Group, job: Job): Unit = {
93 | val periods = StatusChecker.periods(job, now)
94 |
95 | val currentClockCounter = clockCounter()
96 | val expireAt = currentClockCounter + 1000 *
97 | refreshValidityInSeconds
98 | self ! BatchRun(job.cmd, periods, job.env.map(_.tupled),
99 | group.groupId, job.jobId, job.prometheusId,
100 | currentClockCounter, expireAt)
101 | }
102 |
103 | private[this] def refresh(now: ZonedDateTime, group: Group): Unit = {
104 | group.jobs.foreach { job => refresh(now, group, job) }
105 | }
106 |
107 | private[this] def refresh(now: ZonedDateTime): Unit = {
108 |
109 | groups.foreach { group =>
110 | refresh(now, group)
111 | }
112 | }
113 |
114 | override def receive: Receive = {
115 | case Refresh(now) =>
116 | refresh(now())
117 | case RefreshGroup(now, groupId) =>
118 | groups.find(_.groupId == groupId) match {
119 | case Some(group) =>
120 | refresh(now(), group)
121 | context.sender ! true
122 | case None =>
123 | context.sender ! false
124 | }
125 | case RefreshJob(now, groupId, jobId) =>
126 | val result = for {
127 | group <- groups.find(_.groupId == groupId)
128 | job <- group.jobs.lift(jobId)
129 | } yield {
130 | refresh(now(), group, job)
131 | }
132 | context.sender ! result.isDefined
133 | case RunResult(periodHealth, groupId, jobId, clockCounter) =>
134 | val bucket = state(groupId)
135 | val currentStatus = bucket.status(jobId)
136 | if(currentStatus.updatedAt < clockCounter) {
137 | bucket.status(jobId) = currentStatus.copy(updatedAt = clockCounter,
138 | periodHealth = periodHealth)
139 | }
140 | case GetMissing => context.sender ! getMissing()
141 | case MaxLag => context.sender ! maxLag()
142 | case AllEntries => context.sender ! allEntries()
143 | case Summary => context.sender ! summary()
144 | case GetGroupStatus(id) =>
145 | context.sender ! getGroupStatus(id)
146 | case GetJobStatus(gid, jid) =>
147 | context.sender ! getJobStatus(gid, jid)
148 | case run: BatchRun =>
149 | router.route(run, context.sender)
150 | }
151 | }
152 |
153 | object StatusChecker {
154 | private[checker] def initState(groups: Seq[Group]): IndexedSeq[GroupStatus] = {
155 | groups.map { group =>
156 | val jobStatus = group.jobs.map { job =>
157 | JobStatus(job, -1, Seq.empty)
158 | }
159 | GroupStatus(group, jobStatus.toArray)
160 | }.toIndexedSeq
161 | }
162 |
163 | private[checker] def periods(entry: Job, now: ZonedDateTime): Seq[String] = {
164 | @tailrec def loop(time: ZonedDateTime, count: Int, acc: Seq[String]): Seq[String] = {
165 | if(time.toEpochSecond < entry.startAt || count == 0) acc.reverse
166 | else
167 | loop(entry.frequency.prev(time), count - 1,
168 | acc :+ time.format(entry.timeFormat))
169 | }
170 |
171 | loop(nowMinusOffset(entry, now),
172 | entry.lookback, Vector.empty[String])
173 | }
174 |
175 | private[checker] def nowMinusOffset(entry: Job,
176 | now: ZonedDateTime): ZonedDateTime =
177 | if(entry.periodCheckOffset == 0)
178 | now.withZoneSameInstant(entry.timezone)
179 | else
180 | (1 to entry.periodCheckOffset)
181 | .foldLeft(now.withZoneSameInstant(entry.timezone))(
182 | (acc, next) => entry.frequency.prev(acc))
183 | }
184 |
--------------------------------------------------------------------------------
/src/main/scala/endpoints/Routes.scala:
--------------------------------------------------------------------------------
1 | package me.amanj.greenish.endpoints
2 |
3 | import java.time.ZonedDateTime
4 | import java.io.File
5 | import akka.actor.ActorRef
6 | import akka.util.Timeout
7 | import akka.pattern.ask
8 | import akka.http.scaladsl.model.{StatusCodes, ContentTypes}
9 | import akka.http.scaladsl.server.Directive
10 | import akka.http.scaladsl.server.Directives._
11 | import scala.concurrent.duration.Duration
12 | import io.circe.syntax._
13 | import io.circe.Printer
14 | import me.amanj.greenish.models._
15 | import me.amanj.greenish.stats._
16 | import me.amanj.greenish.checker._
17 | import akka.http.scaladsl.model.HttpResponse
18 | import scala.util.Success
19 |
20 | class Routes(namespace: Option[String],
21 | scratchDir: File,
22 | statusChecker: ActorRef,
23 | statsActor: ActorRef,
24 | goodRefreshRecency: Long,
25 | now: () => ZonedDateTime = () => ZonedDateTime.now) {
26 | private[this] implicit val timeout = Timeout(Duration.fromNanos(5000000L))
27 | private[this] val jsonPrinter = Printer (
28 | dropNullValues = true,
29 | indent=""
30 | )
31 |
32 | private[this] val maxlag = get {
33 | path("maxlag") {
34 | val lagFuture = (
35 | statusChecker ? MaxLag
36 | ).mapTo[Lag]
37 | onComplete(lagFuture) { lag =>
38 | complete(lag.map(o => jsonPrinter.print(o.asJson)))
39 | }
40 | }
41 | }
42 |
43 | private[this] val summary = get {
44 | path("summary") {
45 | val lagFuture = (
46 | statusChecker ? Summary
47 | ).mapTo[Seq[GroupStatusSummary]]
48 | onComplete(lagFuture) { lag =>
49 | complete(lag.map(o => jsonPrinter.print(o.asJson)))
50 | }
51 | }
52 | }
53 |
54 | private[this] val missing = get {
55 | path("missing") {
56 | val missingFuture = (
57 | statusChecker ? GetMissing
58 | ).mapTo[Seq[GroupStatus]]
59 | onComplete(missingFuture) { missing =>
60 | complete(missing.map(o => jsonPrinter.print(o.asJson)))
61 | }
62 | }
63 | }
64 |
65 | private[this] val state = get {
66 | path("state") {
67 | val allFuture = (
68 | statusChecker ? AllEntries
69 | ).mapTo[Seq[GroupStatus]]
70 | onComplete(allFuture) { completed =>
71 | complete(completed.map(o => jsonPrinter.print(o.asJson)))
72 | }
73 | }
74 | }
75 |
76 | private[this] val getGroup = get {
77 | path("group" / IntNumber) { id =>
78 | val groupFuture = (
79 | statusChecker ? GetGroupStatus(id)
80 | ).mapTo[Option[GroupStatus]]
81 |
82 | onComplete(groupFuture) {
83 | case Success(Some(group)) =>
84 | complete(jsonPrinter.print(group.asJson))
85 | case _ =>
86 | val error = jsonPrinter.print(errorJson("Group id does not exist"))
87 | complete(HttpResponse(StatusCodes.BadRequest, entity = error))
88 | }
89 | }
90 | }
91 |
92 | private[this] val getJob = get {
93 | path("group" / IntNumber / "job" / IntNumber) {
94 | (gid, jid) =>
95 | val jobFuture = (
96 | statusChecker ? GetJobStatus(gid, jid)
97 | ).mapTo[Option[JobStatus]]
98 | onComplete(jobFuture) {
99 | case Success(Some(job)) =>
100 | complete(jsonPrinter.print(job.asJson))
101 | case _ =>
102 | val error = jsonPrinter
103 | .print(errorJson("Group id and/or job id does not exist"))
104 | complete(HttpResponse(StatusCodes.BadRequest, entity = error))
105 | }
106 | }
107 | }
108 |
109 | private[this] val getJobOutput = get {
110 | path("group" / IntNumber / "job" / IntNumber / "stdout") {
111 | (gid, jid) =>
112 | getFromFile(new File(debugFile(scratchDir, gid, jid)),
113 | ContentTypes.`text/plain(UTF-8)`)
114 | }
115 | }
116 |
117 | private[this] val refreshState = get {
118 | path("state" / "refresh") {
119 | statusChecker ! Refresh(now)
120 | complete(jsonPrinter.print(okJson("State refresh is scheduled")))
121 | }
122 | }
123 |
124 | private[this] val refreshGroup = get {
125 | path("group" / IntNumber / "refresh") { id =>
126 | val statusFuture = (
127 | statusChecker ? RefreshGroup(now, id)
128 | ).mapTo[Boolean]
129 |
130 | onComplete(statusFuture) {
131 | case Success(true) =>
132 | complete(jsonPrinter.print(okJson("Group status refresh is scheduled")))
133 | case _ =>
134 | val error = jsonPrinter.print(errorJson("Group id does not exist"))
135 | complete(HttpResponse(StatusCodes.BadRequest, entity = error))
136 | }
137 | }
138 | }
139 |
140 | private[this] val refreshJob = get {
141 | path("group" / IntNumber / "job" / IntNumber / "refresh") {
142 | (gid, jid) =>
143 | val statusFuture = (
144 | statusChecker ? RefreshJob(now, gid, jid)
145 | ).mapTo[Boolean]
146 | onComplete(statusFuture) {
147 | case Success(true) =>
148 | complete(jsonPrinter.print(okJson("Job status refresh is scheduled")))
149 | case _ =>
150 | val error = jsonPrinter
151 | .print(errorJson("Group id and/or job id does not exist"))
152 | complete(HttpResponse(StatusCodes.BadRequest, entity = error))
153 | }
154 | }
155 | }
156 |
157 | private[this] val dashboard =
158 | (get & pathPrefix("dashboard")) {
159 | (pathEndOrSingleSlash &
160 | redirectToTrailingSlashIfMissing(StatusCodes.TemporaryRedirect)) {
161 | getFromResource("dashboard/index.html")
162 | } ~ {
163 | getFromResourceDirectory("dashboard")
164 | }
165 | }
166 |
167 | private[this] val system = get {
168 | path("system") {
169 | val json = jsonPrinter.print(sysinfo(namespace))
170 | complete(json)
171 | }
172 | }
173 |
174 | private[this] val prometheus = get {
175 | path("prometheus") {
176 | val statsFuture =
177 | (statsActor ? GetPrometheus)
178 | .mapTo[StatsCollector.MetricsEntity]
179 | onComplete(statsFuture) { entity =>
180 | complete(entity)
181 | }
182 | }
183 | }
184 |
185 | private[this] val health = get {
186 | path("health") {
187 | val entriesFuture = (statusChecker ? AllEntries)
188 | .mapTo[Seq[GroupStatus]]
189 |
190 | onComplete(entriesFuture) { entity =>
191 | val health = entity.map( groups =>
192 | Routes.isHealthy(groups, goodRefreshRecency)).getOrElse(false)
193 | val json = jsonPrinter.print(healthJson(health))
194 | complete(json)
195 | }
196 | }
197 | }
198 |
199 | val routes =
200 | getJob ~ getJobOutput ~ getGroup ~ refreshState ~ refreshGroup ~
201 | refreshJob ~ maxlag ~ summary ~ missing ~ state ~ dashboard ~
202 | system ~ prometheus ~ health
203 | }
204 |
205 | object Routes {
206 | private[endpoints] def isHealthy(groups: Seq[GroupStatus],
207 | recency: Long): Boolean = {
208 | val now = System.currentTimeMillis
209 | groups.map { group =>
210 | group.status.filterNot { job =>
211 | (now - job.updatedAt) > recency || job.periodHealth.isEmpty
212 | }.length
213 | }.exists(_ > 0)
214 | }
215 | }
216 |
--------------------------------------------------------------------------------
/src/test/scala/stats/StatsCollectorSpec.scala:
--------------------------------------------------------------------------------
1 | package me.amanj.greenish.stats
2 |
3 | import akka.actor.{ActorSystem, Props}
4 | import akka.testkit.{ ImplicitSender, TestKit }
5 | import org.scalatest.BeforeAndAfterAll
6 | import org.scalatest.matchers.should.Matchers
7 | import org.scalatest.wordspec.AnyWordSpecLike
8 | import org.scalatest.concurrent.Eventually
9 | import java.time.{ZoneId, ZonedDateTime}
10 | import scala.concurrent.duration._
11 | import scala.language.postfixOps
12 | import me.amanj.greenish.models._
13 | import io.prometheus.client.Collector.MetricFamilySamples
14 | import java.io.File
15 | import scala.jdk.CollectionConverters._
16 |
17 | class StatsCollectorSpec()
18 | extends TestKit(ActorSystem("StatsCollectorSpec"))
19 | with ImplicitSender
20 | with AnyWordSpecLike
21 | with Matchers
22 | with BeforeAndAfterAll
23 | with Eventually {
24 |
25 | import StatsCollectorSpec._
26 |
27 | "StatsCollector" must {
28 | "initialize labels upon instantiation" in {
29 | val jobs = Set("p1", "p2")
30 | val stats = system.actorOf(
31 | Props(new StatsCollector(jobs)))
32 |
33 | stats ! GetPrometheus
34 |
35 | val received = receiveOne(2 seconds)
36 | assert(received.isInstanceOf[StatsCollector.MetricsEntity])
37 |
38 | val prometheus = received
39 | .asInstanceOf[StatsCollector.MetricsEntity]
40 | .samples
41 | .asScala
42 | .toList
43 |
44 | prometheus.isEmpty shouldBe false
45 | prometheus.foreach { prom =>
46 | val labels = prom.samples.asScala
47 | .flatMap(_.labelValues.asScala)
48 | .filter(jobs.contains(_))
49 | .toSet
50 | labels shouldBe jobs
51 | }
52 | }
53 |
54 | "properly handle IncExpiredRefresh message" in {
55 | val jobs = Set("p1", "p2")
56 | val stats = system.actorOf(
57 | Props(new StatsCollector(jobs)))
58 |
59 | stats ! IncExpiredRefresh("p2")
60 | stats ! GetPrometheus
61 |
62 | val expected = Seq(
63 | (Seq("p1"), 0.0),
64 | (Seq("p2"), 1.0),
65 | )
66 |
67 | val prom = receiveOne(2 seconds)
68 | .asInstanceOf[StatsCollector.MetricsEntity]
69 | .samples.asScala.toList
70 |
71 | checkSamples(prom, "greenish_state_refresh_total", expected)
72 | checkSamples(prom, "greenish_state_refresh_expired_total", expected)
73 | }
74 |
75 | "properly handle IncRefresh message" in {
76 | val jobs = Set("p1", "p2")
77 | val stats = system.actorOf(
78 | Props(new StatsCollector(jobs)))
79 |
80 | stats ! IncRefresh("p2")
81 | stats ! GetPrometheus
82 |
83 | val expected = Seq(
84 | (Seq("p1"), 0.0),
85 | (Seq("p2"), 1.0),
86 | )
87 |
88 | val prom = receiveOne(2 seconds)
89 | .asInstanceOf[StatsCollector.MetricsEntity]
90 | .samples.asScala.toList
91 |
92 | checkSamples(prom, "greenish_state_refresh_total", expected)
93 | checkSamples(prom, "greenish_active_refresh_tasks", expected)
94 | }
95 |
96 | "properly handle DecRefresh message" in {
97 | val jobs = Set("p1", "p2")
98 | val stats = system.actorOf(
99 | Props(new StatsCollector(jobs)))
100 |
101 | stats ! IncRefresh("p1")
102 | stats ! DecRefresh("p1")
103 | stats ! GetPrometheus
104 |
105 | val expectedTotal = Seq(
106 | (Seq("p1"), 1.0),
107 | (Seq("p2"), 0.0),
108 | )
109 |
110 | val expectedActive = Seq(
111 | (Seq("p1"), 0.0),
112 | (Seq("p2"), 0.0),
113 | )
114 |
115 | val prom = receiveOne(2 seconds)
116 | .asInstanceOf[StatsCollector.MetricsEntity]
117 | .samples.asScala.toList
118 |
119 | checkSamples(prom, "greenish_state_refresh_total", expectedTotal)
120 | checkSamples(prom, "greenish_active_refresh_tasks", expectedActive)
121 | }
122 |
123 | "properly handle IncBadRefresh message" in {
124 | val jobs = Set("p1", "p2")
125 | val stats = system.actorOf(
126 | Props(new StatsCollector(jobs)))
127 |
128 | stats ! IncBadRefresh("p1")
129 | stats ! GetPrometheus
130 |
131 | val expected = Seq(
132 | (Seq("p1"), 1.0),
133 | (Seq("p2"), 0.0),
134 | )
135 |
136 | val prom = receiveOne(2 seconds)
137 | .asInstanceOf[StatsCollector.MetricsEntity]
138 | .samples.asScala.toList
139 |
140 | checkSamples(prom, "greenish_state_refresh_failed_total", expected)
141 | }
142 |
143 | "properly handle OldestMissingPeriod message" in {
144 | val jobs = Set("p1", "p2")
145 | val stats = system.actorOf(
146 | Props(new StatsCollector(jobs)))
147 |
148 | stats ! OldestMissingPeriod("p1", 3)
149 | stats ! GetPrometheus
150 |
151 | val expected = Seq(
152 | (Seq("p1"), 3.0),
153 | (Seq("p2"), 0.0),
154 | )
155 |
156 | val prom = receiveOne(2 seconds)
157 | .asInstanceOf[StatsCollector.MetricsEntity]
158 | .samples.asScala.toList
159 |
160 | checkSamples(prom, "greenish_oldest_missing_period", expected)
161 | }
162 |
163 | "properly handle MissingPeriods message" in {
164 | val jobs = Set("p1", "p2")
165 | val stats = system.actorOf(
166 | Props(new StatsCollector(jobs)))
167 |
168 | stats ! MissingPeriods("p1", 3)
169 | stats ! GetPrometheus
170 |
171 | val expected = Seq(
172 | (Seq("p1"), 3.0),
173 | (Seq("p2"), 0.0),
174 | )
175 |
176 | val prom = receiveOne(2 seconds)
177 | .asInstanceOf[StatsCollector.MetricsEntity]
178 | .samples.asScala.toList
179 |
180 | checkSamples(prom, "greenish_missing_periods_total", expected)
181 | }
182 |
183 | "properly handle RefreshTime message" in {
184 | val jobs = Set("p1", "p2")
185 | val stats = system.actorOf(
186 | Props(new StatsCollector(jobs)))
187 |
188 | stats ! RefreshTime("p1", 3)
189 | stats ! GetPrometheus
190 |
191 | val expected = Set("p1")
192 |
193 | val prom = receiveOne(2 seconds)
194 | .asInstanceOf[StatsCollector.MetricsEntity]
195 | .samples.asScala.toList
196 |
197 | val actual =
198 | getNoneZeroHistogramLabels(prom,
199 | "greenish_state_refresh_time_seconds")
200 | actual shouldBe expected
201 | }
202 | }
203 | }
204 |
205 | object StatsCollectorSpec extends Matchers {
206 | def getNoneZeroHistogramLabels(
207 | prom: List[MetricFamilySamples],
208 | name: String): Set[String] =
209 | prom.filter { prom =>
210 | prom.name == name
211 | }.flatMap { metric =>
212 | metric.samples.asScala
213 | .map(sample => (sample.labelValues.asScala, sample.value))
214 | }.filter { case (seq, num) =>
215 | // Only keep what is set
216 | num != 0
217 | }.map { case (seq, num) => seq.head }
218 | .toSet
219 |
220 |
221 | def checkSamples(
222 | prom: List[MetricFamilySamples],
223 | name: String,
224 | expected: Seq[(Seq[String], Double)]): Unit = {
225 |
226 | val actual = prom
227 | .filter { prom =>
228 | prom.name == name
229 | }.flatMap { metric =>
230 | metric.samples.asScala
231 | .map(sample => (sample.labelValues.asScala, sample.value))
232 | }
233 |
234 | actual shouldBe expected
235 | }
236 | }
237 |
--------------------------------------------------------------------------------
/src/main/scala/AppConfig.scala:
--------------------------------------------------------------------------------
1 | package me.amanj.greenish
2 |
3 | import com.typesafe.config.{Config, ConfigFactory}
4 | import java.time.ZoneId
5 | import java.io.File
6 | import scala.util.Try
7 | import models._
8 | import scala.jdk.CollectionConverters._
9 |
10 | case class AppConfig(groups: Seq[Group], namespace: Option[String],
11 | scratchDir: File, refreshInSeconds: Int,
12 | address: String, port: Int,
13 | )
14 | object AppConfig {
15 | def apply(): AppConfig = {
16 | val config = ConfigFactory.load()
17 | val appConfig = config.getConfig("check-groups")
18 | val refreshRate = appConfig.getInt("refresh-in-seconds")
19 | val port = appConfig.getInt("port")
20 | val namespace = Try(appConfig.getString("namespace")).toOption
21 | val scratchDir = new File(appConfig.getString("scratch-dir"))
22 | scratchDir.delete
23 | val address = appConfig.getString("binding-address")
24 | new AppConfig(readEntries(appConfig), namespace, scratchDir,
25 | refreshRate, address, port)
26 | }
27 |
28 | private[this] def readEntries(config: Config): Seq[Group] = {
29 | val defaultOwner = config.getOptionStringWithDefault("default-owner", None)
30 | val defaultPeriodCheckOffset = config.getInt("default-period-check-offset")
31 | val defaultTimePattern = config.getString("default-period-pattern")
32 | val defaultFrequency = config.getString("default-job-run-frequency")
33 | val defaultTimezone = config.getString("default-timezone")
34 | val defaultLookback = config.getInt("default-lookback")
35 | val defaultGreatAt = config.getInt("default-great-at")
36 | val defaultNormalAt = config.getInt("default-normal-at")
37 | val defaultWarnAt = config.getInt("default-warn-at")
38 | val defaultErrorAt = config.getInt("default-error-at")
39 | val defaultStartAt = config.getLong("default-start-at")
40 | val defaultInfo = config.getOptionStringWithDefault("default-info", None)
41 | val globalEnv = config.getEnv("env", Seq.empty)
42 |
43 | config.getConfigList("groups").asScala.zipWithIndex.map { case (groupConfig, index) =>
44 | val groupName = groupConfig.getString("group-name")
45 | val groupOwner = groupConfig.getOptionStringWithDefault("group-owner", defaultOwner)
46 | val groupPeriodCheckOffset =
47 | groupConfig.getIntWithDefault("group-period-check-offset", defaultPeriodCheckOffset)
48 | val groupTimePattern = groupConfig.getStringWithDefault(
49 | "group-period-pattern", defaultTimePattern)
50 | val groupFrequency = groupConfig.getStringWithDefault(
51 | "group-job-run-frequency", defaultFrequency)
52 | val groupTimezone = groupConfig.getStringWithDefault(
53 | "group-timezone", defaultTimezone)
54 | val groupLookback = groupConfig.getIntWithDefault(
55 | "group-lookback", defaultLookback)
56 | val groupGreatAt = groupConfig.getIntWithDefault(
57 | "group-great-at", defaultGreatAt)
58 | val groupNormalAt = groupConfig.getIntWithDefault(
59 | "group-normal-at", defaultNormalAt)
60 | val groupWarnAt = groupConfig.getIntWithDefault(
61 | "group-warn-at", defaultWarnAt)
62 | val groupErrorAt = groupConfig.getIntWithDefault(
63 | "group-error-at", defaultErrorAt)
64 | val groupStartAt = groupConfig.getLongWithDefault(
65 | "group-start-at", defaultStartAt)
66 | val groupInfo = groupConfig.getOptionStringWithDefault("group-info", defaultInfo)
67 | val groupEnv = groupConfig.getEnv("env", globalEnv)
68 |
69 | val checkEntries = groupConfig.getConfigList("job-entries")
70 | .asScala.zipWithIndex.map { case (jobConfig, index) =>
71 | val jobName = jobConfig.getString("job-name")
72 | val jobOwner = jobConfig.getOptionStringWithDefault("job-owner", groupOwner)
73 | val prometheusId = normalizePrometheusId(
74 | jobConfig.getStringWithDefault(
75 | "prometheus-id", s"$groupName $jobName"))
76 | val cmd = jobConfig.getString("check-command")
77 | val jobPeriodCheckOffset = jobConfig.getIntWithDefault(
78 | "job-period-check-offset", groupPeriodCheckOffset)
79 | val timePattern = jobConfig.getStringWithDefault(
80 | "period-pattern", groupTimePattern)
81 | val frequency = toFrequency(
82 | jobConfig.getStringWithDefault(
83 | "job-run-frequency", groupFrequency))
84 | val timezone = ZoneId.of(
85 | jobConfig.getStringWithDefault("timezone", groupTimezone))
86 | val lookback = jobConfig.getIntWithDefault(
87 | "lookback", groupLookback)
88 | val greatAt = jobConfig.getIntWithDefault(
89 | "great-at", groupGreatAt)
90 | val normalAt = jobConfig.getIntWithDefault(
91 | "normal-at", groupNormalAt)
92 | val warnAt = jobConfig.getIntWithDefault(
93 | "warn-at", groupWarnAt)
94 | val errorAt = jobConfig.getIntWithDefault(
95 | "error-at", groupErrorAt)
96 | val startAt = jobConfig.getLongWithDefault(
97 | "start-at", groupStartAt)
98 | val jobInfo = jobConfig.getOptionStringWithDefault("job-info", groupInfo)
99 | val jobEnv = jobConfig.getEnv("env", groupEnv).map {
100 | case (name, value) => EnvVar(name, value)
101 | }
102 |
103 | Job(
104 | index,
105 | jobName,
106 | jobOwner,
107 | prometheusId,
108 | cmd,
109 | timePattern,
110 | frequency,
111 | jobPeriodCheckOffset,
112 | timezone,
113 | lookback,
114 | startAt,
115 | AlertLevels(greatAt, normalAt, warnAt, errorAt),
116 | jobInfo.map(_.stripMargin),
117 | jobEnv,
118 | )
119 | }.toSeq
120 | Group(index, groupName, checkEntries)
121 | }.toSeq
122 | }
123 |
124 | private[greenish] def normalizePrometheusId(id: String): String = {
125 | val spacelessId = id.replaceAll("(\\s|-)+","_").toLowerCase
126 | val pattern = "[a-zA-Z_][a-zA-Z0-9_]*"
127 | if(!spacelessId.matches(pattern)) {
128 | throw new Exception(
129 | s"""|$id: Invalid prometheus label ID, please provide a valid one.
130 | |Prometheus label names should match: "$pattern"""".stripMargin)
131 | }
132 | spacelessId
133 | }
134 |
135 | private[greenish] def toFrequency(freq: String): CheckFrequency = {
136 | freq.toLowerCase match {
137 | case "hourly" => Hourly
138 | case "daily" => Daily
139 | case "monthly" => Monthly
140 | case "annually" => Annually
141 | case _ =>
142 | try {
143 | Cron(freq)
144 | } catch {
145 | case e: IllegalArgumentException =>
146 | throw new Exception(
147 | s"""|${e.getMessage}
148 | |$freq: unsupported frequency, supported frequenices are:
149 | |hourly, daily, monthly, annually and Unix cron syntax"""
150 | .stripMargin)
151 | }
152 | }
153 | }
154 |
155 | implicit class ConfigExt[C <: Config](self: Config) {
156 | def getStringWithDefault(path: String, default: String): String =
157 | if(self.hasPath(path))
158 | self.getString(path)
159 | else default
160 |
161 | def getIntWithDefault(path: String, default: Int): Int =
162 | if(self.hasPath(path))
163 | self.getInt(path)
164 | else default
165 |
166 | def getLongWithDefault(path: String, default: Long): Long =
167 | if(self.hasPath(path))
168 | self.getLong(path)
169 | else default
170 |
171 | def getOptionStringWithDefault(path: String,
172 | default: Option[String]): Option[String] =
173 | if(self.hasPath(path))
174 | Some(self.getString(path))
175 | else default
176 |
177 |
178 | def getEnv(path: String, parent: Seq[(String, String)]): Seq[(String, String)] =
179 | if(self.hasPath(path)) {
180 | val localEnv = self.getConfig("env")
181 | .entrySet.asScala
182 | .map(e => (e.getKey, e.getValue.unwrapped.asInstanceOf[String]))
183 | .toMap
184 |
185 | val overriddenParent = parent.filterNot { case (k, _) =>
186 | localEnv.contains(k)
187 | }
188 |
189 | (localEnv.toSeq ++ overriddenParent).sorted
190 | } else parent
191 | }
192 | }
193 |
--------------------------------------------------------------------------------
/src/test/scala/AppConfigSpec.scala:
--------------------------------------------------------------------------------
1 | package me.amanj.greenish
2 |
3 | import org.scalatest.matchers.should.Matchers
4 | import org.scalatest.wordspec.AnyWordSpecLike
5 | import java.time.ZoneId
6 | import models._
7 | import java.io.File
8 |
9 | class AppConfigSpec() extends Matchers
10 | with AnyWordSpecLike {
11 |
12 | "AppConfig" must {
13 | "read config file correctly" in {
14 | val actual = AppConfig()
15 | val expected = new AppConfig(
16 | Seq(
17 | Group(0, "Group1", Seq(
18 | Job(0, "Job1", Some("Data"), "job_1", "/tmp/first_script",
19 | "yyyy-MM-dd-HH", Hourly, 3,
20 | ZoneId.of("UTC"), 24, 2,
21 | AlertLevels(0, 1, 2, 3),
22 | Some("Job info"),
23 | Seq(EnvVar("VAR1", "baz"), EnvVar("VAR2", "bazomba"),
24 | EnvVar("VAR3", "bada"), EnvVar("VAR4", "badam")),
25 | ),
26 | Job(1, "Job2", Some("Reporting"), "job_2", "/tmp/second_script job2",
27 | "yyyy-MM-dd-HH", Daily, 2,
28 | ZoneId.of("UTC"), 24, 1,
29 | AlertLevels(0, 1, 2, 3),
30 | Some("Group info"),
31 | Seq(EnvVar("VAR1", "baz"), EnvVar("VAR2", "secure(bar)"),
32 | EnvVar("VAR3", "bazooka")),
33 | ),
34 | Job(2, "Job5", Some("Reporting"), "group1_job5", "/tmp/second_script job5",
35 | "yyyy-MM-dd-HH", Hourly, 2,
36 | ZoneId.of("US/Alaska"), 24, 1,
37 | AlertLevels(0, 1, 2, 3),
38 | Some("Group info"),
39 | Seq(EnvVar("VAR1", "baz"), EnvVar("VAR2", "secure(bar)"),
40 | EnvVar("VAR3", "bazooka")),
41 | ),
42 | Job(3, "Job7", Some("Reporting"), "group1_job7", "/tmp/second_script job7",
43 | "yyyy-MM-dd-HH", Cron("0 * * * *"), 2,
44 | ZoneId.of("US/Alaska"), 24, 1,
45 | AlertLevels(0, 1, 2, 3),
46 | Some("Group info"),
47 | Seq(EnvVar("VAR1", "baz"), EnvVar("VAR2", "secure(bar)"),
48 | EnvVar("VAR3", "bazooka")),
49 | ),
50 | )),
51 | Group(1, "Group2", Seq(
52 | Job(0, "Job3", Some("SRE"), "job_3", "/tmp/third_script",
53 | "yyyy-MM-dd", Monthly, 1,
54 | ZoneId.of("UTC"), 3, 0,
55 | AlertLevels(0, 1, 2, 3),
56 | Some("""|
57 | |Link
58 | |""".stripMargin),
59 | Seq(EnvVar("VAR1", "foo"), EnvVar("VAR2", "secure(bar)")),
60 | ),
61 | Job(1, "Job4", Some("SRE"), "job_4", "/tmp/fourth_script",
62 | "yyyy-01-01", Annually, 1,
63 | ZoneId.of("UTC"), 3, 0,
64 | AlertLevels(0, 1, 2, 3),
65 | Some("""|
66 | |Link
67 | |""".stripMargin),
68 | Seq(EnvVar("VAR1", "foo"), EnvVar("VAR2", "secure(bar)")),
69 | ),
70 | Job(2, "Job6", Some("SRE"), "group2_job6", "/tmp/second_script job6",
71 | "yyyy-MM-dd-HH-mm", Daily, 1,
72 | ZoneId.of("US/Samoa"), 270, 0,
73 | AlertLevels(30, 40, 50, 60),
74 | Some("""|
75 | |Link
76 | |""".stripMargin),
77 | Seq(EnvVar("VAR1", "baz"), EnvVar("VAR2", "secure(bar)"),
78 | EnvVar("VAR3", "bazooka")),
79 | ),
80 | )),
81 | ),
82 | Some("Test dashboard"),
83 | new File("/tmp/greenish/stdout"),
84 | 30,
85 | "127.0.0.1",
86 | 8080,
87 | )
88 | actual shouldBe expected
89 | }
90 | }
91 |
92 | "toFrequency" must {
93 | import AppConfig.toFrequency
94 | "handle Unix cron syntax" in {
95 | val patterns = Seq(
96 | "* * * * *",
97 | "1-2 * * * *",
98 | "*/5 * * * *",
99 | "0 23 * * MON-FRI",
100 | "1-5 0 * JAN-DEC 0-4",
101 | )
102 | patterns.foreach { pattern =>
103 | toFrequency(pattern) shouldBe Cron(pattern)
104 | }
105 | }
106 |
107 | "handle both lower and upper case frequencies" in {
108 | toFrequency("hOURly") shouldBe Hourly
109 | toFrequency("AnnuaLLy") shouldBe Annually
110 | toFrequency("monthly") shouldBe Monthly
111 | toFrequency("DAILY") shouldBe Daily
112 | }
113 |
114 | "throw an exception when it doesn't recognize a frequency" in {
115 | intercept[Exception](toFrequency("kkk"))
116 | intercept[Exception](toFrequency("weekly"))
117 | intercept[Exception](toFrequency("minutes"))
118 | }
119 | }
120 |
121 | "normalizePrometheusId" must {
122 | import AppConfig.normalizePrometheusId
123 | "convert prometheus_id to all lowercase" in {
124 | normalizePrometheusId("ABC") shouldBe "abc"
125 | }
126 |
127 | "replace - characters in prometheus_id to _" in {
128 | normalizePrometheusId("a---b") shouldBe "a_b"
129 | }
130 |
131 | "replace whitesapce characters in prometheus_id to _" in {
132 | normalizePrometheusId("a b\nc\td\t") shouldBe "a_b_c_d_"
133 | }
134 |
135 | "throw exception when prometheus_id starts with a digit" in {
136 | intercept[Exception](normalizePrometheusId("9a b\nc\td\t"))
137 | }
138 |
139 | "throw exception when prometheus_id contains anything but [a-zA-Z0-9_]" in {
140 | intercept[Exception](normalizePrometheusId("a;a"))
141 | }
142 |
143 | "throw exception when prometheus_id is empty string" in {
144 | intercept[Exception](normalizePrometheusId(""))
145 | }
146 | "accept valid characters in the begining prometheus_id" in {
147 | normalizePrometheusId("a") shouldBe "a"
148 | normalizePrometheusId("A") shouldBe "a"
149 | normalizePrometheusId("_") shouldBe "_"
150 | }
151 | }
152 |
153 | "getIntWithDefault" must {
154 | import com.typesafe.config.ConfigFactory
155 | import AppConfig._
156 | val config = ConfigFactory.load()
157 | val appConfig = config.getConfig("check-groups")
158 | "get what the value of the property if the key exists" in {
159 | val actual = appConfig.getIntWithDefault("default-error-at", 100)
160 | val expected = 60
161 | actual shouldBe expected
162 | }
163 |
164 | "return default value if the key doesn't exists" in {
165 | val actual = appConfig.getIntWithDefault("naaah", 100)
166 | val expected = 100
167 | actual shouldBe expected
168 | }
169 | }
170 |
171 | "getStringWithDefault" must {
172 | import com.typesafe.config.ConfigFactory
173 | import AppConfig._
174 | val config = ConfigFactory.load()
175 | val appConfig = config.getConfig("check-groups")
176 | "get what the value of the property if the key exists" in {
177 | val actual = appConfig.getStringWithDefault("default-period-pattern", "kkkk")
178 | val expected = "yyyy-MM-dd-HH-mm"
179 | actual shouldBe expected
180 | }
181 |
182 | "return default value if the key doesn't exists" in {
183 | val actual = appConfig.getStringWithDefault("naaah", "kkkk")
184 | val expected = "kkkk"
185 | actual shouldBe expected
186 | }
187 | }
188 |
189 | "getEnv" must {
190 | import com.typesafe.config.ConfigFactory
191 | import AppConfig._
192 | val config = ConfigFactory.load()
193 | val appConfig = config.getConfig("check-groups")
194 | val groupConfig = appConfig.getConfigList("groups").iterator.next()
195 | val jobConfig = groupConfig.getConfigList("job-entries").iterator.next()
196 | val appEnv = appConfig.getEnv("env", Seq.empty)
197 |
198 | "get value if parent is empty, and key exists" in {
199 | appEnv shouldBe Seq("VAR1" -> "foo", "VAR2" -> "secure(bar)")
200 | }
201 |
202 | "properly dedup parent and child lists, if key exists" in {
203 | val actualGroup = groupConfig.getEnv("env", appEnv)
204 | val expectedGroup = Seq("VAR1" -> "baz", "VAR2" -> "secure(bar)",
205 | "VAR3" -> "bazooka")
206 |
207 | actualGroup shouldBe expectedGroup
208 |
209 | val actualJob = jobConfig.getEnv("env", expectedGroup)
210 | val expectedJob = Seq("VAR1" -> "baz", "VAR2" -> "bazomba",
211 | "VAR3" -> "bada", "VAR4" -> "badam")
212 |
213 | actualJob shouldBe expectedJob
214 | }
215 |
216 | "return parent env if the key doesn't exists" in {
217 | val actual = appConfig.getEnv("naaah", appEnv)
218 | val expected = appEnv
219 | actual shouldBe expected
220 | }
221 | }
222 | }
223 |
224 |
225 |
--------------------------------------------------------------------------------
/src/main/resources/dashboard/greenish-favicon.svg:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/doc/api.md:
--------------------------------------------------------------------------------
1 | # The API
2 |
3 | ## REST
4 |
5 | Greenish provides a few REST endpoints:
6 |
7 | ### Display the maximum number of missing datasets
8 |
9 | Basically, for all the jobs, find the job that misses the most number of
10 | period datasets, and return the number.
11 |
12 | ```
13 | $ curl --silent -G http://0.0.0.0:8080/maxlag | jq .
14 | {
15 | "lag": 0
16 | }
17 | ```
18 |
19 | ### Summary
20 |
21 | Display the summary of all the monitoring tasks. Very good for a quick glance:
22 |
23 | ```
24 | $ curl --silent -G http://0.0.0.0:8080/summary | jq .
25 | [
26 | {
27 | "group_id": 0,
28 | "name": "Group1",
29 | "status": [
30 | {
31 | "job_id": 0,
32 | "name": "Job1",
33 | "missing": 4,
34 | "oldest_mising_period": 10,
35 | "alert_level": "warn"
36 | },
37 | {
38 | "job_id": 1,
39 | "name": "Job2",
40 | "missing": 2,
41 | "oldest_mising_period": 3,
42 | "alert_level": "normal"
43 | }
44 | ]
45 | },
46 | {
47 | "group_id": 0,
48 | "name": "Group2",
49 | "status": [
50 | {
51 | "job_id": 0,
52 | "name": "Job3",
53 | "missing": 6,
54 | "oldest_mising_period": 6,
55 | "alert_level": "critical"
56 | },
57 | {
58 | "job_id": 1,
59 | "name": "Job4",
60 | "missing": 0,
61 | "oldest_mising_period": 0,
62 | "alert_level": "great"
63 | }
64 | ]
65 | }
66 | ]
67 | ```
68 |
69 | ### Display all the periods that are missing for all the jobs
70 |
71 | ```
72 | $ curl --silent -G http://0.0.0.0:8080/missing | jq .
73 | [
74 | {
75 | "group": {
76 | "group_id": 0,
77 | "name": "Group1",
78 | "jobs": [
79 | {
80 | "job_id": 0,
81 | "name": "Job1",
82 | "cmd": "/tmp/first_script",
83 | "time_pattern": "yyyy-MM-dd-HH",
84 | "frequency": "hourly",
85 | "timezone": {
86 | "zone_id": "UTC"
87 | },
88 | "lookback": 24,
89 | "start_at": 1593093930,
90 | "alert_levels": {
91 | "great": 0,
92 | "normal": 1,
93 | "warn": 2,
94 | "critical": 3
95 | }
96 | env: []
97 | }
98 | ]
99 | },
100 | "status": [
101 | {
102 | "job": {
103 | "job_id": 0,
104 | "name": "Job1",
105 | "cmd": "/tmp/first_script",
106 | "time_pattern": "yyyy-MM-dd-HH",
107 | "frequency": "hourly",
108 | "timezone": {
109 | "zone_id": "UTC"
110 | },
111 | "lookback": 24,
112 | "start_at": 1593093930,
113 | "alert_levels": {
114 | "great": 0,
115 | "normal": 1,
116 | "warn": 2,
117 | "critical": 3
118 | }
119 | env: []
120 | },
121 | "updated_at": 1593567901,
122 | "period_health": [
123 | {
124 | "period": "2020-06-27-20",
125 | "ok": false
126 | }
127 |
128 | ...
129 | ```
130 |
131 | ### Display the current state
132 |
133 | A very detailed view for all monitoring tasks:
134 |
135 | ```
136 | $ curl --silent -G http://0.0.0.0:8080/state | jq .
137 | [
138 | {
139 | "group": {
140 | "group_id": 0,
141 | "name": "Group1",
142 | "jobs": [
143 | {
144 | "job_id": 0,
145 | "name": "Job1",
146 | "owner": "Reporting Team"
147 | "cmd": "/tmp/first_script",
148 | "time_pattern": "yyyy-MM-dd-HH",
149 | "frequency": "hourly",
150 | "timezone": {
151 | "zone_id": "UTC"
152 | },
153 | "lookback": 24,
154 | "start_at": 1593093930,
155 | "alert_levels": {
156 | "great": 0,
157 | "normal": 1,
158 | "warn": 2,
159 | "critical": 3
160 | }
161 | env: []
162 | }
163 | ]
164 | },
165 | "status": [
166 | {
167 | "job": {
168 | "job_id": 0,
169 | "name": "Job1",
170 | "cmd": "/tmp/first_script",
171 | "time_pattern": "yyyy-MM-dd-HH",
172 | "frequency": "hourly",
173 | "timezone": {
174 | "zone_id": "UTC"
175 | },
176 | "lookback": 24,
177 | "start_at": 1593093930,
178 | "alert_levels": {
179 | "great": 0,
180 | "normal": 1,
181 | "warn": 2,
182 | "critical": 3
183 | }
184 | env: []
185 | },
186 | "updated_at": 1593567901,
187 | "period_health": [
188 | {
189 | "period": "2020-06-27-20",
190 | "ok": true
191 | },
192 | {
193 | "period": "2020-06-27-21",
194 | "ok": true
195 | },
196 |
197 | ...
198 | ```
199 |
200 | ### Get job and group by id
201 |
202 | You can query a single group by its id:
203 |
204 | ```
205 | $ curl --silent -G localhost:8080/group/1 | jq .
206 | {
207 | "group": {
208 | "group_id": 1,
209 | "name": "Group2",
210 | "jobs": [
211 | {
212 | "job_id": 0,
213 | "name": "Job3",
214 | "cmd": "/tmp/third_script",
215 | "time_pattern": "yyyy-MM-dd",
216 | "frequency": "monthly",
217 | "timezone": {
218 | ...
219 | ```
220 |
221 | You can also focus on a single job, and query it:
222 |
223 | ```
224 | $ curl --silent -G localhost:8080/group/1/job/0 | jq .
225 | {
226 | "job": {
227 | "job_id": 0,
228 | "name": "Job3",
229 | "cmd": "/tmp/third_script",
230 | "time_pattern": "yyyy-MM-dd",
231 | "frequency": "monthly",
232 | "timezone": {
233 | "zone_id": "UTC"
234 | },
235 | "lookback": 3,
236 | "start_at": 1593093930,
237 | "alert_levels": {
238 | "great": 0,
239 | "normal": 1,
240 | "warn": 2,
241 | "critical": 3
242 | }
243 | env: []
244 | },
245 | "updated_at": 1593585049298,
246 | "period_health": [
247 | {
248 | "period": "2020-05-01",
249 | "ok": true
250 | },
251 | {
252 | "period": "2020-06-01",
253 | "ok": true
254 | },
255 | {
256 | "period": "2020-07-01",
257 | "ok": true
258 | }
259 | ]
260 | }
261 | ```
262 |
263 | Finally, you can alos get the output of a check (monitoring script) for a job:
264 |
265 | ```
266 | $ curl --silent -G localhost:8080/group/0/job/0/stdout
267 | ```
268 |
269 | This is useful when debugging the monitoring script, or if there are further
270 | information in the stdout that is not expressed in the UI/API.
271 |
272 |
273 | ### Refresh the state
274 |
275 | You can refresh the entire at once:
276 |
277 | ```
278 | $ curl --silent -G localhost:8080/state/refresh | jq .
279 | {
280 | "ok": "State refresh is scheduled"
281 | }
282 | ```
283 |
284 | You can point refresh the state of a single group by its id:
285 |
286 | ```
287 | $ curl --silent -G localhost:8080/group/0/refresh | jq .
288 | {
289 | "ok": "Group status refresh is scheduled"
290 | }
291 | ```
292 |
293 | You can also point refresh the state of a single job by its id:
294 |
295 | ```
296 | $ curl --silent -G localhost:8080/group/0/job/0/refresh | jq .
297 | {
298 | "ok": "Job status refresh is scheduled"
299 | }
300 | ```
301 | ### Health-check
302 |
303 | Checks if any of the last 5 state refreshes succeeded, if yes, then it is
304 | considered a good health.
305 |
306 | ```
307 | $ curl --silent -G http://0.0.0.0:8080/health | jq .
308 | {
309 | "health": "good"
310 | }
311 | ```
312 |
313 | ### System info
314 |
315 | Print basic information about the service.
316 |
317 | ```
318 | $ curl --silent -G localhost:8080/system | jq .
319 | {
320 | "service": "Greenish",
321 | "namespace": "Staging",
322 | "version": "1.4.0-SNAPSHOT",
323 | "uptime": 1784338
324 | }
325 | ```
326 |
327 | ## Prometheus
328 |
329 | Greenish can also export data to Prometheus. These are the supported metrics:
330 |
331 | ```
332 | TYPE: GAUGE
333 | NAME: greenish_active_refresh_tasks
334 | HELP: Current number active state refresh tasks
335 | LABELS: job_id
336 |
337 | TYPE: HISTOGRAM
338 | NAME: greenish_state_refresh_time_seconds
339 | HELP: Job state refreshing time
340 | LABELS: job_id
341 |
342 | TYPE: COUNTER
343 | NAME: greenish_state_refresh_total
344 | HELP: Total number of job state refresh instances
345 | LABELS: job_id
346 |
347 | TYPE: COUNTER
348 | NAME: greenish_state_refresh_expired_total
349 | HELP: Total number of expired job state refresh instances
350 | LABELS: job_id
351 |
352 | TYPE: COUNTER
353 | NAME: greenish_state_refresh_failed_total
354 | HELP: Total number of failed job state refresh instances
355 | LABELS: job_id
356 |
357 | TYPE: GAUGE
358 | NAME: greenish_missing_periods_total
359 | HELP: Current number of missing dataset periods
360 | LABELS: job_id
361 |
362 | TYPE: GAUGE
363 | NAME: greenish_oldest_missing_period
364 | HELP: The oldest missing period
365 | LABELS: job_id
366 |
367 | ```
368 |
369 | Prometheus metrics can be accessed at `/prometheus` endpoint:
370 |
371 | ```
372 | $ curl --silent -G localhost:8080/prometheus
373 | # HELP greenish_active_refresh_tasks Current number active state refresh tasks
374 | # TYPE greenish_active_refresh_tasks gauge
375 | greenish_active_refresh_tasks{job_id="job_2",} 1.0
376 | greenish_active_refresh_tasks{job_id="job_1",} 0.0
377 | greenish_active_refresh_tasks{job_id="job_4",} 1.0
378 | greenish_active_refresh_tasks{job_id="job_3",} 1.0
379 | ...
380 | ```
381 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
73 |
80 |
81 |
104 |
105 | )
106 | })
107 | }
108 | 74 | `${loc.pathname}?page=group&gid=${gid}`} 75 | onClick={() => {this.handleGroupClick(gid)}} className="link"> 76 | {group.name} 77 | 78 |
79 || Job name | 83 |# Missing data sets | 84 |
|---|---|
| 91 | `${loc.pathname}?page=job&gid=${gid}&jid=${jid}`} 92 | onClick={() => {this.handleJobClick(gid, jid)}} className="link"> 93 | {job.name} 94 | 95 | | 96 |{job.missing} | 98 |
2 |
3 | [](https://travis-ci.org/amanjpro/greenish)
4 | [](https://codecov.io/gh/amanjpro/greenish) [](https://gitter.im/greenish-monitoring/greenish?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
5 |
6 | **Greenish** is a monitoring tool that checks datasets for existence.
7 |
8 | Greenish understands _periods;_ for example, for an hourly job, it can
9 | verify that all datasets for the past _N_ hours exist.
10 |
11 | Configuration files use the [HOCON] syntax (a superset of [JSON];
12 | similar to [YAML]):
13 |
14 | * [annotated example](src/test/resources/application.conf);
15 | * [default values](src/main/resources/reference.conf).
16 |
17 | [HOCON]: https://github.com/lightbend/config/blob/master/HOCON.md
18 | [JSON]: https://en.wikipedia.org/wiki/JSON
19 | [YAML]: https://en.wikipedia.org/wiki/YAML
20 |
21 |
22 | Greenish runs [monitoring jobs] to collect information about which
23 | datasets are available and which are missing. Those are individual
24 | scripts that can be written in any language.
25 |
26 | [monitoring jobs]: (#monitoring-jobs)
27 |
28 |
29 | ## Greenish dashboard
30 |
31 | Greenish provides a basic HTML dashboard to visualise the state of the
32 | monitored jobs. The dashboard can be accessed at `/dashboard`.
33 |
34 | Here is a screenshot:
35 |
36 | 
37 |
38 | ## API
39 |
40 | [The Greenish API is documented in `api.md`.](doc/api.md)
41 |
42 | ## Who uses Greenish?
43 |
44 | Greenish is still new. As of now, [Samsung
45 | Ads](https://www.samsung.com/us/business/samsungads/) uses Greenish to monitor
46 | _business-critical datasets_.
47 |
48 | ## Greenish vs others
49 |
50 | * **Nagios** is a monitoring tool for systems, network and
51 | infrastructure. It is very good to keep track of the instantaneous
52 | state of a system. But it has no notion of datasets that follow a
53 | periodic pattern (e.g., daily jobs or hourly jobs). Making Nagios
54 | aware of periods is entirely on the shoulder of the check writers,
55 | which can be very tricky to do (or even impossible?).
56 |
57 | * **Prometheus** is another great tool for monitoring metrics, and the
58 | health of other systems, but again it doesn't know about datasets
59 | that follow periodic patterns. It is worth mentioning that Greenish
60 | provides an endpoint to export metrics to Prometheus.
61 |
62 | * **Airflow** knows about periods, but it is not a monitoring
63 | tool. Airflow can alert when a run fails, but if an existing dataset
64 | gets deleted accidentally, Airflow stays unaware.
65 |
66 | What sets Greenish apart is that it knows about periods, and keeps checking
67 | datasets for existence.
68 |
69 | ## Monitoring Jobs
70 |
71 | As mentioned earlier, monitoring scripts are stand-alone programs,
72 | written in any language, that respect the following contract:
73 |
74 | * The scripts must be executable.
75 |
76 | * The scripts must accept an arbitrary number of `period` arguments at
77 | the end of their parameter list; e.g., for a script named
78 | `monitor-foo`, running on the `staging` environment, asked to check
79 | the status of three hourly periods:
80 |
81 | ```shell
82 | monitor-foo staging 2020-20-06-10 2020-20-06-11 2020-20-06-12
83 | ```
84 |
85 | The `check-command` entry for the example above could be:
86 |
87 | ```yaml
88 | check-command: "monitor-foo staging"
89 | period-pattern: "yyyy-MM-dd-HH"
90 | ```
91 |
92 | - The scripts must print one diagnostic line per provided period in
93 | one of the following two formats, where `1` indicates a successful
94 | period, and `0` indicates a failed period:
95 |
96 | ```text
97 | greenish-period