├── notifiers ├── notifier.go ├── slack-notifier_test.go ├── mock_Notifier.go └── slack-notifier.go ├── .gitignore ├── .travis.yml ├── checks ├── checks_test.go ├── check-suspended.go ├── checks.go ├── check-suspended_test.go ├── check-min-healthy.go ├── check-min-instances.go ├── check-min-healthy_test.go └── check-min-instances_test.go ├── glide.yaml ├── mock_Checker_test.go ├── marathon.json.conf ├── Makefile ├── glide.lock ├── app-checker_test.go ├── routes ├── routes.go └── routes_test.go ├── app-checker.go ├── main.go ├── alert-manager.go ├── README.md ├── alert-manager_test.go ├── LICENSE └── mock_Marathon_test.go /notifiers/notifier.go: -------------------------------------------------------------------------------- 1 | package notifiers 2 | 3 | import "github.com/ashwanthkumar/marathon-alerts/checks" 4 | 5 | type Notifier interface { 6 | Notify(check checks.AppCheck) 7 | Name() string 8 | } 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | marathon-alerts 2 | marathon-alerts-* 3 | .DS_Store 4 | vendor/ 5 | PID 6 | cover.out 7 | coverage.html 8 | coverage.txt 9 | cov_total.out 10 | checks.txt 11 | main.txt 12 | notifiers.txt 13 | routes.txt 14 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.8.x 5 | - 1.7.x 6 | - 1.6.x 7 | 8 | # Install glide 9 | addons: 10 | apt: 11 | sources: 12 | - sourceline: 'ppa:masterminds/glide' 13 | packages: 14 | - glide 15 | 16 | install: 17 | - make setup 18 | 19 | script: 20 | - make test-ci 21 | - make build-all 22 | 23 | -------------------------------------------------------------------------------- /checks/checks_test.go: -------------------------------------------------------------------------------- 1 | package checks 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestResultToString(t *testing.T) { 10 | assert.Equal(t, "Passed", CheckStatusToString(Pass)) 11 | assert.Equal(t, "Warning", CheckStatusToString(Warning)) 12 | assert.Equal(t, "Critical", CheckStatusToString(Critical)) 13 | assert.Equal(t, "Resolved", CheckStatusToString(Resolved)) 14 | assert.Equal(t, "Unknown", CheckStatusToString(127)) 15 | } 16 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: . 2 | import: 3 | - package: github.com/ashwanthkumar/golang-utils 4 | subpackages: 5 | - maps 6 | - sets 7 | - package: github.com/ashwanthkumar/slack-go-webhook 8 | - package: github.com/gambol99/go-marathon 9 | - package: github.com/spf13/pflag 10 | - package: github.com/stretchr/testify 11 | subpackages: 12 | - assert 13 | - mock 14 | - package: github.com/rcrowley/go-metrics 15 | - package: github.com/ryanuber/go-glob 16 | - package: github.com/wadey/gocovmerge 17 | -------------------------------------------------------------------------------- /notifiers/slack-notifier_test.go: -------------------------------------------------------------------------------- 1 | package notifiers 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/ashwanthkumar/marathon-alerts/checks" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestResultToColor(t *testing.T) { 11 | slack := Slack{} 12 | assert.Equal(t, "good", *slack.resultToColor(checks.Pass)) 13 | assert.Equal(t, "good", *slack.resultToColor(checks.Resolved)) 14 | assert.Equal(t, "warning", *slack.resultToColor(checks.Warning)) 15 | assert.Equal(t, "danger", *slack.resultToColor(checks.Critical)) 16 | assert.Equal(t, "black", *slack.resultToColor(127)) 17 | } 18 | -------------------------------------------------------------------------------- /notifiers/mock_Notifier.go: -------------------------------------------------------------------------------- 1 | package notifiers 2 | 3 | import "github.com/stretchr/testify/mock" 4 | 5 | import "github.com/ashwanthkumar/marathon-alerts/checks" 6 | 7 | type MockNotifier struct { 8 | mock.Mock 9 | } 10 | 11 | // Notify provides a mock function with given fields: check 12 | func (_m *MockNotifier) Notify(check checks.AppCheck) { 13 | _m.Called(check) 14 | } 15 | 16 | // Name provides a mock function with given fields: 17 | func (_m *MockNotifier) Name() string { 18 | ret := _m.Called() 19 | 20 | var r0 string 21 | if rf, ok := ret.Get(0).(func() string); ok { 22 | r0 = rf() 23 | } else { 24 | r0 = ret.Get(0).(string) 25 | } 26 | 27 | return r0 28 | } 29 | -------------------------------------------------------------------------------- /checks/check-suspended.go: -------------------------------------------------------------------------------- 1 | package checks 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/gambol99/go-marathon" 8 | ) 9 | 10 | type SuspendedCheck struct{} 11 | 12 | func (s *SuspendedCheck) Name() string { 13 | return "suspended" 14 | } 15 | 16 | func (n *SuspendedCheck) Check(app marathon.Application) AppCheck { 17 | var result CheckStatus 18 | var message string 19 | if app.Instances == 0 { 20 | result = Critical 21 | message = fmt.Sprintf("%s is suspended.", app.ID) 22 | } else { 23 | result = Pass 24 | message = fmt.Sprintf("%s is not suspended.", app.ID) 25 | } 26 | return AppCheck{ 27 | App: app.ID, 28 | Labels: app.Labels, 29 | CheckName: n.Name(), 30 | Result: result, 31 | Message: message, 32 | Timestamp: time.Now(), 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /mock_Checker_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/ashwanthkumar/marathon-alerts/checks" 5 | "github.com/stretchr/testify/mock" 6 | ) 7 | 8 | import "github.com/gambol99/go-marathon" 9 | 10 | type MockChecker struct { 11 | mock.Mock 12 | } 13 | 14 | // Name provides a mock function with given fields: 15 | func (_m *MockChecker) Name() string { 16 | ret := _m.Called() 17 | 18 | var r0 string 19 | if rf, ok := ret.Get(0).(func() string); ok { 20 | r0 = rf() 21 | } else { 22 | r0 = ret.Get(0).(string) 23 | } 24 | 25 | return r0 26 | } 27 | 28 | // Check provides a mock function with given fields: _a0 29 | func (_m *MockChecker) Check(_a0 marathon.Application) checks.AppCheck { 30 | ret := _m.Called(_a0) 31 | 32 | var r0 checks.AppCheck 33 | if rf, ok := ret.Get(0).(func(marathon.Application) checks.AppCheck); ok { 34 | r0 = rf(_a0) 35 | } else { 36 | r0 = ret.Get(0).(checks.AppCheck) 37 | } 38 | 39 | return r0 40 | } 41 | -------------------------------------------------------------------------------- /checks/checks.go: -------------------------------------------------------------------------------- 1 | package checks 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/gambol99/go-marathon" 7 | ) 8 | 9 | type AppCheck struct { 10 | App string 11 | CheckName string 12 | Result CheckStatus 13 | Message string 14 | Timestamp time.Time 15 | Labels map[string]string 16 | Times int 17 | } 18 | 19 | type Checker interface { 20 | Name() string 21 | Check(marathon.Application) AppCheck 22 | } 23 | 24 | type CheckStatus uint8 25 | 26 | const ( 27 | Pass = CheckStatus(99) 28 | Resolved = CheckStatus(98) 29 | Warning = CheckStatus(2) 30 | Critical = CheckStatus(1) 31 | ) 32 | 33 | var CheckLevels = [...]CheckStatus{Warning, Critical} 34 | 35 | func CheckStatusToString(result CheckStatus) string { 36 | value := "Unknown" 37 | switch result { 38 | case Pass: 39 | value = "Passed" 40 | case Resolved: 41 | value = "Resolved" 42 | case Warning: 43 | value = "Warning" 44 | case Critical: 45 | value = "Critical" 46 | } 47 | 48 | return value 49 | } 50 | -------------------------------------------------------------------------------- /checks/check-suspended_test.go: -------------------------------------------------------------------------------- 1 | package checks 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/gambol99/go-marathon" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestSuspendedCheckWhenEverythingIsFine(t *testing.T) { 11 | check := SuspendedCheck{} 12 | app := marathon.Application{ 13 | ID: "/foo", 14 | Instances: 10, 15 | } 16 | 17 | appCheck := check.Check(app) 18 | assert.Equal(t, Pass, appCheck.Result) 19 | assert.Equal(t, "suspended", appCheck.CheckName) 20 | assert.Equal(t, "/foo", appCheck.App) 21 | assert.Equal(t, "/foo is not suspended.", appCheck.Message) 22 | } 23 | 24 | func TestSuspendedCheckWhenAppIsSuspended(t *testing.T) { 25 | check := SuspendedCheck{} 26 | app := marathon.Application{ 27 | ID: "/foo", 28 | Instances: 0, 29 | } 30 | 31 | appCheck := check.Check(app) 32 | assert.Equal(t, Critical, appCheck.Result) 33 | assert.Equal(t, "suspended", appCheck.CheckName) 34 | assert.Equal(t, "/foo", appCheck.App) 35 | assert.Equal(t, "/foo is suspended.", appCheck.Message) 36 | } 37 | -------------------------------------------------------------------------------- /marathon.json.conf: -------------------------------------------------------------------------------- 1 | { 2 | "id": "{{ .DEPLOY_ENV }}.malerts", 3 | "cpus": 0.1, 4 | "mem": 64.0, 5 | "instances": 1, 6 | "backoffSeconds": 1, 7 | "backoffFactor": 1.01, 8 | "maxLaunchDelaySeconds": 30, 9 | "ports": [], 10 | "cmd": "chmod +x marathon-alerts-linux-amd64 && ./marathon-alerts-linux-amd64 --uri ${MARATHON_URI} --slack-webhook ${SLACK_WEBHOOK} --pid PID", 11 | "uris": [ 12 | "https://github.com/ashwanthkumar/marathon-alerts/releases/download/v0.3.5/marathon-alerts-linux-amd64" 13 | ], 14 | "upgradeStrategy": { 15 | "minimumHealthCapacity": 0.9, 16 | "maximumOverCapacity": 0.1 17 | }, 18 | "env": { 19 | "MARATHON_URI": "{{ .Env.MARATHON_URI }}", 20 | "SLACK_WEBHOOK": "{{ .Env.SLACK_WEBHOOK }}" 21 | }, 22 | "healthChecks": [ 23 | { 24 | "protocol": "COMMAND", 25 | "command": { "value": "ps -p $(cat PID) > /dev/null" }, 26 | "gracePeriodSeconds": 240, 27 | "intervalSeconds": 60, 28 | "maxConsecutiveFailures": 3, 29 | "timeoutSeconds": 20 30 | } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | APPNAME = marathon-alerts 2 | VERSION=0.0.1-dev 3 | TESTFLAGS=-v -cover -covermode=atomic -bench=. 4 | TEST_COVERAGE_THRESHOLD=55.0 5 | 6 | build: 7 | go build -tags netgo -ldflags "-w" -o ${APPNAME} . 8 | 9 | build-linux: 10 | GOOS=linux GOARCH=amd64 go build -tags netgo -ldflags "-w -s -X main.APP_VERSION=${VERSION}" -v -o ${APPNAME}-linux-amd64 . 11 | shasum -a256 ${APPNAME}-linux-amd64 12 | 13 | build-mac: 14 | GOOS=darwin GOARCH=amd64 go build -tags netgo -ldflags "-w -s -X main.APP_VERSION=${VERSION}" -v -o ${APPNAME}-darwin-amd64 . 15 | shasum -a256 ${APPNAME}-darwin-amd64 16 | 17 | build-all: build-mac build-linux 18 | 19 | all: setup 20 | build 21 | install 22 | 23 | setup: 24 | go get github.com/wadey/gocovmerge 25 | glide install 26 | 27 | test-only: 28 | go test ${TESTFLAGS} github.com/ashwanthkumar/marathon-alerts/${name} 29 | 30 | test: 31 | go test ${TESTFLAGS} -coverprofile=main.txt github.com/ashwanthkumar/marathon-alerts/ 32 | go test ${TESTFLAGS} -coverprofile=checks.txt github.com/ashwanthkumar/marathon-alerts/checks 33 | go test ${TESTFLAGS} -coverprofile=notifiers.txt github.com/ashwanthkumar/marathon-alerts/notifiers 34 | go test ${TESTFLAGS} -coverprofile=routes.txt github.com/ashwanthkumar/marathon-alerts/routes 35 | 36 | test-ci: test 37 | gocovmerge *.txt > coverage.txt 38 | @go tool cover -html=coverage.txt -o coverage.html 39 | @go tool cover -func=coverage.txt | grep "total:" | awk '{print $$3}' | sed -e 's/%//' > cov_total.out 40 | @bash -c 'COVERAGE=$$(cat cov_total.out); \ 41 | echo "Current Coverage % is $$COVERAGE, expected is ${TEST_COVERAGE_THRESHOLD}."; \ 42 | exit $$(echo $$COVERAGE"<${TEST_COVERAGE_THRESHOLD}" | bc -l)' 43 | -------------------------------------------------------------------------------- /checks/check-min-healthy.go: -------------------------------------------------------------------------------- 1 | package checks 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | maps "github.com/ashwanthkumar/golang-utils/maps" 8 | "github.com/gambol99/go-marathon" 9 | ) 10 | 11 | // Checks for minimum healthy instances of an app running with respect to total # of instances that is 12 | // supposed to run 13 | type MinHealthyTasks struct { 14 | // DefaultWarningThreshold - overriden using alerts.min-instances.warning 15 | DefaultWarningThreshold float32 16 | // DefaultCriticalThreshold - overriden using alerts.min-instances.fail 17 | DefaultCriticalThreshold float32 18 | } 19 | 20 | func (n *MinHealthyTasks) Name() string { 21 | return "min-healthy" 22 | } 23 | 24 | func (n *MinHealthyTasks) Check(app marathon.Application) AppCheck { 25 | failThreshold := maps.GetFloat32(app.Labels, "alerts.min-healthy.critical.threshold", n.DefaultCriticalThreshold) 26 | warnThreshold := maps.GetFloat32(app.Labels, "alerts.min-healthy.warn.threshold", n.DefaultWarningThreshold) 27 | result := Pass 28 | currentlyRunning := float32(app.TasksHealthy) 29 | message := fmt.Sprintf("Only %d are healthy out of total %d", int(currentlyRunning), app.Instances) 30 | 31 | if currentlyRunning == 0.0 && app.Instances > 0 { 32 | result = Critical 33 | } else if currentlyRunning > 0.0 && currentlyRunning < failThreshold*float32(app.Instances) { 34 | result = Critical 35 | } else if currentlyRunning < warnThreshold*float32(app.Instances) { 36 | result = Warning 37 | } else { 38 | message = fmt.Sprintf("We now have %d healthy out of total %d", app.TasksHealthy, app.Instances) 39 | } 40 | 41 | return AppCheck{ 42 | App: app.ID, 43 | Labels: app.Labels, 44 | CheckName: n.Name(), 45 | Result: result, 46 | Message: message, 47 | Timestamp: time.Now(), 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /checks/check-min-instances.go: -------------------------------------------------------------------------------- 1 | package checks 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | maps "github.com/ashwanthkumar/golang-utils/maps" 8 | "github.com/gambol99/go-marathon" 9 | ) 10 | 11 | // Checks for minimum instances of an app running with respect to total # of instances that is 12 | // supposed to run 13 | type MinInstances struct { 14 | // DefaultWarningThreshold - overriden using alerts.min-instances.warning 15 | DefaultWarningThreshold float32 16 | // DefaultCriticalThreshold - overriden using alerts.min-instances.fail 17 | DefaultCriticalThreshold float32 18 | } 19 | 20 | func (n *MinInstances) Name() string { 21 | return "min-instances" 22 | } 23 | 24 | func (n *MinInstances) Check(app marathon.Application) AppCheck { 25 | failThreshold := maps.GetFloat32(app.Labels, "alerts.min-instances.critical.threshold", n.DefaultCriticalThreshold) 26 | warnThreshold := maps.GetFloat32(app.Labels, "alerts.min-instances.warn.threshold", n.DefaultWarningThreshold) 27 | result := Pass 28 | currentlyRunning := float32(app.TasksHealthy + app.TasksStaged) 29 | message := fmt.Sprintf("Only %d are healthy out of total %d", int(currentlyRunning), app.Instances) 30 | 31 | if currentlyRunning == 0.0 && app.Instances > 0 { 32 | result = Critical 33 | } else if currentlyRunning > 0.0 && currentlyRunning < failThreshold*float32(app.Instances) { 34 | result = Critical 35 | } else if currentlyRunning < warnThreshold*float32(app.Instances) { 36 | result = Warning 37 | } else { 38 | message = fmt.Sprintf("We now have %d healthy out of total %d", app.TasksHealthy, app.Instances) 39 | } 40 | 41 | return AppCheck{ 42 | App: app.ID, 43 | Labels: app.Labels, 44 | CheckName: n.Name(), 45 | Result: result, 46 | Message: message, 47 | Timestamp: time.Now(), 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /glide.lock: -------------------------------------------------------------------------------- 1 | hash: d6f3fa685c83d156955e0105f284124288e0a8dc2e4a4e2d017a622517d6c7ad 2 | updated: 2016-03-10T15:36:04.179319521+05:30 3 | imports: 4 | - name: github.com/ashwanthkumar/golang-utils 5 | version: 6362d7ff76b62be7ac4ef53aad5812791b390974 6 | subpackages: 7 | - maps 8 | - sets 9 | - name: github.com/ashwanthkumar/slack-go-webhook 10 | version: 8716eb370d7fc459f4ad13035cdfba3fe36fab30 11 | - name: github.com/davecgh/go-spew 12 | version: 5215b55f46b2b919f50a1df0eaa5886afe4e3b3d 13 | subpackages: 14 | - spew 15 | - name: github.com/donovanhide/eventsource 16 | version: c3f57f280ec708df24886d9e62f2fd178d69d8e8 17 | - name: github.com/gambol99/go-marathon 18 | version: de3f7b297e825ac61224ad3a3c0b88ba814a23cb 19 | - name: github.com/google/go-querystring 20 | version: 6bb77fe6f42b85397288d4f6f67ac72f8f400ee7 21 | subpackages: 22 | - query 23 | - name: github.com/moul/http2curl 24 | version: 1812aee76a1ce98d604a44200c6a23c689b17a89 25 | - name: github.com/parnurzeal/gorequest 26 | version: c73179dd31355d86bd7692de9b47623d6d0fa696 27 | - name: github.com/pmezard/go-difflib 28 | version: d8ed2627bdf02c080bf22230dbb337003b7aba2d 29 | subpackages: 30 | - difflib 31 | - name: github.com/rcrowley/go-metrics 32 | version: eeba7bd0dd01ace6e690fa833b3f22aaec29af43 33 | - name: github.com/ryanuber/go-glob 34 | version: 572520ed46dbddaed19ea3d9541bdd0494163693 35 | - name: github.com/spf13/pflag 36 | version: 7f60f83a2c81bc3c3c0d5297f61ddfa68da9d3b7 37 | - name: github.com/stretchr/objx 38 | version: cbeaeb16a013161a98496fad62933b1d21786672 39 | - name: github.com/stretchr/testify 40 | version: 6fe211e493929a8aac0469b93f28b1d0688a9a3a 41 | subpackages: 42 | - assert 43 | - mock 44 | - http 45 | - name: github.com/wadey/gocovmerge 46 | version: c01c9239511539a811674313e55794e950f3c7ab 47 | - name: golang.org/x/net 48 | version: a4bbce9fcae005b22ae5443f6af064d80a6f5a55 49 | subpackages: 50 | - publicsuffix 51 | - name: golang.org/x/tools 52 | version: 093d7650abf0a26882924de03a6c840a00356572 53 | subpackages: 54 | - cover 55 | devImports: [] 56 | -------------------------------------------------------------------------------- /app-checker_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/url" 5 | "testing" 6 | "time" 7 | 8 | "github.com/ashwanthkumar/marathon-alerts/checks" 9 | marathon "github.com/gambol99/go-marathon" 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/mock" 12 | ) 13 | 14 | func TestProcessCheckForAllSubscribers(t *testing.T) { 15 | appLabels := make(map[string]string) 16 | client := new(MockMarathon) 17 | apps := marathon.Applications{ 18 | Apps: []marathon.Application{marathon.Application{Labels: appLabels}}, 19 | } 20 | var urlValues url.Values 21 | client.On("Applications", urlValues).Return(&apps, nil) 22 | 23 | alertChan := make(chan checks.AppCheck, 1) 24 | check, now := CreateMockChecker(appLabels) 25 | 26 | appChecker := AppChecker{ 27 | Client: client, 28 | AlertsChannel: alertChan, 29 | Checks: []checks.Checker{check}, 30 | } 31 | 32 | expectedCheck := checks.AppCheck{Result: checks.Critical, Timestamp: now, App: "/foo-app", Labels: appLabels} 33 | err := appChecker.processChecks() 34 | assert.Nil(t, err) 35 | assert.Len(t, alertChan, 1) 36 | actualCheck := <-alertChan 37 | assert.Equal(t, actualCheck, expectedCheck) 38 | } 39 | 40 | func TestProcessCheckForWithNoSubscribers(t *testing.T) { 41 | appLabels := make(map[string]string) 42 | appLabels["alerts.checks.subscribe"] = "check-that-does-not-exist" 43 | client := new(MockMarathon) 44 | apps := marathon.Applications{ 45 | Apps: []marathon.Application{marathon.Application{Labels: appLabels}}, 46 | } 47 | var urlValues url.Values 48 | client.On("Applications", urlValues).Return(&apps, nil) 49 | 50 | alertChan := make(chan checks.AppCheck, 1) 51 | check, _ := CreateMockChecker(appLabels) 52 | 53 | appChecker := AppChecker{ 54 | Client: client, 55 | AlertsChannel: alertChan, 56 | Checks: []checks.Checker{check}, 57 | } 58 | 59 | err := appChecker.processChecks() 60 | assert.Nil(t, err) 61 | assert.Len(t, alertChan, 0) 62 | } 63 | 64 | func CreateMockChecker(appLabels map[string]string) (checks.Checker, time.Time) { 65 | now := time.Now() 66 | check := new(MockChecker) 67 | check.On("Name").Return("mock-check") 68 | check.On("Check", mock.AnythingOfType("Application")).Return(checks.AppCheck{ 69 | Result: checks.Critical, 70 | Timestamp: now, 71 | App: "/foo-app", 72 | Labels: appLabels, 73 | }) 74 | 75 | return check, now 76 | } 77 | -------------------------------------------------------------------------------- /routes/routes.go: -------------------------------------------------------------------------------- 1 | package routes 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/ashwanthkumar/marathon-alerts/checks" 8 | "github.com/ryanuber/go-glob" 9 | ) 10 | 11 | var DefaultRoutes = "*/warning/*;*/critical/*;*/resolved/*" 12 | 13 | // Routes holds the routing information for every checks, alert level combination which Notifier 14 | // should be used. 15 | // Routes are of the form 16 | // `check-name/check-result/notifier-name` 17 | // Ex. min-healthy/warning/slack 18 | // The default Route(s) are of the form 19 | // `*/warning/*` and `*/critical/*` and `*/resolved/*` 20 | type Route struct { 21 | Check string 22 | CheckLevel checks.CheckStatus 23 | Notifier string 24 | } 25 | 26 | func (r *Route) Match(check checks.AppCheck) bool { 27 | nameMatches := glob.Glob(r.Check, check.CheckName) 28 | checkLevelMatches := r.CheckLevel == check.Result 29 | return nameMatches && checkLevelMatches 30 | } 31 | 32 | func (r *Route) MatchNotifier(notifier string) bool { 33 | return glob.Glob(r.Notifier, notifier) 34 | } 35 | 36 | func (r *Route) MatchCheckResult(level checks.CheckStatus) bool { 37 | return r.CheckLevel == level 38 | } 39 | 40 | func ParseRoutes(routes string) ([]Route, error) { 41 | var finalRoutes []Route 42 | routesAsString := strings.Split(routes, ";") 43 | for _, routeAsString := range routesAsString { 44 | if routes != "" && routeAsString == "" { 45 | continue 46 | } 47 | segments := strings.Split(routeAsString, "/") 48 | if len(segments) != 3 { 49 | return nil, fmt.Errorf("Expected 3 parts in %s, separated by `/` but %d found", routeAsString, len(segments)) 50 | } 51 | checkLevel, err := parseCheckLevel(segments[1]) 52 | if err != nil { 53 | return nil, err 54 | } 55 | route := Route{ 56 | Check: segments[0], 57 | CheckLevel: checkLevel, 58 | Notifier: segments[2], 59 | } 60 | finalRoutes = append(finalRoutes, route) 61 | } 62 | return finalRoutes, nil 63 | } 64 | 65 | func parseCheckLevel(checkLevel string) (checks.CheckStatus, error) { 66 | switch strings.ToLower(checkLevel) { 67 | case "warning": 68 | return checks.Warning, nil 69 | case "critical": 70 | return checks.Critical, nil 71 | case "pass": 72 | return checks.Pass, nil 73 | case "resolved": 74 | return checks.Resolved, nil 75 | default: 76 | return checks.Critical, fmt.Errorf("Expected one of warning / critical / pass / resolved but %s found", strings.ToLower(checkLevel)) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /notifiers/slack-notifier.go: -------------------------------------------------------------------------------- 1 | package notifiers 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | maps "github.com/ashwanthkumar/golang-utils/maps" 8 | "github.com/ashwanthkumar/marathon-alerts/checks" 9 | "github.com/ashwanthkumar/slack-go-webhook" 10 | ) 11 | 12 | type Slack struct { 13 | Webhook string 14 | Channel string 15 | Owners string 16 | } 17 | 18 | func (s *Slack) Name() string { 19 | return "slack" 20 | } 21 | 22 | func (s *Slack) Notify(check checks.AppCheck) { 23 | attachment := slack.Attachment{ 24 | Text: &check.Message, 25 | Color: s.resultToColor(check.Result), 26 | } 27 | attachment. 28 | AddField(slack.Field{Title: "App", Value: check.App, Short: true}). 29 | AddField(slack.Field{Title: "Check", Value: check.CheckName, Short: true}). 30 | AddField(slack.Field{Title: "Result", Value: checks.CheckStatusToString(check.Result), Short: true}). 31 | AddField(slack.Field{Title: "Times", Value: fmt.Sprintf("%d", check.Times), Short: true}) 32 | 33 | destination := maps.GetString(check.Labels, "alerts.slack.channel", s.Channel) 34 | 35 | appSpecificOwners := maps.GetString(check.Labels, "alerts.slack.owners", s.Owners) 36 | var owners []string 37 | if appSpecificOwners != "" { 38 | owners = strings.Split(appSpecificOwners, ",") 39 | } else { 40 | owners = []string{"@here"} 41 | } 42 | 43 | alertSuffix := "Please check!" 44 | if check.Result == checks.Resolved { 45 | alertSuffix = "Check Resolved, thanks!" 46 | } else if check.Result == checks.Pass { 47 | alertSuffix = "Check Passed" 48 | } 49 | mainText := fmt.Sprintf("%s, %s", s.parseOwners(owners), alertSuffix) 50 | 51 | payload := slack.Payload(mainText, 52 | "marathon-alerts", 53 | "", 54 | destination, 55 | []slack.Attachment{attachment}) 56 | 57 | webhooks := strings.Split(maps.GetString(check.Labels, "alerts.slack.webhook", s.Webhook), ",") 58 | 59 | for _, webhook := range webhooks { 60 | err := slack.Send(webhook, "", payload) 61 | if err != nil { 62 | fmt.Printf("Unexpected Error - %v", err) 63 | } 64 | } 65 | } 66 | 67 | func (s *Slack) resultToColor(result checks.CheckStatus) *string { 68 | color := "black" 69 | switch { 70 | case checks.Pass == result || checks.Resolved == result: 71 | color = "good" 72 | case checks.Warning == result: 73 | color = "warning" 74 | case checks.Critical == result: 75 | color = "danger" 76 | } 77 | 78 | return &color 79 | } 80 | 81 | func (s *Slack) parseOwners(owners []string) string { 82 | parsedOwners := []string{} 83 | for _, owner := range owners { 84 | if owner != "@here" { 85 | owner = fmt.Sprintf("@%s", owner) 86 | } 87 | parsedOwners = append(parsedOwners, owner) 88 | } 89 | 90 | return strings.Join(parsedOwners, ", ") 91 | } 92 | -------------------------------------------------------------------------------- /app-checker.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "strings" 6 | "sync" 7 | "time" 8 | 9 | maps "github.com/ashwanthkumar/golang-utils/maps" 10 | sets "github.com/ashwanthkumar/golang-utils/sets" 11 | "github.com/ashwanthkumar/marathon-alerts/checks" 12 | marathon "github.com/gambol99/go-marathon" 13 | "github.com/rcrowley/go-metrics" 14 | ) 15 | 16 | const ( 17 | CheckSubscriptionLabel = "alerts.checks.subscribe" 18 | SubscribeAllChecks = "all" 19 | ) 20 | 21 | type AppChecker struct { 22 | Client marathon.Marathon 23 | RunWaitGroup sync.WaitGroup 24 | CheckInterval time.Duration 25 | stopChannel chan bool 26 | Checks []checks.Checker 27 | AlertsChannel chan checks.AppCheck 28 | // Snooze the entire system for some Time 29 | // Useful if we don't want to SPAM the notifications 30 | // when doing maintenance of mesos cluster 31 | // TODO - Enable this feature via API endpoint 32 | IsSnoozed bool 33 | SnoozedAt time.Time 34 | SnoozedFor time.Duration 35 | } 36 | 37 | func (a *AppChecker) Start() { 38 | log.Println("Starting App Checker...") 39 | a.RunWaitGroup.Add(1) 40 | a.stopChannel = make(chan bool) 41 | a.AlertsChannel = make(chan checks.AppCheck) 42 | 43 | a.IsSnoozed = false 44 | 45 | go a.run() 46 | log.Println("App Checker Started.") 47 | log.Printf("App Checker - Checking the status of all the apps every %v\n", a.CheckInterval) 48 | } 49 | 50 | func (a *AppChecker) Stop() { 51 | log.Println("Stopping App Checker...") 52 | close(a.stopChannel) 53 | a.RunWaitGroup.Done() 54 | } 55 | 56 | func (a *AppChecker) run() { 57 | running := true 58 | for running { 59 | select { 60 | case <-time.After(a.CheckInterval): 61 | err := a.processChecks() 62 | if err != nil { 63 | log.Fatalf("Unexpected error - %v\n", err) 64 | } 65 | case <-a.stopChannel: 66 | metrics.GetOrRegisterCounter("apps-checker-stopped", DebugMetricsRegistry).Inc(1) 67 | running = false 68 | } 69 | time.Sleep(1 * time.Second) 70 | } 71 | } 72 | 73 | func (a *AppChecker) processChecks() error { 74 | var apps *marathon.Applications 75 | var err error 76 | metrics.GetOrRegisterTimer("marathon-all-apps-response-time", nil).Time(func() { 77 | apps, err = a.Client.Applications(nil) 78 | }) 79 | metrics.GetOrRegisterCounter("apps-checker-marathon-all-apps-api", DebugMetricsRegistry).Inc(1) 80 | if err != nil { 81 | return err 82 | } 83 | for _, app := range apps.Apps { 84 | checksSubscribed := sets.FromSlice( 85 | strings.Split(maps.GetString(app.Labels, CheckSubscriptionLabel, SubscribeAllChecks), 86 | ",")) 87 | for _, check := range a.Checks { 88 | if checksSubscribed.Contains(check.Name()) || checksSubscribed.Contains(SubscribeAllChecks) { 89 | result := check.Check(app) 90 | a.AlertsChannel <- result 91 | metrics.GetOrRegisterCounter("apps-checker-alerts-sent", DebugMetricsRegistry).Inc(1) 92 | metrics.GetOrRegisterCounter("apps-checker-check-"+check.Name(), DebugMetricsRegistry).Inc(1) 93 | metrics.GetOrRegisterCounter("apps-checker-app-"+app.ID, DebugMetricsRegistry).Inc(1) 94 | metrics.GetOrRegisterCounter("apps-checker-"+app.ID+"-"+check.Name(), DebugMetricsRegistry).Inc(1) 95 | } 96 | } 97 | } 98 | 99 | return nil 100 | } 101 | -------------------------------------------------------------------------------- /checks/check-min-healthy_test.go: -------------------------------------------------------------------------------- 1 | package checks 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/gambol99/go-marathon" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | // === MinHealthyTasks === 11 | func TestMinHealthyTasksWhenEverythingIsFine(t *testing.T) { 12 | check := MinHealthyTasks{ 13 | DefaultCriticalThreshold: 0.5, 14 | DefaultWarningThreshold: 0.6, 15 | } 16 | app := marathon.Application{ 17 | ID: "/foo", 18 | Instances: 100, 19 | TasksHealthy: 100, 20 | } 21 | 22 | appCheck := check.Check(app) 23 | assert.Equal(t, Pass, appCheck.Result) 24 | assert.Equal(t, "min-healthy", appCheck.CheckName) 25 | assert.Equal(t, "/foo", appCheck.App) 26 | assert.Equal(t, "We now have 100 healthy out of total 100", appCheck.Message) 27 | } 28 | 29 | func TestMinHealthyTasksWhenWarningThresholdIsMet(t *testing.T) { 30 | check := MinHealthyTasks{ 31 | DefaultCriticalThreshold: 0.5, 32 | DefaultWarningThreshold: 0.6, 33 | } 34 | app := marathon.Application{ 35 | ID: "/foo", 36 | Instances: 100, 37 | TasksHealthy: 59, 38 | } 39 | 40 | appCheck := check.Check(app) 41 | assert.Equal(t, Warning, appCheck.Result) 42 | assert.Equal(t, "min-healthy", appCheck.CheckName) 43 | assert.Equal(t, "/foo", appCheck.App) 44 | assert.Equal(t, "Only 59 are healthy out of total 100", appCheck.Message) 45 | } 46 | 47 | func TestMinHealthyTasksWhenWarningThresholdIsMetButOverridenFromAppLabels(t *testing.T) { 48 | check := MinHealthyTasks{ 49 | DefaultCriticalThreshold: 0.4, 50 | DefaultWarningThreshold: 0.6, 51 | } 52 | appLabels := make(map[string]string) 53 | appLabels["alerts.min-healthy.warn.threshold"] = "0.5" 54 | app := marathon.Application{ 55 | ID: "/foo", 56 | Instances: 100, 57 | TasksHealthy: 59, 58 | Labels: appLabels, 59 | } 60 | 61 | appCheck := check.Check(app) 62 | assert.Equal(t, Pass, appCheck.Result) 63 | assert.Equal(t, "min-healthy", appCheck.CheckName) 64 | assert.Equal(t, "/foo", appCheck.App) 65 | assert.Equal(t, "We now have 59 healthy out of total 100", appCheck.Message) 66 | } 67 | 68 | func TestMinHealthyTasksWhenFailThresholdIsMet(t *testing.T) { 69 | check := MinHealthyTasks{ 70 | DefaultCriticalThreshold: 0.5, 71 | DefaultWarningThreshold: 0.6, 72 | } 73 | app := marathon.Application{ 74 | ID: "/foo", 75 | Instances: 100, 76 | TasksHealthy: 49, 77 | } 78 | 79 | appCheck := check.Check(app) 80 | assert.Equal(t, Critical, appCheck.Result) 81 | assert.Equal(t, "min-healthy", appCheck.CheckName) 82 | assert.Equal(t, "/foo", appCheck.App) 83 | assert.Equal(t, "Only 49 are healthy out of total 100", appCheck.Message) 84 | } 85 | 86 | func TestMinHealthyTasksWhenFailThresholdIsMetButOverridenFromAppLabels(t *testing.T) { 87 | check := MinHealthyTasks{ 88 | DefaultCriticalThreshold: 0.5, 89 | DefaultWarningThreshold: 0.6, 90 | } 91 | appLabels := make(map[string]string) 92 | appLabels["alerts.min-healthy.critical.threshold"] = "0.4" 93 | app := marathon.Application{ 94 | ID: "/foo", 95 | Instances: 100, 96 | TasksHealthy: 49, 97 | Labels: appLabels, 98 | } 99 | 100 | appCheck := check.Check(app) 101 | assert.Equal(t, Warning, appCheck.Result) 102 | assert.Equal(t, "min-healthy", appCheck.CheckName) 103 | assert.Equal(t, "/foo", appCheck.App) 104 | assert.Equal(t, "Only 49 are healthy out of total 100", appCheck.Message) 105 | } 106 | 107 | func TestMinHealthyTasksWhenNoTasksAreRunning(t *testing.T) { 108 | check := MinHealthyTasks{ 109 | DefaultCriticalThreshold: 0.5, 110 | DefaultWarningThreshold: 0.6, 111 | } 112 | app := marathon.Application{ 113 | ID: "/foo", 114 | Instances: 1, 115 | TasksHealthy: 0, 116 | } 117 | 118 | appCheck := check.Check(app) 119 | assert.Equal(t, Critical, appCheck.Result) 120 | assert.Equal(t, "min-healthy", appCheck.CheckName) 121 | assert.Equal(t, "/foo", appCheck.App) 122 | assert.Equal(t, "Only 0 are healthy out of total 1", appCheck.Message) 123 | } 124 | -------------------------------------------------------------------------------- /checks/check-min-instances_test.go: -------------------------------------------------------------------------------- 1 | package checks 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/gambol99/go-marathon" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | // === MinInstances === 11 | func TestMinInstancesWhenEverythingIsFine(t *testing.T) { 12 | check := MinInstances{ 13 | DefaultCriticalThreshold: 0.5, 14 | DefaultWarningThreshold: 0.6, 15 | } 16 | app := marathon.Application{ 17 | ID: "/foo", 18 | Instances: 100, 19 | TasksHealthy: 100, 20 | } 21 | 22 | appCheck := check.Check(app) 23 | assert.Equal(t, Pass, appCheck.Result) 24 | assert.Equal(t, "min-instances", appCheck.CheckName) 25 | assert.Equal(t, "/foo", appCheck.App) 26 | assert.Equal(t, "We now have 100 healthy out of total 100", appCheck.Message) 27 | } 28 | 29 | func TestMinInstancesWhenWarningThresholdIsMet(t *testing.T) { 30 | check := MinInstances{ 31 | DefaultCriticalThreshold: 0.5, 32 | DefaultWarningThreshold: 0.6, 33 | } 34 | app := marathon.Application{ 35 | ID: "/foo", 36 | Instances: 100, 37 | TasksHealthy: 59, 38 | } 39 | 40 | appCheck := check.Check(app) 41 | assert.Equal(t, Warning, appCheck.Result) 42 | assert.Equal(t, "min-instances", appCheck.CheckName) 43 | assert.Equal(t, "/foo", appCheck.App) 44 | assert.Equal(t, "Only 59 are healthy out of total 100", appCheck.Message) 45 | } 46 | 47 | func TestMinInstancesWhenWarningThresholdIsMetButOverridenFromAppLabels(t *testing.T) { 48 | check := MinInstances{ 49 | DefaultCriticalThreshold: 0.4, 50 | DefaultWarningThreshold: 0.6, 51 | } 52 | appLabels := make(map[string]string) 53 | appLabels["alerts.min-instances.warn.threshold"] = "0.5" 54 | app := marathon.Application{ 55 | ID: "/foo", 56 | Instances: 100, 57 | TasksHealthy: 59, 58 | Labels: appLabels, 59 | } 60 | 61 | appCheck := check.Check(app) 62 | assert.Equal(t, Pass, appCheck.Result) 63 | assert.Equal(t, "min-instances", appCheck.CheckName) 64 | assert.Equal(t, "/foo", appCheck.App) 65 | assert.Equal(t, "We now have 59 healthy out of total 100", appCheck.Message) 66 | } 67 | 68 | func TestMinInstancesWhenFailThresholdIsMet(t *testing.T) { 69 | check := MinInstances{ 70 | DefaultCriticalThreshold: 0.5, 71 | DefaultWarningThreshold: 0.6, 72 | } 73 | app := marathon.Application{ 74 | ID: "/foo", 75 | Instances: 100, 76 | TasksHealthy: 47, 77 | TasksStaged: 2, 78 | } 79 | 80 | appCheck := check.Check(app) 81 | assert.Equal(t, Critical, appCheck.Result) 82 | assert.Equal(t, "min-instances", appCheck.CheckName) 83 | assert.Equal(t, "/foo", appCheck.App) 84 | assert.Equal(t, "Only 49 are healthy out of total 100", appCheck.Message) 85 | } 86 | 87 | func TestMinInstancesWhenFailThresholdIsMetButOverridenFromAppLabels(t *testing.T) { 88 | check := MinInstances{ 89 | DefaultCriticalThreshold: 0.5, 90 | DefaultWarningThreshold: 0.6, 91 | } 92 | appLabels := make(map[string]string) 93 | appLabels["alerts.min-instances.critical.threshold"] = "0.4" 94 | app := marathon.Application{ 95 | ID: "/foo", 96 | Instances: 100, 97 | TasksHealthy: 48, 98 | TasksStaged: 1, 99 | Labels: appLabels, 100 | } 101 | 102 | appCheck := check.Check(app) 103 | assert.Equal(t, Warning, appCheck.Result) 104 | assert.Equal(t, "min-instances", appCheck.CheckName) 105 | assert.Equal(t, "/foo", appCheck.App) 106 | assert.Equal(t, "Only 49 are healthy out of total 100", appCheck.Message) 107 | } 108 | 109 | func TestMinInstancesWhenNoTasksAreRunning(t *testing.T) { 110 | check := MinInstances{ 111 | DefaultCriticalThreshold: 0.5, 112 | DefaultWarningThreshold: 0.6, 113 | } 114 | app := marathon.Application{ 115 | ID: "/foo", 116 | Instances: 1, 117 | TasksHealthy: 0, 118 | TasksStaged: 0, 119 | } 120 | 121 | appCheck := check.Check(app) 122 | assert.Equal(t, Critical, appCheck.Result) 123 | assert.Equal(t, "min-instances", appCheck.CheckName) 124 | assert.Equal(t, "/foo", appCheck.App) 125 | assert.Equal(t, "Only 0 are healthy out of total 1", appCheck.Message) 126 | } 127 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "log" 7 | "net/http" 8 | "os" 9 | "time" 10 | 11 | "github.com/ashwanthkumar/marathon-alerts/checks" 12 | "github.com/ashwanthkumar/marathon-alerts/notifiers" 13 | flag "github.com/spf13/pflag" 14 | 15 | marathon "github.com/gambol99/go-marathon" 16 | "github.com/rcrowley/go-metrics" 17 | ) 18 | 19 | var appChecker AppChecker 20 | var alertManager AlertManager 21 | 22 | // Check settings 23 | var minHealthyWarningThreshold float32 24 | var minHealthyCriticalThreshold float32 25 | var minInstancesWarningThreshold float32 26 | var minInstancesCriticalThreshold float32 27 | 28 | // Required flags 29 | var marathonURI string 30 | var checkInterval time.Duration 31 | var alertSuppressDuration time.Duration 32 | var debugMode bool 33 | var pidFile string 34 | 35 | // Slack flags 36 | var slackWebhooks string 37 | var slackChannel string 38 | var slackOwners string 39 | 40 | // DebugMetricsRegistry is used for pushing debug level metrics by rest of the app 41 | var DebugMetricsRegistry metrics.Registry 42 | 43 | func main() { 44 | log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds | log.LUTC | log.Lshortfile) 45 | log.SetOutput(os.Stdout) 46 | os.Args[0] = "marathon-alerts" 47 | defineFlags() 48 | flag.Parse() 49 | pid := []byte(fmt.Sprintf("%d\n", os.Getpid())) 50 | err := ioutil.WriteFile(pidFile, pid, 0644) 51 | if err != nil { 52 | fmt.Println("Unable to write pid file. ") 53 | log.Fatalf("Error - %v\n", err) 54 | } 55 | 56 | client, err := marathonClient(marathonURI) 57 | if err != nil { 58 | fmt.Printf("%v\n", err) 59 | os.Exit(1) 60 | } 61 | DebugMetricsRegistry = metrics.NewPrefixedRegistry("debug") 62 | 63 | minHealthyTasks := &checks.MinHealthyTasks{ 64 | DefaultCriticalThreshold: minHealthyCriticalThreshold, 65 | DefaultWarningThreshold: minHealthyWarningThreshold, 66 | } 67 | minInstances := &checks.MinInstances{ 68 | DefaultCriticalThreshold: minHealthyCriticalThreshold, 69 | DefaultWarningThreshold: minHealthyWarningThreshold, 70 | } 71 | suspendedCheck := &checks.SuspendedCheck{} 72 | checks := []checks.Checker{minHealthyTasks, minInstances, suspendedCheck} 73 | 74 | appChecker = AppChecker{ 75 | Client: client, 76 | CheckInterval: checkInterval, 77 | Checks: checks, 78 | } 79 | appChecker.Start() 80 | 81 | var allNotifiers []notifiers.Notifier 82 | slack := notifiers.Slack{ 83 | Webhook: slackWebhooks, 84 | Channel: slackChannel, 85 | Owners: slackOwners, 86 | } 87 | allNotifiers = append(allNotifiers, &slack) 88 | 89 | alertManager = AlertManager{ 90 | CheckerChan: appChecker.AlertsChannel, 91 | SuppressDuration: alertSuppressDuration, 92 | Notifiers: allNotifiers, 93 | } 94 | alertManager.Start() 95 | 96 | metrics.RegisterDebugGCStats(DebugMetricsRegistry) 97 | metrics.RegisterRuntimeMemStats(DebugMetricsRegistry) 98 | go metrics.CaptureDebugGCStats(DebugMetricsRegistry, 15*time.Minute) 99 | go metrics.CaptureRuntimeMemStats(DebugMetricsRegistry, 5*time.Minute) 100 | go metrics.Log(metrics.DefaultRegistry, 60*time.Second, log.New(os.Stderr, "metrics: ", log.Lmicroseconds)) 101 | if debugMode { 102 | go metrics.Log(DebugMetricsRegistry, 300*time.Second, log.New(os.Stderr, "debug-metrics: ", log.Lmicroseconds)) 103 | } 104 | appChecker.RunWaitGroup.Wait() 105 | // Handle signals and cleanup all routines 106 | } 107 | 108 | func marathonClient(uri string) (marathon.Marathon, error) { 109 | config := marathon.NewDefaultConfig() 110 | config.URL = uri 111 | config.HTTPClient = &http.Client{ 112 | Timeout: (30 * time.Second), 113 | } 114 | 115 | return marathon.NewClient(config) 116 | } 117 | 118 | func defineFlags() { 119 | flag.StringVar(&marathonURI, "uri", "", "Marathon URI to connect") 120 | flag.StringVar(&pidFile, "pid", "PID", "File to write PID file") 121 | flag.BoolVar(&debugMode, "debug", false, "Enable debug mode. More counters for now.") 122 | flag.DurationVar(&checkInterval, "check-interval", 60*time.Second, "Check runs periodically on this interval") 123 | flag.DurationVar(&alertSuppressDuration, "alerts-suppress-duration", 30*time.Minute, "Suppress alerts for this duration once notified") 124 | 125 | // Check flags 126 | flag.Float32Var(&minHealthyWarningThreshold, "check-min-healthy-warn-threshold", 0.75, "Min Healthy instances check warning threshold") 127 | flag.Float32Var(&minHealthyCriticalThreshold, "check-min-healthy-critical-threshold", 0.5, "Min Healthy instances check fail threshold") 128 | flag.Float32Var(&minInstancesWarningThreshold, "check-min-instances-warn-threshold", 0.75, "Min Instances check warning threshold") 129 | flag.Float32Var(&minInstancesCriticalThreshold, "check-min-instances-critical-threshold", 0.5, "Min Instances check fail threshold") 130 | 131 | // Slack flags 132 | flag.StringVar(&slackWebhooks, "slack-webhook", "", "Comma list of Slack webhooks to post the alert") 133 | flag.StringVar(&slackChannel, "slack-channel", "", "#Channel / @User to post the alert (defaults to webhook configuration)") 134 | flag.StringVar(&slackOwners, "slack-owner", "", "Comma list of owners who should be alerted on the post") 135 | } 136 | -------------------------------------------------------------------------------- /routes/routes_test.go: -------------------------------------------------------------------------------- 1 | package routes 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/ashwanthkumar/marathon-alerts/checks" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestSimpleParseRoutes(t *testing.T) { 11 | routeString := "min-healthy/warning/slack" 12 | routes, err := ParseRoutes(routeString) 13 | assert.NoError(t, err) 14 | assert.Len(t, routes, 1) 15 | route := routes[0] 16 | expectedRoute := Route{ 17 | Check: "min-healthy", 18 | CheckLevel: checks.Warning, 19 | Notifier: "slack", 20 | } 21 | 22 | assert.Equal(t, expectedRoute, route) 23 | } 24 | 25 | func TestSimpleParseRoutesEndingWithSemiColon(t *testing.T) { 26 | routeString := "min-healthy/warning/slack;" 27 | routes, err := ParseRoutes(routeString) 28 | assert.NoError(t, err) 29 | assert.Len(t, routes, 1) 30 | route := routes[0] 31 | expectedRoute := Route{ 32 | Check: "min-healthy", 33 | CheckLevel: checks.Warning, 34 | Notifier: "slack", 35 | } 36 | 37 | assert.Equal(t, expectedRoute, route) 38 | } 39 | 40 | func TestParseRoutesForEmptyString(t *testing.T) { 41 | routes, err := ParseRoutes("") 42 | assert.Error(t, err) 43 | assert.Nil(t, routes) 44 | } 45 | 46 | func TestParseRoutesForMultipleRoutes(t *testing.T) { 47 | routeString := "min-healthy/warning/slack;min-healthy/critical/slack" 48 | routes, err := ParseRoutes(routeString) 49 | assert.NoError(t, err) 50 | assert.Len(t, routes, 2) 51 | route := routes[0] 52 | expectedRoute := Route{ 53 | Check: "min-healthy", 54 | CheckLevel: checks.Warning, 55 | Notifier: "slack", 56 | } 57 | assert.Equal(t, expectedRoute, route) 58 | 59 | route = routes[1] 60 | expectedRoute = Route{ 61 | Check: "min-healthy", 62 | CheckLevel: checks.Critical, 63 | Notifier: "slack", 64 | } 65 | assert.Equal(t, expectedRoute, route) 66 | } 67 | 68 | func TestParseInvalidCheckLevel(t *testing.T) { 69 | routeString := "min-healthy/blahblah/slack" 70 | _, err := ParseRoutes(routeString) 71 | assert.Error(t, err) 72 | } 73 | 74 | func TestParseCheckLevel(t *testing.T) { 75 | expected := make(map[string]checks.CheckStatus) 76 | expected["Warning"] = checks.Warning 77 | expected["WARNING"] = checks.Warning 78 | expected["warning"] = checks.Warning 79 | expected["WaRnInG"] = checks.Warning 80 | expected["Critical"] = checks.Critical 81 | expected["CRITICAL"] = checks.Critical 82 | expected["critical"] = checks.Critical 83 | expected["CrItIcAl"] = checks.Critical 84 | expected["Pass"] = checks.Pass 85 | expected["pass"] = checks.Pass 86 | expected["PASS"] = checks.Pass 87 | expected["PaSs"] = checks.Pass 88 | expected["Resolved"] = checks.Resolved 89 | expected["resolved"] = checks.Resolved 90 | expected["RESOLVED"] = checks.Resolved 91 | expected["ReSoLvEd"] = checks.Resolved 92 | for input, expectedOutput := range expected { 93 | output, err := parseCheckLevel(input) 94 | assert.NoError(t, err) 95 | assert.Equal(t, expectedOutput, output) 96 | } 97 | 98 | _, err := parseCheckLevel("invalid-check-level") 99 | assert.Error(t, err) 100 | } 101 | 102 | func TestDefaultRoutes(t *testing.T) { 103 | defaultRoutes, err := ParseRoutes(DefaultRoutes) 104 | assert.NoError(t, err) 105 | assert.Len(t, defaultRoutes, 3) 106 | allWarningRoute := defaultRoutes[0] 107 | expectedWarningRoute := Route{ 108 | Check: "*", 109 | CheckLevel: checks.Warning, 110 | Notifier: "*", 111 | } 112 | assert.Equal(t, expectedWarningRoute, allWarningRoute) 113 | 114 | allCriticalRoute := defaultRoutes[1] 115 | expectedCriticalRoute := Route{ 116 | Check: "*", 117 | CheckLevel: checks.Critical, 118 | Notifier: "*", 119 | } 120 | assert.Equal(t, expectedCriticalRoute, allCriticalRoute) 121 | 122 | allResolvedRoute := defaultRoutes[2] 123 | expectedResolvedRoute := Route{ 124 | Check: "*", 125 | CheckLevel: checks.Resolved, 126 | Notifier: "*", 127 | } 128 | assert.Equal(t, expectedResolvedRoute, allResolvedRoute) 129 | } 130 | 131 | func TestRouteMatch(t *testing.T) { 132 | defaultRoutes, err := ParseRoutes(DefaultRoutes) 133 | assert.NoError(t, err) 134 | assert.Len(t, defaultRoutes, 3) 135 | 136 | allWarningRoute := defaultRoutes[0] 137 | warningCheck := checks.AppCheck{ 138 | CheckName: "check-name", 139 | Result: checks.Warning, 140 | } 141 | warningCheckMatch := allWarningRoute.Match(warningCheck) 142 | assert.True(t, warningCheckMatch) 143 | 144 | allCriticalRoute := defaultRoutes[1] 145 | criticalCheck := checks.AppCheck{ 146 | CheckName: "check-name", 147 | Result: checks.Critical, 148 | } 149 | criticalCheckMatch := allCriticalRoute.Match(criticalCheck) 150 | assert.True(t, criticalCheckMatch) 151 | } 152 | 153 | func TestRouteMatchDoesNotWork(t *testing.T) { 154 | defaultRoutes, err := ParseRoutes(DefaultRoutes) 155 | assert.NoError(t, err) 156 | assert.Len(t, defaultRoutes, 3) 157 | 158 | allWarningRoute := defaultRoutes[0] 159 | resolvedCheck := checks.AppCheck{ 160 | CheckName: "check-name", 161 | Result: checks.Resolved, 162 | } 163 | resolvedCheckMatch := allWarningRoute.Match(resolvedCheck) 164 | assert.False(t, resolvedCheckMatch) 165 | } 166 | 167 | func TestRouteMatchNotifier(t *testing.T) { 168 | route := Route{ 169 | Notifier: "*", 170 | } 171 | assert.True(t, route.MatchNotifier("slack")) 172 | } 173 | 174 | func TestRouteMatchCheckResult(t *testing.T) { 175 | route := Route{ 176 | CheckLevel: checks.Warning, 177 | } 178 | assert.True(t, route.MatchCheckResult(checks.Warning)) 179 | assert.False(t, route.MatchCheckResult(checks.Pass)) 180 | } 181 | 182 | func BenchmarkSimpleParseRoutes(b *testing.B) { 183 | routeString := "min-healthy/warning/slack" 184 | for i := 0; i < b.N; i++ { 185 | ParseRoutes(routeString) 186 | } 187 | } 188 | 189 | func BenchmarkParseRoutesFor2Routes(b *testing.B) { 190 | routeString := "min-healthy/warning/slack;min-healthy/critical/slack" 191 | for i := 0; i < b.N; i++ { 192 | ParseRoutes(routeString) 193 | } 194 | } 195 | 196 | func BenchmarkParseRoutesFor3Routes(b *testing.B) { 197 | routeString := "min-healthy/warning/slack;min-healthy/critical/slack;min-healthy/resolved/slack" 198 | for i := 0; i < b.N; i++ { 199 | ParseRoutes(routeString) 200 | } 201 | } 202 | 203 | func BenchmarkParseRoutesFor4Routes(b *testing.B) { 204 | routeString := "min-healthy/warning/slack;min-healthy/critical/slack;min-healthy/resolved/slack;*/pass/*" 205 | for i := 0; i < b.N; i++ { 206 | ParseRoutes(routeString) 207 | } 208 | } 209 | -------------------------------------------------------------------------------- /alert-manager.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "sync" 7 | "time" 8 | 9 | maps "github.com/ashwanthkumar/golang-utils/maps" 10 | "github.com/ashwanthkumar/marathon-alerts/checks" 11 | "github.com/ashwanthkumar/marathon-alerts/notifiers" 12 | "github.com/ashwanthkumar/marathon-alerts/routes" 13 | "github.com/rcrowley/go-metrics" 14 | ) 15 | 16 | const ( 17 | AlertsEnabledLabel = "alerts.enabled" 18 | AppRoutesLabel = "alerts.routes" 19 | ) 20 | 21 | type AlertManager struct { 22 | CheckerChan chan checks.AppCheck // channel to get app check results 23 | AppSuppress map[string]time.Time // Key - AppName-CheckName-CheckResult 24 | AlertCount map[string]int // Key - AppName-CheckName -> Consecutive # of failures 25 | SuppressDuration time.Duration 26 | Notifiers []notifiers.Notifier 27 | RunWaitGroup sync.WaitGroup 28 | stopChannel chan bool 29 | supressMutex sync.Mutex 30 | } 31 | 32 | func (a *AlertManager) Start() { 33 | log.Println("Starting Alert Manager...") 34 | a.RunWaitGroup.Add(1) 35 | a.stopChannel = make(chan bool) 36 | a.AppSuppress = make(map[string]time.Time) 37 | a.AlertCount = make(map[string]int) 38 | go a.run() 39 | log.Println("Alert Manager Started.") 40 | } 41 | 42 | func (a *AlertManager) Stop() { 43 | log.Println("Stopping Alert Manager...") 44 | close(a.stopChannel) 45 | a.RunWaitGroup.Done() 46 | } 47 | 48 | func (a *AlertManager) cleanUpSupressedAlerts() { 49 | a.supressMutex.Lock() 50 | for key, suppressedOn := range a.AppSuppress { 51 | if time.Now().Sub(suppressedOn) > a.SuppressDuration { 52 | metrics.GetOrRegisterCounter("alerts-suppressed-cleaned", nil).Inc(int64(1)) 53 | delete(a.AppSuppress, key) 54 | } 55 | } 56 | a.supressMutex.Unlock() 57 | } 58 | 59 | func (a *AlertManager) run() { 60 | running := true 61 | for running { 62 | select { 63 | case <-time.After(5 * time.Second): 64 | metrics.GetOrRegisterCounter("alerts-suppressed-called", DebugMetricsRegistry).Inc(int64(1)) 65 | a.cleanUpSupressedAlerts() 66 | case check := <-a.CheckerChan: 67 | metrics.GetOrRegisterCounter("alerts-process-check-called", DebugMetricsRegistry).Inc(int64(1)) 68 | a.processCheck(check) 69 | case <-a.stopChannel: 70 | metrics.GetOrRegisterCounter("alerts-manager-stopped", DebugMetricsRegistry).Inc(int64(1)) 71 | running = false 72 | } 73 | } 74 | } 75 | 76 | func (a *AlertManager) processCheck(check checks.AppCheck) { 77 | a.supressMutex.Lock() 78 | defer a.supressMutex.Unlock() 79 | 80 | alertEnabled := maps.GetBoolean(check.Labels, AlertsEnabledLabel, true) 81 | 82 | if alertEnabled { 83 | allRoutes, err := routes.ParseRoutes(maps.GetString(check.Labels, AppRoutesLabel, routes.DefaultRoutes)) 84 | if err != nil { 85 | log.Printf("Error - %v\n", err) 86 | return 87 | } 88 | checkExists, keyPrefixIfCheckExists, keyIfCheckExists, previousCheckLevel := a.checkExist(check) 89 | 90 | if checkExists && check.Result == checks.Pass { 91 | a.AlertCount[keyPrefixIfCheckExists]++ 92 | check.Times = a.AlertCount[keyPrefixIfCheckExists] 93 | check.Result = checks.Resolved 94 | previousRouteExists := a.checkForRouteWithCheckLevel(previousCheckLevel, allRoutes) 95 | delete(a.AppSuppress, keyIfCheckExists) 96 | delete(a.AlertCount, keyPrefixIfCheckExists) 97 | if previousRouteExists { 98 | a.notifyCheck(check, allRoutes) 99 | a.incNotifCounter(check) 100 | } 101 | } else if checkExists && check.Result != previousCheckLevel { 102 | delete(a.AppSuppress, keyIfCheckExists) 103 | key := a.key(check, check.Result) 104 | a.AppSuppress[key] = check.Timestamp 105 | a.AlertCount[keyPrefixIfCheckExists]++ 106 | check.Times = a.AlertCount[keyPrefixIfCheckExists] 107 | a.notifyCheck(check, allRoutes) 108 | a.incNotifCounter(check) 109 | } else if !checkExists && check.Result != checks.Pass { 110 | keyPrefix := a.keyPrefix(check) 111 | key := a.key(check, check.Result) 112 | a.AppSuppress[key] = check.Timestamp 113 | _, present := a.AlertCount[keyPrefix] 114 | if present { 115 | a.AlertCount[keyPrefix]++ 116 | } else { 117 | a.AlertCount[keyPrefix] = 1 118 | } 119 | check.Times = a.AlertCount[keyPrefix] 120 | a.notifyCheck(check, allRoutes) 121 | a.incNotifCounter(check) 122 | } else if !checkExists && check.Result == checks.Pass { 123 | keyPrefix := a.keyPrefix(check) 124 | delete(a.AlertCount, keyPrefix) 125 | } 126 | } 127 | // TODO - Add a log message that runs only once per every new app / if app state has changed 128 | } 129 | 130 | func (a *AlertManager) checkForRouteWithCheckLevel(level checks.CheckStatus, allRoutes []routes.Route) bool { 131 | for _, route := range allRoutes { 132 | if route.MatchCheckResult(level) { 133 | return true 134 | } 135 | } 136 | 137 | return false 138 | } 139 | 140 | func (a *AlertManager) notifyCheck(check checks.AppCheck, allRoutes []routes.Route) { 141 | log.Printf("[NotifyCheck] App: %s, Result: %s, Check: %s, Reason: %s \n", check.App, checks.CheckStatusToString(check.Result), check.CheckName, check.Message) 142 | for _, route := range allRoutes { 143 | if route.Match(check) { 144 | for _, notifier := range a.Notifiers { 145 | if route.MatchNotifier(notifier.Name()) { 146 | notifier.Notify(check) 147 | } 148 | } 149 | } 150 | } 151 | } 152 | 153 | func (a *AlertManager) checkExist(check checks.AppCheck) (bool, string, string, checks.CheckStatus) { 154 | for _, level := range checks.CheckLevels { 155 | keyPrefix := a.keyPrefix(check) 156 | key := a.key(check, level) 157 | _, present := a.AppSuppress[key] 158 | if present { 159 | return true, keyPrefix, key, level 160 | } 161 | } 162 | 163 | return false, "", "", checks.Pass 164 | } 165 | 166 | func (a *AlertManager) key(check checks.AppCheck, level checks.CheckStatus) string { 167 | return fmt.Sprintf("%s-%d", a.keyPrefix(check), level) 168 | } 169 | 170 | func (a *AlertManager) keyPrefix(check checks.AppCheck) string { 171 | return fmt.Sprintf("%s-%s", check.App, check.CheckName) 172 | } 173 | 174 | func (a *AlertManager) incNotifCounter(check checks.AppCheck) { 175 | metrics.GetOrRegisterCounter("notifications-total", nil).Inc(1) 176 | metrics.GetOrRegisterMeter("notifications-rate", nil).Mark(1) 177 | if check.Result == checks.Warning { 178 | metrics.GetOrRegisterCounter("notifications-warning", nil).Inc(1) 179 | metrics.GetOrRegisterMeter("notifications-warning-rate", DebugMetricsRegistry).Mark(1) 180 | } else if check.Result == checks.Critical { 181 | metrics.GetOrRegisterCounter("notifications-critical", nil).Inc(1) 182 | metrics.GetOrRegisterMeter("notifications-critical-rate", DebugMetricsRegistry).Mark(1) 183 | } else if check.Result == checks.Pass { 184 | metrics.GetOrRegisterCounter("notifications-pass", nil).Inc(1) 185 | metrics.GetOrRegisterMeter("notifications-pass-rate", DebugMetricsRegistry).Mark(1) 186 | } else if check.Result == checks.Resolved { 187 | metrics.GetOrRegisterCounter("notifications-resolved", nil).Inc(1) 188 | metrics.GetOrRegisterMeter("notifications-resolved-rate", DebugMetricsRegistry).Mark(1) 189 | } else { 190 | panic("Calling incCheckCounter for " + fmt.Sprintf("%v", check)) 191 | } 192 | } 193 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/ashwanthkumar/marathon-alerts.svg?branch=master)](https://travis-ci.org/ashwanthkumar/marathon-alerts) 2 | [![codecov.io](https://codecov.io/github/ashwanthkumar/marathon-alerts/coverage.svg?branch=master)](https://codecov.io/github/ashwanthkumar/marathon-alerts?branch=master) 3 | 4 | # marathon-alerts 5 | 6 | Marathon Alerts is a tool for monitoring the apps running on marathon. Inspired from [kubernetes-alerts](https://github.com/AcalephStorage/kubernetes-alerts) and [consul-alerts](https://github.com/AcalephStorage/consul-alerts). 7 | 8 | This was initially built for Marathon 0.8.0, hence we don't use the event bus. 9 | 10 | ## Usage 11 | ``` 12 | $ marathon-alerts --help 13 | Usage of marathon-alerts: 14 | --alerts-suppress-duration duration Suppress alerts for this duration once notified (default 30m0s) 15 | --check-interval duration Check runs periodically on this interval (default 30s) 16 | --check-min-healthy-critical-threshold value Min Healthy instances check fail threshold (default 0.5) 17 | --check-min-healthy-warn-threshold value Min Healthy instances check warning threshold (default 0.75) 18 | --check-min-instances-critical-threshold value Min Instances check fail threshold (default 0.5) 19 | --check-min-instances-warn-threshold value Min Instances check warning threshold (default 0.75) 20 | --debug Enable debug mode. More counters for now. 21 | --pid string File to write PID file (default "PID") 22 | --slack-channel string #Channel / @User to post the alert (defaults to webhook configuration) 23 | --slack-owner string Comma list of owners who should be alerted on the post 24 | --slack-webhook string Comma list of Slack webhooks to post the alert 25 | --uri string Marathon URI to connect 26 | ``` 27 | 28 | Example invocation would be like the following 29 | ``` 30 | $ marathon-alerts --uri http://marathon1:8080,marathon2:8080 \ 31 | --slack-webhook https://hooks.slack.com/services/..../ \ 32 | --slack-owner ashwanthkumar,slackbot 33 | ``` 34 | 35 | ## App Labels 36 | Apart from the flags that are used while starting up, the functionality can be controlled at an app level using labels in the app specification. The following table explains the properties and it's usage. 37 | 38 | | Property | Description | Example | 39 | | --- | --- | --- | 40 | | alerts.enabled | Controls if the alerts for the app should be enabled or disabled. Defaults - true | false | 41 | | alerts.checks.subscribe | Comma separated list of checks that needs to be run. Defaults - all | all | 42 | | alerts.routes | Ability to route different checks to different notifiers based on their level. See the section below on Routes to understand how you can add routes to your apps. Defaults - `*/resolved/*;*/warning/*;*/critical/*` | min-healthy/critical/pagerduty;min-healthy/warning/slack | 43 | | alerts.min-healthy.critical.threshold | Failure threshold for min-healthy check. Defaults - `--check-min-healthy-critical-threshold` | 0.5 | 44 | | alerts.min-healthy.warn.threshold | Warning threshold for min-healthy check. Defaults - `--check-min-healthy-warn-threshold` | 0.4 | 45 | | alerts.min-instances.critical.threshold | Failure threshold for min-instances check. Defaults - `--check-min-instances-critical-threshold` | 0.5 | 46 | | alerts.min-instances.warn.threshold | Warning threshold for min-instances check. Defaults - `--check-min-instances-warn-threshold` | 0.4 | 47 | | alerts.slack.webhook | Comma separated list of Slack webhooks to send slack notifications. Overrides - `--slack-webhook` | http://hooks.slack.com/.../ | 48 | | alerts.slack.channel | #Channel / @User to post the alert into. Overrides - `--slack-channel` | z_development | 49 | | alerts.slack.owners | Comma separated list of users who should be tagged in the alert. Overrides - `--slack-owner` | ashwanthkumar,slackbot | 50 | 51 | ## Metrics 52 | We collect some metrics internally in marathon-alerts. They're dumped periodically to STDERR. You can find the list of metrics and it's usage in the following table 53 | 54 | | Metric | Description | 55 | | :------------- | :------------- | 56 | | alerts-suppressed-cleaned | Number of alerts we cleaned up because they got expired from suppress duration. | 57 | | marathon-all-apps-response-time | Response time of marathon's /v2/apps API call | 58 | | notifications-total | Total number of notifications we sent from AlertManager to NotificationManager | 59 | | notifications-warning | Number of Warning check notifications we sent from AlertManager to NotificationManager | 60 | | notifications-critical | Number of Critical check notifications we sent from AlertManager to NotificationManager | 61 | | notifications-resolved | Number of Pass (aka Resolved) check notifications we sent from AlertManager to NotificationManager | 62 | | notifications-rate | Meter metric that denotes the rate at which notifications are being sent | 63 | 64 | ## Debug Metrics 65 | Apart from the standard metrics above, we also collect quite a few other metrics, mostly for debugging purposes. You can enable these metrics if run `marathon-alerts` with a `--debug` flag. 66 | 67 | | Metric | Description | 68 | | :------------- | :------------- | 69 | | alerts-suppressed-called | Number of times we called AlertManager.cleanUpSupressedAlerts() | 70 | | alerts-process-check-called | Number of times we called AlertManager.processCheck() | 71 | | alerts-manager-stopped | Number of times we called AlertManager.Stop() | 72 | | apps-checker-stopped | Number of times we called AppChecker.Stop() | 73 | | apps-checker-marathon-all-apps-api | Number of times we called Marathon's /v2/apps API | 74 | | apps-checker-alerts-sent | Number of checks we sent to AlertManager from AppChecker | 75 | | apps-checker-check-<name> | Number of checks identified by <name> we sent to AlertManager | 76 | | apps-checker-app-<id> | Number of checks for an app identified by <id> we sent to AlertManager | 77 | | apps-checker-<id>-<name> | Number of checks identified by <name> for an app identified by <id> we sent to AlertManager | 78 | | notifications-warning-rate | Meter metric that denotes the rate at which warning notifications are being sent | 79 | | notifications-critical-rate | Meter metric that denotes the rate at which critical notifications are being sent | 80 | | notifications-resolved-rate | Meter metric that denotes the rate at which resolved notifications are being sent | 81 | 82 | ## Routes 83 | From v0.3.0-RC7 onwards we've an ability to route different check alerts to different notifiers. On a per-app basis you can control the routes using `alerts.routes` label. The format of the value should be as following - 84 | ``` 85 | //;[//] 86 | ``` 87 | 88 | ### Rules 89 | 1. Check name and Notifier names can be glob patterns. No complicated regex allowed as of now. 90 | 2. Check level has to be one of warning / pass / critical / resolved. 91 | 3. Multiple routes can be defined by separating them using `;`. 92 | 93 | Default routes if none specified is - `"*/warning/*;*/critical/*;*/resolved/*"`. It means we'll route all check's warning / critical / resolved notifications to all available notifiers. 94 | 95 | ## Releases 96 | Binaries are available [here](https://github.com/ashwanthkumar/marathon-alerts/releases). 97 | 98 | ## Deployment 99 | We've a sample `marathon.json.conf` that we use in our organization along with [`marathonctl deploy`](https://github.com/ashwanthkumar/marathonctl). 100 | 101 | ## Building 102 | To build from source, you need [`glide`](http://glide.sh/) tool in `$PATH`. 103 | 104 | ``` 105 | $ cd $GOPATH/src 106 | $ mkdir -p github.com/ashwanthkumar/marathon-alerts 107 | $ git clone https://github.com/ashwanthkumar/marathon-alerts.git github.com/ashwanthkumar/marathon-alerts 108 | $ cd github.com/ashwanthkumar/marathon-alerts 109 | $ make setup # Downloads the required dependencies 110 | $ make test # Runs the test 111 | $ make build # Builds the distribution specific binary 112 | ``` 113 | 114 | ## Available Checks 115 | - [x] `min-healthy` - Minimum % of Task instances that should be healthy else this check is fired. 116 | - [x] `min-instances` - Minimum % of Task instances that should be healthy or staged, else this check is fired. 117 | - [ ] `max-instances` - If the number of instances goes beyond some % of the pre-defined max limit 118 | - [x] `suspended` - If the service was suspended by mistake or unintentionally. `min-healthy` doesn't catch suspended services today. 119 | 120 | ## Notifiers 121 | - [x] Slack 122 | - [ ] Influx 123 | - [ ] Pager Duty 124 | - [ ] Email 125 | 126 | ## Contribute 127 | If you've any feature requests or issues, please open a Github issue. We accept PRs. Fork away! 128 | 129 | ## License 130 | http://www.apache.org/licenses/LICENSE-2.0 131 | -------------------------------------------------------------------------------- /alert-manager_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/ashwanthkumar/marathon-alerts/checks" 8 | "github.com/ashwanthkumar/marathon-alerts/notifiers" 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/mock" 11 | ) 12 | 13 | func TestKey(t *testing.T) { 14 | check := checks.AppCheck{ 15 | App: "/foo", 16 | CheckName: "check-name", 17 | } 18 | mgr := AlertManager{} 19 | key := mgr.key(check, checks.Pass) 20 | assert.Equal(t, "/foo-check-name-99", key) 21 | } 22 | 23 | func TestKeyPrefix(t *testing.T) { 24 | check := checks.AppCheck{ 25 | App: "/foo", 26 | CheckName: "check-name", 27 | } 28 | mgr := AlertManager{} 29 | key := mgr.keyPrefix(check) 30 | assert.Equal(t, "/foo-check-name", key) 31 | } 32 | 33 | func TestCheckExists(t *testing.T) { 34 | suppressedApps := make(map[string]time.Time) 35 | suppressedApps["/foo-check-name-2"] = time.Now() 36 | check := checks.AppCheck{ 37 | App: "/foo", 38 | CheckName: "check-name", 39 | Result: checks.Warning, 40 | } 41 | mgr := AlertManager{ 42 | AppSuppress: suppressedApps, 43 | AlertCount: make(map[string]int), 44 | } 45 | exist, keyPrefixIfExist, keyIfExist, checkLevel := mgr.checkExist(check) 46 | assert.True(t, true, exist) 47 | assert.Equal(t, "/foo-check-name", keyPrefixIfExist) 48 | assert.Equal(t, "/foo-check-name-2", keyIfExist) 49 | assert.Equal(t, checks.Warning, checkLevel) 50 | } 51 | 52 | func TestProcessCheckWhenNewCheckArrives(t *testing.T) { 53 | suppressedApps := make(map[string]time.Time) 54 | mockNotifier := new(notifiers.MockNotifier) 55 | mockNotifier.On("Name").Return("mock-notifer") 56 | mockNotifier.On("Notify", mock.AnythingOfType("AppCheck")).Return(nil) 57 | check := checks.AppCheck{ 58 | App: "/foo", 59 | CheckName: "check-name", 60 | Result: checks.Warning, 61 | } 62 | mgr := AlertManager{ 63 | AppSuppress: suppressedApps, 64 | AlertCount: make(map[string]int), 65 | Notifiers: []notifiers.Notifier{mockNotifier}, 66 | } 67 | 68 | mgr.processCheck(check) 69 | expectedCheck := checks.AppCheck{ 70 | App: "/foo", 71 | CheckName: "check-name", 72 | Result: checks.Warning, 73 | Times: 1, 74 | } 75 | mockNotifier.AssertCalled(t, "Notify", expectedCheck) 76 | assert.Equal(t, mgr.AlertCount["/foo-check-name"], 1) 77 | } 78 | 79 | func TestProcessCheckWhenNewPassCheckArrives(t *testing.T) { 80 | mockNotifier := new(notifiers.MockNotifier) 81 | mockNotifier.On("Name").Return("mock-notifer") 82 | mockNotifier.On("Notify", mock.AnythingOfType("AppCheck")).Return(nil) 83 | suppressedApps := make(map[string]time.Time) 84 | check := checks.AppCheck{ 85 | App: "/foo", 86 | CheckName: "check-name", 87 | Result: checks.Pass, 88 | } 89 | alertCount := make(map[string]int) 90 | alertCount["/foo-check-name"] = 2 91 | mgr := AlertManager{ 92 | AppSuppress: suppressedApps, 93 | AlertCount: alertCount, 94 | Notifiers: []notifiers.Notifier{mockNotifier}, 95 | } 96 | 97 | mgr.processCheck(check) 98 | mockNotifier.AssertNotCalled(t, "Notify", check) 99 | assert.Equal(t, mgr.AlertCount["/foo-check-name"], 0) 100 | } 101 | 102 | func TestProcessCheckWhenExistingCheckOfDifferentLevel(t *testing.T) { 103 | mockNotifier := new(notifiers.MockNotifier) 104 | mockNotifier.On("Name").Return("mock-notifer") 105 | mockNotifier.On("Notify", mock.AnythingOfType("AppCheck")).Return(nil) 106 | 107 | suppressedApps := make(map[string]time.Time) 108 | suppressedApps["/foo-check-name-2"] = time.Now() 109 | check := checks.AppCheck{ 110 | App: "/foo", 111 | CheckName: "check-name", 112 | Result: checks.Critical, 113 | } 114 | alertCount := make(map[string]int) 115 | alertCount["/foo-check-name"] = 1 116 | mgr := AlertManager{ 117 | AppSuppress: suppressedApps, 118 | AlertCount: alertCount, 119 | Notifiers: []notifiers.Notifier{mockNotifier}, 120 | } 121 | 122 | mgr.processCheck(check) 123 | expectedCheck := checks.AppCheck{ 124 | App: "/foo", 125 | CheckName: "check-name", 126 | Result: checks.Critical, 127 | Times: 2, 128 | } 129 | mockNotifier.AssertCalled(t, "Notify", expectedCheck) 130 | assert.Equal(t, mgr.AlertCount["/foo-check-name"], 2) 131 | } 132 | 133 | func TestProcessCheckWhenExistingCheckOfSameLevel(t *testing.T) { 134 | mockNotifier := new(notifiers.MockNotifier) 135 | mockNotifier.On("Name").Return("mock-notifer") 136 | mockNotifier.On("Notify", mock.AnythingOfType("AppCheck")).Return(nil) 137 | 138 | suppressedApps := make(map[string]time.Time) 139 | suppressedApps["/foo-check-name-2"] = time.Now() 140 | alertCount := make(map[string]int) 141 | alertCount["/foo-check-name"] = 1 142 | check := checks.AppCheck{ 143 | App: "/foo", 144 | CheckName: "check-name", 145 | Result: checks.Warning, 146 | } 147 | mgr := AlertManager{ 148 | AppSuppress: suppressedApps, 149 | AlertCount: alertCount, 150 | Notifiers: []notifiers.Notifier{mockNotifier}, 151 | } 152 | 153 | mgr.processCheck(check) 154 | mockNotifier.AssertNotCalled(t, "Notify", check) 155 | assert.Equal(t, mgr.AlertCount["/foo-check-name"], 1) 156 | } 157 | 158 | func TestProcessCheckWhenResolvedCheckArrives(t *testing.T) { 159 | mockNotifier := new(notifiers.MockNotifier) 160 | mockNotifier.On("Name").Return("mock-notifer") 161 | mockNotifier.On("Notify", mock.AnythingOfType("AppCheck")).Return(nil) 162 | 163 | suppressedApps := make(map[string]time.Time) 164 | suppressedApps["/foo-check-name-2"] = time.Now() 165 | check := checks.AppCheck{ 166 | App: "/foo", 167 | CheckName: "check-name", 168 | Result: checks.Pass, 169 | } 170 | alertCount := make(map[string]int) 171 | alertCount["/foo-check-name"] = 1 172 | mgr := AlertManager{ 173 | AppSuppress: suppressedApps, 174 | AlertCount: alertCount, 175 | Notifiers: []notifiers.Notifier{mockNotifier}, 176 | } 177 | 178 | mgr.processCheck(check) 179 | expectedCheck := checks.AppCheck{ 180 | App: "/foo", 181 | CheckName: "check-name", 182 | Result: checks.Resolved, 183 | Times: 2, 184 | } 185 | mockNotifier.AssertCalled(t, "Notify", expectedCheck) 186 | // We remove AlertCount upon Resolved check 187 | assert.Equal(t, mgr.AlertCount["/foo-check-name"], 0) 188 | } 189 | 190 | func TestProcessCheckWhenNewCheckArrivesButDisabledViaLabels(t *testing.T) { 191 | mockNotifier := new(notifiers.MockNotifier) 192 | mockNotifier.On("Name").Return("mock-notifer") 193 | mockNotifier.On("Notify", mock.AnythingOfType("AppCheck")).Return(nil) 194 | 195 | suppressedApps := make(map[string]time.Time) 196 | appLabels := make(map[string]string) 197 | appLabels["alerts.enabled"] = "false" 198 | check := checks.AppCheck{ 199 | App: "/foo", 200 | CheckName: "check-name", 201 | Result: checks.Warning, 202 | Labels: appLabels, 203 | } 204 | mgr := AlertManager{ 205 | AppSuppress: suppressedApps, 206 | AlertCount: make(map[string]int), 207 | Notifiers: []notifiers.Notifier{mockNotifier}, 208 | } 209 | 210 | mgr.processCheck(check) 211 | mockNotifier.AssertNotCalled(t, "Notify", check) 212 | } 213 | 214 | func TestCleanUpSupressedAlerts(t *testing.T) { 215 | suppressedApps := make(map[string]time.Time) 216 | suppressedApps["/foo-check-name-2"] = time.Now().Add(-5 * time.Minute) 217 | mgr := AlertManager{ 218 | AppSuppress: suppressedApps, 219 | AlertCount: make(map[string]int), 220 | SuppressDuration: 1 * time.Minute, 221 | } 222 | 223 | assert.Equal(t, 1, len(mgr.AppSuppress)) 224 | mgr.cleanUpSupressedAlerts() 225 | assert.Equal(t, 0, len(mgr.AppSuppress)) 226 | } 227 | 228 | func TestCleanUpSupressedAlertsIgnoreIfLessThanSuppressDuration(t *testing.T) { 229 | suppressedApps := make(map[string]time.Time) 230 | suppressedApps["/foo-check-name-2"] = time.Now().Add(-5 * time.Minute) 231 | mgr := AlertManager{ 232 | AppSuppress: suppressedApps, 233 | AlertCount: make(map[string]int), 234 | SuppressDuration: 10 * time.Minute, 235 | } 236 | 237 | assert.Equal(t, 1, len(mgr.AppSuppress)) 238 | mgr.cleanUpSupressedAlerts() 239 | assert.Equal(t, 1, len(mgr.AppSuppress)) 240 | } 241 | 242 | func TestTimesCountAfterTheCheckHasBeenIdleForSuppressedDuration(t *testing.T) { 243 | mockNotifier := new(notifiers.MockNotifier) 244 | mockNotifier.On("Name").Return("mock-notifer") 245 | mockNotifier.On("Notify", mock.AnythingOfType("AppCheck")).Return(nil) 246 | 247 | alertCount := make(map[string]int) 248 | alertCount["/foo-check-name"] = 1 249 | suppressedApps := make(map[string]time.Time) 250 | suppressedApps["/foo-check-name-2"] = time.Now().Add(-15 * time.Minute) 251 | mgr := AlertManager{ 252 | AppSuppress: suppressedApps, 253 | Notifiers: []notifiers.Notifier{mockNotifier}, 254 | AlertCount: alertCount, 255 | SuppressDuration: 10 * time.Minute, 256 | } 257 | check := checks.AppCheck{ 258 | App: "/foo", 259 | CheckName: "check-name", 260 | Result: checks.Critical, 261 | } 262 | 263 | // When Times 1 264 | mgr.processCheck(check) 265 | assert.Equal(t, 2, mgr.AlertCount["/foo-check-name"]) 266 | // After cleaning up supressed alerts 267 | mgr.cleanUpSupressedAlerts() 268 | mgr.processCheck(check) 269 | assert.Equal(t, 3, mgr.AlertCount["/foo-check-name"]) 270 | } 271 | 272 | func TestShouldNotNotifyResolvedBecauseRoutesDoesNotHaveWarning(t *testing.T) { 273 | mockNotifier := new(notifiers.MockNotifier) 274 | mockNotifier.On("Name").Return("mock-notifer") 275 | mockNotifier.On("Notify", mock.AnythingOfType("AppCheck")).Return(nil) 276 | 277 | suppressedApps := make(map[string]time.Time) 278 | suppressedApps["/foo-check-name-2"] = time.Now() 279 | appLabels := make(map[string]string) 280 | appLabels["alerts.routes"] = "*/resolved/*;*/critical/*" 281 | check := checks.AppCheck{ 282 | App: "/foo", 283 | CheckName: "check-name", 284 | Result: checks.Pass, 285 | Labels: appLabels, 286 | } 287 | alertCount := make(map[string]int) 288 | alertCount["/foo-check-name"] = 1 289 | mgr := AlertManager{ 290 | AppSuppress: suppressedApps, 291 | AlertCount: alertCount, 292 | Notifiers: []notifiers.Notifier{mockNotifier}, 293 | } 294 | 295 | mgr.processCheck(check) 296 | mockNotifier.AssertNotCalled(t, "Notify", mock.AnythingOfType("AppCheck")) 297 | // We remove AlertCount upon Resolved check 298 | assert.Equal(t, mgr.AlertCount["/foo-check-name"], 0) 299 | } 300 | 301 | func TestShouldNotifyResolvedBecauseRoutesHaveWarning(t *testing.T) { 302 | mockNotifier := new(notifiers.MockNotifier) 303 | mockNotifier.On("Name").Return("mock-notifer") 304 | mockNotifier.On("Notify", mock.AnythingOfType("AppCheck")).Return(nil) 305 | 306 | suppressedApps := make(map[string]time.Time) 307 | suppressedApps["/foo-check-name-2"] = time.Now() 308 | appLabels := make(map[string]string) 309 | appLabels["alerts.routes"] = "*/resolved/*;*/warning/*" 310 | check := checks.AppCheck{ 311 | App: "/foo", 312 | CheckName: "check-name", 313 | Result: checks.Pass, 314 | Labels: appLabels, 315 | } 316 | alertCount := make(map[string]int) 317 | alertCount["/foo-check-name"] = 1 318 | mgr := AlertManager{ 319 | AppSuppress: suppressedApps, 320 | AlertCount: alertCount, 321 | Notifiers: []notifiers.Notifier{mockNotifier}, 322 | } 323 | 324 | mgr.processCheck(check) 325 | expectedCheck := checks.AppCheck{ 326 | App: "/foo", 327 | CheckName: "check-name", 328 | Result: checks.Resolved, 329 | Times: 2, 330 | Labels: appLabels, 331 | } 332 | mockNotifier.AssertCalled(t, "Notify", expectedCheck) 333 | // We remove AlertCount upon Resolved check 334 | assert.Equal(t, mgr.AlertCount["/foo-check-name"], 0) 335 | } 336 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /mock_Marathon_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/gambol99/go-marathon" 4 | import "github.com/stretchr/testify/mock" 5 | 6 | import "net/url" 7 | 8 | import "time" 9 | 10 | type MockMarathon struct { 11 | mock.Mock 12 | } 13 | 14 | // ListApplications provides a mock function with given fields: _a0 15 | func (_m *MockMarathon) ListApplications(_a0 url.Values) ([]string, error) { 16 | ret := _m.Called(_a0) 17 | 18 | var r0 []string 19 | if rf, ok := ret.Get(0).(func(url.Values) []string); ok { 20 | r0 = rf(_a0) 21 | } else { 22 | if ret.Get(0) != nil { 23 | r0 = ret.Get(0).([]string) 24 | } 25 | } 26 | 27 | var r1 error 28 | if rf, ok := ret.Get(1).(func(url.Values) error); ok { 29 | r1 = rf(_a0) 30 | } else { 31 | r1 = ret.Error(1) 32 | } 33 | 34 | return r0, r1 35 | } 36 | 37 | // ApplicationVersions provides a mock function with given fields: name 38 | func (_m *MockMarathon) ApplicationVersions(name string) (*marathon.ApplicationVersions, error) { 39 | ret := _m.Called(name) 40 | 41 | var r0 *marathon.ApplicationVersions 42 | if rf, ok := ret.Get(0).(func(string) *marathon.ApplicationVersions); ok { 43 | r0 = rf(name) 44 | } else { 45 | if ret.Get(0) != nil { 46 | r0 = ret.Get(0).(*marathon.ApplicationVersions) 47 | } 48 | } 49 | 50 | var r1 error 51 | if rf, ok := ret.Get(1).(func(string) error); ok { 52 | r1 = rf(name) 53 | } else { 54 | r1 = ret.Error(1) 55 | } 56 | 57 | return r0, r1 58 | } 59 | 60 | // HasApplicationVersion provides a mock function with given fields: name, version 61 | func (_m *MockMarathon) HasApplicationVersion(name string, version string) (bool, error) { 62 | ret := _m.Called(name, version) 63 | 64 | var r0 bool 65 | if rf, ok := ret.Get(0).(func(string, string) bool); ok { 66 | r0 = rf(name, version) 67 | } else { 68 | r0 = ret.Get(0).(bool) 69 | } 70 | 71 | var r1 error 72 | if rf, ok := ret.Get(1).(func(string, string) error); ok { 73 | r1 = rf(name, version) 74 | } else { 75 | r1 = ret.Error(1) 76 | } 77 | 78 | return r0, r1 79 | } 80 | 81 | // SetApplicationVersion provides a mock function with given fields: name, version 82 | func (_m *MockMarathon) SetApplicationVersion(name string, version *marathon.ApplicationVersion) (*marathon.DeploymentID, error) { 83 | ret := _m.Called(name, version) 84 | 85 | var r0 *marathon.DeploymentID 86 | if rf, ok := ret.Get(0).(func(string, *marathon.ApplicationVersion) *marathon.DeploymentID); ok { 87 | r0 = rf(name, version) 88 | } else { 89 | if ret.Get(0) != nil { 90 | r0 = ret.Get(0).(*marathon.DeploymentID) 91 | } 92 | } 93 | 94 | var r1 error 95 | if rf, ok := ret.Get(1).(func(string, *marathon.ApplicationVersion) error); ok { 96 | r1 = rf(name, version) 97 | } else { 98 | r1 = ret.Error(1) 99 | } 100 | 101 | return r0, r1 102 | } 103 | 104 | // ApplicationOK provides a mock function with given fields: name 105 | func (_m *MockMarathon) ApplicationOK(name string) (bool, error) { 106 | ret := _m.Called(name) 107 | 108 | var r0 bool 109 | if rf, ok := ret.Get(0).(func(string) bool); ok { 110 | r0 = rf(name) 111 | } else { 112 | r0 = ret.Get(0).(bool) 113 | } 114 | 115 | var r1 error 116 | if rf, ok := ret.Get(1).(func(string) error); ok { 117 | r1 = rf(name) 118 | } else { 119 | r1 = ret.Error(1) 120 | } 121 | 122 | return r0, r1 123 | } 124 | 125 | // CreateApplication provides a mock function with given fields: application 126 | func (_m *MockMarathon) CreateApplication(application *marathon.Application) (*marathon.Application, error) { 127 | ret := _m.Called(application) 128 | 129 | var r0 *marathon.Application 130 | if rf, ok := ret.Get(0).(func(*marathon.Application) *marathon.Application); ok { 131 | r0 = rf(application) 132 | } else { 133 | if ret.Get(0) != nil { 134 | r0 = ret.Get(0).(*marathon.Application) 135 | } 136 | } 137 | 138 | var r1 error 139 | if rf, ok := ret.Get(1).(func(*marathon.Application) error); ok { 140 | r1 = rf(application) 141 | } else { 142 | r1 = ret.Error(1) 143 | } 144 | 145 | return r0, r1 146 | } 147 | 148 | // DeleteApplication provides a mock function with given fields: name 149 | func (_m *MockMarathon) DeleteApplication(name string) (*marathon.DeploymentID, error) { 150 | ret := _m.Called(name) 151 | 152 | var r0 *marathon.DeploymentID 153 | if rf, ok := ret.Get(0).(func(string) *marathon.DeploymentID); ok { 154 | r0 = rf(name) 155 | } else { 156 | if ret.Get(0) != nil { 157 | r0 = ret.Get(0).(*marathon.DeploymentID) 158 | } 159 | } 160 | 161 | var r1 error 162 | if rf, ok := ret.Get(1).(func(string) error); ok { 163 | r1 = rf(name) 164 | } else { 165 | r1 = ret.Error(1) 166 | } 167 | 168 | return r0, r1 169 | } 170 | 171 | // UpdateApplication provides a mock function with given fields: application, force 172 | func (_m *MockMarathon) UpdateApplication(application *marathon.Application, force bool) (*marathon.DeploymentID, error) { 173 | ret := _m.Called(application, force) 174 | 175 | var r0 *marathon.DeploymentID 176 | if rf, ok := ret.Get(0).(func(*marathon.Application, bool) *marathon.DeploymentID); ok { 177 | r0 = rf(application, force) 178 | } else { 179 | if ret.Get(0) != nil { 180 | r0 = ret.Get(0).(*marathon.DeploymentID) 181 | } 182 | } 183 | 184 | var r1 error 185 | if rf, ok := ret.Get(1).(func(*marathon.Application, bool) error); ok { 186 | r1 = rf(application, force) 187 | } else { 188 | r1 = ret.Error(1) 189 | } 190 | 191 | return r0, r1 192 | } 193 | 194 | // ApplicationDeployments provides a mock function with given fields: name 195 | func (_m *MockMarathon) ApplicationDeployments(name string) ([]*marathon.DeploymentID, error) { 196 | ret := _m.Called(name) 197 | 198 | var r0 []*marathon.DeploymentID 199 | if rf, ok := ret.Get(0).(func(string) []*marathon.DeploymentID); ok { 200 | r0 = rf(name) 201 | } else { 202 | if ret.Get(0) != nil { 203 | r0 = ret.Get(0).([]*marathon.DeploymentID) 204 | } 205 | } 206 | 207 | var r1 error 208 | if rf, ok := ret.Get(1).(func(string) error); ok { 209 | r1 = rf(name) 210 | } else { 211 | r1 = ret.Error(1) 212 | } 213 | 214 | return r0, r1 215 | } 216 | 217 | // ScaleApplicationInstances provides a mock function with given fields: name, instances, force 218 | func (_m *MockMarathon) ScaleApplicationInstances(name string, instances int, force bool) (*marathon.DeploymentID, error) { 219 | ret := _m.Called(name, instances, force) 220 | 221 | var r0 *marathon.DeploymentID 222 | if rf, ok := ret.Get(0).(func(string, int, bool) *marathon.DeploymentID); ok { 223 | r0 = rf(name, instances, force) 224 | } else { 225 | if ret.Get(0) != nil { 226 | r0 = ret.Get(0).(*marathon.DeploymentID) 227 | } 228 | } 229 | 230 | var r1 error 231 | if rf, ok := ret.Get(1).(func(string, int, bool) error); ok { 232 | r1 = rf(name, instances, force) 233 | } else { 234 | r1 = ret.Error(1) 235 | } 236 | 237 | return r0, r1 238 | } 239 | 240 | // RestartApplication provides a mock function with given fields: name, force 241 | func (_m *MockMarathon) RestartApplication(name string, force bool) (*marathon.DeploymentID, error) { 242 | ret := _m.Called(name, force) 243 | 244 | var r0 *marathon.DeploymentID 245 | if rf, ok := ret.Get(0).(func(string, bool) *marathon.DeploymentID); ok { 246 | r0 = rf(name, force) 247 | } else { 248 | if ret.Get(0) != nil { 249 | r0 = ret.Get(0).(*marathon.DeploymentID) 250 | } 251 | } 252 | 253 | var r1 error 254 | if rf, ok := ret.Get(1).(func(string, bool) error); ok { 255 | r1 = rf(name, force) 256 | } else { 257 | r1 = ret.Error(1) 258 | } 259 | 260 | return r0, r1 261 | } 262 | 263 | // Applications provides a mock function with given fields: _a0 264 | func (_m *MockMarathon) Applications(_a0 url.Values) (*marathon.Applications, error) { 265 | ret := _m.Called(_a0) 266 | 267 | var r0 *marathon.Applications 268 | if rf, ok := ret.Get(0).(func(url.Values) *marathon.Applications); ok { 269 | r0 = rf(_a0) 270 | } else { 271 | if ret.Get(0) != nil { 272 | r0 = ret.Get(0).(*marathon.Applications) 273 | } 274 | } 275 | 276 | var r1 error 277 | if rf, ok := ret.Get(1).(func(url.Values) error); ok { 278 | r1 = rf(_a0) 279 | } else { 280 | r1 = ret.Error(1) 281 | } 282 | 283 | return r0, r1 284 | } 285 | 286 | // Application provides a mock function with given fields: name 287 | func (_m *MockMarathon) Application(name string) (*marathon.Application, error) { 288 | ret := _m.Called(name) 289 | 290 | var r0 *marathon.Application 291 | if rf, ok := ret.Get(0).(func(string) *marathon.Application); ok { 292 | r0 = rf(name) 293 | } else { 294 | if ret.Get(0) != nil { 295 | r0 = ret.Get(0).(*marathon.Application) 296 | } 297 | } 298 | 299 | var r1 error 300 | if rf, ok := ret.Get(1).(func(string) error); ok { 301 | r1 = rf(name) 302 | } else { 303 | r1 = ret.Error(1) 304 | } 305 | 306 | return r0, r1 307 | } 308 | 309 | // WaitOnApplication provides a mock function with given fields: name, timeout 310 | func (_m *MockMarathon) WaitOnApplication(name string, timeout time.Duration) error { 311 | ret := _m.Called(name, timeout) 312 | 313 | var r0 error 314 | if rf, ok := ret.Get(0).(func(string, time.Duration) error); ok { 315 | r0 = rf(name, timeout) 316 | } else { 317 | r0 = ret.Error(0) 318 | } 319 | 320 | return r0 321 | } 322 | 323 | // Tasks provides a mock function with given fields: application 324 | func (_m *MockMarathon) Tasks(application string) (*marathon.Tasks, error) { 325 | ret := _m.Called(application) 326 | 327 | var r0 *marathon.Tasks 328 | if rf, ok := ret.Get(0).(func(string) *marathon.Tasks); ok { 329 | r0 = rf(application) 330 | } else { 331 | if ret.Get(0) != nil { 332 | r0 = ret.Get(0).(*marathon.Tasks) 333 | } 334 | } 335 | 336 | var r1 error 337 | if rf, ok := ret.Get(1).(func(string) error); ok { 338 | r1 = rf(application) 339 | } else { 340 | r1 = ret.Error(1) 341 | } 342 | 343 | return r0, r1 344 | } 345 | 346 | // AllTasks provides a mock function with given fields: opts 347 | func (_m *MockMarathon) AllTasks(opts *marathon.AllTasksOpts) (*marathon.Tasks, error) { 348 | ret := _m.Called(opts) 349 | 350 | var r0 *marathon.Tasks 351 | if rf, ok := ret.Get(0).(func(*marathon.AllTasksOpts) *marathon.Tasks); ok { 352 | r0 = rf(opts) 353 | } else { 354 | if ret.Get(0) != nil { 355 | r0 = ret.Get(0).(*marathon.Tasks) 356 | } 357 | } 358 | 359 | var r1 error 360 | if rf, ok := ret.Get(1).(func(*marathon.AllTasksOpts) error); ok { 361 | r1 = rf(opts) 362 | } else { 363 | r1 = ret.Error(1) 364 | } 365 | 366 | return r0, r1 367 | } 368 | 369 | // TaskEndpoints provides a mock function with given fields: name, port, healthCheck 370 | func (_m *MockMarathon) TaskEndpoints(name string, port int, healthCheck bool) ([]string, error) { 371 | ret := _m.Called(name, port, healthCheck) 372 | 373 | var r0 []string 374 | if rf, ok := ret.Get(0).(func(string, int, bool) []string); ok { 375 | r0 = rf(name, port, healthCheck) 376 | } else { 377 | if ret.Get(0) != nil { 378 | r0 = ret.Get(0).([]string) 379 | } 380 | } 381 | 382 | var r1 error 383 | if rf, ok := ret.Get(1).(func(string, int, bool) error); ok { 384 | r1 = rf(name, port, healthCheck) 385 | } else { 386 | r1 = ret.Error(1) 387 | } 388 | 389 | return r0, r1 390 | } 391 | 392 | // KillApplicationTasks provides a mock function with given fields: applicationID, opts 393 | func (_m *MockMarathon) KillApplicationTasks(applicationID string, opts *marathon.KillApplicationTasksOpts) (*marathon.Tasks, error) { 394 | ret := _m.Called(applicationID, opts) 395 | 396 | var r0 *marathon.Tasks 397 | if rf, ok := ret.Get(0).(func(string, *marathon.KillApplicationTasksOpts) *marathon.Tasks); ok { 398 | r0 = rf(applicationID, opts) 399 | } else { 400 | if ret.Get(0) != nil { 401 | r0 = ret.Get(0).(*marathon.Tasks) 402 | } 403 | } 404 | 405 | var r1 error 406 | if rf, ok := ret.Get(1).(func(string, *marathon.KillApplicationTasksOpts) error); ok { 407 | r1 = rf(applicationID, opts) 408 | } else { 409 | r1 = ret.Error(1) 410 | } 411 | 412 | return r0, r1 413 | } 414 | 415 | // KillTask provides a mock function with given fields: taskID, opts 416 | func (_m *MockMarathon) KillTask(taskID string, opts *marathon.KillTaskOpts) (*marathon.Task, error) { 417 | ret := _m.Called(taskID, opts) 418 | 419 | var r0 *marathon.Task 420 | if rf, ok := ret.Get(0).(func(string, *marathon.KillTaskOpts) *marathon.Task); ok { 421 | r0 = rf(taskID, opts) 422 | } else { 423 | if ret.Get(0) != nil { 424 | r0 = ret.Get(0).(*marathon.Task) 425 | } 426 | } 427 | 428 | var r1 error 429 | if rf, ok := ret.Get(1).(func(string, *marathon.KillTaskOpts) error); ok { 430 | r1 = rf(taskID, opts) 431 | } else { 432 | r1 = ret.Error(1) 433 | } 434 | 435 | return r0, r1 436 | } 437 | 438 | // KillTasks provides a mock function with given fields: taskIDs, opts 439 | func (_m *MockMarathon) KillTasks(taskIDs []string, opts *marathon.KillTaskOpts) error { 440 | ret := _m.Called(taskIDs, opts) 441 | 442 | var r0 error 443 | if rf, ok := ret.Get(0).(func([]string, *marathon.KillTaskOpts) error); ok { 444 | r0 = rf(taskIDs, opts) 445 | } else { 446 | r0 = ret.Error(0) 447 | } 448 | 449 | return r0 450 | } 451 | 452 | // Groups provides a mock function with given fields: 453 | func (_m *MockMarathon) Groups() (*marathon.Groups, error) { 454 | ret := _m.Called() 455 | 456 | var r0 *marathon.Groups 457 | if rf, ok := ret.Get(0).(func() *marathon.Groups); ok { 458 | r0 = rf() 459 | } else { 460 | if ret.Get(0) != nil { 461 | r0 = ret.Get(0).(*marathon.Groups) 462 | } 463 | } 464 | 465 | var r1 error 466 | if rf, ok := ret.Get(1).(func() error); ok { 467 | r1 = rf() 468 | } else { 469 | r1 = ret.Error(1) 470 | } 471 | 472 | return r0, r1 473 | } 474 | 475 | // Group provides a mock function with given fields: name 476 | func (_m *MockMarathon) Group(name string) (*marathon.Group, error) { 477 | ret := _m.Called(name) 478 | 479 | var r0 *marathon.Group 480 | if rf, ok := ret.Get(0).(func(string) *marathon.Group); ok { 481 | r0 = rf(name) 482 | } else { 483 | if ret.Get(0) != nil { 484 | r0 = ret.Get(0).(*marathon.Group) 485 | } 486 | } 487 | 488 | var r1 error 489 | if rf, ok := ret.Get(1).(func(string) error); ok { 490 | r1 = rf(name) 491 | } else { 492 | r1 = ret.Error(1) 493 | } 494 | 495 | return r0, r1 496 | } 497 | 498 | // CreateGroup provides a mock function with given fields: group 499 | func (_m *MockMarathon) CreateGroup(group *marathon.Group) error { 500 | ret := _m.Called(group) 501 | 502 | var r0 error 503 | if rf, ok := ret.Get(0).(func(*marathon.Group) error); ok { 504 | r0 = rf(group) 505 | } else { 506 | r0 = ret.Error(0) 507 | } 508 | 509 | return r0 510 | } 511 | 512 | // DeleteGroup provides a mock function with given fields: name 513 | func (_m *MockMarathon) DeleteGroup(name string) (*marathon.DeploymentID, error) { 514 | ret := _m.Called(name) 515 | 516 | var r0 *marathon.DeploymentID 517 | if rf, ok := ret.Get(0).(func(string) *marathon.DeploymentID); ok { 518 | r0 = rf(name) 519 | } else { 520 | if ret.Get(0) != nil { 521 | r0 = ret.Get(0).(*marathon.DeploymentID) 522 | } 523 | } 524 | 525 | var r1 error 526 | if rf, ok := ret.Get(1).(func(string) error); ok { 527 | r1 = rf(name) 528 | } else { 529 | r1 = ret.Error(1) 530 | } 531 | 532 | return r0, r1 533 | } 534 | 535 | // UpdateGroup provides a mock function with given fields: id, group 536 | func (_m *MockMarathon) UpdateGroup(id string, group *marathon.Group) (*marathon.DeploymentID, error) { 537 | ret := _m.Called(id, group) 538 | 539 | var r0 *marathon.DeploymentID 540 | if rf, ok := ret.Get(0).(func(string, *marathon.Group) *marathon.DeploymentID); ok { 541 | r0 = rf(id, group) 542 | } else { 543 | if ret.Get(0) != nil { 544 | r0 = ret.Get(0).(*marathon.DeploymentID) 545 | } 546 | } 547 | 548 | var r1 error 549 | if rf, ok := ret.Get(1).(func(string, *marathon.Group) error); ok { 550 | r1 = rf(id, group) 551 | } else { 552 | r1 = ret.Error(1) 553 | } 554 | 555 | return r0, r1 556 | } 557 | 558 | // HasGroup provides a mock function with given fields: name 559 | func (_m *MockMarathon) HasGroup(name string) (bool, error) { 560 | ret := _m.Called(name) 561 | 562 | var r0 bool 563 | if rf, ok := ret.Get(0).(func(string) bool); ok { 564 | r0 = rf(name) 565 | } else { 566 | r0 = ret.Get(0).(bool) 567 | } 568 | 569 | var r1 error 570 | if rf, ok := ret.Get(1).(func(string) error); ok { 571 | r1 = rf(name) 572 | } else { 573 | r1 = ret.Error(1) 574 | } 575 | 576 | return r0, r1 577 | } 578 | 579 | // WaitOnGroup provides a mock function with given fields: name, timeout 580 | func (_m *MockMarathon) WaitOnGroup(name string, timeout time.Duration) error { 581 | ret := _m.Called(name, timeout) 582 | 583 | var r0 error 584 | if rf, ok := ret.Get(0).(func(string, time.Duration) error); ok { 585 | r0 = rf(name, timeout) 586 | } else { 587 | r0 = ret.Error(0) 588 | } 589 | 590 | return r0 591 | } 592 | 593 | // Deployments provides a mock function with given fields: 594 | func (_m *MockMarathon) Deployments() ([]*marathon.Deployment, error) { 595 | ret := _m.Called() 596 | 597 | var r0 []*marathon.Deployment 598 | if rf, ok := ret.Get(0).(func() []*marathon.Deployment); ok { 599 | r0 = rf() 600 | } else { 601 | if ret.Get(0) != nil { 602 | r0 = ret.Get(0).([]*marathon.Deployment) 603 | } 604 | } 605 | 606 | var r1 error 607 | if rf, ok := ret.Get(1).(func() error); ok { 608 | r1 = rf() 609 | } else { 610 | r1 = ret.Error(1) 611 | } 612 | 613 | return r0, r1 614 | } 615 | 616 | // DeleteDeployment provides a mock function with given fields: id, force 617 | func (_m *MockMarathon) DeleteDeployment(id string, force bool) (*marathon.DeploymentID, error) { 618 | ret := _m.Called(id, force) 619 | 620 | var r0 *marathon.DeploymentID 621 | if rf, ok := ret.Get(0).(func(string, bool) *marathon.DeploymentID); ok { 622 | r0 = rf(id, force) 623 | } else { 624 | if ret.Get(0) != nil { 625 | r0 = ret.Get(0).(*marathon.DeploymentID) 626 | } 627 | } 628 | 629 | var r1 error 630 | if rf, ok := ret.Get(1).(func(string, bool) error); ok { 631 | r1 = rf(id, force) 632 | } else { 633 | r1 = ret.Error(1) 634 | } 635 | 636 | return r0, r1 637 | } 638 | 639 | // HasDeployment provides a mock function with given fields: id 640 | func (_m *MockMarathon) HasDeployment(id string) (bool, error) { 641 | ret := _m.Called(id) 642 | 643 | var r0 bool 644 | if rf, ok := ret.Get(0).(func(string) bool); ok { 645 | r0 = rf(id) 646 | } else { 647 | r0 = ret.Get(0).(bool) 648 | } 649 | 650 | var r1 error 651 | if rf, ok := ret.Get(1).(func(string) error); ok { 652 | r1 = rf(id) 653 | } else { 654 | r1 = ret.Error(1) 655 | } 656 | 657 | return r0, r1 658 | } 659 | 660 | // WaitOnDeployment provides a mock function with given fields: id, timeout 661 | func (_m *MockMarathon) WaitOnDeployment(id string, timeout time.Duration) error { 662 | ret := _m.Called(id, timeout) 663 | 664 | var r0 error 665 | if rf, ok := ret.Get(0).(func(string, time.Duration) error); ok { 666 | r0 = rf(id, timeout) 667 | } else { 668 | r0 = ret.Error(0) 669 | } 670 | 671 | return r0 672 | } 673 | 674 | // Subscriptions provides a mock function with given fields: 675 | func (_m *MockMarathon) Subscriptions() (*marathon.Subscriptions, error) { 676 | ret := _m.Called() 677 | 678 | var r0 *marathon.Subscriptions 679 | if rf, ok := ret.Get(0).(func() *marathon.Subscriptions); ok { 680 | r0 = rf() 681 | } else { 682 | if ret.Get(0) != nil { 683 | r0 = ret.Get(0).(*marathon.Subscriptions) 684 | } 685 | } 686 | 687 | var r1 error 688 | if rf, ok := ret.Get(1).(func() error); ok { 689 | r1 = rf() 690 | } else { 691 | r1 = ret.Error(1) 692 | } 693 | 694 | return r0, r1 695 | } 696 | 697 | // AddEventsListener provides a mock function with given fields: channel, filter 698 | func (_m *MockMarathon) AddEventsListener(channel marathon.EventsChannel, filter int) error { 699 | ret := _m.Called(channel, filter) 700 | 701 | var r0 error 702 | if rf, ok := ret.Get(0).(func(marathon.EventsChannel, int) error); ok { 703 | r0 = rf(channel, filter) 704 | } else { 705 | r0 = ret.Error(0) 706 | } 707 | 708 | return r0 709 | } 710 | 711 | // RemoveEventsListener provides a mock function with given fields: channel 712 | func (_m *MockMarathon) RemoveEventsListener(channel marathon.EventsChannel) { 713 | _m.Called(channel) 714 | } 715 | 716 | // Unsubscribe provides a mock function with given fields: _a0 717 | func (_m *MockMarathon) Unsubscribe(_a0 string) error { 718 | ret := _m.Called(_a0) 719 | 720 | var r0 error 721 | if rf, ok := ret.Get(0).(func(string) error); ok { 722 | r0 = rf(_a0) 723 | } else { 724 | r0 = ret.Error(0) 725 | } 726 | 727 | return r0 728 | } 729 | 730 | // GetMarathonURL provides a mock function with given fields: 731 | func (_m *MockMarathon) GetMarathonURL() string { 732 | ret := _m.Called() 733 | 734 | var r0 string 735 | if rf, ok := ret.Get(0).(func() string); ok { 736 | r0 = rf() 737 | } else { 738 | r0 = ret.Get(0).(string) 739 | } 740 | 741 | return r0 742 | } 743 | 744 | // Ping provides a mock function with given fields: 745 | func (_m *MockMarathon) Ping() (bool, error) { 746 | ret := _m.Called() 747 | 748 | var r0 bool 749 | if rf, ok := ret.Get(0).(func() bool); ok { 750 | r0 = rf() 751 | } else { 752 | r0 = ret.Get(0).(bool) 753 | } 754 | 755 | var r1 error 756 | if rf, ok := ret.Get(1).(func() error); ok { 757 | r1 = rf() 758 | } else { 759 | r1 = ret.Error(1) 760 | } 761 | 762 | return r0, r1 763 | } 764 | 765 | // Info provides a mock function with given fields: 766 | func (_m *MockMarathon) Info() (*marathon.Info, error) { 767 | ret := _m.Called() 768 | 769 | var r0 *marathon.Info 770 | if rf, ok := ret.Get(0).(func() *marathon.Info); ok { 771 | r0 = rf() 772 | } else { 773 | if ret.Get(0) != nil { 774 | r0 = ret.Get(0).(*marathon.Info) 775 | } 776 | } 777 | 778 | var r1 error 779 | if rf, ok := ret.Get(1).(func() error); ok { 780 | r1 = rf() 781 | } else { 782 | r1 = ret.Error(1) 783 | } 784 | 785 | return r0, r1 786 | } 787 | 788 | // Leader provides a mock function with given fields: 789 | func (_m *MockMarathon) Leader() (string, error) { 790 | ret := _m.Called() 791 | 792 | var r0 string 793 | if rf, ok := ret.Get(0).(func() string); ok { 794 | r0 = rf() 795 | } else { 796 | r0 = ret.Get(0).(string) 797 | } 798 | 799 | var r1 error 800 | if rf, ok := ret.Get(1).(func() error); ok { 801 | r1 = rf() 802 | } else { 803 | r1 = ret.Error(1) 804 | } 805 | 806 | return r0, r1 807 | } 808 | 809 | // AbdicateLeader provides a mock function with given fields: 810 | func (_m *MockMarathon) AbdicateLeader() (string, error) { 811 | ret := _m.Called() 812 | 813 | var r0 string 814 | if rf, ok := ret.Get(0).(func() string); ok { 815 | r0 = rf() 816 | } else { 817 | r0 = ret.Get(0).(string) 818 | } 819 | 820 | var r1 error 821 | if rf, ok := ret.Get(1).(func() error); ok { 822 | r1 = rf() 823 | } else { 824 | r1 = ret.Error(1) 825 | } 826 | 827 | return r0, r1 828 | } 829 | --------------------------------------------------------------------------------