├── VERSION ├── marathon ├── testdata │ ├── no_groups.json │ ├── one_root_group.json │ ├── nested_groups_empty.json │ └── nested_groups_two_empty_one_with_app.json ├── config.go ├── group.go ├── group_test.go ├── marathon_stub.go ├── app.go ├── app_test.go ├── marathon.go └── marathon_test.go ├── web ├── config.go ├── health_handler.go ├── sse.go ├── sse_handler.go ├── event.go ├── event_test.go └── event_handler.go ├── mgc ├── config.go ├── date.go ├── mgc.go └── mgc_test.go ├── score ├── config.go ├── score.go └── score_test.go ├── debian ├── appcop.upstart └── appcop.service ├── glide.yaml ├── golangcilinter.yaml ├── .gitignore ├── config ├── testdata │ └── config.json ├── config_test.go └── config.go ├── metrics ├── config.go ├── system_metrics_test.go ├── system_metrics.go ├── metrics.go └── metrics_test.go ├── .travis.yml ├── glide.lock ├── Dockerfile ├── main.go ├── Makefile ├── README.md └── LICENSE /VERSION: -------------------------------------------------------------------------------- 1 | 0.13.0 2 | -------------------------------------------------------------------------------- /marathon/testdata/no_groups.json: -------------------------------------------------------------------------------- 1 | 2 | {"groups": [ 3 | ] 4 | } 5 | -------------------------------------------------------------------------------- /marathon/testdata/one_root_group.json: -------------------------------------------------------------------------------- 1 | {"groups": [ 2 | {"apps": [], "groups": [], "id": "idgroup0", "version": "1234"} 3 | ] 4 | } 5 | -------------------------------------------------------------------------------- /web/config.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | // Config specific to web package 4 | type Config struct { 5 | Listen string 6 | Location string 7 | QueueSize int 8 | WorkersCount int 9 | MyLeader string 10 | } 11 | -------------------------------------------------------------------------------- /mgc/config.go: -------------------------------------------------------------------------------- 1 | package mgc 2 | 3 | import "time" 4 | 5 | // Config specific to mgc module 6 | type Config struct { 7 | MaxSuspendTime time.Duration 8 | Interval time.Duration 9 | AppCopOnly bool 10 | Enabled bool 11 | } 12 | -------------------------------------------------------------------------------- /web/health_handler.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | ) 7 | 8 | // HealthHandler is standart health check for appcop 9 | func HealthHandler(w http.ResponseWriter, r *http.Request) { 10 | _, err := fmt.Fprint(w, "OK") 11 | if err != nil { 12 | return 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /score/config.go: -------------------------------------------------------------------------------- 1 | package score 2 | 3 | import "time" 4 | 5 | // Config contains specific configuration to score module 6 | type Config struct { 7 | DryRun bool 8 | ScaleDownScore int 9 | UpdateInterval time.Duration 10 | ResetInterval time.Duration 11 | EvaluateInterval time.Duration 12 | ScaleLimit int 13 | } 14 | -------------------------------------------------------------------------------- /debian/appcop.upstart: -------------------------------------------------------------------------------- 1 | description "AppCop - Marathon applications law enforcement" 2 | 3 | start on runlevel [2345] 4 | stop on runlevel [!2345] 5 | 6 | respawn 7 | 8 | script 9 | echo $$ > /var/run/appcop.pid 10 | exec appcop --config-file=/etc/appcop/appcop.json 11 | end script 12 | 13 | post-stop script 14 | rm -f /var/run/appcop.pid 15 | end script 16 | -------------------------------------------------------------------------------- /debian/appcop.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=AppCop - Marathon applications law enforcement 3 | Requires=network-online.target 4 | After=network-online.target 5 | 6 | [Service] 7 | ExecStart=/usr/bin/appcop --config-file=/etc/appcop/appcop.json 8 | ExecReload=/bin/kill -HUP $MAINPID 9 | Restart=on-failure 10 | KillSignal=SIGINT 11 | 12 | [Install] 13 | WantedBy=multi-user.target 14 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: github.com/allegro/marathon-appcop 2 | import: 3 | - package: github.com/Sirupsen/logrus 4 | - package: github.com/cyberdelia/go-metrics-graphite 5 | - package: github.com/ogier/pflag 6 | - package: github.com/rcrowley/go-metrics 7 | - package: github.com/sethgrid/pester 8 | testImport: 9 | - package: github.com/stretchr/testify 10 | subpackages: 11 | - assert 12 | - require 13 | -------------------------------------------------------------------------------- /golangcilinter.yaml: -------------------------------------------------------------------------------- 1 | run: 2 | aggregate: true 3 | concurrency: 2 4 | cyclo: 14 5 | deadline: "300s" 6 | skip-files: 7 | - ".*_string.go" 8 | - ".*_test.go" 9 | issues: 10 | max-same-issues: 0 11 | linters: 12 | disable-all: true 13 | enable: 14 | - stylecheck 15 | - deadcode 16 | - errcheck 17 | - gocyclo 18 | - goimports 19 | - golint 20 | - gosimple 21 | - govet 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.deb 2 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 3 | *.o 4 | *.a 5 | *.so 6 | 7 | # Folders 8 | .idea 9 | _obj 10 | _test 11 | build 12 | vendor 13 | bin 14 | dist 15 | 16 | # Architecture specific extensions/prefixes 17 | *.[568vq] 18 | [568vq].out 19 | 20 | *.cgo1.go 21 | *.cgo2.c 22 | _cgo_defun.c 23 | _cgo_gotypes.go 24 | _cgo_export.* 25 | 26 | _testmain.go 27 | 28 | *.exe 29 | *.test 30 | *.prof 31 | marathon-forwarder 32 | 33 | # goxc 34 | .goxc.local.json 35 | 36 | # coverage 37 | /coverage 38 | 39 | tags 40 | -------------------------------------------------------------------------------- /marathon/config.go: -------------------------------------------------------------------------------- 1 | package marathon 2 | 3 | import "time" 4 | 5 | // Config contains marathon module specific configuration 6 | type Config struct { 7 | Location string 8 | Protocol string 9 | Username string 10 | Password string 11 | // AppIDPrefix is a part of application id preferably present 12 | // in all applications in marathon, if found it is removed for the sake of 13 | // making applications paths shorter. 14 | // By default this string is empty and no prefix is considered. 15 | AppIDPrefix string 16 | VerifySsl bool 17 | Timeout time.Duration 18 | } 19 | -------------------------------------------------------------------------------- /mgc/date.go: -------------------------------------------------------------------------------- 1 | package mgc 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // MarathonDate parses and hold marathon date in marathon specific format 8 | type MarathonDate struct { 9 | timeFormat string 10 | inTime time.Time 11 | } 12 | 13 | func toMarathonDate(dateStr string) (*MarathonDate, error) { 14 | timeFormat := "2006-01-02T15:04:05.000Z" 15 | inTime, err := time.Parse(timeFormat, dateStr) 16 | if err != nil { 17 | return nil, err 18 | } 19 | return &MarathonDate{timeFormat: timeFormat, inTime: inTime}, nil 20 | } 21 | 22 | func (d *MarathonDate) elapsed() time.Duration { 23 | return time.Since(d.inTime) 24 | } 25 | -------------------------------------------------------------------------------- /config/testdata/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "Web": { 3 | "Listen": ":4444", 4 | "MyLeader": "" 5 | }, 6 | "Marathon": { 7 | "Location": "example.com:8080", 8 | "Protocol": "http", 9 | "Username": "", 10 | "Password": "", 11 | "VerifySsl": true 12 | }, 13 | "Score": { 14 | "ScaleDownScore": 200, 15 | "UpdateInterval": 2000000000, 16 | "ResetInterval": 86400000000000, 17 | "EvaluateInterval": 20000000000, 18 | "DryRun": false 19 | }, 20 | "Metrics": { 21 | "Target": "stdout", 22 | "Prefix": "default", 23 | "Interval": 30000000000, 24 | "Addr": "" 25 | }, 26 | "Log": { 27 | "Level": "info", 28 | "Format": "text" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /metrics/config.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import "time" 4 | 5 | // Config specific to metrics package 6 | type Config struct { 7 | Target string 8 | //Prefix is the begining of metric, it is prepended 9 | // in each and every published metric. 10 | Prefix string 11 | Interval time.Duration 12 | Addr string 13 | Instance string 14 | // SystemSubPrefix it is part of a metric that is appended to the 15 | // main Prefix, representing appcop internal metrics 16 | // essential to appcop admins, e.g runtime metrics, event processing time, 17 | // event queue size etc. 18 | SystemSubPrefix string 19 | // AppSubPrefix it is part of a metric that is appended to the 20 | // main Prefix, representing applications specific metric, e.g task_running, 21 | // task_staging, task_failed. 22 | AppSubPrefix string 23 | } 24 | -------------------------------------------------------------------------------- /marathon/group.go: -------------------------------------------------------------------------------- 1 | package marathon 2 | 3 | import "encoding/json" 4 | 5 | // GroupsResponse json returned from Marathon with apps definitions 6 | type GroupsResponse struct { 7 | Groups []*Group `json:"groups"` 8 | } 9 | 10 | // Group represents application returned in Marathon json 11 | type Group struct { 12 | Apps []*App `json:"apps"` 13 | Groups []*Group `json:"groups"` 14 | ID GroupID `json:"id"` 15 | Version string `json:"version"` 16 | } 17 | 18 | // IsEmpty checks if group is an empty group 19 | func (g *Group) IsEmpty() bool { 20 | return len(g.Apps) == 0 && len(g.Groups) == 0 21 | } 22 | 23 | // GroupID represents group id from marathon 24 | type GroupID string 25 | 26 | //// String stringer for group 27 | func (id GroupID) String() string { 28 | return string(id) 29 | } 30 | 31 | // ParseGroups json 32 | func ParseGroups(jsonBlob []byte) ([]*Group, error) { 33 | groups := &GroupsResponse{} 34 | err := json.Unmarshal(jsonBlob, groups) 35 | 36 | return groups.Groups, err 37 | } 38 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | dist: trusty 3 | sudo: false 4 | services: 5 | - docker 6 | go: 7 | - 1.12 8 | before_install: 9 | - go get github.com/mattn/goveralls 10 | script: 11 | - make pack 12 | - goveralls -coverprofile=coverage/gover.coverprofile -service travis-ci 13 | deploy: 14 | provider: releases 15 | skip_cleanup: true 16 | api_key: 17 | secure: Rp+trThfG+pljpISsxtYlmGkjbnqjbKHztcJJcvYDfkctxTGwCOjbk0GsbC0g+23BTB8fvAoKdD+jx2yuSIVbEM0StYrhVaztuye/1lA/L9w8Ib9pqJIH/kvZhw5aC8PyJXdZrEpxLwteJsHQP/hgwmjUHnHVGaG0VJJHpaGAYNBmng1NNiVmGdIRvAhbW3+RT+W56AGbPFsLJavsPYGK7ecYcge4SP/1V0652VjIE1saH+LpMx5ndPQ01aj2tpV5/vyWfPPKzo5/au9JaMBw7L8/L1mkZUaqFHQMLehM9rcXs7Hgcir9Tjt7faHDeIrZt+W320QXO3skmsr2Kgt49HU9Gz3v6j4XJ9wxflupWmP0b51BIhVP/qpv2V+oIFZVen5hr3P8nTPtdgrOT9kZJCK6BTGaKzoTJdOki21FerW4hwHiq5/rvjuK9/yuBn7s0IOe4c2YMy83S1ZbAw1ukLOUfbonhKy4q2Fv9WIjROOfx4AOMxQU4zJfjme3KzwoNe0UWpIpzX1eOdULMzlVU/BEVLqSAuKVhFLtu6FEf3QmJzJS0+W7jhkeHGcGg4TDRufrpspZi8tyZqKBIAXpjg5xJfNcoKnfIfgom6kHOePScu+QAp5R/UXkkoutigLcnUwGGZdgjuZnLnYxxR1+4Bz7aICajodlRtfRMbcyJo= 18 | file_glob: true 19 | file: dist/appcop* 20 | on: 21 | tags: true 22 | -------------------------------------------------------------------------------- /metrics/system_metrics_test.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "testing" 5 | 6 | "runtime" 7 | 8 | "github.com/rcrowley/go-metrics" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestMetricShouldBeGauge(t *testing.T) { 13 | t.Parallel() 14 | 15 | // expect 16 | assert.Implements(t, (*metrics.Gauge)(nil), baseGauge{}) 17 | } 18 | 19 | func TestGaugeReturnsValueFromGivenFunction(t *testing.T) { 20 | t.Parallel() 21 | 22 | // given 23 | var counter int64 24 | bg := baseGauge{value: func(_ runtime.MemStats) int64 { 25 | counter++ 26 | return counter 27 | }} 28 | 29 | //when 30 | bg.Value() 31 | bg.Value() 32 | bg.Value() 33 | 34 | //then 35 | assert.Equal(t, (int64)(3), counter) 36 | } 37 | 38 | func TestMetricsRegistered(t *testing.T) { 39 | t.Parallel() 40 | 41 | //when 42 | collectSystemMetrics() 43 | 44 | //then 45 | assert.NotNil(t, metrics.Get(systemMetric(allocGauge))) 46 | assert.NotNil(t, metrics.Get(systemMetric(heapObjectsGauge))) 47 | assert.NotNil(t, metrics.Get(systemMetric(totalPauseGauge))) 48 | assert.NotNil(t, metrics.Get(systemMetric(lastPauseGauge))) 49 | } 50 | -------------------------------------------------------------------------------- /glide.lock: -------------------------------------------------------------------------------- 1 | hash: cdbab15413ea5a796fb7d267bc1c283de88eb10d352d0cbec4fd9cecedb936ed 2 | updated: 2017-03-15T13:57:12.272147094+01:00 3 | imports: 4 | - name: github.com/cyberdelia/go-metrics-graphite 5 | version: 7e54b5c2aa6eaff4286c44129c3def899dff528c 6 | - name: github.com/ogier/pflag 7 | version: 32a05c62658bd1d7c7e75cbc8195de5d585fde0f 8 | - name: github.com/rcrowley/go-metrics 9 | version: bdb33529eca3e55eac7328e07c57012a797af602 10 | - name: github.com/sethgrid/pester 11 | version: ab3a58f3fbc32058b25cf9e416272cf0097be523 12 | - name: github.com/Sirupsen/logrus 13 | version: c078b1e43f58d563c74cebe63c85789e76ddb627 14 | - name: golang.org/x/sys 15 | version: 478fcf54317e52ab69f40bb4c7a1520288d7f7ea 16 | subpackages: 17 | - unix 18 | testImports: 19 | - name: github.com/davecgh/go-spew 20 | version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9 21 | subpackages: 22 | - spew 23 | - name: github.com/pmezard/go-difflib 24 | version: d8ed2627bdf02c080bf22230dbb337003b7aba2d 25 | subpackages: 26 | - difflib 27 | - name: github.com/stretchr/testify 28 | version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0 29 | subpackages: 30 | - assert 31 | - require 32 | -------------------------------------------------------------------------------- /marathon/group_test.go: -------------------------------------------------------------------------------- 1 | package marathon 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestParseGroupsWhenMalformedJSONSIPassedShouldReturnErrorAndNilGroups(t *testing.T) { 11 | t.Parallel() 12 | // given 13 | var jsonBlob = []byte(`{ 14 | "groups": [ 15 | } 16 | `) 17 | 18 | // when 19 | groups, err := ParseGroups(jsonBlob) 20 | 21 | // then 22 | require.Error(t, err) 23 | assert.Nil(t, groups) 24 | 25 | } 26 | 27 | func TestParseGroupsWhenProperJSONIsProvidedWithGroupAndEmptyAppsSholdReturnGroupAndNoError(t *testing.T) { 28 | t.Parallel() 29 | // given 30 | var jsonBlob = []byte(`{ 31 | "groups": [ 32 | { 33 | "apps": [], 34 | "dependencies": [], 35 | "groups": [], 36 | "id": "/com.example.tech.maas", 37 | "version": "2017-01-24T15:37:58.780Z" 38 | } 39 | ] 40 | } 41 | `) 42 | 43 | expectedGroups := []*Group{ 44 | {Apps: []*App{}, 45 | Groups: []*Group{}, 46 | ID: "/com.example.tech.maas", 47 | Version: "2017-01-24T15:37:58.780Z"}, 48 | } 49 | // when 50 | groups, err := ParseGroups(jsonBlob) 51 | 52 | // then 53 | require.NoError(t, err) 54 | assert.Equal(t, expectedGroups, groups) 55 | 56 | } 57 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:14.04 2 | # 3 | # Docker file used to build packages for .rpm and .deb 4 | # based distributions. 5 | # To propertly build package run in project directory: 6 | # 7 | # $ make build && make pack 8 | # or 9 | # $ make all 10 | # 11 | # For details refer to README.md file. 12 | # 13 | # Requires installed docker. 14 | # 15 | ENV GOPATH /golang 16 | ENV DESCRIPTION 'AppCop - Marathon applications law enforcement' 17 | ENV MAINTAINER "Allegro" 18 | 19 | RUN apt-get update && apt-get install -y software-properties-common 20 | RUN apt-add-repository -y ppa:brightbox/ruby-ng 21 | RUN apt-get update && apt-get install -y \ 22 | python-software-properties \ 23 | build-essential ruby2.1 \ 24 | ruby-switch \ 25 | ruby2.1-dev \ 26 | rpm 27 | 28 | RUN gem install fpm 29 | 30 | ADD . /work 31 | 32 | ENTRYPOINT cd /work && fpm -s dir -t deb \ 33 | --deb-upstart debian/appcop.upstart \ 34 | --deb-systemd debian/appcop.service \ 35 | -n appcop \ 36 | -v `cat ./VERSION` \ 37 | -m "$MAINTAINER" \ 38 | --description "$DESCRIPTION" \ 39 | build/appcop=/usr/bin/ \ 40 | && mv *deb dist/ && \ 41 | fpm -s dir -t rpm \ 42 | -n appcop \ 43 | -v `cat ./VERSION` \ 44 | -m "$MAINTAINER" \ 45 | --description "$DESCRIPTION" \ 46 | build/appcop=/usr/bin/ \ 47 | debian/appcop.service=/etc/systemd/system/appcop.service \ 48 | debian/appcop.service=/etc/init/appcop.conf \ 49 | && mv *rpm dist/ 50 | 51 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | 6 | log "github.com/Sirupsen/logrus" 7 | "github.com/allegro/marathon-appcop/config" 8 | "github.com/allegro/marathon-appcop/marathon" 9 | "github.com/allegro/marathon-appcop/metrics" 10 | "github.com/allegro/marathon-appcop/mgc" 11 | "github.com/allegro/marathon-appcop/score" 12 | "github.com/allegro/marathon-appcop/web" 13 | ) 14 | 15 | // Version variable provided at build time 16 | var Version string 17 | 18 | func main() { 19 | 20 | log.Infof("Appcop Version: %s", Version) 21 | config, err := config.NewConfig() 22 | if err != nil { 23 | log.Fatal(err.Error()) 24 | } 25 | 26 | err = metrics.Init(config.Metrics) 27 | if err != nil { 28 | log.Fatal(err.Error()) 29 | } 30 | 31 | remote, err := marathon.New(config.Marathon) 32 | if err != nil { 33 | log.Fatal(err.Error()) 34 | } 35 | 36 | scores, err := score.New(config.Score, remote) 37 | if err != nil { 38 | log.Fatal(err.Error()) 39 | } 40 | updates := scores.ScoreManager() 41 | 42 | gc, err := mgc.New(config.MGC, remote) 43 | if err != nil { 44 | log.Fatal(err.Error()) 45 | } 46 | stop := web.NewHandler(config.Web, remote, gc, updates) 47 | defer stop() 48 | 49 | // set up routes 50 | http.HandleFunc("/health", web.HealthHandler) 51 | 52 | log.WithField("Port", config.Web.Listen).Info("Listening") 53 | log.Fatal(http.ListenAndServe(config.Web.Listen, nil)) 54 | 55 | } 56 | -------------------------------------------------------------------------------- /marathon/testdata/nested_groups_empty.json: -------------------------------------------------------------------------------- 1 | { 2 | 3 | "id": "/", 4 | "dependencies": [ ], 5 | "version": "", 6 | "apps": [ ], 7 | "groups": [ 8 | { 9 | "id": "/grpa", 10 | "dependencies": [ ], 11 | "version": "", 12 | "apps": [ ], 13 | "groups": [ 14 | { 15 | "id": "/grpa/grpb", 16 | "dependencies": [ ], 17 | "version": "", 18 | "apps": [ ], 19 | "groups": [ 20 | { 21 | "id": "/grpa/grpb/grpc0", 22 | "dependencies": [ ], 23 | "version": "", 24 | "apps": [ ], 25 | "groups": [ ] 26 | }, 27 | { 28 | "id": "/grpa/grpb/grpc1", 29 | "dependencies": [ ], 30 | "version": "", 31 | "apps": [ ], 32 | "groups": [ ] 33 | } 34 | ] 35 | }, 36 | { 37 | "id": "/grpa/grpb1", 38 | "dependencies": [ ], 39 | "version": "", 40 | "apps": [ ], 41 | "groups": [ ] 42 | } 43 | ] 44 | } 45 | ] 46 | 47 | } 48 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | VERSION := `cat VERSION` 2 | LDFLAGS := -X "main.Version=$(VERSION)" 3 | GO_BUILD := go build -v -ldflags "$(LDFLAGS)" 4 | 5 | CURRENTDIR = $(shell pwd) 6 | 7 | COVERAGEDIR = $(CURRENTDIR)/coverage 8 | PACKAGES = $(shell go list ./... | grep -v /vendor/) 9 | TEST_TARGETS = $(PACKAGES) 10 | 11 | all: lint test build 12 | 13 | build: deps 14 | $(GO_BUILD) -o build/appcop . 15 | 16 | build_linux: 17 | env GOOS=linux GOARCH=amd64 GO111MODULE=on $(GO_BUILD) -o build/appcop . 18 | 19 | clean: 20 | go clean -v . 21 | rm -rf build 22 | 23 | debug: deps 24 | $(GO_BUILD) -race -tags 'debug' -o build/appcop . 25 | 26 | deps: 27 | @mkdir -p $(COVERAGEDIR) 28 | @go get github.com/modocache/gover 29 | @go get -u github.com/Masterminds/glide 30 | @glide install 31 | 32 | lint: deps lint-deps onlylint 33 | 34 | lint-deps: 35 | @which golangci-lint > /dev/null || \ 36 | (go get -u github.com/golangci/golangci-lint/cmd/golangci-lint) 37 | 38 | release: lint test 39 | GOARCH=amd64 GOOS=linux $(GO_BUILD) -o build/appcop . 40 | 41 | test: deps $(SOURCES) $(TEST_TARGETS) 42 | gover $(COVERAGEDIR) $(COVERAGEDIR)/gover.coverprofile 43 | 44 | $(TEST_TARGETS): 45 | go test -coverprofile=coverage/$(shell basename $@).coverprofile $@ 46 | 47 | pack: test lint build_linux 48 | docker build -t appcop . && mkdir -p dist && docker run -v ${PWD}/dist:/work/dist appcop 49 | 50 | onlylint: build 51 | golangci-lint run --config=golangcilinter.yaml web marathon metrics mgc score config 52 | 53 | version: deps 54 | echo -n $(v) > VERSION 55 | git add VERSION 56 | git commit -m "Release $(v)" 57 | -------------------------------------------------------------------------------- /metrics/system_metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "runtime" 5 | 6 | "github.com/rcrowley/go-metrics" 7 | ) 8 | 9 | const allocGauge = "runtime.mem.bytes_allocated_and_not_yet_freed" 10 | const heapObjectsGauge = "runtime.mem.total_number_of_allocated_objects" 11 | const totalPauseGauge = "runtime.mem.pause_total_ns" 12 | const lastPauseGauge = "runtime.mem.last_pause" 13 | 14 | func collectSystemMetrics() { 15 | err := metrics.Register( 16 | systemMetric(allocGauge), baseGauge{value: func(memStats runtime.MemStats) int64 { return int64(memStats.Alloc) }}) 17 | if err != nil { 18 | return 19 | } 20 | err = metrics.Register( 21 | systemMetric(heapObjectsGauge), baseGauge{value: func(memStats runtime.MemStats) int64 { return int64(memStats.HeapObjects) }}) 22 | if err != nil { 23 | return 24 | } 25 | err = metrics.Register( 26 | systemMetric(totalPauseGauge), baseGauge{value: func(memStats runtime.MemStats) int64 { return int64(memStats.PauseTotalNs) }}) 27 | if err != nil { 28 | return 29 | } 30 | err = metrics.Register( 31 | systemMetric(lastPauseGauge), baseGauge{value: func(memStats runtime.MemStats) int64 { return int64(memStats.PauseNs[(memStats.NumGC+255)%256]) }}) 32 | if err != nil { 33 | return 34 | } 35 | } 36 | 37 | type baseGauge struct { 38 | value func(runtime.MemStats) int64 39 | } 40 | 41 | func (g baseGauge) Value() int64 { 42 | var memStats runtime.MemStats 43 | runtime.ReadMemStats(&memStats) 44 | return g.value(memStats) 45 | } 46 | 47 | func (g baseGauge) Snapshot() metrics.Gauge { return metrics.GaugeSnapshot(g.Value()) } 48 | 49 | func (baseGauge) Update(int64) {} 50 | -------------------------------------------------------------------------------- /web/sse.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "time" 5 | 6 | log "github.com/Sirupsen/logrus" 7 | "github.com/allegro/marathon-appcop/marathon" 8 | "github.com/allegro/marathon-appcop/metrics" 9 | "github.com/allegro/marathon-appcop/mgc" 10 | "github.com/allegro/marathon-appcop/score" 11 | ) 12 | 13 | // Stop all channels 14 | type Stop func() 15 | 16 | // NewHandler is main initialization function 17 | func NewHandler(config Config, marathon marathon.Marathoner, gc *mgc.MarathonGC, 18 | scoreUpdate chan score.Update) Stop { 19 | 20 | // TODO implement proper leader election 21 | // Right now this part of code highly rely on marathon v2/leader endpoint 22 | leaderPoll(marathon, config.MyLeader) 23 | 24 | stopChannels := make([]chan<- stopEvent, config.WorkersCount) 25 | eventQueue := make(chan Event, config.QueueSize) 26 | 27 | for i := 0; i < config.WorkersCount; i++ { 28 | handler := newEventHandler(i, marathon, eventQueue, scoreUpdate) 29 | stopChannels[i] = handler.Start() 30 | } 31 | 32 | // start dispatcher 33 | sse := newSSEHandler(eventQueue, marathon.AuthGet(), marathon.LocationGet()) 34 | dispatcherStop := sse.start() 35 | stopChannels = append(stopChannels, dispatcherStop) 36 | 37 | // schedule marathon GC job 38 | go gc.StartMarathonGCJob() 39 | 40 | return stop(stopChannels) 41 | } 42 | 43 | func leaderPoll(service marathon.Marathoner, myLeader string) { 44 | pollTicker := time.NewTicker(5 * time.Second) 45 | for { 46 | 47 | leader, err := service.LeaderGet() 48 | if err != nil { 49 | log.WithError(err).Error("Error while getting leader") 50 | continue 51 | } 52 | if leader == myLeader { 53 | break 54 | } 55 | metrics.UpdateGauge("leader", int64(0)) 56 | <-pollTicker.C 57 | } 58 | metrics.UpdateGauge("leader", int64(1)) 59 | 60 | } 61 | 62 | func stop(channels []chan<- stopEvent) Stop { 63 | return func() { 64 | for _, channel := range channels { 65 | channel <- stopEvent{} 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /web/sse_handler.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "io" 7 | "net/http" 8 | 9 | "github.com/sethgrid/pester" 10 | 11 | "net/url" 12 | 13 | log "github.com/Sirupsen/logrus" 14 | ) 15 | 16 | // SSEHandler defines handler for marathon event stream, opening and closing 17 | // subscription 18 | type SSEHandler struct { 19 | eventQueue chan Event 20 | loc string 21 | client *pester.Client 22 | close context.CancelFunc 23 | req *http.Request 24 | } 25 | 26 | func close(r *http.Response) { 27 | err := r.Body.Close() 28 | if err != nil { 29 | log.WithError(err).Error("Can't close response") 30 | } 31 | } 32 | 33 | func newSSEHandler(eventQueue chan Event, auth *url.Userinfo, loc string) *SSEHandler { 34 | 35 | subURL := subscribeURL(auth, loc) 36 | req, err := http.NewRequest("GET", subURL, nil) 37 | if err != nil { 38 | log.WithError(err).Fatalf("Unable to generate sse request %v", err) 39 | return nil 40 | } 41 | 42 | req.Header.Set("Accept", "text/event-stream") 43 | ctx, cancel := context.WithCancel(context.Background()) 44 | req = req.WithContext(ctx) 45 | 46 | client := pester.New() 47 | client.Concurrency = 1 48 | client.MaxRetries = 3 49 | client.Backoff = pester.ExponentialBackoff 50 | client.KeepLog = true 51 | 52 | return &SSEHandler{ 53 | eventQueue: eventQueue, 54 | loc: loc, 55 | client: client, 56 | close: cancel, 57 | req: req, 58 | } 59 | } 60 | 61 | // Open connection to marathon v2/events 62 | func (h *SSEHandler) start() chan<- stopEvent { 63 | res, err := h.client.Do(h.req) 64 | if err != nil { 65 | log.WithFields(log.Fields{ 66 | "Location": h.loc, 67 | "Method": "GET", 68 | }).Fatalf("error performing request : %v", err) 69 | } 70 | if res.StatusCode != http.StatusOK { 71 | log.WithFields(log.Fields{ 72 | "Location": h.loc, 73 | "Method": "GET", 74 | }).Errorf("Got status code : %d", res.StatusCode) 75 | } 76 | log.WithFields(log.Fields{ 77 | "Location": h.loc, 78 | "Method": "GET", 79 | }).Debug("Subsciption success") 80 | stopChan := make(chan stopEvent) 81 | go func() { 82 | <-stopChan 83 | h.stop() 84 | }() 85 | 86 | go func() { 87 | defer close(res) 88 | 89 | reader := bufio.NewReader(res.Body) 90 | for { 91 | e, err := parseEvent(reader) 92 | if err != nil { 93 | if err == io.EOF { 94 | h.eventQueue <- e 95 | } 96 | log.Fatalf("Error processing parsing event %s", err) 97 | } 98 | h.eventQueue <- e 99 | } 100 | }() 101 | return stopChan 102 | } 103 | 104 | // Close connections managed by context 105 | func (h *SSEHandler) stop() { 106 | h.close() 107 | } 108 | -------------------------------------------------------------------------------- /marathon/marathon_stub.go: -------------------------------------------------------------------------------- 1 | package marathon 2 | 3 | import ( 4 | "errors" 5 | "net/url" 6 | ) 7 | 8 | // MStub is a stub for marathon functionality 9 | type MStub struct { 10 | Apps []*App 11 | Groups []*Group 12 | // AppGetFail - Set to true and this Stub get method will return errors 13 | AppsGetFail bool 14 | // AppDelFail - Set to true and this Stub methods will return error 15 | AppDelFail bool 16 | // AppDelHalfFail - Set to true and this Stub methods will return error at each second call 17 | AppDelHalfFail bool 18 | GroupDelFail bool 19 | AppScaleDownFail bool 20 | FailCounter *FailCounter 21 | ScaleCounter *ScaleCounter 22 | } 23 | 24 | // FailCounter is structure to hold state between failures 25 | type FailCounter struct { 26 | Counter int 27 | } 28 | 29 | // ScaleCounter is counting scaling operations 30 | type ScaleCounter struct { 31 | Counter int 32 | } 33 | 34 | // AppsGet get stubbed apps 35 | func (m MStub) AppsGet() ([]*App, error) { 36 | if m.AppsGetFail { 37 | return nil, errors.New("unable to get applications from marathon") 38 | } 39 | return m.Apps, nil 40 | } 41 | 42 | // AppGet get stubbed app 43 | func (m MStub) AppGet(appID AppID) (*App, error) { 44 | for _, app := range m.Apps { 45 | if app.ID == appID { 46 | return app, nil 47 | } 48 | } 49 | return &App{ID: appID}, nil 50 | } 51 | 52 | // GroupsGet get stubbed groups 53 | func (m MStub) GroupsGet() ([]*Group, error) { 54 | return m.Groups, nil 55 | } 56 | 57 | // TasksGet get stubed Tasks 58 | func (m MStub) TasksGet(appID AppID) ([]*Task, error) { 59 | return []*Task{ 60 | {AppID: appID}, 61 | }, nil 62 | } 63 | 64 | // AuthGet get stubbed auth 65 | func (m MStub) AuthGet() *url.Userinfo { 66 | return &url.Userinfo{} 67 | } 68 | 69 | // LocationGet get stubbed location 70 | func (m MStub) LocationGet() string { 71 | return "" 72 | } 73 | 74 | // LeaderGet get stubbed leader 75 | func (m MStub) LeaderGet() (string, error) { 76 | return "", nil 77 | } 78 | 79 | // AppScaleDown by one instance 80 | func (m MStub) AppScaleDown(app *App) error { 81 | if m.AppScaleDownFail { 82 | return errors.New("unable to scale down") 83 | } 84 | m.ScaleCounter.Counter = 1 85 | return nil 86 | } 87 | 88 | // AppDelete application by provided AppID 89 | func (m MStub) AppDelete(appID AppID) error { 90 | if m.AppDelFail { 91 | return errors.New("unable to delete app") 92 | } 93 | if m.AppDelHalfFail { 94 | if m.FailCounter.Counter%2 == 0 { 95 | m.FailCounter.Counter++ 96 | return errors.New("unable to delete app") 97 | } 98 | m.FailCounter.Counter++ 99 | } 100 | return nil 101 | } 102 | 103 | // GroupDelete by provided GroupID 104 | func (m MStub) GroupDelete(groupID GroupID) error { 105 | if m.GroupDelFail { 106 | return errors.New("unable to delete group") 107 | } 108 | return nil 109 | } 110 | 111 | // GetEmptyLeafGroups returns groups from marathon which are leafs in group tree 112 | func (m MStub) GetEmptyLeafGroups() ([]*Group, error) { 113 | return []*Group{}, nil 114 | } 115 | 116 | func (m MStub) GetAppIDPrefix() string { 117 | return "" 118 | } 119 | -------------------------------------------------------------------------------- /marathon/testdata/nested_groups_two_empty_one_with_app.json: -------------------------------------------------------------------------------- 1 | { 2 | 3 | "id": "/", 4 | "dependencies": [ ], 5 | "version": "", 6 | "apps": [ ], 7 | "groups": [ 8 | { 9 | "id": "/grpa", 10 | "dependencies": [ ], 11 | "version": "", 12 | "apps": [ ], 13 | "groups": [ 14 | { 15 | "id": "/grpa/grpb", 16 | "dependencies": [ ], 17 | "version": "", 18 | "apps": [ ], 19 | "groups": [ 20 | { 21 | "id": "/grpa/grpb/grpc0", 22 | "dependencies": [ ], 23 | "version": "", 24 | "apps": [ 25 | { 26 | "id": "/grpa/grpb/grpc0/testappc0", 27 | "cmd": "python -m SimpleHTTPServer $PORT0", 28 | "instances": 1, 29 | "cpus": 0.1, 30 | "mem": 32, 31 | "disk": 0, 32 | "gpus": 0, 33 | "backoffSeconds": 1, 34 | "backoffFactor": 1.15, 35 | "maxLaunchDelaySeconds": 3600, 36 | "acceptedResourceRoles": null, 37 | "taskKillGracePeriodSeconds": null, 38 | "ports": [ 39 | 10000 40 | ], 41 | "portDefinitions": [ 42 | { 43 | "port": 10000, 44 | "protocol": "tcp", 45 | "labels": { } 46 | } 47 | ], 48 | "requirePorts": false, 49 | "versionInfo": { 50 | "lastScalingAt": "", 51 | "lastConfigChangeAt": "" 52 | } 53 | } 54 | ], 55 | "groups": [ ] 56 | }, 57 | { 58 | "id": "/grpa/grpb/grpc1", 59 | "dependencies": [ ], 60 | "version": "", 61 | "apps": [ ], 62 | "groups": [ ] 63 | } 64 | ] 65 | }, 66 | { 67 | "id": "/grpa/grpb1", 68 | "dependencies": [ ], 69 | "version": "", 70 | "apps": [ ], 71 | "groups": [ ] 72 | } 73 | ] 74 | } 75 | ] 76 | } 77 | -------------------------------------------------------------------------------- /config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "os" 5 | "reflect" 6 | "testing" 7 | "time" 8 | 9 | "github.com/allegro/marathon-appcop/marathon" 10 | "github.com/allegro/marathon-appcop/metrics" 11 | "github.com/allegro/marathon-appcop/score" 12 | "github.com/allegro/marathon-appcop/web" 13 | "github.com/stretchr/testify/assert" 14 | ) 15 | 16 | func TestConfig_NewReturnsErrorWhenFileNotExist(t *testing.T) { 17 | clear() 18 | 19 | // given 20 | os.Args = []string{"./appcop", "--config-file=unknown.json"} 21 | 22 | // when 23 | _, err := NewConfig() 24 | 25 | // then 26 | assert.Error(t, err) 27 | } 28 | 29 | func TestConfig_NewReturnsErrorWhenFileIsNotJson(t *testing.T) { 30 | clear() 31 | 32 | // given 33 | os.Args = []string{"./appcop", "--config-file=config.go"} 34 | 35 | // when 36 | _, err := NewConfig() 37 | 38 | // then 39 | assert.Error(t, err) 40 | } 41 | 42 | func TestConfig_ShouldReturnErrorForBadLogLevel(t *testing.T) { 43 | clear() 44 | 45 | // given 46 | os.Args = []string{"./appcop", "--log-level=bad"} 47 | 48 | // when 49 | _, err := NewConfig() 50 | 51 | // then 52 | assert.Error(t, err) 53 | } 54 | 55 | func TestConfig_ShouldParseFlags(t *testing.T) { 56 | clear() 57 | 58 | // given 59 | os.Args = []string{"./appcop", "--log-level=debug", "--marathon-location=test.host:8080", "--log-format=json", "--my-leader=marathon.dev:8080"} 60 | 61 | // when 62 | actual, err := NewConfig() 63 | 64 | // then 65 | assert.NoError(t, err) 66 | assert.Equal(t, "debug", actual.Log.Level) 67 | assert.Equal(t, "json", actual.Log.Format) 68 | assert.Equal(t, "test.host:8080", actual.Marathon.Location) 69 | } 70 | 71 | func TestConfig_ShouldUseTextFormatterWhenFormatterIsUnknown(t *testing.T) { 72 | clear() 73 | 74 | // given 75 | os.Args = []string{"./appcop", "--log-level=debug", "--log-format=unknown", "--workers-pool-size=10", "--my-leader=marathon.dev:8080"} 76 | 77 | // when 78 | _, err := NewConfig() 79 | 80 | // then 81 | assert.NoError(t, err) 82 | } 83 | 84 | func TestConfig_ShouldBeMergedWithFileDefaultsAndFlags(t *testing.T) { 85 | clear() 86 | expected := &Config{ 87 | Web: web.Config{ 88 | Listen: ":4444", 89 | QueueSize: 0, 90 | WorkersCount: 10, 91 | }, 92 | Marathon: marathon.Config{ 93 | Location: "example.com:8080", 94 | Protocol: "http", 95 | Username: "", 96 | Password: "", 97 | VerifySsl: true, 98 | }, 99 | Score: score.Config{ 100 | ScaleDownScore: 200, 101 | UpdateInterval: 2 * time.Second, 102 | ResetInterval: 24 * time.Hour, 103 | EvaluateInterval: 20 * time.Second, 104 | ScaleLimit: 0, 105 | DryRun: false, 106 | }, 107 | Metrics: metrics.Config{Target: "stdout", 108 | Prefix: "default", 109 | Interval: 30 * time.Second, 110 | Addr: ""}, 111 | Log: struct{ Level, Format, File string }{ 112 | Level: "info", 113 | Format: "text", 114 | File: "", 115 | }, 116 | configFile: "testdata/config.json", 117 | } 118 | 119 | os.Args = []string{"./appcop", "--log-level=debug", "--config-file=testdata/config.json", "--marathon-location=example.com:8080", "--workers-pool-size=10", "--my-leader=marathon.dev:8080"} 120 | actual, err := NewConfig() 121 | 122 | assert.NoError(t, err) 123 | assert.Equal(t, expected, actual) 124 | } 125 | 126 | // http://stackoverflow.com/a/29169727/1387612 127 | func clear() { 128 | p := reflect.ValueOf(config).Elem() 129 | p.Set(reflect.Zero(p.Type())) 130 | } 131 | -------------------------------------------------------------------------------- /web/event.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "errors" 7 | "io" 8 | "net/url" 9 | "time" 10 | ) 11 | 12 | // Event holds state of parsed fields from marathon EventStream 13 | type Event struct { 14 | timestamp time.Time 15 | eventType string 16 | body []byte 17 | id string 18 | } 19 | 20 | func (e *Event) parseLine(line []byte) bool { 21 | 22 | // https://www.w3.org/TR/2011/WD-eventsource-20110208/ 23 | // Quote: Lines must be separated by either a U+000D CARRIAGE RETURN U+000A 24 | // LINE FEED (CRLF) character pair, a single U+000A LINE FEED (LF) character, 25 | // or a single U+000D CARRIAGE RETURN (CR) character. 26 | line = bytes.TrimSuffix(line, []byte{'\n'}) 27 | line = bytes.TrimSuffix(line, []byte{'\r'}) 28 | 29 | //If the line is empty (a blank line) 30 | if len(line) == 0 { 31 | //Dispatch the event, as defined below. 32 | return !e.isEmpty() 33 | } 34 | 35 | //If the line starts with a U+003A COLON character (:) 36 | if bytes.HasPrefix(line, []byte{':'}) { 37 | //Ignore the line. 38 | return false 39 | } 40 | 41 | var field string 42 | var value []byte 43 | //If the line contains a U+003A COLON character (:) 44 | //Collect the characters on the line before the first U+003A COLON character (:), and let field be that string. 45 | split := bytes.SplitN(line, []byte{':'}, 2) 46 | if len(split) == 2 { 47 | field = string(split[0]) 48 | //Collect the characters on the line after the first U+003A COLON character (:), and let value be that string. 49 | //If value starts with a U+0020 SPACE character, remove it from value. 50 | value = bytes.TrimPrefix(split[1], []byte{' '}) 51 | } else { 52 | //Otherwise, the string is not empty but does not contain a U+003A COLON character (:) 53 | //Process the field using the steps described below, using the whole line as the field name, 54 | //and the empty string as the field value. 55 | field = string(line) 56 | value = []byte{} 57 | 58 | } 59 | stringValue := string(value) 60 | //If the field name is 61 | switch field { 62 | case "event": 63 | //Set the event name buffer to field value. 64 | e.eventType = stringValue 65 | case "data": 66 | //If the data buffer is not the empty string, 67 | if len(value) != 0 { 68 | //Append the field value to the data buffer, 69 | //then append a single U+000A LINE FEED (LF) character to the data buffer. 70 | e.body = append(e.body, value...) 71 | e.body = append(e.body, '\n') 72 | } 73 | case "id": 74 | //Set the last event ID buffer to the field value. 75 | e.id = stringValue 76 | case "retry": 77 | // TODO consider reconnection delay 78 | } 79 | 80 | return false 81 | } 82 | 83 | func (e *Event) isEmpty() bool { 84 | return e.eventType == "" && e.body == nil && e.id == "" 85 | } 86 | 87 | func parseEvent(reader *bufio.Reader) (Event, error) { 88 | e := Event{} 89 | for dispatch := false; !dispatch; { 90 | //TODO: Use scanner use ReadLine 91 | line, err := reader.ReadBytes('\n') 92 | if err == io.EOF { 93 | dispatch = e.parseLine(line) 94 | if !dispatch { 95 | return e, errors.New("unexpected EOF") 96 | } 97 | return e, io.EOF 98 | } 99 | if err != nil { 100 | return e, err 101 | } 102 | dispatch = e.parseLine(line) 103 | } 104 | return e, nil 105 | } 106 | 107 | func subscribeURL(auth *url.Userinfo, location string) string { 108 | 109 | marathonurl := url.URL{ 110 | Scheme: "http", 111 | User: auth, 112 | Host: location, 113 | Path: "/v2/events", 114 | } 115 | query := marathonurl.Query() 116 | 117 | marathonurl.RawQuery = query.Encode() 118 | return marathonurl.String() 119 | } 120 | -------------------------------------------------------------------------------- /metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | //All credits to https://github.com/eBay/fabio/tree/master/metrics 4 | import ( 5 | "errors" 6 | "fmt" 7 | logger "log" 8 | "net" 9 | "os" 10 | "path/filepath" 11 | "strings" 12 | "time" 13 | 14 | log "github.com/Sirupsen/logrus" 15 | graphite "github.com/cyberdelia/go-metrics-graphite" 16 | "github.com/rcrowley/go-metrics" 17 | ) 18 | 19 | const ( 20 | // PathSeparator is separator of groups in app name 21 | PathSeparator = "/" 22 | // MetricSeparator is separator of groups in metrics system 23 | MetricSeparator = "." 24 | ) 25 | 26 | var ( 27 | prefix string 28 | instance string 29 | systemSubPrefix string 30 | appSubPrefix string 31 | ) 32 | 33 | // FilterOutEmptyStrings filters empty strings 34 | func FilterOutEmptyStrings(data []string) []string { 35 | var parts []string 36 | for _, part := range data { 37 | if part != "" { 38 | parts = append(parts, part) 39 | } 40 | } 41 | return parts 42 | } 43 | 44 | func systemMetric(name string) string { 45 | parts := FilterOutEmptyStrings([]string{systemSubPrefix, instance, name}) 46 | return strings.Join(parts, MetricSeparator) 47 | } 48 | 49 | func appMetric(name string) string { 50 | parts := FilterOutEmptyStrings([]string{appSubPrefix, name}) 51 | return strings.Join(parts, MetricSeparator) 52 | } 53 | 54 | // Mark or register Meter on graphite 55 | func Mark(name string) { 56 | meter := metrics.GetOrRegisterMeter( 57 | systemMetric(name), 58 | metrics.DefaultRegistry, 59 | ) 60 | meter.Mark(1) 61 | } 62 | 63 | // MarkApp marks or register Meter on graphite 64 | func MarkApp(name string) { 65 | meter := metrics.GetOrRegisterMeter( 66 | appMetric(name), 67 | metrics.DefaultRegistry, 68 | ) 69 | meter.Mark(1) 70 | } 71 | 72 | // Time execution of function 73 | func Time(name string, function func()) { 74 | timer := metrics.GetOrRegisterTimer( 75 | systemMetric(name), 76 | metrics.DefaultRegistry, 77 | ) 78 | timer.Time(function) 79 | } 80 | 81 | // UpdateGauge for provided metric 82 | func UpdateGauge(name string, value int64) { 83 | gauge := metrics.GetOrRegisterGauge( 84 | systemMetric(name), 85 | metrics.DefaultRegistry, 86 | ) 87 | gauge.Update(value) 88 | } 89 | 90 | // Init Metrics 91 | func Init(cfg Config) error { 92 | prefix = cfg.Prefix 93 | if prefix == "default" { 94 | pfx, err := defaultPrefix() 95 | if err != nil { 96 | return err 97 | } 98 | prefix = pfx 99 | } 100 | 101 | instance = cfg.Instance 102 | if instance == "" { 103 | ins, err := hostname() 104 | if err != nil { 105 | ins = "localhost" 106 | } 107 | instance = ins 108 | } 109 | 110 | systemSubPrefix = cfg.SystemSubPrefix 111 | appSubPrefix = cfg.AppSubPrefix 112 | 113 | collectSystemMetrics() 114 | 115 | switch cfg.Target { 116 | case "stdout": 117 | log.Info("Sending metrics to stdout") 118 | return initStdout(cfg.Interval) 119 | case "graphite": 120 | if cfg.Addr == "" { 121 | return errors.New("metrics: graphite addr missing") 122 | } 123 | 124 | log.Infof("Sending metrics to Graphite on %s as %q", cfg.Addr, prefix) 125 | return initGraphite(cfg.Addr, cfg.Interval) 126 | case "": 127 | log.Infof("Metrics disabled") 128 | return nil 129 | default: 130 | return fmt.Errorf("invalid metrics target %s", cfg.Target) 131 | } 132 | } 133 | 134 | func clean(s string) string { 135 | if s == "" { 136 | return "_" 137 | } 138 | s = strings.Replace(s, ".", "_", -1) 139 | s = strings.Replace(s, ":", "_", -1) 140 | return strings.ToLower(s) 141 | } 142 | 143 | // stubbed out for testing 144 | var hostname = os.Hostname 145 | 146 | func defaultPrefix() (string, error) { 147 | host, err := hostname() 148 | if err != nil { 149 | log.WithError(err).Error("Problem with detecting prefix") 150 | return "", err 151 | } 152 | exe := filepath.Base(os.Args[0]) 153 | return clean(host) + "." + clean(exe), nil 154 | } 155 | 156 | func initStdout(interval time.Duration) error { 157 | logger := logger.New(os.Stderr, "localhost: ", logger.Lmicroseconds) 158 | go metrics.Log(metrics.DefaultRegistry, interval, logger) 159 | return nil 160 | } 161 | 162 | func initGraphite(addr string, interval time.Duration) error { 163 | a, err := net.ResolveTCPAddr("tcp", addr) 164 | if err != nil { 165 | return fmt.Errorf("metrics: cannot connect to Graphite: %s", err) 166 | } 167 | 168 | go graphite.Graphite(metrics.DefaultRegistry, interval, prefix, a) 169 | return nil 170 | } 171 | -------------------------------------------------------------------------------- /web/event_test.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "bufio" 5 | "errors" 6 | "strings" 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestEventIfEventIsEmptyReturnsFalse(t *testing.T) { 14 | t.Parallel() 15 | // given 16 | event := &Event{timestamp: time.Now(), 17 | eventType: "status_update_event", 18 | body: []byte(`{"id": "simpleId"}`), 19 | id: "id", 20 | } 21 | // when 22 | expected := false 23 | actual := event.isEmpty() 24 | // then 25 | assert.Equal(t, expected, actual) 26 | } 27 | 28 | func TestEventIfEventIsEmptyReturnsTrue(t *testing.T) { 29 | t.Parallel() 30 | // given 31 | event := &Event{} 32 | // when 33 | expected := true 34 | actual := event.isEmpty() 35 | // then 36 | assert.Equal(t, expected, actual) 37 | } 38 | 39 | func TestParseLineWhenStautsUpdateEventPassed(t *testing.T) { 40 | t.Parallel() 41 | // given 42 | event := &Event{} 43 | line0 := []byte("id: 0\n") 44 | line1 := []byte("event: status_update_event\n") 45 | line2 := []byte("data: testData\n") 46 | expected0 := "0" 47 | expected1 := "status_update_event" 48 | expected2 := []byte("testData\n") 49 | // when 50 | event.parseLine(line0) 51 | event.parseLine(line1) 52 | event.parseLine(line2) 53 | // then 54 | assert.Equal(t, expected0, event.id) 55 | assert.Equal(t, expected1, event.eventType) 56 | assert.Equal(t, expected2, event.body) 57 | } 58 | 59 | func TestParseLineWhenGarbageIsProvidedBodyShouldBeNil(t *testing.T) { 60 | t.Parallel() 61 | // given 62 | event := &Event{} 63 | line := []byte("garbage data\n") 64 | expectedBody := []byte(nil) 65 | // when 66 | _ = event.parseLine(line) 67 | // then 68 | assert.Equal(t, expectedBody, event.body) 69 | } 70 | 71 | func TestParseEventWhenOneStatusUpdateEventIsProvided(t *testing.T) { 72 | t.Parallel() 73 | // given 74 | sreader := strings.NewReader("event: status_update_event\ndata: testData\n") 75 | reader := bufio.NewReader(sreader) 76 | expectedEvent := "status_update_event" 77 | // when 78 | event, _ := parseEvent(reader) 79 | // then 80 | assert.Equal(t, expectedEvent, event.eventType) 81 | } 82 | 83 | func TestParseEventWhenSimpleDataIsProvidedShouldReturnEOFError(t *testing.T) { 84 | t.Parallel() 85 | // given 86 | sreader := strings.NewReader("event: status_update_event\ndata: testData\n") 87 | reader := bufio.NewReader(sreader) 88 | expectedEvent := "status_update_event" 89 | expectedError := errors.New("EOF") 90 | // when 91 | event, err := parseEvent(reader) 92 | // then 93 | assert.Equal(t, expectedError, err) 94 | assert.Equal(t, expectedEvent, event.eventType) 95 | } 96 | 97 | func TestParseEventWhenSimpleDataIsProvidedAndNotCompleteEventIsProvidedShouldReturnUnexpectedEOFError(t *testing.T) { 98 | t.Parallel() 99 | // given 100 | sreader := strings.NewReader("event: status_update_event\ndata: testData\nlkajsd") 101 | reader := bufio.NewReader(sreader) 102 | expectedEvent := "status_update_event" 103 | expectedError := errors.New("unexpected EOF") 104 | // when 105 | event, err := parseEvent(reader) 106 | // then 107 | assert.Equal(t, expectedError, err) 108 | assert.Equal(t, expectedEvent, event.eventType) 109 | } 110 | 111 | func TestParseEventWhenNotCompleteDataIsProvidedEventShouldContainOnlyEventType(t *testing.T) { 112 | t.Parallel() 113 | // given 114 | sreader := strings.NewReader("event: status_update_event\ndata") 115 | reader := bufio.NewReader(sreader) 116 | expectedEvent := "status_update_event" 117 | // when 118 | event, _ := parseEvent(reader) 119 | // then 120 | assert.Equal(t, expectedEvent, event.eventType) 121 | assert.Nil(t, event.body) 122 | } 123 | 124 | func TestParseEventWhenDataIsEmptyProvidedEventShouldContainOnlyEventType(t *testing.T) { 125 | t.Parallel() 126 | // given 127 | sreader := strings.NewReader("event: status_update_event\ndata:\n") 128 | reader := bufio.NewReader(sreader) 129 | expectedEvent := "status_update_event" 130 | // when 131 | event, _ := parseEvent(reader) 132 | // then 133 | assert.Equal(t, expectedEvent, event.eventType) 134 | assert.Nil(t, event.body) 135 | } 136 | 137 | func TestParseEventWhenDataIsProvidedButNoLFShouldContainDataEvenIfNotComplete(t *testing.T) { 138 | t.Parallel() 139 | // given 140 | sreader := strings.NewReader("event: status_update_event\ndata: testEventData") 141 | reader := bufio.NewReader(sreader) 142 | expectedEvent := "status_update_event" 143 | // when 144 | event, _ := parseEvent(reader) 145 | // then 146 | assert.Equal(t, expectedEvent, event.eventType) 147 | assert.Equal(t, []byte("testEventData\n"), event.body) 148 | } 149 | -------------------------------------------------------------------------------- /web/event_handler.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "time" 7 | 8 | log "github.com/Sirupsen/logrus" 9 | "github.com/allegro/marathon-appcop/marathon" 10 | "github.com/allegro/marathon-appcop/metrics" 11 | "github.com/allegro/marathon-appcop/score" 12 | ) 13 | 14 | type eventHandler struct { 15 | id int 16 | marathon marathon.Marathoner 17 | eventQueue <-chan Event 18 | scoreUpdate chan score.Update 19 | } 20 | 21 | type stopEvent struct{} 22 | 23 | const ( 24 | taskFinished = "TASK_FINISHED" 25 | taskFailed = "TASK_FAILED" 26 | taskKilled = "TASK_KILLED" 27 | taskRunning = "TASK_RUNNING" 28 | ) 29 | 30 | func newEventHandler(id int, marathon marathon.Marathoner, eventQueue <-chan Event, 31 | scoreUpdate chan score.Update) *eventHandler { 32 | return &eventHandler{ 33 | id: id, 34 | marathon: marathon, 35 | eventQueue: eventQueue, 36 | scoreUpdate: scoreUpdate, 37 | } 38 | } 39 | 40 | // Start event handler 41 | func (fh *eventHandler) Start() chan<- stopEvent { 42 | var event Event 43 | process := func() { 44 | err := fh.handleEvent(event.eventType, event.body) 45 | if err != nil { 46 | metrics.Mark("events.processing.error") 47 | } else { 48 | metrics.Mark("events.processing.succes") 49 | } 50 | } 51 | 52 | quitChan := make(chan stopEvent) 53 | log.WithField("Id", fh.id).Println("Starting worker") 54 | go func() { 55 | for { 56 | select { 57 | case event = <-fh.eventQueue: 58 | metrics.Mark(fmt.Sprintf("events.handler.%d", fh.id)) 59 | metrics.UpdateGauge("events.queue.len", int64(len(fh.eventQueue))) 60 | metrics.UpdateGauge("events.queue.delay_ns", time.Since(event.timestamp).Nanoseconds()) 61 | metrics.Time("events.processing."+event.eventType, process) 62 | case <-quitChan: 63 | log.WithField("Id", fh.id).Info("Stopping worker") 64 | } 65 | } 66 | }() 67 | return quitChan 68 | } 69 | 70 | func (fh *eventHandler) handleEvent(eventType string, body []byte) error { 71 | 72 | body = replaceTaskIDWithID(body) 73 | 74 | switch eventType { 75 | case "status_update_event": 76 | return fh.handleStatusEvent(body) 77 | case "unhealthy_task_kill_event": 78 | return fh.handleUnhealthyTaskKillEvent(body) 79 | default: 80 | log.WithField("EventType", eventType).Debug("Not handled event type") 81 | return nil 82 | } 83 | } 84 | 85 | func (fh *eventHandler) handleStatusEvent(body []byte) error { 86 | task, err := marathon.ParseTask(body) 87 | 88 | if err != nil { 89 | log.WithField("Body", body).Error("Could not parse event body") 90 | return err 91 | } 92 | 93 | log.WithFields(log.Fields{ 94 | "Id": task.ID, 95 | "TaskStatus": task.TaskStatus, 96 | }).Debug("Got StatusEvent") 97 | 98 | appMetric := task.GetMetric(fh.marathon.GetAppIDPrefix()) 99 | metrics.MarkApp(appMetric) 100 | 101 | switch task.TaskStatus { 102 | case taskFinished, taskFailed, taskKilled: 103 | appID := task.AppID 104 | app, err := fh.marathon.AppGet(appID) 105 | if err != nil { 106 | return err 107 | } 108 | fh.scoreUpdate <- score.Update{App: app, Update: 1} 109 | return nil 110 | case taskRunning: 111 | log.WithFields(log.Fields{ 112 | "Id": task.AppID, 113 | "Host": task.Host, 114 | "Ports": task.Ports, 115 | }).Info("Got task running status") 116 | return nil 117 | default: 118 | log.WithFields(log.Fields{ 119 | "Id": task.ID, 120 | "taskStatus": task.TaskStatus, 121 | }).Debug("Not handled task status") 122 | return nil 123 | } 124 | } 125 | 126 | func (fh *eventHandler) handleUnhealthyTaskKillEvent(body []byte) error { 127 | task, err := marathon.ParseTask(body) 128 | 129 | if err != nil { 130 | log.WithField("Body", body).Error("Could not parse event body") 131 | return err 132 | } 133 | 134 | log.WithFields(log.Fields{ 135 | "Id": task.ID, 136 | }).Debug("Got Unhealthy TaskKilled Event") 137 | 138 | // update score killed app 139 | appID := task.AppID 140 | app, err := fh.marathon.AppGet(appID) 141 | if err != nil { 142 | log.WithField("appID", appID).Error("Could not get app by id") 143 | return err 144 | } 145 | fh.scoreUpdate <- score.Update{App: app, Update: 1} 146 | return nil 147 | } 148 | 149 | // for every other use of Tasks, Marathon uses the "id" field for the task ID. 150 | // Here, it uses "taskId", with most of the other fields being equal. We'll 151 | // just swap "taskId" for "id" in the body so that we can successfully parse 152 | // incoming events. 153 | func replaceTaskIDWithID(body []byte) []byte { 154 | return bytes.Replace(body, []byte("taskId"), []byte("id"), -1) 155 | } 156 | -------------------------------------------------------------------------------- /marathon/app.go: -------------------------------------------------------------------------------- 1 | package marathon 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/allegro/marathon-appcop/metrics" 9 | ) 10 | 11 | const ApplicationImmunityLabel = "APP_IMMUNITY" 12 | 13 | // AppWrapper json returned from marathon with app definition 14 | type AppWrapper struct { 15 | App App `json:"app"` 16 | } 17 | 18 | // AppsResponse json returned from marathon with apps definitions 19 | type AppsResponse struct { 20 | Apps []*App `json:"apps"` 21 | } 22 | 23 | // App represents application returned in marathon json 24 | type App struct { 25 | Labels map[string]string `json:"labels"` 26 | ID AppID `json:"id"` 27 | Tasks []Task `json:"tasks"` 28 | Instances int `json:"instances"` 29 | VersionInfo VersionInfo `json:"versionInfo"` 30 | } 31 | 32 | // HasImmunity check if application behavior is tolerated without consequence 33 | func (app App) HasImmunity() bool { 34 | if val, ok := app.Labels[ApplicationImmunityLabel]; ok && val == "true" { 35 | return true 36 | } 37 | return false 38 | } 39 | 40 | func (app *App) penalize() error { 41 | 42 | if app.Instances >= 1 { 43 | app.Instances-- 44 | } else { 45 | return fmt.Errorf("unable to scale down, zero instance") 46 | } 47 | 48 | if app.Instances == 0 { 49 | app.Labels["appcop"] = "suspend" 50 | } else { 51 | app.Labels["appcop"] = "scaleDown" 52 | } 53 | 54 | return nil 55 | 56 | } 57 | 58 | // VersionInfo represents json field of this name, inside marathon app 59 | // definition 60 | type VersionInfo struct { 61 | LastScalingAt string `json:"lastScalingAt"` 62 | LastConfigChangeAt string `json:"lastConfigChangeAt"` 63 | } 64 | 65 | // AppID Marathon Application Id (aka PathId) 66 | // Usually in the form of /rootGroup/subGroup/subSubGroup/name 67 | // allowed characters: lowercase letters, digits, hyphens, slash 68 | type AppID string 69 | 70 | // String stringer for app 71 | func (id AppID) String() string { 72 | return string(id) 73 | } 74 | 75 | // ParseApps json 76 | func ParseApps(jsonBlob []byte) ([]*App, error) { 77 | apps := &AppsResponse{} 78 | err := json.Unmarshal(jsonBlob, apps) 79 | 80 | return apps.Apps, err 81 | } 82 | 83 | // ParseApp json 84 | func ParseApp(jsonBlob []byte) (*App, error) { 85 | wrapper := &AppWrapper{} 86 | err := json.Unmarshal(jsonBlob, wrapper) 87 | 88 | return &wrapper.App, err 89 | } 90 | 91 | // Task definition returned in marathon event 92 | type Task struct { 93 | ID TaskID 94 | TaskStatus string `json:"taskStatus"` 95 | AppID AppID `json:"appId"` 96 | Host string `json:"host"` 97 | Ports []int `json:"ports"` 98 | HealthCheckResults []HealthCheckResult `json:"healthCheckResults"` 99 | } 100 | 101 | // GetMetric returns a string indicating where this applications metric should be placed 102 | // in graphite in defined prefix. It is done by triming begining prefix (if defined) 103 | // from application id and replacing appID separators with 104 | // metrics separators appropriate for graphite. 105 | func (t Task) GetMetric(prefix string) string { 106 | taskAppID := string(t.AppID) 107 | 108 | var appID string 109 | if prefix == "" { 110 | appID = taskAppID 111 | } else { 112 | appID = strings.Replace(taskAppID, prefix, "", 1) 113 | } 114 | noRootAppID := strings.TrimPrefix(appID, "/") 115 | metricPath := strings.Replace(noRootAppID, metrics.PathSeparator, metrics.MetricSeparator, -1) 116 | taskStatus := strings.ToLower(t.TaskStatus) 117 | 118 | filteredPathParts := metrics.FilterOutEmptyStrings([]string{metricPath, taskStatus}) 119 | return strings.Join(filteredPathParts, metrics.MetricSeparator) 120 | 121 | } 122 | 123 | // TaskID from marathon 124 | // Usually in the form of AppID.uuid with '/' replaced with '_' 125 | type TaskID string 126 | 127 | func (id TaskID) String() string { 128 | return string(id) 129 | } 130 | 131 | // AppID contains string defining application in marathon 132 | func (id TaskID) AppID() AppID { 133 | index := strings.LastIndex(id.String(), ".") 134 | return AppID("/" + strings.Replace(id.String()[0:index], "_", "/", -1)) 135 | } 136 | 137 | // HealthCheckResult returned from marathon api 138 | type HealthCheckResult struct { 139 | Alive bool `json:"alive"` 140 | } 141 | 142 | // TasksResponse response to TasksGet call 143 | type TasksResponse struct { 144 | Tasks []*Task `json:"tasks"` 145 | } 146 | 147 | // ParseTasks try to convert raw Tasks data to json 148 | func ParseTasks(jsonBlob []byte) ([]*Task, error) { 149 | tasks := &TasksResponse{} 150 | err := json.Unmarshal(jsonBlob, tasks) 151 | 152 | return tasks.Tasks, err 153 | } 154 | 155 | // ParseTask try to convert raw Task data to json 156 | func ParseTask(event []byte) (*Task, error) { 157 | task := &Task{} 158 | err := json.Unmarshal(event, task) 159 | return task, err 160 | } 161 | -------------------------------------------------------------------------------- /mgc/mgc.go: -------------------------------------------------------------------------------- 1 | package mgc 2 | 3 | import ( 4 | "time" 5 | 6 | log "github.com/Sirupsen/logrus" 7 | "github.com/allegro/marathon-appcop/marathon" 8 | "github.com/allegro/marathon-appcop/metrics" 9 | ) 10 | 11 | // MarathonGC is Marathon Garbage Collector receiever, mainly holds applications registry 12 | // and marathon client 13 | type MarathonGC struct { 14 | config Config 15 | marathon marathon.Marathoner 16 | apps []*marathon.App 17 | lastRefresh time.Time 18 | } 19 | 20 | // New instantiates MarathonGC reciever 21 | func New(config Config, marathon marathon.Marathoner) (*MarathonGC, error) { 22 | 23 | return &MarathonGC{ 24 | config: config, 25 | marathon: marathon, 26 | apps: nil, 27 | lastRefresh: time.Time{}, 28 | }, nil 29 | } 30 | 31 | // StartMarathonGCJob is highest control element of MarathonGC module, 32 | // which starts job goroutine for periodic: 33 | // - collection of suspended apps, 34 | // - collection of empty groups. 35 | func (mgc *MarathonGC) StartMarathonGCJob() { 36 | if !mgc.config.Enabled { 37 | log.Info("Marathon Garbage Collection enabled") 38 | return 39 | } 40 | log.WithFields(log.Fields{ 41 | "Interval": mgc.config.Interval, 42 | }).Info("Marathon GC job started") 43 | 44 | go func() { 45 | var err error 46 | ticker := time.NewTicker(mgc.config.Interval) 47 | for range ticker.C { 48 | metrics.Time("mgc.refresh", func() { err = mgc.refresh() }) 49 | if err != nil { 50 | metrics.Mark("mgc.refresh.error") 51 | continue 52 | } 53 | mgc.gcSuspended() 54 | mgc.gcEmptyGroups() 55 | } 56 | }() 57 | } 58 | 59 | // gcSuspended commits garbage collection for suspended apps 60 | func (mgc *MarathonGC) gcSuspended() { 61 | log.Info("Staring GC on suspended apps") 62 | apps := mgc.getOldSuspended() 63 | if len(apps) == 0 { 64 | log.Info("No suspended apps to gc") 65 | return 66 | } 67 | 68 | var deletedCount int 69 | metrics.Time("mgc.delete.suspended", func() { 70 | deletedCount = mgc.deleteSuspended(apps) 71 | }) 72 | if deletedCount == 0 { 73 | metrics.UpdateGauge("mgc.delete.suspended.count", int64(deletedCount)) 74 | log.Info("Nothing GC'ed for long suspend") 75 | } 76 | 77 | } 78 | 79 | // gcEmptyGroups is starting GC jobs on groups 80 | // It is evaluating time of last group update 81 | func (mgc *MarathonGC) gcEmptyGroups() { 82 | log.Info("Staring GC on empty groups") 83 | groups, err := mgc.marathon.GetEmptyLeafGroups() 84 | if err != nil { 85 | log.WithError(err).Error("Ending GCEmptyGroups") 86 | return 87 | } 88 | 89 | for _, group := range groups { 90 | t, err := toMarathonDate(group.Version) 91 | if err != nil { 92 | log.WithError(err).Error("Unable to parse date") 93 | continue 94 | } 95 | if group.IsEmpty() && (t.elapsed() > mgc.config.MaxSuspendTime) { 96 | metrics.Time("mgc.groups.delete", func() { 97 | err = mgc.groupDelete(group.ID) 98 | }) 99 | if err != nil { 100 | metrics.Mark("mgc.groups.delete.error") 101 | continue 102 | } 103 | } 104 | } 105 | } 106 | 107 | func (mgc *MarathonGC) groupDelete(groupID marathon.GroupID) error { 108 | log.Infof("Deleting group %s", groupID) 109 | return mgc.marathon.GroupDelete(groupID) 110 | } 111 | 112 | func (mgc *MarathonGC) refresh() error { 113 | log.WithFields(log.Fields{ 114 | "LastUpdate": mgc.lastRefresh, 115 | }).Info("Refreshing local app registry") 116 | 117 | // get apps 118 | apps, err := mgc.marathon.AppsGet() 119 | if err != nil { 120 | log.WithFields(log.Fields{ 121 | "LastUpdate": mgc.lastRefresh, 122 | }).Error("Refresh fail") 123 | 124 | return err 125 | } 126 | mgc.apps = apps 127 | mgc.lastRefresh = time.Now() 128 | 129 | return nil 130 | } 131 | 132 | func (mgc *MarathonGC) shouldBeCollected(app *marathon.App) bool { 133 | if app.Instances > 0 { 134 | return false 135 | } 136 | scaleDate := app.VersionInfo.LastScalingAt 137 | t, err := toMarathonDate(scaleDate) 138 | 139 | if err != nil { 140 | log.WithError(err).Error("Unable to parse provided date") 141 | return false 142 | } 143 | return t.elapsed() > mgc.config.MaxSuspendTime 144 | } 145 | 146 | func (mgc *MarathonGC) getOldSuspended() []*marathon.App { 147 | var ret []*marathon.App 148 | 149 | for _, app := range mgc.apps { 150 | if mgc.shouldBeCollected(app) && (!mgc.config.AppCopOnly || appCopped(app)) { 151 | ret = append(ret, app) 152 | } 153 | } 154 | return ret 155 | } 156 | 157 | // deleteSuspended returns number (int) of successfully deleted applications 158 | func (mgc *MarathonGC) deleteSuspended(apps []*marathon.App) int { 159 | 160 | n := 0 161 | var err error 162 | for _, app := range apps { 163 | err = mgc.marathon.AppDelete(app.ID) 164 | if err != nil { 165 | log.WithError(err).Errorf("Error while deleting suspended app: %s", app.ID) 166 | continue 167 | } 168 | n++ 169 | } 170 | return n 171 | } 172 | 173 | func appCopped(app *marathon.App) bool { 174 | _, ok := app.Labels["appcop"] 175 | 176 | return ok 177 | 178 | } 179 | -------------------------------------------------------------------------------- /score/score.go: -------------------------------------------------------------------------------- 1 | package score 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "sync" 7 | "time" 8 | 9 | log "github.com/Sirupsen/logrus" 10 | "github.com/allegro/marathon-appcop/marathon" 11 | "github.com/allegro/marathon-appcop/metrics" 12 | ) 13 | 14 | // Score contains score value to update, struct keeped inside Scorer as value as 15 | // value as value as value 16 | type Score struct { 17 | score int 18 | lastUpdate time.Time 19 | } 20 | 21 | // Scorer keeps records of all applications behaviour on marathon 22 | type Scorer struct { 23 | mutex sync.RWMutex 24 | ScaleDownScore int 25 | ResetInterval time.Duration 26 | UpdateInterval time.Duration 27 | EvaluateInterval time.Duration 28 | DryRun bool 29 | ScaleLimit int 30 | service marathon.Marathoner 31 | scores map[marathon.AppID]*Score 32 | } 33 | 34 | // Update struct for scoring specific app 35 | type Update struct { 36 | // TODO(tz) to consider, store only AppID 37 | App *marathon.App 38 | Update int 39 | } 40 | 41 | // New creates new scorer instance 42 | func New(config Config, m marathon.Marathoner) (*Scorer, error) { 43 | 44 | if config.ResetInterval <= config.UpdateInterval { 45 | return nil, errors.New("UpdateInterval should be lower than ResetInterval") 46 | } 47 | 48 | if config.ResetInterval <= config.EvaluateInterval { 49 | return nil, errors.New("ResetInterval should be lower than EvaluateInterval") 50 | } 51 | 52 | return &Scorer{ 53 | ScaleDownScore: config.ScaleDownScore, 54 | ResetInterval: config.ResetInterval, 55 | UpdateInterval: config.UpdateInterval, 56 | EvaluateInterval: config.EvaluateInterval, 57 | ScaleLimit: config.ScaleLimit, 58 | DryRun: config.DryRun, 59 | service: m, 60 | scores: make(map[marathon.AppID]*Score), 61 | }, nil 62 | } 63 | 64 | // ScoreManager starts Scorer job 65 | func (s *Scorer) ScoreManager() chan Update { 66 | updates := make(chan Update) 67 | 68 | log.Info("Starting ScoreManager") 69 | if s.DryRun { 70 | log.Info("DryRun, NOOP mode") 71 | } 72 | printTicker := time.NewTicker(s.UpdateInterval) 73 | evaluateTicker := time.NewTicker(s.EvaluateInterval) 74 | resetTimer := time.NewTicker(s.ResetInterval) 75 | 76 | go func() { 77 | for { 78 | select { 79 | case <-evaluateTicker.C: 80 | metrics.Mark("score.evaluates") 81 | go s.EvaluateApps() 82 | case <-printTicker.C: 83 | // Only used for debug purposes 84 | go s.printScores() 85 | case <-resetTimer.C: 86 | metrics.Mark("score.resets") 87 | go s.resetScores() 88 | case u := <-updates: 89 | metrics.UpdateGauge("score.updateQueue", int64(len(updates))) 90 | go s.initOrUpdateScore(u) 91 | } 92 | } 93 | }() 94 | return updates 95 | } 96 | 97 | func (s *Scorer) initOrUpdateScore(u Update) { 98 | log.WithFields(log.Fields{ 99 | "appId": u.App.ID, 100 | "scoreUpdate": u.Update, 101 | }).Debug("Score update") 102 | 103 | s.mutex.Lock() 104 | 105 | su := u.Update 106 | now := time.Now() 107 | 108 | if appScore, isScored := s.scores[u.App.ID]; isScored { 109 | appScore.score += su 110 | appScore.lastUpdate = now 111 | } else { 112 | s.scores[u.App.ID] = &Score{score: su, lastUpdate: now} 113 | } 114 | s.mutex.Unlock() 115 | } 116 | 117 | // if no such key, resetScores is noop 118 | func (s *Scorer) resetScore(appID marathon.AppID) { 119 | s.mutex.Lock() 120 | 121 | delete(s.scores, appID) 122 | s.mutex.Unlock() 123 | } 124 | 125 | // Substracts score by configured treshold 126 | // Noop if appID not exists 127 | func (s *Scorer) subtractScore(appID marathon.AppID) { 128 | s.mutex.Lock() 129 | defer s.mutex.Unlock() 130 | 131 | var ok bool 132 | var score *Score 133 | if score, ok = s.scores[appID]; !ok { 134 | return 135 | } 136 | 137 | score.score -= s.ScaleDownScore 138 | } 139 | 140 | func (s *Scorer) resetScores() { 141 | log.WithFields(log.Fields{ 142 | "ScoresRecorded": len(s.scores), 143 | }).Debug("Reseting scores") 144 | 145 | s.mutex.Lock() 146 | 147 | s.scores = make(map[marathon.AppID]*Score) 148 | s.mutex.Unlock() 149 | } 150 | 151 | // EvaluateApps checks apps scores and if any is higher on score than limit, 152 | // scale them down by one instance 153 | func (s *Scorer) EvaluateApps() { 154 | 155 | i, err := s.evaluateApps() 156 | if err != nil && i == 0 { 157 | log.WithError(err).Error("Failed to evaluate") 158 | } 159 | log.Debugf("%d apps qualified for penalty", i) 160 | } 161 | 162 | func (s *Scorer) evaluateApps() (int, error) { 163 | limit := 2 164 | i := 0 165 | var lastErr error 166 | 167 | for appID, score := range s.scores { 168 | 169 | curScore := score.score 170 | // TODO(tz) - implement proper rate limiter with shared state accross goroutines 171 | // and configurable 172 | // https://gobyexample.com/rate-limiting 173 | if !(curScore > s.ScaleDownScore && i <= limit) { 174 | continue 175 | } 176 | 177 | err := s.scaleDown(appID) 178 | if err != nil { 179 | lastErr = err 180 | log.WithFields(log.Fields{ 181 | "appId": appID, 182 | "Limit": i, 183 | }).Error(err) 184 | metrics.Mark("score.scale_fail") 185 | s.resetScore(appID) 186 | 187 | continue 188 | } 189 | 190 | metrics.Mark("score.scale_success") 191 | s.subtractScore(appID) 192 | 193 | i++ 194 | } 195 | return i, lastErr 196 | } 197 | 198 | func (s *Scorer) scaleDown(appID marathon.AppID) error { 199 | s.mutex.Lock() 200 | defer s.mutex.Unlock() 201 | 202 | log.WithFields(log.Fields{ 203 | "appId": appID, 204 | "score": s.scores[appID].score, 205 | }).Info("Scaling down application") 206 | 207 | app, err := s.service.AppGet(appID) 208 | if err != nil { 209 | return err 210 | } 211 | 212 | // dry-run flag 213 | if s.DryRun { 214 | log.WithFields(log.Fields{ 215 | "appId": appID, 216 | "score": s.scores[appID].score, 217 | }).Info("NOOP - App Scale Down") 218 | return nil 219 | } 220 | 221 | if app.HasImmunity() { 222 | // returning error up makes sure rate limiting works, 223 | // otherwise AppCop could loop over immune apps 224 | return fmt.Errorf("app: %s has immunity", app.ID) 225 | } 226 | 227 | err = s.service.AppScaleDown(app) 228 | return err 229 | 230 | } 231 | 232 | func (s *Scorer) printScores() { 233 | for app, score := range s.scores { 234 | log.WithFields(log.Fields{ 235 | "app": app, 236 | "score": score.score}).Debug("Output Scores") 237 | } 238 | } 239 | -------------------------------------------------------------------------------- /metrics/metrics_test.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "testing" 7 | 8 | "github.com/rcrowley/go-metrics" 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | var filterOutEmptyStringsTestCases = []struct { 14 | input, expectedOutput []string 15 | }{ 16 | { 17 | input: []string{"a", ""}, 18 | expectedOutput: []string{"a"}, 19 | }, 20 | { 21 | input: []string{"a", "b"}, 22 | expectedOutput: []string{"a", "b"}, 23 | }, 24 | { 25 | input: []string{"", ""}, 26 | expectedOutput: nil, 27 | }, 28 | } 29 | 30 | func TestFilterOutEmptyStrings(t *testing.T) { 31 | t.Parallel() 32 | 33 | for _, testCase := range filterOutEmptyStringsTestCases { 34 | actualOutput := FilterOutEmptyStrings(testCase.input) 35 | assert.Equal(t, testCase.expectedOutput, actualOutput) 36 | } 37 | } 38 | 39 | var systemMetricTestcases = []struct { 40 | config Config 41 | metric string 42 | expectedMetric string 43 | }{ 44 | { 45 | config: Config{SystemSubPrefix: "system"}, 46 | metric: "metric", 47 | expectedMetric: "system.localhost.metric", 48 | }, 49 | { 50 | config: Config{SystemSubPrefix: ""}, 51 | metric: "metric", 52 | expectedMetric: "localhost.metric", 53 | }, 54 | { 55 | config: Config{SystemSubPrefix: ""}, 56 | metric: "", 57 | expectedMetric: "localhost", 58 | }, 59 | } 60 | 61 | func TestSystemMetricTestCases(t *testing.T) { 62 | hostname = func() (string, error) { return "localhost", nil } 63 | for _, testCase := range systemMetricTestcases { 64 | err := Init(testCase.config) 65 | require.NoError(t, err) 66 | 67 | actualMetric := systemMetric(testCase.metric) 68 | assert.Equal(t, testCase.expectedMetric, actualMetric) 69 | } 70 | } 71 | 72 | var appMetricTestcases = []struct { 73 | config Config 74 | metric string 75 | expectedMetric string 76 | }{ 77 | { 78 | config: Config{AppSubPrefix: "applications"}, 79 | metric: "metric", 80 | expectedMetric: "applications.metric", 81 | }, 82 | { 83 | config: Config{AppSubPrefix: ""}, 84 | metric: "metric", 85 | expectedMetric: "metric", 86 | }, 87 | { 88 | config: Config{AppSubPrefix: ""}, 89 | metric: "", 90 | expectedMetric: "", 91 | }, 92 | } 93 | 94 | func TestAppMetricTestCases(t *testing.T) { 95 | hostname = func() (string, error) { return "localhost", nil } 96 | for _, testCase := range appMetricTestcases { 97 | err := Init(testCase.config) 98 | require.NoError(t, err) 99 | 100 | actualMetric := appMetric(testCase.metric) 101 | assert.Equal(t, testCase.expectedMetric, actualMetric) 102 | } 103 | } 104 | 105 | func TestMark(t *testing.T) { 106 | // given 107 | err := Init(Config{Target: "stdout", Prefix: ""}) 108 | systemMarker := systemMetric("marker") 109 | 110 | // expect 111 | assert.Nil(t, metrics.Get(systemMarker)) 112 | 113 | // when 114 | Mark("marker") 115 | 116 | // then 117 | mark, _ := metrics.Get(systemMarker).(metrics.Meter) 118 | assert.Nil(t, err) 119 | assert.Equal(t, int64(1), mark.Count()) 120 | 121 | // when 122 | Mark("marker") 123 | 124 | // then 125 | assert.Equal(t, int64(2), mark.Count()) 126 | } 127 | 128 | func TestMarkApp(t *testing.T) { 129 | // given 130 | err := Init(Config{Target: "stdout", Prefix: "prefix", AppSubPrefix: "applications"}) 131 | appMarker := "applications.marker" 132 | 133 | // expect 134 | assert.Nil(t, metrics.Get(appMarker)) 135 | 136 | // when 137 | MarkApp("marker") 138 | 139 | // then 140 | mark, _ := metrics.Get(appMarker).(metrics.Meter) 141 | assert.Nil(t, err) 142 | assert.Equal(t, int64(1), mark.Count()) 143 | 144 | // when 145 | MarkApp("marker") 146 | 147 | // then 148 | assert.Equal(t, int64(2), mark.Count()) 149 | } 150 | 151 | func TestTime(t *testing.T) { 152 | // given 153 | err := Init(Config{Target: "stdout", Prefix: ""}) 154 | systemTimer := systemMetric("timer") 155 | 156 | // expect 157 | assert.Nil(t, metrics.Get(systemTimer)) 158 | 159 | // when 160 | Time("timer", func() {}) 161 | 162 | // then 163 | time, _ := metrics.Get(systemTimer).(metrics.Timer) 164 | assert.Equal(t, int64(1), time.Count()) 165 | 166 | // when 167 | Time("timer", func() {}) 168 | 169 | // then 170 | assert.Nil(t, err) 171 | assert.Equal(t, int64(2), time.Count()) 172 | } 173 | 174 | func TestUpdateGauge(t *testing.T) { 175 | // given 176 | err := Init(Config{Target: "stdout", Prefix: ""}) 177 | systemCounter := systemMetric("counter") 178 | 179 | // expect 180 | assert.Nil(t, metrics.Get(systemCounter)) 181 | 182 | // when 183 | UpdateGauge("counter", 2) 184 | 185 | // then 186 | gauge := metrics.Get(systemCounter).(metrics.Gauge) 187 | assert.Equal(t, int64(2), gauge.Value()) 188 | 189 | // when 190 | UpdateGauge("counter", 123) 191 | 192 | // then 193 | assert.Equal(t, int64(123), gauge.Value()) 194 | assert.Nil(t, err) 195 | } 196 | 197 | func TestMetricsInit_ForGraphiteWithNoAddress(t *testing.T) { 198 | err := Init(Config{Target: "graphite", Addr: ""}) 199 | assert.Error(t, err) 200 | } 201 | 202 | func TestMetricsInit_ForGraphiteWithBadAddress(t *testing.T) { 203 | err := Init(Config{Target: "graphite", Addr: "localhost"}) 204 | assert.Error(t, err) 205 | } 206 | 207 | func TestMetricsInit_ForGraphit(t *testing.T) { 208 | err := Init(Config{Target: "graphite", Addr: "localhost:81"}) 209 | assert.NoError(t, err) 210 | } 211 | 212 | func TestMetricsInit_ForUnknownTarget(t *testing.T) { 213 | err := Init(Config{Target: "unknown"}) 214 | assert.Error(t, err) 215 | } 216 | 217 | func TestMetricsInit(t *testing.T) { 218 | // when 219 | err := Init(Config{Prefix: "prefix"}) 220 | 221 | // then 222 | assert.Equal(t, "prefix", prefix) 223 | assert.NoError(t, err) 224 | } 225 | 226 | func TestInit_DefaultPrefix(t *testing.T) { 227 | // given 228 | hostname = func() (string, error) { return "", fmt.Errorf("Some error") } 229 | 230 | // when 231 | err := Init(Config{Prefix: "default"}) 232 | 233 | // then 234 | assert.Error(t, err) 235 | } 236 | 237 | func TestInit_DefaultPrefix_WithErrors(t *testing.T) { 238 | // given 239 | hostname = func() (string, error) { return "myhost", nil } 240 | os.Args = []string{"./myapp"} 241 | 242 | // when 243 | err := Init(Config{Prefix: "default"}) 244 | 245 | // then 246 | assert.NoError(t, err) 247 | assert.Equal(t, "myhost.myapp", prefix) 248 | } 249 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | "os" 7 | "strings" 8 | "time" 9 | 10 | log "github.com/Sirupsen/logrus" 11 | "github.com/allegro/marathon-appcop/marathon" 12 | "github.com/allegro/marathon-appcop/metrics" 13 | "github.com/allegro/marathon-appcop/mgc" 14 | "github.com/allegro/marathon-appcop/score" 15 | "github.com/allegro/marathon-appcop/web" 16 | flag "github.com/ogier/pflag" 17 | ) 18 | 19 | // Config specific config 20 | type Config struct { 21 | Web web.Config 22 | Marathon marathon.Config 23 | Score score.Config 24 | MGC mgc.Config 25 | Metrics metrics.Config 26 | Log struct { 27 | Level string 28 | Format string 29 | File string 30 | } 31 | configFile string 32 | } 33 | 34 | var config = &Config{} 35 | 36 | // NewConfig config instance 37 | func NewConfig() (*Config, error) { 38 | 39 | if !flag.Parsed() { 40 | config.parseFlags() 41 | } 42 | flag.Parse() 43 | err := config.loadConfigFromFile() 44 | 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | err = config.setLogOutput() 50 | if err != nil { 51 | return nil, err 52 | } 53 | 54 | config.setLogFormat() 55 | err = config.setLogLevel() 56 | if err != nil { 57 | return nil, err 58 | } 59 | 60 | return config, err 61 | } 62 | 63 | func (config *Config) parseFlags() { 64 | // Web 65 | flag.StringVar(&config.Web.Listen, "listen", ":4444", "Port to listen on, at this point only for health checking") 66 | flag.StringVar(&config.Web.Location, "event-stream", "http://example.com:8080/v2/events", "Get events from this stream") 67 | flag.IntVar(&config.Web.QueueSize, "events-queue-size", 1000, "Size of events queue") 68 | flag.IntVar(&config.Web.WorkersCount, "workers-pool-size", 10, "Number of concurrent workers processing events") 69 | flag.StringVar(&config.Web.MyLeader, "my-leader", "example.com:8080", "My leader, when marathon /v2/leader endpoint return the same string as this one, make subscription to event stream") 70 | 71 | // Marathon 72 | flag.StringVar(&config.Marathon.Location, 73 | "marathon-location", "example.com:8080", "Marathon URL") 74 | flag.StringVar(&config.Marathon.Protocol, 75 | "marathon-protocol", "http", "Marathon protocol (http or https)") 76 | flag.StringVar(&config.Marathon.Username, 77 | "marathon-username", "marathon", "Marathon username for basic auth") 78 | flag.StringVar(&config.Marathon.Password, 79 | "marathon-password", "marathon", "Marathon password for basic auth") 80 | flag.BoolVar(&config.Marathon.VerifySsl, 81 | "marathon-ssl-verify", true, "Verify certificates when connecting via SSL") 82 | flag.DurationVar(&config.Marathon.Timeout, 83 | "marathon-timeout", 30*time.Second, 84 | "Time limit for requests made by the Marathon HTTP client. A Timeout of zero means no timeout") 85 | flag.StringVar(&config.Marathon.AppIDPrefix, "appid-prefix", "", 86 | "Prefix common to all fully qualified application ID's. Remove this preffix from applications id's (reffer to README to get an idea when this id is removed)") 87 | 88 | // Score 89 | flag.BoolVar(&config.Score.DryRun, 90 | "dry-run", false, 91 | "Perform a trial run with no changes made to marathon.") 92 | flag.IntVar(&config.Score.ScaleDownScore, 93 | "scale-down-score", 200, 94 | "Score for application to scale it one instance down.") 95 | flag.IntVar(&config.Score.ScaleLimit, 96 | "scale-limit", 2, 97 | "How many application scale down actions to commit in one EvaluateInterval.") 98 | flag.DurationVar(&config.Score.UpdateInterval, 99 | "update-interval", 2*time.Second, 100 | "Interval of updating app scores.") 101 | flag.DurationVar(&config.Score.ResetInterval, 102 | "reset-interval", 60*time.Minute, 103 | "Interval when apps are scored, after interval passes scores are reset.") 104 | flag.DurationVar(&config.Score.EvaluateInterval, 105 | "evaluate-interval", 2*time.Minute, 106 | "Interval when apps are scored, after interval passes scores are reset.") 107 | 108 | // Marathon GC 109 | flag.BoolVar(&config.MGC.Enabled, 110 | "mgc-enabled", true, 111 | "Enable garbage collecting of marathon, old suspended applications will be deleted.") 112 | flag.DurationVar(&config.MGC.MaxSuspendTime, 113 | "mgc-max-suspend-time", 7*24*time.Hour, 114 | "How long application should be suspended before deleting it.") 115 | flag.DurationVar(&config.MGC.Interval, 116 | "mgc-interval", 8*time.Hour, 117 | "Marathon GC interval.") 118 | flag.BoolVar(&config.MGC.AppCopOnly, 119 | "mgc-appcop-only", true, 120 | "Delete only applications suspended by appcop.") 121 | 122 | // Metrics 123 | flag.StringVar(&config.Metrics.Target, "metrics-target", "stdout", 124 | "Metrics destination stdout or graphite (empty string disables metrics)") 125 | flag.StringVar(&config.Metrics.Prefix, "metrics-prefix", "default", 126 | "Metrics prefix (default is resolved to .") 127 | flag.StringVar(&config.Metrics.SystemSubPrefix, "metrics-system-sub-prefix", "appcop-internal", 128 | "System specific metrics. Append to metric-prefix") 129 | flag.StringVar(&config.Metrics.AppSubPrefix, "metrics-app-sub-prefix", "applications", 130 | "Applications specific metrics. Appended to metric-prefix") 131 | flag.DurationVar(&config.Metrics.Interval, "metrics-interval", 30*time.Second, 132 | "Metrics reporting interval") 133 | flag.StringVar(&config.Metrics.Addr, "metrics-location", "", 134 | "Graphite URL (used when metrics-target is set to graphite)") 135 | flag.StringVar(&config.Metrics.Addr, "metrics-instance", "", 136 | "Part of Graphite metric, used to distinguish between AppCop instances internal metrics.") 137 | 138 | // Log 139 | flag.StringVar(&config.Log.Level, "log-level", "info", 140 | "Log level: panic, fatal, error, warn, info, or debug") 141 | flag.StringVar(&config.Log.Format, "log-format", "text", 142 | "Log format: JSON, text") 143 | flag.StringVar(&config.Log.File, "log-file", "", 144 | "Save logs to file (e.g.: `/var/log/appcop.log`). If empty logs are published to STDERR") 145 | 146 | // General 147 | flag.StringVar(&config.configFile, "config-file", "", 148 | "Path to a JSON file to read configuration from. Note: Will override options set earlier on the command line") 149 | } 150 | 151 | func (config *Config) loadConfigFromFile() error { 152 | if config.configFile == "" { 153 | return nil 154 | } 155 | jsonBlob, err := ioutil.ReadFile(config.configFile) 156 | if err != nil { 157 | return err 158 | } 159 | return json.Unmarshal(jsonBlob, config) 160 | } 161 | 162 | func (config *Config) setLogLevel() error { 163 | level, err := log.ParseLevel(config.Log.Level) 164 | if err != nil { 165 | log.WithError(err).WithField("Level", config.Log.Level).Error("Bad level") 166 | return err 167 | } 168 | log.SetLevel(level) 169 | return nil 170 | } 171 | 172 | func (config *Config) setLogOutput() error { 173 | path := config.Log.File 174 | 175 | if len(path) == 0 { 176 | log.SetOutput(os.Stderr) 177 | return nil 178 | } 179 | 180 | f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) 181 | if err != nil { 182 | log.WithError(err).Errorf("error opening file: %s", path) 183 | return err 184 | } 185 | 186 | log.SetOutput(f) 187 | return nil 188 | } 189 | 190 | func (config *Config) setLogFormat() { 191 | format := strings.ToUpper(config.Log.Format) 192 | if format == "JSON" { 193 | log.SetFormatter(&log.JSONFormatter{}) 194 | } else if format == "TEXT" { 195 | log.SetFormatter(&log.TextFormatter{}) 196 | } else { 197 | log.WithField("Format", format).Error("Unknown log format") 198 | } 199 | } 200 | -------------------------------------------------------------------------------- /marathon/app_test.go: -------------------------------------------------------------------------------- 1 | package marathon 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestParseAppsRecievesMalformedJSONBlob(t *testing.T) { 12 | t.Parallel() 13 | // given 14 | var appsJSON = []byte(`{"apps": {}`) 15 | // // when 16 | apps, err := ParseApps(appsJSON) 17 | // //then 18 | require.Error(t, err) 19 | assert.Nil(t, apps) 20 | } 21 | 22 | func TestParseTaskRecievesCorrectJSONBlob(t *testing.T) { 23 | t.Parallel() 24 | // given 25 | var taskJSON = []byte(`{"taskStatus": "healthy", 26 | "appId": "appid", 27 | "host": "fqdn" } 28 | `) 29 | // // when 30 | task, err := ParseTask(taskJSON) 31 | // //then 32 | expected := &Task{ID: "", 33 | TaskStatus: "healthy", 34 | AppID: "appid", 35 | Host: "fqdn", 36 | Ports: []int(nil), 37 | HealthCheckResults: []HealthCheckResult(nil), 38 | } 39 | require.NoError(t, err) 40 | assert.Equal(t, expected, task) 41 | } 42 | 43 | func TestParseTaskRecievesMalformedJSONBlob(t *testing.T) { 44 | t.Parallel() 45 | // given 46 | var taskJSON = []byte(`{"taskStatus": {`) 47 | // // when 48 | _, err := ParseTask(taskJSON) 49 | // //then 50 | require.Error(t, err) 51 | } 52 | 53 | func TestParseTasksRecievesCorrectJSONBlob(t *testing.T) { 54 | t.Parallel() 55 | // given 56 | var tasksJSON = []byte(`{"tasks": [ 57 | { "taskStatus": "healthy", 58 | "appId": "appid", 59 | "host": "fqdn" }, 60 | { "taskStatus": "unlealthy", 61 | "appId": "appid", 62 | "host": "fqdn" } 63 | ] 64 | }`) 65 | // // when 66 | task, err := ParseTasks(tasksJSON) 67 | // //then 68 | expected := []*Task{ 69 | {ID: "", 70 | TaskStatus: "healthy", 71 | AppID: "appid", 72 | Host: "fqdn", 73 | Ports: []int(nil), 74 | HealthCheckResults: []HealthCheckResult(nil)}, 75 | {ID: "", 76 | TaskStatus: "unlealthy", 77 | AppID: "appid", 78 | Host: "fqdn", 79 | Ports: []int(nil), 80 | HealthCheckResults: []HealthCheckResult(nil)}, 81 | } 82 | require.NoError(t, err) 83 | assert.Equal(t, expected, task) 84 | } 85 | 86 | func TestParseTasksRecievesMalformedJSONBlob(t *testing.T) { 87 | t.Parallel() 88 | // given 89 | var taskJSON = []byte(`"tasks": [ {}]tus": {`) 90 | // // when 91 | _, err := ParseTasks(taskJSON) 92 | // //then 93 | require.Error(t, err) 94 | } 95 | 96 | var getMetricTestCases = []struct { 97 | task *Task 98 | prefix string 99 | expectedTaskMetric string 100 | }{ 101 | { 102 | task: &Task{ 103 | TaskStatus: "task_running", 104 | AppID: "com.example.domain.context/app-name", 105 | }, 106 | prefix: "com.example.", 107 | expectedTaskMetric: "domain.context.app-name.task_running", 108 | }, 109 | { 110 | task: &Task{ 111 | TaskStatus: "task_running", 112 | AppID: "/com.example.domain.context/app-name", 113 | }, 114 | prefix: "com.example.", 115 | expectedTaskMetric: "domain.context.app-name.task_running", 116 | }, 117 | { 118 | task: &Task{ 119 | TaskStatus: "task_running", 120 | AppID: "com.example.domain.context/app-name", 121 | }, 122 | prefix: "", 123 | expectedTaskMetric: "com.example.domain.context.app-name.task_running", 124 | }, 125 | { 126 | task: &Task{ 127 | TaskStatus: "task_running", 128 | AppID: "com.example.domain.context/group/app-name", 129 | }, 130 | prefix: "com.example.", 131 | expectedTaskMetric: "domain.context.group.app-name.task_running", 132 | }, 133 | { 134 | task: &Task{ 135 | TaskStatus: "task_staging", 136 | AppID: "com.example.domain.context/group/app-name", 137 | }, 138 | prefix: "com.example.", 139 | expectedTaskMetric: "domain.context.group.app-name.task_staging", 140 | }, 141 | { 142 | task: &Task{ 143 | TaskStatus: "task_staging", 144 | AppID: "com.example.domain.context/group/nested-group/app-name", 145 | }, 146 | prefix: "com.example.", 147 | expectedTaskMetric: "domain.context.group.nested-group.app-name.task_staging", 148 | }, 149 | { 150 | task: &Task{ 151 | TaskStatus: "task_running", 152 | AppID: "app-name", 153 | }, 154 | prefix: "com.example.", 155 | expectedTaskMetric: "app-name.task_running", 156 | }, 157 | { 158 | task: &Task{ 159 | TaskStatus: "task_running", 160 | AppID: "app-name", 161 | }, 162 | prefix: "", 163 | expectedTaskMetric: "app-name.task_running", 164 | }, 165 | { 166 | task: &Task{ 167 | TaskStatus: "task_running", 168 | AppID: "com.example.domain.context/app-name", 169 | }, 170 | prefix: "", 171 | expectedTaskMetric: "com.example.domain.context.app-name.task_running", 172 | }, 173 | { 174 | task: &Task{ 175 | TaskStatus: "task_running", 176 | AppID: "", 177 | }, 178 | prefix: "com.example.", 179 | expectedTaskMetric: "task_running", 180 | }, 181 | { 182 | task: &Task{ 183 | TaskStatus: "", 184 | AppID: "com.example.domain.context/app-name", 185 | }, 186 | prefix: "com.example.", 187 | expectedTaskMetric: "domain.context.app-name", 188 | }, 189 | { 190 | task: &Task{ 191 | TaskStatus: "", 192 | AppID: "", 193 | }, 194 | prefix: "com.example.", 195 | expectedTaskMetric: "", 196 | }, 197 | } 198 | 199 | func TestTaskGetMetricTestCases(t *testing.T) { 200 | t.Parallel() 201 | for _, testCase := range getMetricTestCases { 202 | taskMetric := testCase.task.GetMetric(testCase.prefix) 203 | assert.Equal(t, testCase.expectedTaskMetric, taskMetric) 204 | } 205 | } 206 | 207 | var penalizeTestCases = []struct { 208 | app *App 209 | expectedApp *App 210 | expectedErr error 211 | }{ 212 | { 213 | app: &App{ID: "testApp0", Instances: 1, Labels: map[string]string{}}, 214 | expectedApp: &App{ID: "testApp0", 215 | Instances: 0, 216 | Labels: map[string]string{"appcop": "suspend"}}, 217 | expectedErr: nil, 218 | }, 219 | { 220 | app: &App{ID: "testApp1", 221 | Instances: 2, 222 | Labels: map[string]string{}, 223 | }, 224 | expectedApp: &App{ID: "testApp1", 225 | Instances: 1, 226 | Labels: map[string]string{"appcop": "scaleDown"}}, 227 | expectedErr: nil, 228 | }, 229 | { 230 | app: &App{ID: "testApp2", 231 | Instances: 2, 232 | Labels: map[string]string{}, 233 | }, 234 | expectedApp: &App{ID: "testApp2", 235 | Instances: 1, 236 | Labels: map[string]string{"appcop": "scaleDown"}}, 237 | expectedErr: nil, 238 | }, 239 | { 240 | app: &App{ID: "testApp3", 241 | Instances: 2, 242 | Labels: map[string]string{"APPLABEL": "true"}}, 243 | expectedApp: &App{ID: "testApp3", 244 | Instances: 1, 245 | Labels: map[string]string{"appcop": "scaleDown", "APPLABEL": "true"}}, 246 | expectedErr: nil, 247 | }, 248 | { 249 | app: &App{ID: "testApp4", 250 | Instances: 0, 251 | Labels: map[string]string{"APPLABEL": "true"}}, 252 | expectedApp: &App{ID: "testApp4", 253 | Instances: 0, 254 | Labels: map[string]string{"APPLABEL": "true"}}, 255 | expectedErr: fmt.Errorf("unable to scale down, zero instance"), 256 | }, 257 | } 258 | 259 | func TestPenalizeTestCases(t *testing.T) { 260 | for _, testCase := range penalizeTestCases { 261 | err := testCase.app.penalize() 262 | require.Equal(t, testCase.expectedErr, err) 263 | assert.Equal(t, testCase.app, testCase.expectedApp) 264 | } 265 | } 266 | 267 | func TestIsImmuneShouldReturnTrueWhenImmunityLabelIsSetTrue(t *testing.T) { 268 | // given 269 | app := &App{ 270 | Labels: map[string]string{ApplicationImmunityLabel: "true"}, 271 | } 272 | // when 273 | actualExcused := app.HasImmunity() 274 | expectedExcused := true 275 | // then 276 | assert.Equal(t, expectedExcused, actualExcused) 277 | } 278 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AppCop [![Build](https://travis-ci.org/allegro/marathon-appcop.svg?branch=master)](https://travis-ci.org/allegro/marathon-appcop.svg?branch=master) [![Coverage Status](https://coveralls.io/repos/github/allegro/marathon-appcop/badge.svg?branch=master)](https://coveralls.io/github/allegro/marathon-appcop?branch=master) 2 | 3 | `Marathon AppCop` - Marathon applications law enforcement. 4 | 5 | In large [Mesos](mesos.apache.org) deployments there could be thousands of applications running and deploying every day. 6 | Sometimes they happen to be broken, forgotten and unmaintained which could exert pressure on cluster in numerous ways. 7 | 8 | To address that AppCop clears [Marathon](https://github.com/mesosphere/marathon) from broken application deployments. 9 | 10 | ## How it works 11 | 12 | `AppCop` takes information provided by the [Marathon event-stream](https://mesosphere.github.io/marathon/docs/event-bus.html) 13 | related to applications failures and scales them down. 14 | 15 | ### Scoring Mechanism 16 | 17 | Based on Marathon events (TASK_KILL, TASK_FAIL, TASK_FINISHED), 18 | AppCop is building score registry for each application event emited. 19 | Each score is incremented by each app event, so if events related to failures are comming it 20 | is constantly raising. 21 | When application passes treshold, then AppCop scales application one instance down forcefully and put appcop label in app definition. After that, score for this application is reset. 22 | When there is only one instance, then and score is pass theshold then application is suspended. 23 | Scores are periodically reset. 24 | 25 | ### GarbageCollection 26 | 27 | AppCop is periodically fetching applications and groups from Marathon. 28 | When application is suspended or group is empty for long (configurable) time then it is deleted. 29 | 30 | 31 | ### Metrics 32 | 33 | `AppCop` provides set of standard system metrics as well as application based metrics. 34 | 35 | 36 | #### Metric Types 37 | 38 | `System Metrics` - `AppCop` specific telemetry (e.g - queue Size, Event delays etc). Location equals, `metrics-prefix` append `metrics-system-sub-prefix`. 39 | 40 | `Applications Metrics` - Applications telemetry calculated based on events provided by marathon 41 | (like: task_killed, task_finished counters). Location equals, `metrics-prefix` (append) `metrics-app-sub-prefix`. 42 | 43 | Please note the existance of `appid-prefix` config option, if set, removes matching string from 44 | application id when it comes to metric publication. For example, assumming 45 | 46 | ``` 47 | appid-prefix = com.example. 48 | appID = com.example.exampleapp 49 | ``` 50 | your applications metric will be placed under: 51 | ``` 52 | {prefix}.{metrics-app-sub-prefix}.exampleapp 53 | ``` 54 | 55 | 56 | ## Installation 57 | 58 | ### Installing from source code 59 | 60 | To simply compile and run the source code: 61 | 62 | ``` 63 | go run main.go [options] 64 | ``` 65 | 66 | To run the tests: 67 | 68 | ``` 69 | make test 70 | ``` 71 | 72 | To build the binary: 73 | 74 | ``` 75 | make build 76 | ``` 77 | 78 | To build deb package: 79 | ``` 80 | make pack 81 | ``` 82 | 83 | Check dist/ dir. 84 | 85 | ## Setting up `AppCop` 86 | 87 | AppCcop should be installed on all Marathon masters. 88 | The event subscription should be set to `localhost` to reduce network traffic. 89 | Please refer to options section for more. 90 | 91 | 92 | ## `Marathon Labels` 93 | 94 | `AppCop` is using `Marathon` labels to communicate actions or to tune execution logic. 95 | 96 | Used labels: 97 | 98 | Name | Possible values | r/w | Description 99 | --------------------------|---------------------------|----------|------------------ 100 | appcop | `suspend`, `scaleDown` | w | Every time `AppCop` scales or suspend application, put appropriate label in app definition 101 | APP_IMMUNITY | `false`, `true` | r | When AppCop encounters this label in app definition, treats it as immune to all penalties (excused from all criminal acts on cluster). Use this feature wisely, because if applied to often it could defeat whole purpose for using AppCop 102 | 103 | r - label is taken from app definition, not altered, 104 | w - label is manipulated by `AppCop`. 105 | 106 | ### Options 107 | 108 | Argument | Default | Description 109 | ----------------------------|-------------------|------------------------------------------------------ 110 | config-file | | Path to a JSON file to read configuration from. Note: Will override options set earlier on the command line 111 | event-stream-location | /v2/events | Get events from this stream 112 | my-leader | marathon-dev | My leader, when Marathon /v2/leader endpoint return the same string as this one, make subscription to event stream and launch jobs. 113 | events-queue-size | `1000` | Size of events queue 114 | listen | `:4444` | Accept connections at this address 115 | log-file | | Save logs to file (e.g.: `/var/log/appcop.log`). If empty logs are published to STDERR 116 | log-format | `text` | Log format: JSON, text 117 | log-level | `info` | Log level: panic, fatal, error, warn, info or debug 118 | marathon-location | `example.com:8080`| Marathon URL 119 | marathon-password | | Marathon password for basic auth 120 | marathon-protocol | `http` | Marathon protocol (http or https) 121 | marathon-ssl-verify | `true` | Verify certificates when connecting via SSL 122 | marathon-timeout | `30s` | Time limit for requests made by the Marathon HTTP client. A timeout of zero means no timeout 123 | appid-prefix | | Prefix common to all fully qualified application ID's. Remove this preffix from applications id's ([Metric Types](#metric types)) 124 | marathon-username | | Marathon username for basic auth 125 | scale-down-score | `30` | Score for application to scale it one instance down 126 | scale-limit | `2` | How many scale down actions to commit in one scaling down iteration 127 | update-interval | `2s` | Interval for updating app scores 128 | reset-interval | `1d` | How often collected scores are reset 129 | evaluate-interval | `30s` | How often collected scores are compared against scale-down-score 130 | metrics-interval | `30s` | Metrics reporting interval 131 | metrics-location | | Graphite URL (used when metrics-target is set to graphite) 132 | metrics-prefix | `default` | Metrics prefix (default is resolved to . 133 | metrics-system-sub-prefix | `appcop-internal` | System specific metrics. Append to metric-prefix 134 | metrics-app-sub-prefix | `applications` | Applications specific metrics. Appended to metric-prefix 135 | metrics-target | `stdout` | Metrics destination stdout or graphite (empty string disables metrics) 136 | workers-pool-size | `10` | Number of concurrent workers processing events 137 | mgc-enabled | `true` | Enable garbage collecting of Marathon, old suspended applications will be deleted 138 | mgc-max-suspend-time | `7 days` | How long application should be suspended before deleting it 139 | mgc-interval | `8 hours` | Marathon GC interval 140 | mgc-appcop-only | `true` | Delete only applications suspended by AppCop 141 | dry-run | `false` | Perform a trial run with no changes made to marathon 142 | 143 | 144 | ### Endpoints 145 | 146 | Endpoint | Description 147 | ----------|------------------------------------------------------------------------------------ 148 | `/health` | healthcheck - returns `OK` 149 | -------------------------------------------------------------------------------- /mgc/mgc_test.go: -------------------------------------------------------------------------------- 1 | package mgc 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/allegro/marathon-appcop/marathon" 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestMGCConstructorShouldReturnStruct(t *testing.T) { 13 | t.Parallel() 14 | // given 15 | marathon := marathon.Marathon{} 16 | config := Config{} 17 | timeNow := time.Now() 18 | given := MarathonGC{config: config, 19 | marathon: marathon, 20 | apps: nil, 21 | lastRefresh: timeNow, 22 | } 23 | 24 | // when 25 | mgc, err := New(config, marathon) 26 | 27 | // then 28 | require.NoError(t, err) 29 | assert.ObjectsAreEqual(given, mgc) 30 | 31 | } 32 | 33 | func TestMGCConstructorShouldReturnDifferentStruct(t *testing.T) { 34 | t.Parallel() 35 | // given 36 | marathon := marathon.Marathon{} 37 | config := Config{} 38 | timeNow := time.Now() 39 | given := MarathonGC{config: config, 40 | marathon: nil, 41 | apps: nil, 42 | lastRefresh: timeNow, 43 | } 44 | 45 | // when 46 | mgc, err := New(config, marathon) 47 | 48 | // then 49 | require.NoError(t, err) 50 | assert.NotEqual(t, given, mgc) 51 | 52 | } 53 | 54 | func TestAppCoppedApplicationRetrunsTrue(t *testing.T) { 55 | t.Parallel() 56 | // given 57 | labels := map[string]string{ 58 | "appcop": "suspend", 59 | "consul": "true", 60 | } 61 | app := &marathon.App{ 62 | ID: "/test/app", 63 | Labels: labels, 64 | } 65 | 66 | // when 67 | appcoped := appCopped(app) 68 | //then 69 | assert.True(t, appcoped) 70 | } 71 | 72 | func TestAppCoppedApplicationRetrunsFalse(t *testing.T) { 73 | t.Parallel() 74 | // given 75 | labels := map[string]string{ 76 | "consul": "true", 77 | } 78 | app := &marathon.App{ 79 | ID: "/test/app", 80 | Labels: labels, 81 | } 82 | 83 | // when 84 | appcoped := appCopped(app) 85 | //then 86 | assert.True(t, !appcoped) 87 | } 88 | 89 | func TestAppCoppedApplicationNilLabels(t *testing.T) { 90 | t.Parallel() 91 | // given 92 | app := &marathon.App{ 93 | ID: "/test/app", 94 | } 95 | // when 96 | appcoped := appCopped(app) 97 | //then 98 | assert.True(t, !appcoped) 99 | } 100 | 101 | func TestGetOldSuspendedWhenMarathonReturnsOneOldSuspendedApp(t *testing.T) { 102 | t.Parallel() 103 | //given 104 | m := marathon.Marathon{} 105 | config := Config{} 106 | given, _ := New(config, m) 107 | wayBack := "2006-01-02T15:04:05.000Z" 108 | given.apps = []*marathon.App{ 109 | {VersionInfo: marathon.VersionInfo{ 110 | LastScalingAt: wayBack, 111 | LastConfigChangeAt: wayBack}, 112 | }, 113 | } 114 | // when 115 | apps := given.getOldSuspended() 116 | // then 117 | assert.Equal(t, 1, len(apps)) 118 | assert.NotNil(t, apps) 119 | 120 | } 121 | 122 | func TestGetOldSuspendedWhenMarathonReturnsTwoOldSuspendedApps(t *testing.T) { 123 | t.Parallel() 124 | //given 125 | m := marathon.Marathon{} 126 | config := Config{} 127 | given, _ := New(config, m) 128 | wayBack := "2006-01-02T15:04:05.000Z" 129 | alsoWayBack := "2007-01-02T15:04:05.000Z" 130 | given.apps = []*marathon.App{ 131 | {VersionInfo: marathon.VersionInfo{ 132 | LastScalingAt: wayBack, 133 | LastConfigChangeAt: wayBack}, 134 | }, 135 | {VersionInfo: marathon.VersionInfo{ 136 | LastScalingAt: alsoWayBack, 137 | LastConfigChangeAt: alsoWayBack}, 138 | }, 139 | } 140 | // when 141 | apps := given.getOldSuspended() 142 | // then 143 | assert.Equal(t, 2, len(apps)) 144 | assert.NotNil(t, apps) 145 | 146 | } 147 | 148 | func TestGetOneOldSuspendedWhenMarathonReturnsTwoOldSuspendedApps(t *testing.T) { 149 | t.Parallel() 150 | //given 151 | m := marathon.Marathon{} 152 | config := Config{} 153 | given, _ := New(config, m) 154 | ti := time.Now() 155 | timeNow := ti.Format("2006-01-02T15:04:05.000Z") 156 | 157 | wayBack := "2006-01-02T15:04:05.000Z" 158 | given.apps = []*marathon.App{ 159 | {VersionInfo: marathon.VersionInfo{ 160 | LastScalingAt: wayBack, 161 | LastConfigChangeAt: wayBack}, 162 | }, 163 | {VersionInfo: marathon.VersionInfo{ 164 | LastScalingAt: timeNow, 165 | LastConfigChangeAt: timeNow}, 166 | }, 167 | } 168 | given.config.MaxSuspendTime = 1300000000000 169 | // when 170 | apps := given.getOldSuspended() 171 | // then 172 | assert.Equal(t, 1, len(apps)) 173 | assert.NotNil(t, apps) 174 | 175 | } 176 | 177 | func TestGetOneOldSuspendedWhenMGCIsConfiguredToSuspendOnlyAppCopped(t *testing.T) { 178 | t.Parallel() 179 | //given 180 | m := marathon.Marathon{} 181 | config := Config{AppCopOnly: true} 182 | given, _ := New(config, m) 183 | wayBack := "2006-01-02T15:04:05.000Z" 184 | given.apps = []*marathon.App{ 185 | {VersionInfo: marathon.VersionInfo{ 186 | LastScalingAt: wayBack, 187 | LastConfigChangeAt: wayBack}, 188 | }, 189 | { 190 | VersionInfo: marathon.VersionInfo{ 191 | LastScalingAt: wayBack, 192 | LastConfigChangeAt: wayBack}, 193 | Labels: map[string]string{"appcop": "suspended"}, 194 | }, 195 | } 196 | // when 197 | apps := given.getOldSuspended() 198 | // then 199 | assert.Equal(t, 1, len(apps)) 200 | assert.NotNil(t, apps) 201 | } 202 | 203 | func TestGetOldSuspendedReturnsNothingWhenMGCIsConfiguredToSuspendOnlyAppCopped(t *testing.T) { 204 | t.Parallel() 205 | //given 206 | m := marathon.Marathon{} 207 | config := Config{AppCopOnly: true} 208 | given, _ := New(config, m) 209 | wayBack := "2006-01-02T15:04:05.000Z" 210 | given.apps = []*marathon.App{ 211 | {VersionInfo: marathon.VersionInfo{ 212 | LastScalingAt: wayBack, 213 | LastConfigChangeAt: wayBack}, 214 | }, 215 | {VersionInfo: marathon.VersionInfo{ 216 | LastScalingAt: wayBack, 217 | LastConfigChangeAt: wayBack}, 218 | }, 219 | } 220 | // when 221 | apps := given.getOldSuspended() 222 | // then 223 | assert.Equal(t, 0, len(apps)) 224 | assert.Nil(t, apps) 225 | 226 | } 227 | 228 | func TestGCAbleReturnsFalseWhenInstanceNumIsGreaterThanZero(t *testing.T) { 229 | t.Parallel() 230 | //given 231 | m := marathon.Marathon{} 232 | mgc, err := New(Config{}, m) 233 | app := &marathon.App{Instances: 0} 234 | // when 235 | able := mgc.shouldBeCollected(app) 236 | // then 237 | require.NoError(t, err) 238 | assert.False(t, able) 239 | } 240 | 241 | func TestGCAbleReturnsTrue(t *testing.T) { 242 | t.Parallel() 243 | //given 244 | m := marathon.Marathon{} 245 | mgc, err := New(Config{}, m) 246 | wayBack := "2006-01-02T15:04:05.000Z" 247 | app := &marathon.App{ 248 | VersionInfo: marathon.VersionInfo{ 249 | LastScalingAt: wayBack, 250 | LastConfigChangeAt: wayBack}, 251 | Instances: 0, 252 | } 253 | // when 254 | able := mgc.shouldBeCollected(app) 255 | // then 256 | require.NoError(t, err) 257 | assert.True(t, able) 258 | } 259 | 260 | func TestGCAbleReturnsFalseWhenParsingErrorOccures(t *testing.T) { 261 | t.Parallel() 262 | //given 263 | m := marathon.Marathon{} 264 | mgc, _ := New(Config{}, m) 265 | wayBack := "200aaa6-01-02T15:04:05.000Z" 266 | app := &marathon.App{ 267 | VersionInfo: marathon.VersionInfo{ 268 | LastScalingAt: wayBack, 269 | LastConfigChangeAt: wayBack}, 270 | Instances: 0, 271 | } 272 | // when 273 | able := mgc.shouldBeCollected(app) 274 | // then 275 | assert.False(t, able) 276 | } 277 | 278 | func TestMGCRefreshSuccessWhenMarathonReturnsTwoApps(t *testing.T) { 279 | t.Parallel() 280 | 281 | // given 282 | apps := []*marathon.App{ 283 | {ID: "firstApp", Instances: 1}, 284 | {ID: "secondApp", Instances: 2}, 285 | } 286 | m := marathon.MStub{Apps: apps} 287 | mgc, _ := New(Config{}, m) 288 | 289 | // when 290 | err := mgc.refresh() 291 | 292 | // then 293 | require.NoError(t, err) 294 | assert.NotNil(t, mgc) 295 | assert.Equal(t, mgc.apps, apps) 296 | } 297 | 298 | func TestMGCRefreshErrorWhenMarathonGetAppsReturnsError(t *testing.T) { 299 | t.Parallel() 300 | // given 301 | apps := []*marathon.App{ 302 | {ID: "firstApp", Instances: 1}, 303 | {ID: "secondApp", Instances: 2}, 304 | } 305 | m := marathon.MStub{Apps: apps, AppsGetFail: true} 306 | mgc, _ := New(Config{}, m) 307 | // when 308 | err := mgc.refresh() 309 | // then 310 | require.Error(t, err) 311 | assert.NotNil(t, mgc) 312 | assert.Equal(t, []*marathon.App(nil), mgc.apps) 313 | } 314 | 315 | func TestMGCGroupDeleteWhenMrathonReturnsSuccess(t *testing.T) { 316 | t.Parallel() 317 | // given 318 | m := marathon.MStub{} 319 | mgc, _ := New(Config{}, m) 320 | // when 321 | err := mgc.groupDelete("testgroup") 322 | //then 323 | require.NoError(t, err) 324 | } 325 | 326 | func TestMGCGroupDeleteWhenMrathonReturnsError(t *testing.T) { 327 | t.Parallel() 328 | // given 329 | m := marathon.MStub{GroupDelFail: true} 330 | mgc, _ := New(Config{}, m) 331 | // when 332 | err := mgc.groupDelete("testgroup") 333 | //then 334 | require.Error(t, err) 335 | } 336 | 337 | func TestMGCDeleteSuspendedAppsWhenThereIsOneAppToDelete(t *testing.T) { 338 | t.Parallel() 339 | // given 340 | apps := []*marathon.App{ 341 | {ID: "testapp0"}, 342 | } 343 | m := marathon.MStub{Apps: apps} 344 | mgc, _ := New(Config{}, m) 345 | // when 346 | i := mgc.deleteSuspended(apps) 347 | // then 348 | assert.Equal(t, 1, i) 349 | } 350 | 351 | func TestMGCDeleteSuspendedAppsWhenThereIsTwoAppsToDelete(t *testing.T) { 352 | t.Parallel() 353 | // given 354 | apps := []*marathon.App{ 355 | {ID: "testapp0"}, 356 | {ID: "testapp1"}, 357 | } 358 | m := marathon.MStub{Apps: apps} 359 | mgc, _ := New(Config{}, m) 360 | // when 361 | i := mgc.deleteSuspended(apps) 362 | // then 363 | assert.Equal(t, 2, i) 364 | } 365 | 366 | func TestMGCDeleteSuspendedAppsWhenMarathonReturnsErrorsOnSomeDeletes(t *testing.T) { 367 | t.Parallel() 368 | // given 369 | apps := []*marathon.App{ 370 | {ID: "testapp0"}, 371 | {ID: "testapp1"}, 372 | {ID: "testapp2"}, 373 | {ID: "testapp3"}, 374 | } 375 | failCounter := &marathon.FailCounter{Counter: 1} 376 | m := marathon.MStub{Apps: apps, AppDelHalfFail: true, FailCounter: failCounter} 377 | mgc, _ := New(Config{}, m) 378 | // when 379 | i := mgc.deleteSuspended(apps) 380 | // then 381 | assert.Equal(t, 2, i) 382 | } 383 | -------------------------------------------------------------------------------- /score/score_test.go: -------------------------------------------------------------------------------- 1 | package score 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/allegro/marathon-appcop/marathon" 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func newTestScorer() (*Scorer, error) { 13 | return New(Config{false, 1, 1, 3, 2, 1}, nil) 14 | } 15 | 16 | func TestNewProvidedConfigContainsUnsensibleValuesReturnsErrorAndNilScorer(t *testing.T) { 17 | t.Parallel() 18 | // given 19 | c := Config{ 20 | ScaleDownScore: 1, 21 | UpdateInterval: 1, 22 | ResetInterval: 1, 23 | EvaluateInterval: 1, 24 | ScaleLimit: 1, 25 | } 26 | // when 27 | scorer, err := New(c, nil) 28 | //then 29 | assert.Error(t, err) 30 | assert.Equal(t, scorer, (*Scorer)(nil)) 31 | } 32 | 33 | func TestNewReturnsCorrectlyInitializedScorerAndNoError(t *testing.T) { 34 | t.Parallel() 35 | // given 36 | c := Config{ 37 | ScaleDownScore: 1, 38 | UpdateInterval: 1, 39 | ResetInterval: 3, 40 | EvaluateInterval: 2, 41 | ScaleLimit: 1, 42 | DryRun: false, 43 | } 44 | // when 45 | expectedScorer := &Scorer{ 46 | ScaleDownScore: 1, 47 | ResetInterval: 3, 48 | UpdateInterval: 1, 49 | EvaluateInterval: 2, 50 | ScaleLimit: 1, 51 | scores: map[marathon.AppID]*Score{}, 52 | } 53 | actualScorer, err := New(c, nil) 54 | //then 55 | assert.Equal(t, expectedScorer, actualScorer) 56 | assert.Nil(t, err) 57 | } 58 | 59 | var initOrUpdateTestCases = []struct { 60 | updates []Update 61 | expectedScores map[marathon.AppID]*Score 62 | }{ 63 | { 64 | updates: []Update{ 65 | {App: &marathon.App{ID: "appid"}, Update: 1}, 66 | }, 67 | expectedScores: map[marathon.AppID]*Score{ 68 | marathon.AppID("appid"): {1, time.Now()}, 69 | }, 70 | }, 71 | { 72 | updates: []Update{ 73 | {App: &marathon.App{ID: "appid"}, Update: 1}, 74 | {App: &marathon.App{ID: "appid"}, Update: 1}, 75 | {App: &marathon.App{ID: "appid"}, Update: 1}, 76 | {App: &marathon.App{ID: "appid"}, Update: 1}, 77 | }, 78 | expectedScores: map[marathon.AppID]*Score{ 79 | marathon.AppID("appid"): {4, time.Now()}, 80 | }, 81 | }, 82 | { 83 | updates: []Update{ 84 | {App: &marathon.App{ID: "appid0"}, Update: 1}, 85 | {App: &marathon.App{ID: "appid0"}, Update: 1}, 86 | {App: &marathon.App{ID: "appid1"}, Update: 1}, 87 | {App: &marathon.App{ID: "appid1"}, Update: 1}, 88 | }, 89 | expectedScores: map[marathon.AppID]*Score{ 90 | marathon.AppID("appid0"): {2, time.Now()}, 91 | marathon.AppID("appid1"): {2, time.Now()}, 92 | }, 93 | }, 94 | { 95 | updates: []Update{ 96 | {App: &marathon.App{ID: "appid0"}, Update: -1}, 97 | {App: &marathon.App{ID: "appid0"}, Update: 1}, 98 | {App: &marathon.App{ID: "appid1"}, Update: -1}, 99 | {App: &marathon.App{ID: "appid1"}, Update: -1}, 100 | }, 101 | expectedScores: map[marathon.AppID]*Score{ 102 | marathon.AppID("appid0"): {0, time.Now()}, 103 | marathon.AppID("appid1"): {-2, time.Now()}, 104 | }, 105 | }, 106 | { 107 | updates: []Update{ 108 | {App: &marathon.App{ID: "appid0"}, Update: -1}, 109 | {App: &marathon.App{ID: "appid1"}, Update: 1}, 110 | {App: &marathon.App{ID: "appid2"}, Update: -1}, 111 | {App: &marathon.App{ID: "appid3"}, Update: -1}, 112 | }, 113 | expectedScores: map[marathon.AppID]*Score{ 114 | marathon.AppID("appid0"): {-1, time.Now()}, 115 | marathon.AppID("appid1"): {1, time.Now()}, 116 | marathon.AppID("appid2"): {-1, time.Now()}, 117 | marathon.AppID("appid3"): {-1, time.Now()}, 118 | }, 119 | }, 120 | } 121 | 122 | func TestInitOrUpdateTestCases(t *testing.T) { 123 | t.Parallel() 124 | for _, testCase := range initOrUpdateTestCases { 125 | s, err := newTestScorer() 126 | require.NoError(t, err) 127 | for _, update := range testCase.updates { 128 | s.initOrUpdateScore(update) 129 | } 130 | // check assertions 131 | for appID, score := range testCase.expectedScores { 132 | app, ok := s.scores[appID] 133 | require.True(t, ok) 134 | assert.Equal(t, score.score, app.score) 135 | } 136 | } 137 | } 138 | 139 | func TestResetScoreDeletesSpecifiedAppFromScorerLeaveRestUntoutchCheckScoreMap(t *testing.T) { 140 | t.Parallel() 141 | // given 142 | s, err := newTestScorer() 143 | require.NoError(t, err) 144 | s.scores["testapp0"] = &Score{score: 1, lastUpdate: time.Now()} 145 | // Shouldnt be touch 146 | s.scores["testapp1"] = &Score{score: 2, lastUpdate: time.Now()} 147 | expectedScore := 2 148 | // when 149 | s.resetScore("testapp0") 150 | _, ok0 := s.scores["testapp0"] 151 | app, ok1 := s.scores["testapp1"] 152 | //then 153 | assert.False(t, ok0) 154 | assert.True(t, ok1) 155 | assert.Equal(t, expectedScore, app.score) 156 | } 157 | 158 | var substractScoreTestCases = []struct { 159 | initialScores map[marathon.AppID]int 160 | appsToSubstractScoreFrom []marathon.AppID 161 | expectedScores map[marathon.AppID]int 162 | }{ 163 | { 164 | initialScores: map[marathon.AppID]int{}, 165 | appsToSubstractScoreFrom: []marathon.AppID{}, 166 | expectedScores: map[marathon.AppID]int{}, 167 | }, 168 | { 169 | initialScores: map[marathon.AppID]int{ 170 | marathon.AppID("id1"): 1, 171 | marathon.AppID("id2"): 2, 172 | }, 173 | appsToSubstractScoreFrom: []marathon.AppID{ 174 | marathon.AppID("id1"), marathon.AppID("id2"), 175 | }, 176 | expectedScores: map[marathon.AppID]int{ 177 | marathon.AppID("id1"): 0, 178 | marathon.AppID("id2"): 1, 179 | }, 180 | }, 181 | { 182 | initialScores: map[marathon.AppID]int{ 183 | marathon.AppID("id1"): 20, 184 | marathon.AppID("id2"): 30, 185 | }, 186 | appsToSubstractScoreFrom: []marathon.AppID{ 187 | marathon.AppID("id1"), marathon.AppID("id2"), 188 | }, 189 | expectedScores: map[marathon.AppID]int{ 190 | marathon.AppID("id1"): 19, 191 | marathon.AppID("id2"): 29, 192 | }, 193 | }, 194 | { 195 | initialScores: map[marathon.AppID]int{ 196 | marathon.AppID("id1"): -1, 197 | marathon.AppID("id2"): -2, 198 | }, 199 | appsToSubstractScoreFrom: []marathon.AppID{ 200 | marathon.AppID("id1"), marathon.AppID("id2"), 201 | }, 202 | expectedScores: map[marathon.AppID]int{ 203 | marathon.AppID("id1"): -2, 204 | marathon.AppID("id2"): -3, 205 | }, 206 | }, 207 | } 208 | 209 | func TestSubstractScoresTestCases(t *testing.T) { 210 | t.Parallel() 211 | for _, testCase := range substractScoreTestCases { 212 | scorer, err := newTestScorer() 213 | require.NoError(t, err) 214 | // feed scores 215 | for app, score := range testCase.initialScores { 216 | scorer.scores[app] = &Score{score, time.Now()} 217 | } 218 | // actual substraction 219 | for _, app := range testCase.appsToSubstractScoreFrom { 220 | scorer.subtractScore(app) 221 | } 222 | // check assertions 223 | for app, expectedScore := range testCase.expectedScores { 224 | appScore := scorer.scores[app].score 225 | assert.Equal(t, expectedScore, appScore) 226 | } 227 | } 228 | } 229 | 230 | var evaluateScoresTestCases = []struct { 231 | scaleDownScore int 232 | initialScores map[marathon.AppID]int 233 | expectedAppsToPacify int 234 | }{ 235 | { 236 | scaleDownScore: 20, 237 | initialScores: map[marathon.AppID]int{ 238 | marathon.AppID("id1"): 1, 239 | marathon.AppID("id2"): 2, 240 | }, 241 | expectedAppsToPacify: 0, 242 | }, 243 | { 244 | scaleDownScore: 20, 245 | initialScores: map[marathon.AppID]int{ 246 | marathon.AppID("id1"): 21, 247 | marathon.AppID("id2"): 3, 248 | }, 249 | expectedAppsToPacify: 1, 250 | }, 251 | { 252 | scaleDownScore: 20, 253 | initialScores: map[marathon.AppID]int{ 254 | marathon.AppID("id1"): 21, 255 | marathon.AppID("id2"): 3, 256 | marathon.AppID("id3"): -1, 257 | }, 258 | expectedAppsToPacify: 1, 259 | }, 260 | { 261 | scaleDownScore: 20, 262 | initialScores: map[marathon.AppID]int{ 263 | marathon.AppID("id1"): 1230, 264 | marathon.AppID("id2"): 3, 265 | marathon.AppID("id3"): -1, 266 | }, 267 | expectedAppsToPacify: 1, 268 | }, 269 | } 270 | 271 | func TestEvaluateScoresTestCases(t *testing.T) { 272 | t.Parallel() 273 | for _, testCase := range evaluateScoresTestCases { 274 | scaleCounter := &marathon.ScaleCounter{Counter: 0} 275 | m := marathon.MStub{ScaleCounter: scaleCounter} 276 | scorer, err := New(Config{false, testCase.scaleDownScore, 1, 3, 2, 1}, m) 277 | require.NoError(t, err) 278 | // feed scores 279 | for app, score := range testCase.initialScores { 280 | scorer.scores[app] = &Score{score, time.Now()} 281 | } 282 | // actual evaluation 283 | appsToPacify, _ := scorer.evaluateApps() 284 | // check assertions 285 | assert.Equal(t, testCase.expectedAppsToPacify, appsToPacify) 286 | // expectedAppsToPacify equals ScaleCounter increments 287 | assert.Equal(t, testCase.expectedAppsToPacify, m.ScaleCounter.Counter) 288 | } 289 | } 290 | 291 | func TestEvaluateScoresTestCasesWithDryRunTrue(t *testing.T) { 292 | t.Parallel() 293 | for _, testCase := range evaluateScoresTestCases { 294 | scaleCounter := &marathon.ScaleCounter{Counter: 0} 295 | m := marathon.MStub{ScaleCounter: scaleCounter} 296 | scorer, err := New(Config{true, testCase.scaleDownScore, 1, 3, 2, 1}, m) 297 | require.NoError(t, err) 298 | // feed scores 299 | for app, score := range testCase.initialScores { 300 | scorer.scores[app] = &Score{score, time.Now()} 301 | } 302 | // actual evaluation 303 | appsToPacify, _ := scorer.evaluateApps() 304 | // check assertions 305 | // check how many apps are above threshold 306 | assert.Equal(t, testCase.expectedAppsToPacify, appsToPacify) 307 | // expectedAppsToPacify equals ScaleCounter increments 308 | // when dry run -> never pacify 309 | assert.Equal(t, 0, m.ScaleCounter.Counter) 310 | } 311 | } 312 | 313 | func TestScaleDownShouldReturnErrorWhenApplicationHasImmunityLabelSet(t *testing.T) { 314 | t.Parallel() 315 | // given 316 | m := marathon.MStub{} 317 | app := &marathon.App{ 318 | ID: "testApp0", 319 | Labels: map[string]string{marathon.ApplicationImmunityLabel: "true"}, 320 | Instances: 1, 321 | } 322 | m.Apps = []*marathon.App{app} 323 | scorer, err := New(Config{false, 1, 1, 3, 2, 1}, m) 324 | require.NoError(t, err) 325 | scorer.scores[app.ID] = &Score{1, time.Now()} 326 | // when 327 | err = scorer.scaleDown("testApp0") 328 | // then 329 | assert.Error(t, err) 330 | } 331 | 332 | func TestScaleDownShouldReturnNoErrorAndScaleApplicationDownWhenNoImmunityLabelSet(t *testing.T) { 333 | t.Parallel() 334 | // given 335 | scaleCounter := &marathon.ScaleCounter{Counter: 0} 336 | m := marathon.MStub{ScaleCounter: scaleCounter} 337 | app := &marathon.App{ 338 | ID: "testApp0", 339 | Labels: map[string]string{}, 340 | Instances: 1, 341 | } 342 | m.Apps = []*marathon.App{app} 343 | scorer, err := New(Config{false, 1, 1, 3, 2, 1}, m) 344 | scorer.scores[app.ID] = &Score{1, time.Now()} 345 | require.NoError(t, err) 346 | // when 347 | err = scorer.scaleDown("testApp0") 348 | // then 349 | expectedScale := 1 350 | assert.NoError(t, err) 351 | assert.Equal(t, expectedScale, m.ScaleCounter.Counter) 352 | } 353 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /marathon/marathon.go: -------------------------------------------------------------------------------- 1 | package marathon 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "net/url" 10 | "strings" 11 | 12 | "github.com/sethgrid/pester" 13 | 14 | log "github.com/Sirupsen/logrus" 15 | "github.com/allegro/marathon-appcop/metrics" 16 | ) 17 | 18 | // Marathoner interfacing marathon 19 | type Marathoner interface { 20 | AppGet(AppID) (*App, error) 21 | AppsGet() ([]*App, error) 22 | TasksGet(AppID) ([]*Task, error) 23 | AuthGet() *url.Userinfo 24 | LocationGet() string 25 | LeaderGet() (string, error) 26 | AppScaleDown(*App) error 27 | AppDelete(AppID) error 28 | GroupDelete(GroupID) error 29 | GetEmptyLeafGroups() ([]*Group, error) 30 | GetAppIDPrefix() string 31 | } 32 | 33 | // Marathon reciever 34 | type Marathon struct { 35 | Location string 36 | Protocol string 37 | appIDPrefix string 38 | Auth *url.Userinfo 39 | client *pester.Client 40 | } 41 | 42 | // ScaleData marathon scale json representation 43 | type ScaleData struct { 44 | Instances int `json:"instances"` 45 | Labels map[string]string `json:"labels"` 46 | } 47 | 48 | // ScaleResponse represents marathon response from scaling request 49 | type ScaleResponse struct { 50 | Version string `json:"version"` 51 | DeploymentID string `json:"deploymentId"` 52 | } 53 | 54 | // DeleteResponse represents marathon response from scaling request 55 | type DeleteResponse struct { 56 | Version string `json:"version"` 57 | DeploymentID string `json:"deploymentId"` 58 | } 59 | 60 | // LeaderResponse represents marathon response from /v2/leader request 61 | type LeaderResponse struct { 62 | Leader string `json:"leader"` 63 | } 64 | 65 | type urlParams map[string]string 66 | 67 | // New marathon instance 68 | func New(config Config) (*Marathon, error) { 69 | var auth *url.Userinfo 70 | if len(config.Username) == 0 && len(config.Password) == 0 { 71 | auth = nil 72 | } else { 73 | auth = url.UserPassword(config.Username, config.Password) 74 | } 75 | transport := &http.Transport{ 76 | Proxy: http.ProxyFromEnvironment, 77 | } 78 | pClient := pester.New() 79 | pClient.Concurrency = 3 80 | pClient.MaxRetries = 5 81 | pClient.Backoff = pester.ExponentialBackoff 82 | pClient.KeepLog = true 83 | pClient.Transport = transport 84 | 85 | return &Marathon{ 86 | Location: config.Location, 87 | Protocol: config.Protocol, 88 | appIDPrefix: config.AppIDPrefix, 89 | Auth: auth, 90 | client: pClient, 91 | }, nil 92 | } 93 | 94 | // AppGet get marathons application from v2/apps/ 95 | func (m Marathon) AppGet(appID AppID) (*App, error) { 96 | log.WithField("Location", m.Location).Debugf("Asking Marathon for %s", appID) 97 | 98 | body, err := m.get(m.urlWithQuery(fmt.Sprintf("/v2/apps/%s", appID), urlParams{"embed": "apps.tasks"})) 99 | if err != nil { 100 | return nil, err 101 | } 102 | 103 | return ParseApp(body) 104 | } 105 | 106 | // AppsGet get marathons application from v2/apps/ 107 | func (m Marathon) AppsGet() ([]*App, error) { 108 | log.Debug("Asking Marathon for list of applications") 109 | 110 | body, err := m.get(m.url("/v2/apps/")) 111 | if err != nil { 112 | return nil, err 113 | } 114 | 115 | return ParseApps(body) 116 | } 117 | 118 | // TasksGet lists marathon tasks for specified AppID 119 | func (m Marathon) TasksGet(appID AppID) ([]*Task, error) { 120 | log.WithFields(log.Fields{ 121 | "Location": m.Location, 122 | "Id": appID, 123 | }).Debug("asking Marathon for tasks") 124 | 125 | trimmedAppID := strings.Trim(appID.String(), "/") 126 | body, err := m.get(m.url(fmt.Sprintf("/v2/apps/%s/tasks", trimmedAppID))) 127 | if err != nil { 128 | return nil, err 129 | } 130 | 131 | return ParseTasks(body) 132 | } 133 | 134 | func close(r *http.Response) { 135 | err := r.Body.Close() 136 | if err != nil { 137 | log.WithError(err).Error("Can't close response") 138 | } 139 | } 140 | 141 | func (m Marathon) get(url string) ([]byte, error) { 142 | request, err := http.NewRequest("GET", url, nil) 143 | if err != nil { 144 | log.Error(err.Error()) 145 | return nil, err 146 | } 147 | request.Header.Add("Accept", "application/json") 148 | 149 | log.WithFields(log.Fields{ 150 | "Uri": request.URL.RequestURI(), 151 | "Location": m.Location, 152 | "Protocol": m.Protocol, 153 | }).Debug("Sending GET request to Marathon") 154 | 155 | var response *http.Response 156 | metrics.Time("marathon.get", func() { 157 | response, err = m.client.Do(request) 158 | }) 159 | if err != nil { 160 | metrics.Mark("marathon.get.error") 161 | m.logHTTPError(response, err) 162 | return nil, err 163 | } 164 | defer close(response) 165 | if response.StatusCode != 200 { 166 | metrics.Mark("marathon.get.error") 167 | metrics.Mark(fmt.Sprintf("marathon.get.error.%d", response.StatusCode)) 168 | err = fmt.Errorf("expected 200 but got %d for %s", response.StatusCode, response.Request.URL.Path) 169 | m.logHTTPError(response, err) 170 | return nil, err 171 | } 172 | 173 | return ioutil.ReadAll(response.Body) 174 | } 175 | 176 | func (m Marathon) update(url string, d []byte) ([]byte, error) { 177 | request, err := http.NewRequest("PUT", url, bytes.NewBuffer(d)) 178 | if err != nil { 179 | log.Error(err.Error()) 180 | return nil, err 181 | } 182 | request.Header.Add("Accept", "application/json") 183 | 184 | log.WithFields(log.Fields{ 185 | "Uri": request.URL.RequestURI(), 186 | "Location": m.Location, 187 | "Protocol": m.Protocol, 188 | }).Debug("Sending PUT request to marathon") 189 | 190 | var response *http.Response 191 | metrics.Time("marathon.put", func() { 192 | response, err = m.client.Do(request) 193 | }) 194 | if err != nil { 195 | log.Warn("Updating application failed.") 196 | metrics.Mark("marathon.put.error") 197 | m.logHTTPError(response, err) 198 | return nil, err 199 | } 200 | defer close(response) 201 | 202 | if response.StatusCode != 200 { 203 | metrics.Mark("marathon.put.error") 204 | metrics.Mark(fmt.Sprintf("marathon.put.error.%d", response.StatusCode)) 205 | err = fmt.Errorf("expected 200 but got %d for %s", response.StatusCode, response.Request.URL.Path) 206 | m.logHTTPError(response, err) 207 | return nil, err 208 | } 209 | 210 | return ioutil.ReadAll(response.Body) 211 | } 212 | 213 | func (m Marathon) delete(url string) ([]byte, error) { 214 | request, err := http.NewRequest("DELETE", url, nil) 215 | if err != nil { 216 | log.Error(err.Error()) 217 | return nil, err 218 | } 219 | request.Header.Add("Accept", "application/json") 220 | 221 | log.WithFields(log.Fields{ 222 | "Uri": request.URL.RequestURI(), 223 | "Location": m.Location, 224 | "Protocol": m.Protocol, 225 | }).Debug("Sending DELETE request to marathon") 226 | 227 | var response *http.Response 228 | metrics.Time("marathon.delete", func() { 229 | response, err = m.client.Do(request) 230 | }) 231 | if err != nil { 232 | log.Warn("Deleting application failed.") 233 | metrics.Mark("marathon.delete.error") 234 | m.logHTTPError(response, err) 235 | return nil, err 236 | } 237 | defer close(response) 238 | 239 | if response.StatusCode != 200 { 240 | metrics.Mark("marathon.delete.error") 241 | metrics.Mark(fmt.Sprintf("marathon.delete.error.%d", response.StatusCode)) 242 | err = fmt.Errorf("expected 200 but got %d for %s", response.StatusCode, response.Request.URL.Path) 243 | m.logHTTPError(response, err) 244 | return nil, err 245 | } 246 | 247 | return ioutil.ReadAll(response.Body) 248 | } 249 | 250 | // AppScaleDown scales down app by provided AppID 251 | func (m Marathon) AppScaleDown(app *App) error { 252 | 253 | log.WithFields(log.Fields{ 254 | "AppID": app.ID, 255 | }).Debug("Scaling Down application because of score.") 256 | 257 | err := app.penalize() 258 | if err != nil { 259 | return err 260 | } 261 | 262 | scaleData := &ScaleData{Instances: app.Instances, Labels: app.Labels} 263 | u, err := json.Marshal(scaleData) 264 | if err != nil { 265 | return err 266 | } 267 | 268 | trimmedAppID := strings.Trim(app.ID.String(), "/") 269 | url := m.urlWithQuery(fmt.Sprintf("/v2/apps/%s", trimmedAppID), 270 | urlParams{"force": "true"}) 271 | 272 | body, err := m.update(url, u) 273 | if err != nil { 274 | return err 275 | } 276 | 277 | log.WithFields(log.Fields{ 278 | "URL": url, 279 | "Labels": app.Labels, 280 | "Instances": app.Instances, 281 | }).Debug("Updated app") 282 | 283 | scaleResponse := &ScaleResponse{} 284 | return json.Unmarshal(body, scaleResponse) 285 | } 286 | 287 | // AppDelete scales down app by provided AppID 288 | func (m Marathon) AppDelete(app AppID) error { 289 | 290 | log.WithFields(log.Fields{ 291 | "AppID": app, 292 | }).Info("Deleting application.") 293 | 294 | trimmedAppID := strings.Trim(app.String(), "/") 295 | url := m.url(fmt.Sprintf("/v2/apps/%s", trimmedAppID)) 296 | 297 | log.WithFields(log.Fields{ 298 | "url": url, 299 | }).Debug("Application url.") 300 | 301 | body, err := m.delete(url) 302 | if err != nil { 303 | return err 304 | } 305 | 306 | deleteResponse := &DeleteResponse{} 307 | return json.Unmarshal(body, deleteResponse) 308 | } 309 | 310 | func (m Marathon) logHTTPError(resp *http.Response, err error) { 311 | statusCode := "???" 312 | if resp != nil { 313 | statusCode = fmt.Sprintf("%d", resp.StatusCode) 314 | } 315 | 316 | log.WithFields(log.Fields{ 317 | "Location": m.Location, 318 | "Protocol": m.Protocol, 319 | "statusCode": statusCode, 320 | }).Error(err) 321 | } 322 | 323 | func (m Marathon) url(path string) string { 324 | return m.urlWithQuery(path, nil) 325 | } 326 | 327 | func (m Marathon) urlWithQuery(path string, params urlParams) string { 328 | marathon := url.URL{ 329 | Scheme: m.Protocol, 330 | User: m.Auth, 331 | Host: m.Location, 332 | Path: path, 333 | } 334 | query := marathon.Query() 335 | for key, value := range params { 336 | query.Add(key, value) 337 | } 338 | marathon.RawQuery = query.Encode() 339 | return marathon.String() 340 | } 341 | 342 | // GetEmptyLeafGroups returns groups which are leafs of groups 343 | // directory and only if they are empty (no apps inside). 344 | func (m Marathon) GetEmptyLeafGroups() ([]*Group, error) { 345 | groups, err := m.groupsGet() 346 | if err != nil { 347 | return nil, err 348 | } 349 | 350 | nestedGroups := m.getLeafGroups(groups) 351 | var emptyLeafs []*Group 352 | for _, group := range nestedGroups { 353 | if group.IsEmpty() { 354 | emptyLeafs = append(emptyLeafs, group) 355 | } 356 | } 357 | return emptyLeafs, nil 358 | } 359 | 360 | // groupsGet get marathons application from v2/apps/ 361 | func (m Marathon) groupsGet() ([]*Group, error) { 362 | log.Debug("Asking Marathon for list of groups") 363 | 364 | body, err := m.get(m.url("/v2/groups/")) 365 | if err != nil { 366 | return nil, err 367 | } 368 | 369 | return ParseGroups(body) 370 | } 371 | 372 | // getNestedGroups returns any group inside provided group 373 | // if there is no nested group then returned group should be nil 374 | func (m Marathon) getLeafGroups(groups []*Group) []*Group { 375 | var retGroup []*Group 376 | 377 | for _, group := range groups { 378 | if len(group.Groups) == 0 { 379 | retGroup = append(retGroup, group) 380 | continue 381 | } 382 | retGroup = append(retGroup, m.getLeafGroups(group.Groups)...) 383 | } 384 | 385 | return retGroup 386 | } 387 | 388 | // GroupDelete scales down app by provided AppID 389 | func (m Marathon) GroupDelete(group GroupID) error { 390 | 391 | log.WithFields(log.Fields{ 392 | "GroupID": group, 393 | }).Info("Deleting group.") 394 | 395 | trimmedGroupID := strings.Trim(group.String(), "/") 396 | url := m.url(fmt.Sprintf("/v2/groups/%s", trimmedGroupID)) 397 | 398 | log.WithFields(log.Fields{ 399 | "url": url, 400 | }).Debug("Group url.") 401 | 402 | body, err := m.delete(url) 403 | if err != nil { 404 | return err 405 | } 406 | 407 | deleteResponse := &DeleteResponse{} 408 | return json.Unmarshal(body, deleteResponse) 409 | } 410 | 411 | // AuthGet string from marathon configured instance 412 | func (m Marathon) AuthGet() *url.Userinfo { 413 | return m.Auth 414 | } 415 | 416 | // LocationGet from marathon configured instance 417 | func (m Marathon) LocationGet() string { 418 | return m.Location 419 | } 420 | 421 | // LeaderGet from marathon cluster 422 | func (m Marathon) LeaderGet() (string, error) { 423 | log.WithField("Location", m.Location).Debug("Asking Marathon for leader") 424 | body, err := m.get(m.url("/v2/leader")) 425 | if err != nil { 426 | return "", err 427 | } 428 | leaderResponse := &LeaderResponse{} 429 | err = json.Unmarshal(body, leaderResponse) 430 | 431 | return leaderResponse.Leader, err 432 | } 433 | 434 | func (m Marathon) GetAppIDPrefix() string { 435 | return m.appIDPrefix 436 | } 437 | -------------------------------------------------------------------------------- /marathon/marathon_test.go: -------------------------------------------------------------------------------- 1 | package marathon 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | "net/http/httptest" 8 | "net/url" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/assert" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func TestMarathonTasksWhenMarathonConnectionFailedShouldNotRetry(t *testing.T) { 16 | t.Parallel() 17 | // given 18 | calls := 0 19 | server, transport := mockServer(func(w http.ResponseWriter, r *http.Request) { 20 | calls++ 21 | w.WriteHeader(500) 22 | }) 23 | defer server.Close() 24 | 25 | url, _ := url.Parse(server.URL) 26 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 27 | m.client.Transport = transport 28 | m.client.Concurrency = 1 29 | m.client.MaxRetries = 1 30 | // when 31 | tasks, err := m.TasksGet("/app/id") 32 | //then 33 | m.client.Concurrency = 1 34 | m.client.MaxRetries = 1 35 | assert.Error(t, err) 36 | assert.Empty(t, tasks) 37 | assert.Equal(t, 1, calls) 38 | } 39 | 40 | func TestMarathonAppGetWhenMarathonConnectionFailedShouldNotRetry(t *testing.T) { 41 | t.Parallel() 42 | // given 43 | calls := 0 44 | server, transport := mockServer(func(w http.ResponseWriter, r *http.Request) { 45 | calls++ 46 | w.WriteHeader(500) 47 | }) 48 | defer server.Close() 49 | 50 | url, _ := url.Parse(server.URL) 51 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 52 | m.client.Transport = transport 53 | m.client.Concurrency = 1 54 | m.client.MaxRetries = 1 55 | // when 56 | app, err := m.AppGet("/app/id") 57 | //then 58 | assert.Error(t, err) 59 | assert.Nil(t, app) 60 | assert.Equal(t, 1, calls) 61 | } 62 | 63 | func TestMarathonAppsWhenMarathonReturnMalformedJSONResponse(t *testing.T) { 64 | t.Parallel() 65 | // given 66 | server, transport := stubServer("/v2/apps/testapp?embed=apps.tasks", `{"apps":}`) 67 | defer server.Close() 68 | 69 | url, _ := url.Parse(server.URL) 70 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 71 | m.client.Transport = transport 72 | // when 73 | app, err := m.AppGet("/testapp") 74 | //then 75 | assert.Nil(t, app) 76 | assert.Error(t, err) 77 | } 78 | 79 | func TestMarathonAppWhenMarathonReturnEmptyApp(t *testing.T) { 80 | t.Parallel() 81 | // given 82 | server, transport := stubServer("/v2/apps//test/app?embed=apps.tasks", `{"app": {}}`) 83 | defer server.Close() 84 | 85 | url, _ := url.Parse(server.URL) 86 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 87 | m.client.Transport = transport 88 | // when 89 | app, err := m.AppGet("/test/app") 90 | //then 91 | assert.NoError(t, err) 92 | assert.NotNil(t, app) 93 | } 94 | 95 | func TestMarathonAppWhenMarathonReturnEmptyResponse(t *testing.T) { 96 | t.Parallel() 97 | // given 98 | server, transport := stubServer("/v2/apps//test/app?embed=apps.tasks", ``) 99 | defer server.Close() 100 | 101 | url, _ := url.Parse(server.URL) 102 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 103 | m.client.Transport = transport 104 | // when 105 | app, err := m.AppGet("/test/app") 106 | //then 107 | assert.NotNil(t, app) 108 | assert.Error(t, err) 109 | } 110 | 111 | func TestMarathonTasksWhenMarathonReturnEmptyList(t *testing.T) { 112 | t.Parallel() 113 | // given 114 | server, transport := stubServer("/v2/apps/test/app/tasks", ` 115 | {"tasks": [{ 116 | "appId": "/test", 117 | "host": "192.0.2.114", 118 | "id": "test.47de43bd-1a81-11e5-bdb6-e6cb6734eaf8", 119 | "ports": [31315], 120 | "healthCheckResults":[{ "alive":true }] 121 | }]}`) 122 | defer server.Close() 123 | url, _ := url.Parse(server.URL) 124 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 125 | m.client.Transport = transport 126 | // when 127 | tasks, err := m.TasksGet("/test/app") 128 | //then 129 | assert.NoError(t, err) 130 | assert.NotNil(t, tasks) 131 | } 132 | 133 | func TestMarathonTasksWhenMarathonReturnEmptyResponse(t *testing.T) { 134 | t.Parallel() 135 | // given 136 | server, transport := stubServer("/v2/apps/test/app/tasks", ``) 137 | defer server.Close() 138 | url, _ := url.Parse(server.URL) 139 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 140 | m.client.Transport = transport 141 | // when 142 | tasks, err := m.TasksGet("/test/app") 143 | //then 144 | assert.Nil(t, tasks) 145 | assert.Error(t, err) 146 | } 147 | 148 | func TestMarathonTasksWhenMarathonReturnMalformedJSONResponse(t *testing.T) { 149 | t.Parallel() 150 | // given 151 | server, transport := stubServer("/v2/apps/test/app/tasks", ``) 152 | defer server.Close() 153 | url, _ := url.Parse(server.URL) 154 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 155 | m.client.Transport = transport 156 | // when 157 | tasks, err := m.TasksGet("/test/app") 158 | //then 159 | assert.Nil(t, tasks) 160 | assert.Error(t, err) 161 | } 162 | 163 | func TestURLWithoutAuth(t *testing.T) { 164 | t.Parallel() 165 | // given 166 | config := Config{Location: "example.com:8080", Protocol: "http"} 167 | // when 168 | m, _ := New(config) 169 | // then 170 | assert.Equal(t, "http://example.com:8080/v2/apps", m.url("/v2/apps")) 171 | } 172 | 173 | func TestURLWithAuth(t *testing.T) { 174 | t.Parallel() 175 | // given 176 | config := Config{Location: "example.com:8080", Protocol: "http", Username: "peter", Password: "parker"} 177 | // when 178 | m, _ := New(config) 179 | // then 180 | assert.Equal(t, "http://peter:parker@example.com:8080/v2/apps", m.url("/v2/apps")) 181 | } 182 | 183 | func TestLeaderGetRetrunsNoLeader(t *testing.T) { 184 | t.Parallel() 185 | // given 186 | server, transport := stubServer("/v2/leader", `{"leader":}`) 187 | defer server.Close() 188 | 189 | url, _ := url.Parse(server.URL) 190 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 191 | m.client.Transport = transport 192 | m.client.Transport = transport 193 | m.client.Concurrency = 1 194 | m.client.MaxRetries = 1 195 | 196 | // when 197 | leader, err := m.LeaderGet() 198 | //then 199 | assert.Equal(t, leader, "") 200 | assert.Error(t, err) 201 | } 202 | 203 | func TestLeaderGetRetrunsCorrectLeader(t *testing.T) { 204 | t.Parallel() 205 | // given 206 | server, transport := stubServer("/v2/leader", `{"leader": "marathon-leader"}`) 207 | defer server.Close() 208 | 209 | url, _ := url.Parse(server.URL) 210 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 211 | m.client.Transport = transport 212 | m.client.Concurrency = 1 213 | m.client.MaxRetries = 1 214 | 215 | // when 216 | leader, err := m.LeaderGet() 217 | //then 218 | require.NoError(t, err) 219 | assert.Equal(t, leader, "marathon-leader") 220 | } 221 | 222 | func TestGetLeaderRetrunsMalformedJSON(t *testing.T) { 223 | t.Parallel() 224 | // given 225 | server, transport := stubServer("/v2/leader", `{"leader": `) 226 | defer server.Close() 227 | 228 | url, _ := url.Parse(server.URL) 229 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 230 | m.client.Transport = transport 231 | m.client.Concurrency = 1 232 | m.client.MaxRetries = 1 233 | 234 | // when 235 | leader, err := m.LeaderGet() 236 | //then 237 | require.Error(t, err) 238 | assert.Equal(t, "", leader) 239 | } 240 | 241 | func TestMarathonGetLeaderWhenMarathonConnectionFailedShouldNotRetry(t *testing.T) { 242 | t.Parallel() 243 | // given 244 | calls := 0 245 | server, transport := mockServer(func(w http.ResponseWriter, r *http.Request) { 246 | calls++ 247 | w.WriteHeader(500) 248 | }) 249 | defer server.Close() 250 | 251 | url, _ := url.Parse(server.URL) 252 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 253 | m.client.Transport = transport 254 | m.client.Concurrency = 1 255 | m.client.MaxRetries = 1 256 | // when 257 | leader, err := m.LeaderGet() 258 | //then 259 | assert.Error(t, err) 260 | assert.Equal(t, leader, "") 261 | assert.Equal(t, 1, calls) 262 | } 263 | 264 | func TestGetLocationReturnsLocation(t *testing.T) { 265 | t.Parallel() 266 | // given 267 | m, _ := New(Config{Location: "marathon:8080"}) 268 | // when 269 | location := m.LocationGet() 270 | //then 271 | assert.Equal(t, "marathon:8080", location) 272 | 273 | } 274 | 275 | func TestGetLocationReturnsNoLocation(t *testing.T) { 276 | t.Parallel() 277 | // given 278 | m, _ := New(Config{}) 279 | // when 280 | location := m.LocationGet() 281 | //then 282 | assert.Equal(t, "", location) 283 | } 284 | 285 | func TestGetAuthReturnsCorrectAuth(t *testing.T) { 286 | t.Parallel() 287 | // given 288 | m, _ := New(Config{Username: "test", Password: "test"}) 289 | // when 290 | auth := m.AuthGet() 291 | //then 292 | assert.Equal(t, "test:test", auth.String()) 293 | 294 | } 295 | 296 | func TestGetAuthReturnsNoAuth(t *testing.T) { 297 | t.Parallel() 298 | // given 299 | m, err := New(Config{}) 300 | // when 301 | auth := m.AuthGet() 302 | //then 303 | assert.NoError(t, err) 304 | assert.Nil(t, auth) 305 | } 306 | 307 | func TestMarathonAppsGetWhenMarathonConnectionFailedShouldNotRetry(t *testing.T) { 308 | t.Parallel() 309 | // given 310 | calls := 0 311 | server, transport := mockServer(func(w http.ResponseWriter, r *http.Request) { 312 | calls++ 313 | w.WriteHeader(500) 314 | }) 315 | defer server.Close() 316 | 317 | url, _ := url.Parse(server.URL) 318 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 319 | m.client.Transport = transport 320 | m.client.Concurrency = 1 321 | m.client.MaxRetries = 1 322 | // when 323 | app, err := m.AppsGet() 324 | //then 325 | assert.Error(t, err) 326 | assert.Nil(t, app) 327 | assert.Equal(t, 1, calls) 328 | } 329 | 330 | func TestMarathonAppsGetWhenMarathonReturnEmptyApp(t *testing.T) { 331 | t.Parallel() 332 | // given 333 | server, transport := stubServer("/v2/apps/", `{"apps": {}}`) 334 | defer server.Close() 335 | 336 | url, _ := url.Parse(server.URL) 337 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 338 | m.client.Transport = transport 339 | // when 340 | app, err := m.AppsGet() 341 | //then 342 | assert.Error(t, err) 343 | assert.Nil(t, app) 344 | } 345 | 346 | func TestMarathonAppsGetWhenReturnsMalformedJSON(t *testing.T) { 347 | t.Parallel() 348 | // given 349 | server, transport := stubServer("/v2/apps/", `{"apps": `) 350 | defer server.Close() 351 | 352 | url, _ := url.Parse(server.URL) 353 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 354 | m.client.Transport = transport 355 | // when 356 | app, err := m.AppsGet() 357 | //then 358 | assert.Error(t, err) 359 | assert.Nil(t, app) 360 | } 361 | 362 | func TestMarathonAppScaleDownWhenMarathonConnectionFailedShouldNotRetry(t *testing.T) { 363 | t.Parallel() 364 | // given 365 | calls := 0 366 | server, transport := mockServer(func(w http.ResponseWriter, r *http.Request) { 367 | calls++ 368 | w.WriteHeader(500) 369 | }) 370 | defer server.Close() 371 | 372 | url, _ := url.Parse(server.URL) 373 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 374 | m.client.Transport = transport 375 | m.client.Concurrency = 1 376 | m.client.MaxRetries = 1 377 | // when 378 | app := &App{ 379 | ID: "testapp", Instances: 1, 380 | Labels: make(map[string]string), 381 | } 382 | err := m.AppScaleDown(app) 383 | //then 384 | assert.Error(t, err) 385 | } 386 | 387 | func TestMarathonScaleDownAppsSuccess(t *testing.T) { 388 | t.Parallel() 389 | // given 390 | server, transport := stubServer("/v2/apps/testapp0?force=true", 391 | `{"version": "0", "deploymentId": "a"}`) 392 | defer server.Close() 393 | url, _ := url.Parse(server.URL) 394 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 395 | m.client.Transport = transport 396 | 397 | app := &App{ 398 | ID: "testapp0", Instances: 2, 399 | Labels: make(map[string]string), 400 | } 401 | 402 | // when 403 | err := m.AppScaleDown(app) 404 | //then 405 | assert.Nil(t, err) 406 | 407 | } 408 | 409 | func TestMarathonScaleDownAppsZeroInstances(t *testing.T) { 410 | t.Parallel() 411 | // given 412 | server, transport := stubServer("/v2/apps/testapp0?force=true", 413 | `{"version": "0", 414 | "deploymentId": "a"}`, 415 | ) 416 | defer server.Close() 417 | url, _ := url.Parse(server.URL) 418 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 419 | m.client.Transport = transport 420 | 421 | app := &App{ 422 | ID: "testapp0", Instances: 0, 423 | Labels: make(map[string]string), 424 | } 425 | 426 | // when 427 | err := m.AppScaleDown(app) 428 | //then 429 | assert.Error(t, err) 430 | } 431 | 432 | func TestMarathonAppDeleteSuccess(t *testing.T) { 433 | t.Parallel() 434 | // given 435 | server, transport := stubServer("/v2/apps/testapp", 436 | `{"version": "0", 437 | "deploymentId": "a"}`, 438 | ) 439 | defer server.Close() 440 | url, _ := url.Parse(server.URL) 441 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 442 | m.client.Transport = transport 443 | 444 | // when 445 | err := m.AppDelete("testapp") 446 | //then 447 | assert.Nil(t, err) 448 | } 449 | 450 | func TestMarathonAppDeleteWhenMarathonReturns500(t *testing.T) { 451 | t.Parallel() 452 | // given 453 | calls := 0 454 | server, transport := mockServer(func(w http.ResponseWriter, r *http.Request) { 455 | calls++ 456 | w.WriteHeader(500) 457 | }) 458 | defer server.Close() 459 | 460 | url, _ := url.Parse(server.URL) 461 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 462 | m.client.Transport = transport 463 | m.client.Concurrency = 1 464 | m.client.MaxRetries = 1 465 | // when 466 | err := m.AppDelete("testapp") 467 | //then 468 | assert.Error(t, err) 469 | } 470 | 471 | func TestMarathonGroupsGetSuccessMarathonReturnsOneGroup(t *testing.T) { 472 | t.Parallel() 473 | // given 474 | server, transport := stubServer("/v2/groups/", 475 | `{"groups": [ 476 | {"apps": [], "groups": [], "id": "idgroup0", "version": "1234"} 477 | ]}`, 478 | ) 479 | defer server.Close() 480 | url, _ := url.Parse(server.URL) 481 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 482 | m.client.Transport = transport 483 | 484 | // when 485 | groups, err := m.groupsGet() 486 | //then 487 | require.NoError(t, err) 488 | require.NotNil(t, groups) 489 | assert.Equal(t, groups[0].Version, "1234") 490 | assert.Equal(t, groups[0].ID.String(), "idgroup0") 491 | } 492 | 493 | func TestMarathonGroupsGetWhenMarathonReturns500(t *testing.T) { 494 | t.Parallel() 495 | // given 496 | calls := 0 497 | server, transport := mockServer(func(w http.ResponseWriter, r *http.Request) { 498 | calls++ 499 | w.WriteHeader(500) 500 | }) 501 | defer server.Close() 502 | 503 | url, _ := url.Parse(server.URL) 504 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 505 | m.client.Transport = transport 506 | m.client.Concurrency = 1 507 | m.client.MaxRetries = 1 508 | // when 509 | _, err := m.groupsGet() 510 | //then 511 | assert.Error(t, err) 512 | } 513 | 514 | func TestMarathonGroupDeleteSuccessOnExampleGroup(t *testing.T) { 515 | t.Parallel() 516 | // given 517 | server, transport := stubServer("/v2/groups/testgroup", 518 | `{"version": "0", 519 | "deploymentId": "a"}`, 520 | ) 521 | defer server.Close() 522 | url, _ := url.Parse(server.URL) 523 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 524 | m.client.Transport = transport 525 | 526 | // when 527 | err := m.GroupDelete("testgroup") 528 | //then 529 | require.NoError(t, err) 530 | } 531 | 532 | func TestMarathonGroupDeleteWhenMarathonReturns500(t *testing.T) { 533 | t.Parallel() 534 | // given 535 | calls := 0 536 | server, transport := mockServer(func(w http.ResponseWriter, r *http.Request) { 537 | calls++ 538 | w.WriteHeader(500) 539 | }) 540 | defer server.Close() 541 | 542 | url, _ := url.Parse(server.URL) 543 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 544 | m.client.Transport = transport 545 | m.client.Concurrency = 1 546 | m.client.MaxRetries = 1 547 | // when 548 | err := m.GroupDelete("testgroup") 549 | //then 550 | assert.Error(t, err) 551 | } 552 | 553 | var getEmptyLeafGroupsTestCases = []struct { 554 | testFilename string 555 | expectedIDs []GroupID 556 | }{ 557 | { 558 | testFilename: "testdata/one_root_group.json", 559 | expectedIDs: []GroupID{ 560 | GroupID("idgroup0"), 561 | }, 562 | }, 563 | { 564 | testFilename: "testdata/nested_groups_empty.json", 565 | expectedIDs: []GroupID{ 566 | GroupID("/grpa/grpb/grpc0"), 567 | GroupID("/grpa/grpb/grpc1"), 568 | GroupID("/grpa/grpb1"), 569 | }, 570 | }, 571 | { 572 | testFilename: "testdata/nested_groups_two_empty_one_with_app.json", 573 | expectedIDs: []GroupID{ 574 | GroupID("/grpa/grpb/grpc1"), 575 | GroupID("/grpa/grpb1"), 576 | }, 577 | }, 578 | { 579 | testFilename: "testdata/no_groups.json", 580 | expectedIDs: nil, 581 | }, 582 | } 583 | 584 | func TestMarathonEmptyGetLeafGroupsTestCases(t *testing.T) { 585 | for _, testCase := range getEmptyLeafGroupsTestCases { 586 | buffer, err := ioutil.ReadFile(testCase.testFilename) 587 | require.NoError(t, err) 588 | 589 | server, transport := stubServer("/v2/groups/", 590 | string(buffer), 591 | ) 592 | defer server.Close() 593 | url, _ := url.Parse(server.URL) 594 | m, _ := New(Config{Location: url.Host, Protocol: "HTTP"}) 595 | m.client.Transport = transport 596 | 597 | groups, err := m.GetEmptyLeafGroups() 598 | require.NoError(t, err) 599 | 600 | assert.Equal(t, len(testCase.expectedIDs), len(groups)) 601 | 602 | for i := 0; i < len(testCase.expectedIDs); i++ { 603 | expectedID := testCase.expectedIDs[i] 604 | actualID := groups[i].ID 605 | 606 | assert.Equal(t, expectedID, actualID) 607 | } 608 | } 609 | } 610 | 611 | // http://keighl.com/post/mocking-http-responses-in-golang/ 612 | func stubServer(uri string, body string) (*httptest.Server, *http.Transport) { 613 | return mockServer(func(w http.ResponseWriter, r *http.Request) { 614 | if r.URL.RequestURI() == uri { 615 | w.WriteHeader(200) 616 | w.Header().Set("Content-Type", "application/json") 617 | fmt.Fprintln(w, body) 618 | } else { 619 | w.WriteHeader(404) 620 | } 621 | }) 622 | } 623 | 624 | func mockServer(handle func(w http.ResponseWriter, r *http.Request)) (*httptest.Server, *http.Transport) { 625 | server := httptest.NewServer(http.HandlerFunc(handle)) 626 | 627 | transport := &http.Transport{ 628 | Proxy: func(req *http.Request) (*url.URL, error) { 629 | return url.Parse(server.URL) 630 | }, 631 | } 632 | 633 | return server, transport 634 | } 635 | --------------------------------------------------------------------------------