├── .gitignore ├── README.md ├── anon.go ├── easy ├── easy_example_test.go ├── easy_test.go └── easy.go ├── supervisor_example_test.go ├── group.go ├── doc.go ├── helpers.go ├── supervisor.go ├── LICENSE └── supervisor_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | supervisor.test 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | DEPRECATED: consider using [cirello.io/oversight](https://cirello.io/oversight) 2 | 3 | go get [-u -f] cirello.io/supervisor 4 | 5 | http://godoc.org/cirello.io/supervisor 6 | 7 | -------------------------------------------------------------------------------- /anon.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 github.com/ucirello and https://cirello.io. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to writing, software distributed 9 | // under the License is distributed on a "AS IS" BASIS, WITHOUT WARRANTIES OR 10 | // CONDITIONS OF ANY KIND, either express or implied. 11 | // 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package supervisor 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "sync" 21 | ) 22 | 23 | var ( 24 | universalFuncSvcMu sync.Mutex 25 | universalFuncSvc uint64 26 | ) 27 | 28 | func funcSvcID() uint64 { 29 | universalFuncSvcMu.Lock() 30 | universalFuncSvc++ 31 | v := universalFuncSvc 32 | universalFuncSvcMu.Unlock() 33 | return v 34 | } 35 | 36 | type funcsvc struct { 37 | id uint64 38 | f func(context.Context) 39 | } 40 | 41 | func (a funcsvc) Serve(ctx context.Context) { 42 | a.f(ctx) 43 | } 44 | 45 | func (a funcsvc) String() string { 46 | return fmt.Sprintf("function service %d", a.id) 47 | } 48 | -------------------------------------------------------------------------------- /easy/easy_example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 github.com/ucirello and https://cirello.io. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to writing, software distributed 9 | // under the License is distributed on a "AS IS" BASIS, WITHOUT WARRANTIES OR 10 | // CONDITIONS OF ANY KIND, either express or implied. 11 | // 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package easy_test 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "log" 21 | "sync" 22 | "time" 23 | 24 | supervisor "cirello.io/supervisor/easy" 25 | ) 26 | 27 | func Example() { 28 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 29 | defer cancel() 30 | 31 | var wg sync.WaitGroup 32 | ctx = supervisor.WithContext(ctx) 33 | wg.Add(1) 34 | serviceName, err := supervisor.Add(ctx, func(ctx context.Context) { 35 | select { 36 | case <-ctx.Done(): 37 | return 38 | default: 39 | defer wg.Done() 40 | fmt.Println("executed successfully") 41 | cancel() 42 | } 43 | }) 44 | if err != nil { 45 | log.Fatal(err) 46 | } 47 | 48 | wg.Wait() 49 | 50 | if err := supervisor.Remove(ctx, serviceName); err != nil { 51 | log.Fatal(err) 52 | } 53 | 54 | // Output: 55 | // executed successfully 56 | } 57 | -------------------------------------------------------------------------------- /easy/easy_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 github.com/ucirello and https://cirello.io. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to writing, software distributed 9 | // under the License is distributed on a "AS IS" BASIS, WITHOUT WARRANTIES OR 10 | // CONDITIONS OF ANY KIND, either express or implied. 11 | // 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package easy_test 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "sync" 21 | "testing" 22 | 23 | supervisor "cirello.io/supervisor/easy" 24 | ) 25 | 26 | func TestInvalidContext(t *testing.T) { 27 | ctx := context.Background() 28 | _, err := supervisor.Add(ctx, func(context.Context) {}) 29 | if err != supervisor.ErrNoSupervisorAttached { 30 | t.Errorf("ErrNoSupervisorAttached not found: %v", err) 31 | } 32 | 33 | if err := supervisor.Remove(ctx, "fake name"); err != supervisor.ErrNoSupervisorAttached { 34 | t.Errorf("ErrNoSupervisorAttached not found: %v", err) 35 | } 36 | } 37 | 38 | func TestLogger(t *testing.T) { 39 | ctx, cancel := context.WithCancel(context.Background()) 40 | defer cancel() 41 | 42 | var got string 43 | ctx = supervisor.WithContext(ctx, supervisor.WithLogger(func(a ...interface{}) { 44 | if got == "" { 45 | got = fmt.Sprint(a...) 46 | } 47 | })) 48 | 49 | var wg sync.WaitGroup 50 | wg.Add(1) 51 | supervisor.Add(ctx, func(context.Context) { wg.Done() }, supervisor.Transient) 52 | wg.Wait() 53 | 54 | const expected = "function service 1 starting" 55 | if got != expected { 56 | t.Error("unexpected logged message found. got:", got, "expected:", expected) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /supervisor_example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 github.com/ucirello and https://cirello.io. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to writing, software distributed 9 | // under the License is distributed on a "AS IS" BASIS, WITHOUT WARRANTIES OR 10 | // CONDITIONS OF ANY KIND, either express or implied. 11 | // 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package supervisor_test 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "sync" 21 | "time" 22 | 23 | "cirello.io/supervisor" 24 | ) 25 | 26 | type Simpleservice struct { 27 | id int 28 | sync.WaitGroup 29 | } 30 | 31 | func (s *Simpleservice) Serve(ctx context.Context) { 32 | fmt.Println(s.String()) 33 | s.Done() 34 | <-ctx.Done() 35 | } 36 | 37 | func (s *Simpleservice) String() string { 38 | return fmt.Sprintf("simple service %d", s.id) 39 | } 40 | 41 | func ExampleSupervisor() { 42 | var supervisor supervisor.Supervisor 43 | 44 | svc := &Simpleservice{id: 1} 45 | svc.Add(1) 46 | supervisor.Add(svc) 47 | 48 | ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) 49 | go supervisor.Serve(ctx) 50 | 51 | svc.Wait() 52 | cancel() 53 | } 54 | 55 | func ExampleGroup() { 56 | supervisor := supervisor.Group{ 57 | Supervisor: &supervisor.Supervisor{}, 58 | } 59 | 60 | svc1 := &Simpleservice{id: 1} 61 | svc1.Add(1) 62 | supervisor.Add(svc1) 63 | svc2 := &Simpleservice{id: 2} 64 | svc2.Add(1) 65 | supervisor.Add(svc2) 66 | 67 | ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) 68 | go supervisor.Serve(ctx) 69 | 70 | svc1.Wait() 71 | svc2.Wait() 72 | cancel() 73 | } 74 | -------------------------------------------------------------------------------- /group.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 github.com/ucirello and https://cirello.io. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to writing, software distributed 9 | // under the License is distributed on a "AS IS" BASIS, WITHOUT WARRANTIES OR 10 | // CONDITIONS OF ANY KIND, either express or implied. 11 | // 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package supervisor 16 | 17 | import ( 18 | "context" 19 | "sync" 20 | ) 21 | 22 | // Group is a superset of Supervisor datastructure responsible for offering a 23 | // supervisor tree whose all services are restarted whenever one of them fail or 24 | // is restarted. It assumes that all services rely on each other. It implements 25 | // Service, therefore it can be nested if necessary either with other Group or 26 | // Supervisor. When passing the Group around, remind to do it as reference 27 | // (&group). 28 | type Group struct { 29 | *Supervisor 30 | } 31 | 32 | // Serve starts the Group tree. It can be started only once at a time. If 33 | // stopped (canceled), it can be restarted. In case of concurrent calls, it will 34 | // hang until the current call is completed. 35 | func (g *Group) Serve(ctx context.Context) { 36 | if g.Supervisor == nil { 37 | panic("Supervisor missing for this Group.") 38 | } 39 | g.Supervisor.prepare() 40 | restartCtx, cancel := context.WithCancel(ctx) 41 | 42 | var ( 43 | mu sync.Mutex 44 | processingFailure bool 45 | ) 46 | processFailure := func() { 47 | mu.Lock() 48 | if processingFailure { 49 | mu.Unlock() 50 | return 51 | } 52 | processingFailure = true 53 | mu.Unlock() 54 | 55 | if !g.shouldRestart() { 56 | cancel() 57 | return 58 | } 59 | 60 | g.mu.Lock() 61 | g.log("halting all services after failure") 62 | for _, c := range g.terminations { 63 | c() 64 | } 65 | 66 | g.cancelations = make(map[string]context.CancelFunc) 67 | g.mu.Unlock() 68 | 69 | go func() { 70 | g.log("waiting for all services termination") 71 | g.runningServices.Wait() 72 | g.log("waiting for all services termination - completed") 73 | 74 | mu.Lock() 75 | processingFailure = false 76 | mu.Unlock() 77 | 78 | g.log("triggering group restart") 79 | g.added <- struct{}{} 80 | }() 81 | } 82 | serve(g.Supervisor, restartCtx, processFailure) 83 | } 84 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 github.com/ucirello and https://cirello.io. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to writing, software distributed 9 | // under the License is distributed on a "AS IS" BASIS, WITHOUT WARRANTIES OR 10 | // CONDITIONS OF ANY KIND, either express or implied. 11 | // 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /* 16 | Package supervisor provides supervisor trees for Go applications. 17 | 18 | DEPRECATED: consider using cirello.io/oversight 19 | 20 | This package implements supervisor trees, similar to what Erlang runtime offers. 21 | It is built on top of context package, with all of its advantages, namely the 22 | possibility trickle down context-related values and cancelation signals. 23 | 24 | A supervisor tree can be composed either of services or other supervisors - each 25 | supervisor can have its own set of configurations. Any instance of 26 | supervisor.Service can be added to a tree. 27 | 28 | Supervisor 29 | ├─▶ Supervisor (if one service dies, only one is restarted) 30 | │ ├─▶ Service 31 | │ └─▶ Service 32 | ├─▶ Group (if one service dies, all others are restarted too) 33 | │ └─▶ Service 34 | │ Service 35 | │ Service 36 | └─▶ Service 37 | 38 | Example: 39 | package main 40 | 41 | import ( 42 | "fmt" 43 | "os" 44 | "os/signal" 45 | "time" 46 | 47 | "cirello.io/supervisor" 48 | "context" 49 | ) 50 | 51 | type Simpleservice int 52 | 53 | func (s *Simpleservice) String() string { 54 | return fmt.Sprintf("simple service %d", int(*s)) 55 | } 56 | 57 | func (s *Simpleservice) Serve(ctx context.Context) { 58 | for { 59 | select { 60 | case <-ctx.Done(): 61 | return 62 | default: 63 | fmt.Println("do something...") 64 | time.Sleep(500 * time.Millisecond) 65 | } 66 | } 67 | } 68 | 69 | func main(){ 70 | var supervisor supervisor.Supervisor 71 | 72 | svc := Simpleservice(1) 73 | supervisor.Add(&svc) 74 | 75 | // Simply, if not special context is needed: 76 | // supervisor.Serve() 77 | // Or, using context.Context to propagate behavior: 78 | c := make(chan os.Signal, 1) 79 | signal.Notify(c, os.Interrupt) 80 | ctx, cancel := context.WithCancel(context.Background()) 81 | go func(){ 82 | <-c 83 | fmt.Println("halting supervisor...") 84 | cancel() 85 | }() 86 | 87 | supervisor.Serve(ctx) 88 | } 89 | 90 | TheJerf's blog post about Suture is a very good and helpful read to understand 91 | how this package has been implemented. 92 | 93 | This is package is inspired by github.com/thejerf/suture 94 | 95 | http://www.jerf.org/iri/post/2930 96 | */ 97 | package supervisor // import "cirello.io/supervisor" 98 | -------------------------------------------------------------------------------- /helpers.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 github.com/ucirello and https://cirello.io. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to writing, software distributed 9 | // under the License is distributed on a "AS IS" BASIS, WITHOUT WARRANTIES OR 10 | // CONDITIONS OF ANY KIND, either express or implied. 11 | // 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package supervisor 16 | 17 | import ( 18 | "context" 19 | "sync" 20 | ) 21 | 22 | func serve(s *Supervisor, ctx context.Context, processFailure processFailure) { 23 | s.running.Lock() 24 | defer s.running.Unlock() 25 | 26 | startServices(s, ctx, processFailure) 27 | var wg sync.WaitGroup 28 | wg.Add(1) 29 | go func() { 30 | for { 31 | select { 32 | case <-s.added: 33 | startServices(s, ctx, processFailure) 34 | 35 | case <-ctx.Done(): 36 | wg.Done() 37 | return 38 | } 39 | } 40 | }() 41 | <-ctx.Done() 42 | 43 | wg.Wait() 44 | s.runningServices.Wait() 45 | 46 | s.mu.Lock() 47 | s.cancelations = make(map[string]context.CancelFunc) 48 | s.mu.Unlock() 49 | } 50 | 51 | func startServices(s *Supervisor, supervisorCtx context.Context, processFailure processFailure) { 52 | s.mu.Lock() 53 | defer s.mu.Unlock() 54 | 55 | var wg sync.WaitGroup 56 | for _, name := range s.svcorder { 57 | svc := s.services[name] 58 | if _, ok := s.cancelations[name]; ok { 59 | continue 60 | } 61 | 62 | wg.Add(1) 63 | 64 | terminateCtx, terminate := context.WithCancel(supervisorCtx) 65 | s.cancelations[name] = terminate 66 | s.terminations[name] = terminate 67 | 68 | go func(name string, svc ServiceSpecification) { 69 | s.runningServices.Add(1) 70 | defer s.runningServices.Done() 71 | wg.Done() 72 | retry := true 73 | for retry { 74 | retry = svc.svctype == permanent 75 | s.logf("%s starting", name) 76 | func() { 77 | defer func() { 78 | if r := recover(); r != nil { 79 | s.logf("%s panic: %v", name, r) 80 | retry = svc.svctype == permanent || svc.svctype == transient 81 | } 82 | }() 83 | ctx, cancel := context.WithCancel(terminateCtx) 84 | s.mu.Lock() 85 | s.cancelations[name] = cancel 86 | s.mu.Unlock() 87 | svc.svc.Serve(ctx) 88 | }() 89 | if retry { 90 | processFailure() 91 | } 92 | select { 93 | case <-terminateCtx.Done(): 94 | s.logf("%s restart aborted (terminated)", name) 95 | return 96 | case <-supervisorCtx.Done(): 97 | s.logf("%s restart aborted (supervisor halted)", name) 98 | return 99 | default: 100 | } 101 | switch svc.svctype { 102 | case temporary: 103 | s.logf("%s exited (temporary)", name) 104 | return 105 | case transient: 106 | s.logf("%s exited (transient)", name) 107 | default: 108 | s.logf("%s exited (permanent)", name) 109 | } 110 | } 111 | }(name, svc) 112 | } 113 | wg.Wait() 114 | } 115 | -------------------------------------------------------------------------------- /easy/easy.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 github.com/ucirello and https://cirello.io. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to writing, software distributed 9 | // under the License is distributed on a "AS IS" BASIS, WITHOUT WARRANTIES OR 10 | // CONDITIONS OF ANY KIND, either express or implied. 11 | // 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /* 16 | Package easy is an easier interface to use cirello.io/supervisor. Its lifecycle 17 | is managed through context.Context. Stop a given supervisor by cancelling its 18 | context. 19 | 20 | 21 | package main 22 | 23 | import supervisor "cirello.io/supervisor/easy" 24 | 25 | func main() { 26 | ctx, cancel := context.WithCancel(context.Background()) 27 | defer cancel() 28 | // use cancel() to stop the supervisor 29 | ctx = supervisor.WithContext(ctx) 30 | supervisor.Add(ctx, func(ctx context.Context) { 31 | // ... 32 | }) 33 | } 34 | */ 35 | package easy 36 | 37 | import ( 38 | "context" 39 | "errors" 40 | "fmt" 41 | "math/rand" 42 | "sync" 43 | 44 | "cirello.io/supervisor" 45 | ) 46 | 47 | type ctxKey int 48 | 49 | const supervisorName ctxKey = 0 50 | 51 | var ( 52 | // ErrNoSupervisorAttached means that the given context has not been 53 | // wrapped with WithContext, and thus this package cannot detect 54 | // which supervisore you are referring to. 55 | ErrNoSupervisorAttached = errors.New("no supervisor attached to context") 56 | 57 | mu sync.Mutex 58 | supervisors map[string]*supervisor.Group // map of supervisor name to supervisor.Supervisor 59 | ) 60 | 61 | func init() { 62 | supervisors = make(map[string]*supervisor.Group) 63 | } 64 | 65 | var ( 66 | // Permanent services are always restarted. 67 | Permanent = supervisor.Permanent 68 | 69 | // Transient services are restarted only when panic. 70 | Transient = supervisor.Transient 71 | 72 | // Temporary services are never restarted. 73 | Temporary = supervisor.Temporary 74 | ) 75 | 76 | // Add inserts supervised function to the attached supervisor, it launches 77 | // automatically. If the context is not correctly prepared, it returns an 78 | // ErrNoSupervisorAttached error. By default, the restart policy is Permanent. 79 | func Add(ctx context.Context, f func(context.Context), opts ...supervisor.ServiceOption) (string, error) { 80 | name, ok := extractName(ctx) 81 | if !ok { 82 | return "", ErrNoSupervisorAttached 83 | } 84 | mu.Lock() 85 | svr, ok := supervisors[name] 86 | mu.Unlock() 87 | if !ok { 88 | panic("supervisor not found") 89 | } 90 | opts = append([]supervisor.ServiceOption{Permanent}, opts...) 91 | svcName := svr.AddFunc(f, opts...) 92 | return svcName, nil 93 | } 94 | 95 | // Remove stops and removes the given service from the attached supervisor. If 96 | // the context is not correctly prepared, it returns an ErrNoSupervisorAttached 97 | // error 98 | func Remove(ctx context.Context, name string) error { 99 | name, ok := extractName(ctx) 100 | if !ok { 101 | return ErrNoSupervisorAttached 102 | } 103 | mu.Lock() 104 | svr, ok := supervisors[name] 105 | mu.Unlock() 106 | if !ok { 107 | panic("supervisor not found") 108 | } 109 | svr.Remove(name) 110 | return nil 111 | } 112 | 113 | // WithContext takes a context and prepare it to be used by easy supervisor 114 | // package. Internally, it creates a supervisor in group mode. In this mode, 115 | // every time a service dies, the whole supervisor is restarted. 116 | func WithContext(ctx context.Context, opts ...SupervisorOption) context.Context { 117 | chosenName := fmt.Sprintf("supervisor-%d", rand.Uint64()) 118 | 119 | svr := &supervisor.Supervisor{ 120 | Name: chosenName, 121 | MaxRestarts: supervisor.AlwaysRestart, 122 | Log: func(interface{}) {}, 123 | } 124 | for _, opt := range opts { 125 | opt(svr) 126 | } 127 | group := &supervisor.Group{ 128 | Supervisor: svr, 129 | } 130 | mu.Lock() 131 | supervisors[chosenName] = group 132 | mu.Unlock() 133 | 134 | wrapped := context.WithValue(ctx, supervisorName, chosenName) 135 | go group.Serve(wrapped) 136 | return wrapped 137 | } 138 | 139 | // SupervisorOption reconfigures the supervisor attached to the context. 140 | type SupervisorOption func(*supervisor.Supervisor) 141 | 142 | // WithLogger attaches a log function to the supervisor 143 | func WithLogger(logger func(a ...interface{})) SupervisorOption { 144 | return func(s *supervisor.Supervisor) { 145 | s.Log = func(v interface{}) { 146 | logger(v) 147 | } 148 | } 149 | } 150 | 151 | func extractName(ctx context.Context) (string, bool) { 152 | name, ok := ctx.Value(supervisorName).(string) 153 | return name, ok 154 | } 155 | -------------------------------------------------------------------------------- /supervisor.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 github.com/ucirello and https://cirello.io. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to writing, software distributed 9 | // under the License is distributed on a "AS IS" BASIS, WITHOUT WARRANTIES OR 10 | // CONDITIONS OF ANY KIND, either express or implied. 11 | // 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package supervisor 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "log" 21 | "sync" 22 | "time" 23 | ) 24 | 25 | type processFailure func() 26 | 27 | // AlwaysRestart adjusts the supervisor to never halt in face of failures. 28 | const AlwaysRestart = -1 29 | 30 | type serviceType int 31 | 32 | const ( 33 | permanent serviceType = iota 34 | transient 35 | temporary 36 | ) 37 | 38 | // ServiceSpecification defines how a service is executed by the supervisor. 39 | type ServiceSpecification struct { 40 | svc Service 41 | svctype serviceType 42 | } 43 | 44 | // ServiceOption modifies the service specifications. 45 | type ServiceOption func(*ServiceSpecification) 46 | 47 | // Permanent services are always restarted 48 | func Permanent(s *ServiceSpecification) { 49 | s.svctype = permanent 50 | } 51 | 52 | // Transient services are restarted only when panic. 53 | func Transient(s *ServiceSpecification) { 54 | s.svctype = transient 55 | } 56 | 57 | // Temporary services are never restarted. 58 | func Temporary(s *ServiceSpecification) { 59 | s.svctype = temporary 60 | } 61 | 62 | // Service is the public interface expected by a Supervisor. 63 | // 64 | // This will be internally named after the result of fmt.Stringer, if available. 65 | // Otherwise it is going to use an internal representation for the service 66 | // name. 67 | type Service interface { 68 | // Serve is called by a Supervisor to start the service. It expects the 69 | // service to honor the passed context and its lifetime. Observe 70 | // <-ctx.Done() and ctx.Err(). If the service is stopped by anything 71 | // but the Supervisor, it will get started again. Be careful with shared 72 | // state among restarts. 73 | Serve(ctx context.Context) 74 | } 75 | 76 | // Supervisor is the basic datastructure responsible for offering a supervisor 77 | // tree. It implements Service, therefore it can be nested if necessary. When 78 | // passing the Supervisor around, remind to do it as reference (&supervisor). 79 | // Once the supervisor is started, its attributes are frozen. 80 | type Supervisor struct { 81 | // Name for this supervisor tree, used for logging. 82 | Name string 83 | name string 84 | 85 | // MaxRestarts is the number of maximum restarts given MaxTime. If more 86 | // than MaxRestarts occur in the last MaxTime, then the supervisor 87 | // stops all services and halts. Set this to AlwaysRestart to prevent 88 | // supervisor halt. 89 | MaxRestarts int 90 | maxrestarts int 91 | 92 | // MaxTime is the time period on which the internal restart count will 93 | // be reset. 94 | MaxTime time.Duration 95 | maxtime time.Duration 96 | 97 | // Log is a replaceable function used for overall logging. 98 | // Default: log.Printf. 99 | Log func(interface{}) 100 | log func(interface{}) 101 | 102 | // indicates that supervisor is ready for use. 103 | prepared sync.Once 104 | 105 | // signals that a new service has just been added, so the started 106 | // supervisor picks it up. 107 | added chan struct{} 108 | 109 | // indicates that supervisor has running services. 110 | running sync.Mutex 111 | runningServices sync.WaitGroup 112 | 113 | mu sync.Mutex 114 | svcorder []string // order in which services must be started 115 | services map[string]ServiceSpecification // added services 116 | cancelations map[string]context.CancelFunc // each service cancelation 117 | terminations map[string]context.CancelFunc // each service termination call 118 | lastRestart time.Time 119 | restarts int 120 | } 121 | 122 | func (s *Supervisor) prepare() { 123 | s.prepared.Do(s.reset) 124 | } 125 | 126 | func (s *Supervisor) reset() { 127 | s.mu.Lock() 128 | if s.Name == "" { 129 | s.Name = "supervisor" 130 | } 131 | if s.MaxRestarts == 0 { 132 | s.MaxRestarts = 5 133 | } 134 | if s.MaxTime == 0 { 135 | s.MaxTime = 15 * time.Second 136 | } 137 | if s.Log == nil { 138 | s.Log = func(msg interface{}) { 139 | log.Printf("%s: %v", s.Name, msg) 140 | } 141 | } 142 | 143 | s.name = s.Name 144 | s.maxrestarts = s.MaxRestarts 145 | s.maxtime = s.MaxTime 146 | s.log = s.Log 147 | 148 | s.added = make(chan struct{}) 149 | s.cancelations = make(map[string]context.CancelFunc) 150 | s.services = make(map[string]ServiceSpecification) 151 | s.terminations = make(map[string]context.CancelFunc) 152 | s.mu.Unlock() 153 | } 154 | 155 | func (s *Supervisor) shouldRestart() bool { 156 | if s.maxrestarts == AlwaysRestart { 157 | return true 158 | } 159 | 160 | s.mu.Lock() 161 | defer s.mu.Unlock() 162 | if time.Since(s.lastRestart) > s.maxtime { 163 | s.restarts = 0 164 | } 165 | s.lastRestart = time.Now() 166 | s.restarts++ 167 | return s.restarts < s.maxrestarts 168 | } 169 | 170 | // Cancelations return a list of services names and their cancelation calls. 171 | // These calls be used to force a service restart. 172 | func (s *Supervisor) Cancelations() map[string]context.CancelFunc { 173 | svclist := make(map[string]context.CancelFunc) 174 | s.mu.Lock() 175 | for k, v := range s.cancelations { 176 | svclist[k] = v 177 | } 178 | s.mu.Unlock() 179 | return svclist 180 | } 181 | 182 | // Add inserts into the Supervisor tree a new permanent service. If the 183 | // Supervisor is already started, it will start it automatically. 184 | func (s *Supervisor) Add(service Service, opts ...ServiceOption) { 185 | s.addService(service, opts...) 186 | } 187 | 188 | // AddFunc inserts into the Supervisor tree a new permanent anonymous service. 189 | // If the Supervisor is already started, it will start it automatically. 190 | func (s *Supervisor) AddFunc(f func(context.Context), opts ...ServiceOption) string { 191 | svc := &funcsvc{ 192 | id: funcSvcID(), 193 | f: f, 194 | } 195 | s.addService(svc, opts...) 196 | return svc.String() 197 | } 198 | 199 | func (s *Supervisor) addService(svc Service, opts ...ServiceOption) { 200 | s.prepare() 201 | 202 | name := fmt.Sprintf("%s", svc) 203 | s.mu.Lock() 204 | newsvc := ServiceSpecification{ 205 | svc: svc, 206 | } 207 | for _, opt := range opts { 208 | opt(&newsvc) 209 | } 210 | s.services[name] = newsvc 211 | s.svcorder = append(s.svcorder, name) 212 | s.mu.Unlock() 213 | 214 | go func() { 215 | s.added <- struct{}{} 216 | }() 217 | } 218 | 219 | // Remove stops the service in the Supervisor tree and remove from it. 220 | func (s *Supervisor) Remove(name string) { 221 | s.prepare() 222 | 223 | s.mu.Lock() 224 | defer s.mu.Unlock() 225 | if _, ok := s.services[name]; !ok { 226 | return 227 | } 228 | 229 | delete(s.services, name) 230 | 231 | for i, n := range s.svcorder { 232 | if name == n { 233 | s.svcorder = append(s.svcorder[:i], s.svcorder[i+1:]...) 234 | break 235 | } 236 | } 237 | 238 | if c, ok := s.terminations[name]; ok { 239 | delete(s.terminations, name) 240 | c() 241 | } 242 | 243 | if _, ok := s.cancelations[name]; ok { 244 | delete(s.cancelations, name) 245 | } 246 | } 247 | 248 | // Serve starts the Supervisor tree. It can be started only once at a time. If 249 | // stopped (canceled), it can be restarted. In case of concurrent calls, it will 250 | // hang until the current call is completed. 251 | func (s *Supervisor) Serve(ctx context.Context) { 252 | s.prepare() 253 | restartCtx, cancel := context.WithCancel(ctx) 254 | processFailure := func() { 255 | restart := s.shouldRestart() 256 | if !restart { 257 | cancel() 258 | } 259 | } 260 | serve(s, restartCtx, processFailure) 261 | } 262 | 263 | // Services return a list of services 264 | func (s *Supervisor) Services() map[string]Service { 265 | svclist := make(map[string]Service) 266 | s.mu.Lock() 267 | for k, v := range s.services { 268 | svclist[k] = v.svc 269 | } 270 | s.mu.Unlock() 271 | return svclist 272 | } 273 | 274 | func (s *Supervisor) String() string { 275 | s.prepare() 276 | return s.name 277 | } 278 | 279 | func (s *Supervisor) logf(format string, a ...interface{}) { 280 | s.log(fmt.Sprintf(format, a...)) 281 | } 282 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /supervisor_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 github.com/ucirello and https://cirello.io. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to writing, software distributed 9 | // under the License is distributed on a "AS IS" BASIS, WITHOUT WARRANTIES OR 10 | // CONDITIONS OF ANY KIND, either express or implied. 11 | // 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package supervisor 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "io/ioutil" 21 | "log" 22 | "sync" 23 | "testing" 24 | "time" 25 | ) 26 | 27 | func init() { 28 | log.SetOutput(ioutil.Discard) 29 | } 30 | 31 | func TestAddFuncSupervisor(t *testing.T) { 32 | 33 | var ( 34 | runCount int 35 | wg sync.WaitGroup 36 | ) 37 | 38 | wg.Add(1) 39 | var svr Supervisor 40 | svr.AddFunc(func(ctx context.Context) { 41 | runCount++ 42 | wg.Done() 43 | <-ctx.Done() 44 | }) 45 | 46 | ctx, cancel := context.WithCancel(context.Background()) 47 | go svr.Serve(ctx) 48 | 49 | wg.Wait() 50 | cancel() 51 | 52 | if runCount == 0 { 53 | t.Errorf("anonymous service should have been started") 54 | } 55 | } 56 | 57 | func TestAddServiceAfterServe(t *testing.T) { 58 | 59 | var supervisor Supervisor 60 | supervisor.Name = "TestAddServiceAfterServe supervisor" 61 | svc1 := &holdingservice{id: 1} 62 | svc1.Add(1) 63 | supervisor.Add(svc1) 64 | 65 | ctx, cancel := context.WithCancel(context.Background()) 66 | var wg sync.WaitGroup 67 | wg.Add(1) 68 | go func() { 69 | supervisor.Serve(ctx) 70 | wg.Done() 71 | }() 72 | 73 | svc1.Wait() 74 | 75 | svc2 := &holdingservice{id: 2} 76 | svc2.Add(1) 77 | supervisor.Add(svc2) 78 | svc2.Wait() 79 | 80 | cancel() 81 | <-ctx.Done() 82 | wg.Wait() 83 | 84 | if count := getServiceCount(&supervisor); count != 2 { 85 | t.Errorf("unexpected service count: %v", count) 86 | } 87 | } 88 | 89 | func TestAlwaysRestart(t *testing.T) { 90 | 91 | defer func() { 92 | if r := recover(); r != nil { 93 | t.Errorf("unexpected panic: %v", r) 94 | } 95 | }() 96 | 97 | supervisor := Supervisor{ 98 | Name: "TestAlwaysRestart supervisor", 99 | MaxRestarts: AlwaysRestart, 100 | Log: func(msg interface{}) { 101 | t.Log("supervisor log (always restart):", msg) 102 | }, 103 | } 104 | svc1 := failingservice{id: 1} 105 | supervisor.Add(&svc1) 106 | 107 | ctx, cancel := context.WithCancel(context.Background()) 108 | go supervisor.Serve(ctx) 109 | 110 | for svc1.Count() < 2 { 111 | } 112 | 113 | cancel() 114 | } 115 | 116 | func TestCascaded(t *testing.T) { 117 | 118 | var supervisor Supervisor 119 | supervisor.Name = "TestCascaded root" 120 | svc1 := waitservice{id: 1} 121 | supervisor.Add(&svc1) 122 | svc2 := waitservice{id: 2} 123 | supervisor.Add(&svc2) 124 | 125 | var childSupervisor Supervisor 126 | childSupervisor.Name = "TestCascaded child" 127 | svc3 := waitservice{id: 3} 128 | childSupervisor.Add(&svc3) 129 | svc4 := waitservice{id: 4} 130 | childSupervisor.Add(&svc4) 131 | supervisor.Add(&childSupervisor) 132 | 133 | ctx, cancel := context.WithCancel(context.Background()) 134 | go supervisor.Serve(ctx) 135 | for svc1.Count() == 0 || svc3.Count() == 0 { 136 | } 137 | 138 | cancel() 139 | 140 | if count := getServiceCount(&supervisor); count != 3 { 141 | t.Errorf("unexpected service count: %v", count) 142 | } 143 | 144 | switch { 145 | case svc1.count != 1, svc2.count != 1, svc3.count != 1, svc4.count != 1: 146 | t.Errorf("services should have been executed only once. %d %d %d %d", 147 | svc1.count, svc2.count, svc3.count, svc4.count) 148 | } 149 | } 150 | 151 | func TestFailing(t *testing.T) { 152 | 153 | defer func() { 154 | if r := recover(); r != nil { 155 | t.Errorf("unexpected panic: %v", r) 156 | } 157 | }() 158 | 159 | supervisor := Supervisor{ 160 | Name: "TestFailing supervisor", 161 | Log: func(msg interface{}) { 162 | t.Log("supervisor log (failing):", msg) 163 | }, 164 | } 165 | svc1 := failingservice{id: 1} 166 | supervisor.Add(&svc1) 167 | 168 | ctx, cancel := context.WithCancel(context.Background()) 169 | go supervisor.Serve(ctx) 170 | 171 | for svc1.Count() == 0 { 172 | } 173 | 174 | cancel() 175 | 176 | if svc1.count != 1 { 177 | t.Errorf("the failed service should have been started just once. Got: %d", svc1.count) 178 | } 179 | } 180 | 181 | func TestGroupMaxRestart(t *testing.T) { 182 | 183 | defer func() { 184 | if r := recover(); r != nil { 185 | t.Errorf("unexpected panic: %v", r) 186 | } 187 | }() 188 | 189 | supervisor := Group{ 190 | Supervisor: &Supervisor{ 191 | Name: "TestMaxRestartGroup supervisor", 192 | MaxRestarts: 1, 193 | Log: func(msg interface{}) { 194 | t.Log("supervisor log (max restart group):", msg) 195 | }, 196 | }, 197 | } 198 | svc1 := failingservice{id: 1} 199 | supervisor.Add(&svc1) 200 | 201 | ctx, cancel := context.WithCancel(context.Background()) 202 | 203 | var wg sync.WaitGroup 204 | wg.Add(1) 205 | go func() { 206 | supervisor.Serve(ctx) 207 | wg.Done() 208 | }() 209 | 210 | wg.Wait() 211 | cancel() 212 | 213 | if svc1.count > 1 { 214 | t.Error("the panic service should have not been started more than once.") 215 | } 216 | } 217 | 218 | func TestHaltAfterFailure(t *testing.T) { 219 | 220 | defer func() { 221 | if r := recover(); r != nil { 222 | t.Errorf("unexpected panic: %v", r) 223 | } 224 | }() 225 | 226 | supervisor := Supervisor{ 227 | Name: "TestHaltAfterFailure supervisor", 228 | MaxRestarts: 1, 229 | Log: func(msg interface{}) { 230 | t.Log("supervisor log (halt after failure):", msg) 231 | }, 232 | } 233 | svc1 := failingservice{id: 1} 234 | supervisor.Add(&svc1) 235 | 236 | ctx, cancel := context.WithCancel(context.Background()) 237 | go supervisor.Serve(ctx) 238 | 239 | for svc1.Count() == 0 { 240 | } 241 | 242 | cancel() 243 | 244 | if svc1.count != 1 { 245 | t.Errorf("the failed service should have been started just once. Got: %d", svc1.count) 246 | } 247 | } 248 | 249 | func TestHaltAfterPanic(t *testing.T) { 250 | 251 | defer func() { 252 | if r := recover(); r != nil { 253 | t.Errorf("unexpected panic: %v", r) 254 | } 255 | }() 256 | 257 | supervisor := Supervisor{ 258 | Name: "TestHaltAfterPanic supervisor", 259 | MaxRestarts: AlwaysRestart, 260 | Log: func(msg interface{}) { 261 | t.Log("supervisor log (halt after panic):", msg) 262 | }, 263 | } 264 | 265 | ctx, cancel := context.WithCancel(context.Background()) 266 | 267 | svc1 := &holdingservice{id: 1} 268 | svc1.Add(1) 269 | supervisor.Add(svc1) 270 | 271 | var wg sync.WaitGroup 272 | wg.Add(1) 273 | go func() { 274 | supervisor.Serve(ctx) 275 | wg.Done() 276 | }() 277 | svc1.Wait() 278 | 279 | svc2 := &panicabortsupervisorservice{id: 2, cancel: cancel, supervisor: &supervisor} 280 | supervisor.Add(svc2) 281 | 282 | wg.Wait() 283 | 284 | if svc1.count > 1 { 285 | t.Error("the holding service should have not been started more than once.") 286 | } 287 | } 288 | 289 | func TestInvalidGroup(t *testing.T) { 290 | defer func() { 291 | if r := recover(); r == nil { 292 | t.Error("defer called, but not because of panic") 293 | } 294 | }() 295 | var group Group 296 | ctx, cancel := context.WithCancel(context.Background()) 297 | group.Serve(ctx) 298 | t.Error("this group is invalid and should have had panic()'d") 299 | cancel() 300 | } 301 | 302 | func TestLog(t *testing.T) { 303 | 304 | supervisor := Supervisor{ 305 | Name: "TestLog", 306 | } 307 | svc1 := panicservice{id: 1} 308 | supervisor.Add(&svc1) 309 | 310 | ctx, cancel := context.WithCancel(context.Background()) 311 | go supervisor.Serve(ctx) 312 | for svc1.Count() == 0 { 313 | } 314 | 315 | cancel() 316 | } 317 | 318 | func TestManualCancelation(t *testing.T) { 319 | 320 | defer func() { 321 | if r := recover(); r != nil { 322 | t.Errorf("unexpected panic: %v", r) 323 | } 324 | }() 325 | 326 | supervisor := Supervisor{ 327 | Name: "TestManualCancelation supervisor", 328 | Log: func(msg interface{}) { 329 | t.Log("supervisor log (restartable):", msg) 330 | }, 331 | } 332 | svc1 := &holdingservice{id: 1} 333 | svc1.Add(1) 334 | supervisor.Add(svc1) 335 | svc2 := restartableservice{id: 2, restarted: make(chan struct{})} 336 | supervisor.Add(&svc2) 337 | 338 | ctx, cancel := context.WithCancel(context.Background()) 339 | var wg sync.WaitGroup 340 | wg.Add(1) 341 | go func() { 342 | supervisor.Serve(ctx) 343 | wg.Done() 344 | }() 345 | 346 | svc1.Wait() 347 | 348 | // Testing restart 349 | <-svc2.restarted 350 | svcs := supervisor.Cancelations() 351 | svcancel := svcs[svc2.String()] 352 | svcancel() 353 | <-svc2.restarted 354 | 355 | cancel() 356 | <-ctx.Done() 357 | wg.Wait() 358 | } 359 | 360 | func TestMaxRestart(t *testing.T) { 361 | 362 | defer func() { 363 | if r := recover(); r != nil { 364 | t.Errorf("unexpected panic: %v", r) 365 | } 366 | }() 367 | 368 | supervisor := &Supervisor{ 369 | Name: "TestMaxRestart supervisor", 370 | MaxRestarts: 1, 371 | Log: func(msg interface{}) { 372 | t.Log("supervisor log (max restart):", msg) 373 | }, 374 | } 375 | svc1 := failingservice{id: 1} 376 | supervisor.Add(&svc1) 377 | 378 | ctx, cancel := context.WithCancel(context.Background()) 379 | 380 | var wg sync.WaitGroup 381 | wg.Add(1) 382 | go func() { 383 | supervisor.Serve(ctx) 384 | wg.Done() 385 | }() 386 | 387 | wg.Wait() 388 | cancel() 389 | 390 | if svc1.count > 1 { 391 | t.Error("the panic service should have not been started more than once.") 392 | } 393 | } 394 | 395 | func TestPanic(t *testing.T) { 396 | 397 | defer func() { 398 | if r := recover(); r != nil { 399 | t.Errorf("unexpected panic: %v", r) 400 | } 401 | }() 402 | 403 | supervisor := Supervisor{ 404 | Name: "TestPanic supervisor", 405 | Log: func(msg interface{}) { 406 | t.Log("supervisor log (panic):", msg) 407 | }, 408 | } 409 | svc1 := panicservice{id: 1} 410 | supervisor.Add(&svc1) 411 | 412 | ctx, cancel := context.WithCancel(context.Background()) 413 | go supervisor.Serve(ctx) 414 | for svc1.Count() == 0 { 415 | } 416 | cancel() 417 | if svc1.count == 0 { 418 | t.Errorf("the failed service should have been started at least once. Got: %d", svc1.count) 419 | } 420 | } 421 | 422 | func TestRemovePanicService(t *testing.T) { 423 | 424 | supervisor := Group{ 425 | Supervisor: &Supervisor{ 426 | Name: "TestRemovePanicService supervisor", 427 | Log: func(msg interface{}) { 428 | t.Log("supervisor log (panic bug):", msg) 429 | }, 430 | }, 431 | } 432 | 433 | ctx, cancel := context.WithCancel(context.Background()) 434 | go supervisor.Serve(ctx) 435 | 436 | svc1 := waitservice{id: 1} 437 | supervisor.Add(&svc1) 438 | svc2 := quickpanicservice{id: 2} 439 | supervisor.Add(&svc2) 440 | 441 | supervisor.Remove(svc2.String()) 442 | cancel() 443 | 444 | svcs := supervisor.Services() 445 | if _, ok := svcs[svc2.String()]; ok { 446 | t.Errorf("%s should have been removed.", &svc2) 447 | } 448 | } 449 | 450 | func TestRemoveServiceAfterServe(t *testing.T) { 451 | 452 | var supervisor Supervisor 453 | supervisor.Name = "TestRemoveServiceAfterServe supervisor" 454 | svc1 := &holdingservice{id: 1} 455 | svc1.Add(1) 456 | supervisor.Add(svc1) 457 | svc2 := &holdingservice{id: 2} 458 | svc2.Add(1) 459 | supervisor.Add(svc2) 460 | 461 | ctx, cancel := context.WithCancel(context.Background()) 462 | var wg sync.WaitGroup 463 | wg.Add(1) 464 | go func() { 465 | supervisor.Serve(ctx) 466 | wg.Done() 467 | }() 468 | 469 | lbefore := getServiceCount(&supervisor) 470 | supervisor.Remove("unknown service") 471 | lafter := getServiceCount(&supervisor) 472 | 473 | if lbefore != lafter { 474 | t.Error("the removal of an unknown service shouldn't happen") 475 | } 476 | 477 | svc1.Wait() 478 | svc2.Wait() 479 | 480 | supervisor.Remove(svc1.String()) 481 | lremoved := getServiceCount(&supervisor) 482 | if lbefore == lremoved { 483 | t.Error("the removal of a service should have affected the supervisor:", lbefore, lremoved) 484 | } 485 | 486 | cancel() 487 | <-ctx.Done() 488 | wg.Wait() 489 | } 490 | 491 | func TestRemoveServiceAfterServeBug(t *testing.T) { 492 | 493 | var supervisor Supervisor 494 | supervisor.Name = "TestRemoveServiceAfterServeBug supervisor" 495 | svc1 := &holdingservice{id: 1} 496 | svc1.Add(1) 497 | supervisor.Add(svc1) 498 | 499 | ctx, cancel := context.WithCancel(context.Background()) 500 | go supervisor.Serve(ctx) 501 | 502 | svc1.Wait() 503 | supervisor.Remove(svc1.String()) 504 | cancel() 505 | 506 | if svc1.count > 1 { 507 | t.Errorf("the removal of a service should have terminated it. It was started %v times", svc1.count) 508 | } 509 | } 510 | 511 | func TestServiceList(t *testing.T) { 512 | 513 | var supervisor Supervisor 514 | supervisor.Name = "TestServiceList supervisor" 515 | svc1 := &holdingservice{id: 1} 516 | svc1.Add(1) 517 | supervisor.Add(svc1) 518 | 519 | ctx, cancel := context.WithCancel(context.Background()) 520 | var wg sync.WaitGroup 521 | wg.Add(1) 522 | go func() { 523 | supervisor.Serve(ctx) 524 | }() 525 | svc1.Wait() 526 | 527 | svcs := supervisor.Services() 528 | if svc, ok := svcs[svc1.String()]; !ok || svc1 != svc.(*holdingservice) { 529 | t.Errorf("could not find service when listing them. %s missing", svc1.String()) 530 | } 531 | 532 | cancel() 533 | <-ctx.Done() 534 | wg.Done() 535 | } 536 | 537 | func TestServices(t *testing.T) { 538 | 539 | var supervisor Supervisor 540 | supervisor.Name = "TestServices supervisor" 541 | svc1 := &holdingservice{id: 1} 542 | svc1.Add(1) 543 | supervisor.Add(svc1) 544 | svc2 := &holdingservice{id: 2} 545 | svc2.Add(1) 546 | supervisor.Add(svc2) 547 | 548 | ctx, cancel := context.WithCancel(context.Background()) 549 | var wg sync.WaitGroup 550 | wg.Add(1) 551 | go func() { 552 | supervisor.Serve(ctx) 553 | }() 554 | svc1.Wait() 555 | svc2.Wait() 556 | 557 | svcs := supervisor.Services() 558 | for _, svcname := range []string{svc1.String(), svc2.String()} { 559 | if _, ok := svcs[svcname]; !ok { 560 | t.Errorf("expected service not found: %s", svcname) 561 | } 562 | } 563 | 564 | cancel() 565 | <-ctx.Done() 566 | wg.Done() 567 | } 568 | 569 | func TestString(t *testing.T) { 570 | 571 | const expected = "test" 572 | var supervisor Supervisor 573 | supervisor.Name = expected 574 | 575 | if got := fmt.Sprintf("%s", &supervisor); got != expected { 576 | t.Errorf("error getting supervisor name: %s", got) 577 | } 578 | } 579 | 580 | func TestStringDefaultName(t *testing.T) { 581 | 582 | const expected = "supervisor" 583 | var supervisor Supervisor 584 | supervisor.prepare() 585 | 586 | if got := fmt.Sprintf("%s", &supervisor); got != expected { 587 | t.Errorf("error getting supervisor name: %s", got) 588 | } 589 | } 590 | 591 | func TestSupervisorAbortRestart(t *testing.T) { 592 | supervisor := Supervisor{ 593 | Name: "TestAbortRestart supervisor", 594 | Log: func(msg interface{}) { 595 | t.Log("supervisor log (abort restart):", msg) 596 | }, 597 | } 598 | 599 | svc1 := &holdingservice{id: 1} 600 | svc1.Add(1) 601 | supervisor.Add(svc1) 602 | svc2 := &restartableservice{id: 2} 603 | supervisor.Add(svc2) 604 | 605 | ctx, cancel := context.WithCancel(context.Background()) 606 | var wg sync.WaitGroup 607 | wg.Add(1) 608 | go func() { 609 | supervisor.Serve(ctx) 610 | wg.Done() 611 | }() 612 | svc1.Wait() 613 | 614 | for svc2.Count() < 3 { 615 | } 616 | 617 | cancel() 618 | wg.Wait() 619 | 620 | if svc2.count < 3 { 621 | t.Errorf("the restartable service should have been started twice. Got: %d", svc2.count) 622 | } 623 | } 624 | 625 | func TestTemporaryService(t *testing.T) { 626 | supervisor := Supervisor{ 627 | Name: "TestTemporaryService supervisor", 628 | Log: func(msg interface{}) { 629 | t.Log("supervisor log (termination abort restart):", msg) 630 | }, 631 | } 632 | 633 | ctx, cancel := context.WithCancel(context.Background()) 634 | go supervisor.Serve(ctx) 635 | svc1 := &temporaryservice{id: 1} 636 | supervisor.Add(svc1, Temporary) 637 | 638 | for svc1.Count() < 1 { 639 | } 640 | 641 | svc2 := &temporaryservice{id: 2} 642 | supervisor.Add(svc2, Temporary) 643 | cancel() 644 | 645 | if svc1.count != 1 { 646 | t.Error("the temporary service should have been started just once.", svc1.count) 647 | } 648 | } 649 | 650 | func TestTerminationAfterPanic(t *testing.T) { 651 | 652 | defer func() { 653 | if r := recover(); r != nil { 654 | t.Errorf("unexpected panic: %v", r) 655 | } 656 | }() 657 | 658 | supervisor := Supervisor{ 659 | Name: "TestTerminationAfterPanic supervisor", 660 | MaxRestarts: AlwaysRestart, 661 | Log: func(msg interface{}) { 662 | t.Log("supervisor log (termination after panic):", msg) 663 | }, 664 | } 665 | svc1 := &triggerpanicservice{id: 1} 666 | supervisor.Add(svc1) 667 | svc2 := &holdingservice{id: 2} 668 | svc2.Add(1) 669 | supervisor.Add(svc2) 670 | 671 | ctx, cancel := context.WithCancel(context.Background()) 672 | 673 | var wg sync.WaitGroup 674 | wg.Add(1) 675 | go func() { 676 | supervisor.Serve(ctx) 677 | wg.Done() 678 | }() 679 | svc2.Wait() 680 | 681 | svc3 := &holdingservice{id: 3} 682 | svc3.Add(1) 683 | supervisor.Add(svc3) 684 | svc3.Wait() 685 | 686 | supervisor.Remove(svc1.String()) 687 | 688 | svc4 := &holdingservice{id: 4} 689 | svc4.Add(1) 690 | supervisor.Add(svc4) 691 | svc4.Wait() 692 | 693 | cancel() 694 | 695 | wg.Wait() 696 | 697 | if svc1.count > 1 { 698 | t.Error("the panic service should have not been started more than once.") 699 | } 700 | } 701 | 702 | func TestTransientService(t *testing.T) { 703 | supervisor := Supervisor{ 704 | Name: "TestTemporaryService supervisor", 705 | Log: func(msg interface{}) { 706 | t.Log("supervisor log (termination abort restart):", msg) 707 | }, 708 | } 709 | 710 | svc1 := &transientservice{id: 1} 711 | svc1.Add(1) 712 | supervisor.Add(svc1, Transient) 713 | svc2 := &holdingservice{id: 2} 714 | svc2.Add(1) 715 | supervisor.Add(svc2) 716 | 717 | ctx, cancel := context.WithCancel(context.Background()) 718 | var wg sync.WaitGroup 719 | wg.Add(1) 720 | go func() { 721 | supervisor.Serve(ctx) 722 | wg.Done() 723 | }() 724 | svc1.Wait() 725 | svc2.Wait() 726 | 727 | cancel() 728 | wg.Wait() 729 | 730 | if svc1.count != 2 { 731 | t.Error("the transient service should have been started just twice.") 732 | } 733 | } 734 | 735 | func TestValidGroup(t *testing.T) { 736 | 737 | supervisor := &Group{ 738 | Supervisor: &Supervisor{ 739 | Name: "TestValidGroup supervisor", 740 | Log: func(msg interface{}) { 741 | t.Log("group log:", msg) 742 | }, 743 | }, 744 | } 745 | ctx, cancel := context.WithCancel(context.Background()) 746 | go supervisor.Serve(ctx) 747 | t.Log("supervisor started") 748 | 749 | trigger1 := make(chan struct{}) 750 | listening1 := make(chan struct{}) 751 | svc1 := &triggerfailservice{id: 1, trigger: trigger1, listening: listening1, log: t.Logf} 752 | supervisor.Add(svc1) 753 | t.Log("svc1 added") 754 | 755 | trigger2 := make(chan struct{}) 756 | listening2 := make(chan struct{}) 757 | svc2 := &triggerfailservice{id: 2, trigger: trigger2, listening: listening2, log: t.Logf} 758 | supervisor.Add(svc2) 759 | t.Log("svc2 added") 760 | 761 | <-listening1 762 | <-listening2 763 | trigger1 <- struct{}{} 764 | 765 | <-listening1 766 | <-listening2 767 | 768 | if !(svc1.count == svc2.count && svc1.count == 1) { 769 | t.Errorf("both services should have the same start count") 770 | } 771 | 772 | t.Log("stopping supervisor") 773 | cancel() 774 | } 775 | 776 | func getServiceCount(s *Supervisor) int { 777 | s.mu.Lock() 778 | l := len(s.services) 779 | s.mu.Unlock() 780 | return l 781 | } 782 | 783 | type simpleservice int 784 | 785 | func (s *simpleservice) Serve(ctx context.Context) { 786 | for { 787 | select { 788 | case <-ctx.Done(): 789 | return 790 | default: 791 | time.Sleep(500 * time.Millisecond) 792 | } 793 | } 794 | } 795 | 796 | func (s *simpleservice) String() string { 797 | return fmt.Sprintf("simple service %d", int(*s)) 798 | } 799 | 800 | type failingservice struct { 801 | id int 802 | mu sync.Mutex 803 | count int 804 | } 805 | 806 | func (s *failingservice) Serve(ctx context.Context) { 807 | select { 808 | case <-ctx.Done(): 809 | return 810 | default: 811 | time.Sleep(100 * time.Millisecond) 812 | s.mu.Lock() 813 | s.count++ 814 | s.mu.Unlock() 815 | return 816 | } 817 | } 818 | 819 | func (s *failingservice) String() string { 820 | return fmt.Sprintf("failing service %v", s.id) 821 | } 822 | 823 | func (s *failingservice) Count() int { 824 | s.mu.Lock() 825 | c := s.count 826 | s.mu.Unlock() 827 | return c 828 | } 829 | 830 | type holdingservice struct { 831 | id int 832 | mu sync.Mutex 833 | count int 834 | sync.WaitGroup 835 | } 836 | 837 | func (s *holdingservice) Serve(ctx context.Context) { 838 | s.mu.Lock() 839 | s.count++ 840 | s.mu.Unlock() 841 | s.Done() 842 | <-ctx.Done() 843 | } 844 | 845 | func (s *holdingservice) String() string { 846 | return fmt.Sprintf("holding service %v", s.id) 847 | } 848 | 849 | type panicservice struct { 850 | id int 851 | mu sync.Mutex 852 | count int 853 | } 854 | 855 | func (s *panicservice) Serve(ctx context.Context) { 856 | for { 857 | select { 858 | case <-ctx.Done(): 859 | return 860 | default: 861 | time.Sleep(100 * time.Millisecond) 862 | s.mu.Lock() 863 | s.count++ 864 | s.mu.Unlock() 865 | panic("forcing panic") 866 | } 867 | } 868 | } 869 | 870 | func (s *panicservice) String() string { 871 | return fmt.Sprintf("panic service %v", s.id) 872 | } 873 | 874 | func (s *panicservice) Count() int { 875 | s.mu.Lock() 876 | c := s.count 877 | s.mu.Unlock() 878 | return c 879 | } 880 | 881 | type quickpanicservice struct { 882 | id, count int 883 | } 884 | 885 | func (s *quickpanicservice) Serve(ctx context.Context) { 886 | for { 887 | select { 888 | case <-ctx.Done(): 889 | return 890 | default: 891 | s.count++ 892 | panic("forcing panic") 893 | } 894 | } 895 | } 896 | 897 | func (s *quickpanicservice) String() string { 898 | return fmt.Sprintf("panic service %v", s.id) 899 | } 900 | 901 | type restartableservice struct { 902 | id int 903 | restarted chan struct{} 904 | mu sync.Mutex 905 | count int 906 | } 907 | 908 | func (s *restartableservice) Serve(ctx context.Context) { 909 | for { 910 | s.mu.Lock() 911 | s.count++ 912 | s.mu.Unlock() 913 | select { 914 | case <-ctx.Done(): 915 | return 916 | default: 917 | time.Sleep(500 * time.Millisecond) 918 | select { 919 | case s.restarted <- struct{}{}: 920 | default: 921 | } 922 | } 923 | } 924 | } 925 | 926 | func (s *restartableservice) Count() int { 927 | s.mu.Lock() 928 | c := s.count 929 | s.mu.Unlock() 930 | return c 931 | } 932 | 933 | func (s *restartableservice) String() string { 934 | return fmt.Sprintf("restartable service %v", s.id) 935 | } 936 | 937 | type waitservice struct { 938 | id int 939 | mu sync.Mutex 940 | count int 941 | } 942 | 943 | func (s *waitservice) Serve(ctx context.Context) { 944 | s.mu.Lock() 945 | s.count++ 946 | s.mu.Unlock() 947 | <-ctx.Done() 948 | } 949 | 950 | func (s *waitservice) String() string { 951 | return fmt.Sprintf("wait service %v", s.id) 952 | } 953 | 954 | func (s *waitservice) Count() int { 955 | s.mu.Lock() 956 | c := s.count 957 | s.mu.Unlock() 958 | return c 959 | } 960 | 961 | type temporaryservice struct { 962 | id int 963 | mu sync.Mutex 964 | count int 965 | } 966 | 967 | func (s *temporaryservice) Serve(ctx context.Context) { 968 | s.mu.Lock() 969 | s.count++ 970 | s.mu.Unlock() 971 | } 972 | 973 | func (s *temporaryservice) Count() int { 974 | s.mu.Lock() 975 | c := s.count 976 | s.mu.Unlock() 977 | return c 978 | } 979 | 980 | func (s *temporaryservice) String() string { 981 | return fmt.Sprintf("temporary service %v", s.id) 982 | } 983 | 984 | type triggerpanicservice struct { 985 | id, count int 986 | } 987 | 988 | func (s *triggerpanicservice) Serve(ctx context.Context) { 989 | <-ctx.Done() 990 | s.count++ 991 | panic("forcing panic") 992 | } 993 | 994 | func (s *triggerpanicservice) String() string { 995 | return fmt.Sprintf("iterative panic service %v", s.id) 996 | } 997 | 998 | type panicabortsupervisorservice struct { 999 | id int 1000 | cancel context.CancelFunc 1001 | supervisor *Supervisor 1002 | } 1003 | 1004 | func (s *panicabortsupervisorservice) Serve(ctx context.Context) { 1005 | s.supervisor.mu.Lock() 1006 | defer s.supervisor.mu.Unlock() 1007 | s.supervisor.terminations = nil 1008 | s.cancel() 1009 | panic("forcing panic") 1010 | } 1011 | 1012 | func (s *panicabortsupervisorservice) String() string { 1013 | return fmt.Sprintf("super panic service %v", s.id) 1014 | } 1015 | 1016 | type transientservice struct { 1017 | id int 1018 | mu sync.Mutex 1019 | count int 1020 | sync.WaitGroup 1021 | } 1022 | 1023 | func (s *transientservice) Serve(ctx context.Context) { 1024 | s.mu.Lock() 1025 | defer s.mu.Unlock() 1026 | s.count++ 1027 | if s.count == 1 { 1028 | panic("panic once") 1029 | } 1030 | s.Done() 1031 | <-ctx.Done() 1032 | } 1033 | 1034 | func (s *transientservice) String() string { 1035 | return fmt.Sprintf("transient service %v", s.id) 1036 | } 1037 | 1038 | type triggerfailservice struct { 1039 | id int 1040 | trigger chan struct{} 1041 | listening chan struct{} 1042 | log func(msg string, args ...interface{}) 1043 | count int 1044 | } 1045 | 1046 | func (s *triggerfailservice) Serve(ctx context.Context) { 1047 | s.listening <- struct{}{} 1048 | s.log("listening %d", s.id) 1049 | select { 1050 | case <-s.trigger: 1051 | s.log("triggered %d", s.id) 1052 | s.count++ 1053 | case <-ctx.Done(): 1054 | s.log("context done %d", s.id) 1055 | s.count++ 1056 | } 1057 | } 1058 | 1059 | func (s *triggerfailservice) String() string { 1060 | return fmt.Sprintf("trigger fail service %v", s.id) 1061 | } 1062 | --------------------------------------------------------------------------------