├── .travis.yml ├── go.mod ├── cmd └── sim │ ├── capped.go │ ├── lock.go │ └── main.go ├── go.sum ├── stats └── stats.go ├── LICENSE ├── README.md ├── queue.go ├── queue_test.go ├── codel_test.go ├── CODE_OF_CONDUCT.md ├── priority_test.go ├── priority.go └── codel.go /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - "1.14.x" 5 | - master 6 | 7 | script: 8 | - go test -v . -race -sim 9 | - go test -v . -bench . -benchmem 10 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/bohde/codel 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b 7 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a 8 | pgregory.net/rapid v0.4.2 9 | ) 10 | -------------------------------------------------------------------------------- /cmd/sim/capped.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "sync" 6 | ) 7 | 8 | type Capped struct { 9 | mu sync.Mutex 10 | cur int64 11 | cap int64 12 | } 13 | 14 | func (s *Capped) Lock() error { 15 | s.mu.Lock() 16 | if s.cur >= s.cap { 17 | s.mu.Unlock() 18 | return errors.New("too many threads") 19 | } 20 | 21 | s.cur++ 22 | s.mu.Unlock() 23 | 24 | return nil 25 | } 26 | 27 | func (s *Capped) Unlock() { 28 | s.mu.Lock() 29 | s.cur-- 30 | s.mu.Unlock() 31 | } 32 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b h1:AP/Y7sqYicnjGDfD5VcY4CIfh1hRXBUavxrvELjTiOE= 2 | github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= 3 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= 4 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 5 | pgregory.net/rapid v0.4.2 h1:lsi9jhvZTYvzVpeG93WWgimPRmiJQfGFRNTEZh1dtY0= 6 | pgregory.net/rapid v0.4.2/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU= 7 | -------------------------------------------------------------------------------- /stats/stats.go: -------------------------------------------------------------------------------- 1 | package stats 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | 7 | "github.com/bmizerany/perks/quantile" 8 | ) 9 | 10 | type Stats struct { 11 | stream quantile.Stream 12 | lock sync.Mutex 13 | } 14 | 15 | type Timer struct { 16 | start time.Time 17 | stats *Stats 18 | } 19 | 20 | func (t *Timer) Mark() { 21 | elapsed := time.Since(t.start) 22 | t.stats.Insert(float64(elapsed.Nanoseconds())) 23 | } 24 | 25 | func New() Stats { 26 | stream := quantile.NewTargeted(0.5, 0.95, 0.99) 27 | 28 | return Stats{ 29 | stream: *stream, 30 | } 31 | } 32 | 33 | func (s *Stats) Insert(val float64) { 34 | s.lock.Lock() 35 | s.stream.Insert(val) 36 | s.lock.Unlock() 37 | } 38 | 39 | func (s *Stats) Time() Timer { 40 | return Timer{ 41 | start: time.Now(), 42 | stats: s, 43 | } 44 | } 45 | 46 | func (s *Stats) Query(quantile float64) time.Duration { 47 | s.lock.Lock() 48 | val := s.stream.Query(quantile) 49 | s.lock.Unlock() 50 | return time.Duration(val) 51 | } 52 | -------------------------------------------------------------------------------- /cmd/sim/lock.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "sync" 7 | 8 | "github.com/bohde/codel" 9 | "golang.org/x/sync/semaphore" 10 | ) 11 | 12 | type Locker interface { 13 | Acquire(ctx context.Context) error 14 | Release() 15 | } 16 | 17 | type Semaphore struct { 18 | mu sync.Mutex 19 | cur int64 20 | cap int64 21 | limit int64 22 | sem *semaphore.Weighted 23 | } 24 | 25 | func NewSemaphore(opts codel.Options) *Semaphore { 26 | s := semaphore.NewWeighted(int64(opts.MaxOutstanding)) 27 | return &Semaphore{ 28 | cap: int64(opts.MaxPending) + int64(opts.MaxOutstanding), 29 | limit: int64(opts.MaxOutstanding), 30 | sem: s, 31 | } 32 | } 33 | 34 | func (s *Semaphore) Acquire(ctx context.Context) error { 35 | s.mu.Lock() 36 | // Drop if queue is full 37 | if s.cur >= s.cap { 38 | s.mu.Unlock() 39 | return errors.New("dropped") 40 | } 41 | 42 | s.cur++ 43 | s.mu.Unlock() 44 | 45 | return s.sem.Acquire(ctx, 1) 46 | } 47 | 48 | func (s *Semaphore) Release() { 49 | s.mu.Lock() 50 | s.cur-- 51 | s.mu.Unlock() 52 | 53 | s.sem.Release(1) 54 | } 55 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2018, Rowan Bohde 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # codel 2 | 3 | [![GoDoc](https://pkg.go.dev/github.com/bohde/codel?status.svg)](https://pkg.go.dev/github.com/bohde/codel) 4 | 5 | `codel` implements the [Controlled Delay](https://queue.acm.org/detail.cfm?id=2209336) algorithm for overload detection, providing a mechanism to shed load when overloaded. It optimizes for latency while keeping throughput high, even when downstream rates dynamically change. 6 | 7 | `codel` keeps latency low when even severely overloaded, by preemptively shedding some load when wait latency is long. It is comparable to using a queue to handle bursts of load, but improves upon this technique by avoiding the latency required to handle all previous entries in the queue. 8 | 9 | In a simulation of 1000 reqs/sec incoming, 500 reqs/sec outgoing averages for 10 seconds, here's the corresponding throughput and latency profile of both a queue and `codel`. Throughput is slightly higher than the average due to randomness in the simulation. 10 | 11 | | method | throughput | p50 | p95 | p99 | 12 | |--------|------------|--------------|--------------|--------------| 13 | | queue | 507.41 | 963.604953ms | 1.024595796s | 1.041455537s | 14 | | codel | 513.17 | 27.718827ms | 44.085795ms | 62.756499ms | 15 | 16 | 17 | Source code for the simulations are included in the [sim directory](cmd/sim). 18 | 19 | ## Installation 20 | 21 | ``` 22 | $ go get github.com/bohde/codel 23 | ``` 24 | 25 | ## Example 26 | 27 | ``` 28 | import ( 29 | "context" 30 | "github.com/bohde/codel" 31 | ) 32 | 33 | func Example() { 34 | c := codel.New(codel.Options{ 35 | // The maximum number of pending acquires 36 | MaxPending: 100, 37 | // The maximum number of concurrent acquires 38 | MaxOutstanding: 10, 39 | // The target latency to wait for an acquire. 40 | // Acquires that take longer than this can fail. 41 | TargetLatency: 5 * time.Millisecond, 42 | }) 43 | 44 | // Attempt to acquire the lock. 45 | err := c.Acquire(context.Background()) 46 | 47 | // if err is not nil, acquisition failed. 48 | if err != nil { 49 | return 50 | } 51 | 52 | // If acquisition succeeded, we need to release it. 53 | defer c.Release() 54 | 55 | // Do some process with external resources 56 | } 57 | 58 | ``` 59 | 60 | ## Benchmarks 61 | 62 | The `Lock` serializes access, introducing latency overhead. When not overloaded, this overhead should be under 1us. 63 | 64 | ``` 65 | BenchmarkLockUnblocked-4 20000000 73.1 ns/op 0 B/op 0 allocs/op 66 | BenchmarkLockBlocked-4 2000000 665 ns/op 176 B/op 2 allocs/op 67 | ``` 68 | -------------------------------------------------------------------------------- /queue.go: -------------------------------------------------------------------------------- 1 | package codel 2 | 3 | import ( 4 | "container/heap" 5 | ) 6 | 7 | type queue []*prendezvouz 8 | 9 | func (pq queue) Len() int { return len(pq) } 10 | 11 | func (pq queue) Less(i, j int) bool { 12 | // We want Pop to give us the highest, not lowest, priority so we use greater than here. 13 | return pq[i].priority > pq[j].priority 14 | } 15 | 16 | func (pq queue) Swap(i, j int) { 17 | pq[i], pq[j] = pq[j], pq[i] 18 | pq[i].index = i 19 | pq[j].index = j 20 | } 21 | 22 | func (pq *queue) Push(x interface{}) { 23 | n := len(*pq) 24 | item := x.(*prendezvouz) 25 | item.index = n 26 | *pq = append(*pq, item) 27 | } 28 | 29 | func (pq *queue) Pop() interface{} { 30 | old := *pq 31 | n := len(old) 32 | item := old[n-1] 33 | item.index = -1 // for safety 34 | *pq = old[0 : n-1] 35 | return item 36 | } 37 | 38 | func (pq *queue) lowestIndex() int { 39 | old := *pq 40 | n := len(old) 41 | index := n / 2 42 | 43 | lowestIndex := index 44 | priority := maxInt 45 | 46 | for i := index; i < n; i++ { 47 | if old[i].priority < priority { 48 | lowestIndex = i 49 | priority = old[i].priority 50 | } 51 | } 52 | 53 | return lowestIndex 54 | } 55 | 56 | type priorityQueue queue 57 | 58 | func newQueue(capacity int) priorityQueue { 59 | return priorityQueue(make([]*prendezvouz, 0, capacity)) 60 | } 61 | 62 | func (pq *priorityQueue) Len() int { 63 | return len(*pq) 64 | } 65 | 66 | func (pq *priorityQueue) Cap() int { 67 | return cap(*pq) 68 | } 69 | 70 | func (pq *priorityQueue) push(r *prendezvouz) { 71 | heap.Push((*queue)(pq), r) 72 | } 73 | 74 | func (pq *priorityQueue) Push(r *prendezvouz) bool { 75 | // If we're under capacity, push it to the queue 76 | if pq.Len() < pq.Cap() { 77 | pq.push(r) 78 | return true 79 | } 80 | 81 | if pq.Cap() == 0 { 82 | return false 83 | } 84 | 85 | // otherwise, we need to check if this takes priority over the lowest element 86 | lowestIndex := ((*queue)(pq)).lowestIndex() 87 | last := (*pq)[lowestIndex] 88 | if last.priority < r.priority { 89 | (*pq)[lowestIndex] = r 90 | 91 | // Fix the index 92 | r.index = lowestIndex 93 | heap.Fix((*queue)(pq), lowestIndex) 94 | 95 | // For safety 96 | last.index = -1 97 | last.Drop() 98 | 99 | return true 100 | } 101 | 102 | return false 103 | 104 | } 105 | 106 | func (pq *priorityQueue) Empty() bool { 107 | return len(*pq) <= 0 108 | } 109 | 110 | func (pq *priorityQueue) Pop() prendezvouz { 111 | r := heap.Pop((*queue)(pq)).(*prendezvouz) 112 | return *r 113 | } 114 | 115 | func (pq *priorityQueue) Remove(r *prendezvouz) { 116 | heap.Remove((*queue)(pq), r.index) 117 | } 118 | -------------------------------------------------------------------------------- /queue_test.go: -------------------------------------------------------------------------------- 1 | package codel 2 | 3 | import ( 4 | "sync" 5 | "testing" 6 | "time" 7 | 8 | "pgregory.net/rapid" 9 | ) 10 | 11 | type queueMachine struct { 12 | q *priorityQueue // queue being tested 13 | n int // maximum queue size 14 | } 15 | 16 | // Init is an action for initializing a queueMachine instance. 17 | func (m *queueMachine) Init(t *rapid.T) { 18 | n := rapid.IntRange(1, 3).Draw(t, "n").(int) 19 | q := newQueue(n) 20 | m.q = &q 21 | m.n = n 22 | } 23 | 24 | // Model of Push 25 | func (m *queueMachine) Push(t *rapid.T) { 26 | r := prendezvouz{ 27 | priority: rapid.Int().Draw(t, "priority").(int), 28 | enqueuedTime: time.Now(), 29 | errChan: make(chan error, 1), 30 | } 31 | 32 | m.q.Push(&r) 33 | } 34 | 35 | // Model of Remove 36 | func (m *queueMachine) Remove(t *rapid.T) { 37 | if m.q.Empty() { 38 | t.Skip("empty") 39 | } 40 | 41 | r := (*m.q)[rapid.IntRange(0, m.q.Len()-1).Draw(t, "i").(int)] 42 | m.q.Remove(r) 43 | } 44 | 45 | // Model of Drop 46 | func (m *queueMachine) Drop(t *rapid.T) { 47 | if m.q.Empty() { 48 | t.Skip("empty") 49 | } 50 | 51 | r := (*m.q)[rapid.IntRange(0, m.q.Len()-1).Draw(t, "i").(int)] 52 | r.Drop() 53 | } 54 | 55 | // Model of Signal 56 | func (m *queueMachine) Pop(t *rapid.T) { 57 | if m.q.Empty() { 58 | t.Skip("empty") 59 | } 60 | 61 | r := m.q.Pop() 62 | r.Signal() 63 | } 64 | 65 | // validate that invariants hold 66 | func (m *queueMachine) Check(t *rapid.T) { 67 | if m.q.Len() > m.q.Cap() { 68 | t.Fatalf("queue over capacity: %v vs expected %v", m.q.Len(), m.q.Cap()) 69 | } 70 | 71 | for i, r := range *m.q { 72 | if r.index != i { 73 | t.Fatalf("illegal index: expected %d, got %+v ", i, r) 74 | } 75 | } 76 | 77 | } 78 | 79 | func TestPriorityQueue(t *testing.T) { 80 | t.Run("It should meet invariants", func(t *testing.T) { 81 | rapid.Check(t, rapid.Run(&queueMachine{})) 82 | }) 83 | 84 | t.Run("It should not panic", func(t *testing.T) { 85 | q := newQueue(3) 86 | 87 | mu := sync.Mutex{} 88 | 89 | wg := sync.WaitGroup{} 90 | 91 | for i := 0; i < 4; i++ { 92 | wg.Add(1) 93 | priority := i 94 | go func() { 95 | defer wg.Done() 96 | defer func() { 97 | if r := recover(); r != nil { 98 | t.Errorf("Failed with panic %+v", r) 99 | } 100 | }() 101 | 102 | for i := 0; i < 1000; i++ { 103 | r := prendezvouz{ 104 | priority: priority, 105 | enqueuedTime: time.Now(), 106 | errChan: make(chan error, 1), 107 | } 108 | 109 | mu.Lock() 110 | pushed := q.Push(&r) 111 | mu.Unlock() 112 | 113 | if !pushed { 114 | continue 115 | } 116 | 117 | mu.Lock() 118 | if r.index >= 0 { 119 | q.Remove(&r) 120 | } 121 | mu.Unlock() 122 | } 123 | 124 | }() 125 | } 126 | 127 | wg.Wait() 128 | 129 | }) 130 | 131 | } 132 | -------------------------------------------------------------------------------- /codel_test.go: -------------------------------------------------------------------------------- 1 | package codel 2 | 3 | import ( 4 | "context" 5 | "math/rand" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func init() { 11 | rand.Seed(time.Now().UTC().UnixNano()) 12 | } 13 | 14 | func Example() { 15 | c := New(Options{ 16 | // The maximum number of pending acquires 17 | MaxPending: 100, 18 | // The maximum number of concurrent acquires 19 | MaxOutstanding: 10, 20 | // The target latency to wait for an acquire. 21 | // Acquires that take longer than this can fail. 22 | TargetLatency: 5 * time.Millisecond, 23 | }) 24 | 25 | // Attempt to acquire the lock. 26 | err := c.Acquire(context.Background()) 27 | 28 | // if err is not nil, acquisition failed. 29 | if err != nil { 30 | return 31 | } 32 | 33 | // If acquisition succeeded, we need to release it. 34 | defer c.Release() 35 | 36 | // Do some process with external resources 37 | } 38 | 39 | func TestLock(t *testing.T) { 40 | c := New(Options{ 41 | MaxPending: 1, 42 | MaxOutstanding: 1, 43 | TargetLatency: 5 * time.Millisecond, 44 | }) 45 | 46 | err := c.Acquire(context.Background()) 47 | if err != nil { 48 | t.Error("Got an error:", err) 49 | } 50 | 51 | c.Release() 52 | } 53 | 54 | func TestAcquireFailsForCanceledContext(t *testing.T) { 55 | c := New(Options{ 56 | MaxPending: 1, 57 | MaxOutstanding: 0, 58 | TargetLatency: 5 * time.Millisecond, 59 | }) 60 | 61 | ctx, cancel := context.WithCancel(context.Background()) 62 | cancel() 63 | 64 | err := c.Acquire(ctx) 65 | if err == nil { 66 | t.Error("Expected an error:", err) 67 | c.Release() 68 | } 69 | 70 | } 71 | 72 | func TestLockCanHaveMultiple(t *testing.T) { 73 | const concurrent = 4 74 | 75 | c := New(Options{ 76 | MaxPending: 1, 77 | MaxOutstanding: concurrent, 78 | TargetLatency: 5 * time.Millisecond, 79 | }) 80 | 81 | ctx := context.Background() 82 | 83 | for i := 0; i < concurrent; i++ { 84 | err := c.Acquire(ctx) 85 | if err != nil { 86 | t.Error("Got an error:", err) 87 | return 88 | } 89 | } 90 | 91 | for i := 0; i < concurrent; i++ { 92 | c.Release() 93 | } 94 | } 95 | 96 | func BenchmarkLockUnblocked(b *testing.B) { 97 | c := New(Options{ 98 | MaxPending: 1, 99 | MaxOutstanding: 1, 100 | TargetLatency: 5 * time.Millisecond, 101 | }) 102 | 103 | ctx := context.Background() 104 | b.ResetTimer() 105 | 106 | for i := 0; i < b.N; i++ { 107 | err := c.Acquire(ctx) 108 | 109 | if err != nil { 110 | b.Log("Got an error:", err) 111 | return 112 | } 113 | c.Release() 114 | } 115 | b.StopTimer() 116 | } 117 | 118 | func BenchmarkLockBlocked(b *testing.B) { 119 | const concurrent = 4 120 | 121 | c := New(Options{ 122 | MaxPending: 1, 123 | MaxOutstanding: concurrent, 124 | TargetLatency: 5 * time.Millisecond, 125 | }) 126 | 127 | ctx := context.Background() 128 | 129 | // Acquire maximum outstanding to avoid fast path 130 | for i := 0; i < concurrent; i++ { 131 | err := c.Acquire(ctx) 132 | if err != nil { 133 | b.Error("Got an error:", err) 134 | return 135 | } 136 | } 137 | 138 | b.ResetTimer() 139 | 140 | // Race the release and the acquire in order to benchmark slow path 141 | for i := 0; i < b.N; i++ { 142 | go func() { 143 | c.Release() 144 | 145 | }() 146 | err := c.Acquire(ctx) 147 | 148 | if err != nil { 149 | b.Log("Got an error:", err) 150 | return 151 | } 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at rowan.bohde@gmail.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /priority_test.go: -------------------------------------------------------------------------------- 1 | package codel 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "math/rand" 7 | "sync" 8 | "sync/atomic" 9 | "testing" 10 | "time" 11 | ) 12 | 13 | var sim = flag.Bool("sim", false, "run simulation test") 14 | 15 | func msToWait(perSec int64) time.Duration { 16 | ms := rand.ExpFloat64() / (float64(perSec) / 1000) 17 | return time.Duration(ms * float64(time.Millisecond)) 18 | } 19 | 20 | func TestPriority(t *testing.T) { 21 | t.Run("It should drop if zero enqueued", func(t *testing.T) { 22 | limiter := NewPriority(Options{ 23 | MaxPending: 0, 24 | MaxOutstanding: 1, 25 | TargetLatency: 10 * time.Millisecond, 26 | }) 27 | 28 | err := limiter.Acquire(context.Background(), 0) 29 | if err != nil { 30 | t.Errorf("Expected nil err: %s", err) 31 | return 32 | } 33 | 34 | err = limiter.Acquire(context.Background(), 0) 35 | if err == nil { 36 | t.Errorf("Expected non-nil err: %s", err) 37 | } 38 | 39 | }) 40 | 41 | } 42 | 43 | // Simulate 3 priorities of 1000 reqs/second each, fighting for a 44 | // process that can process 15 concurrent at 100 reqs/second. This 45 | // should be enough capacity for the highest priority, The middle 46 | // priority seeing partial unavailability, and the lowest priority 47 | // seeing near full unavailability 48 | func TestConcurrentSimulation(t *testing.T) { 49 | if !(*sim) { 50 | t.Log("Skipping sim since -sim not passed") 51 | t.Skip() 52 | } 53 | 54 | wg := sync.WaitGroup{} 55 | 56 | limiter := NewPriority(Options{ 57 | MaxPending: 100, 58 | MaxOutstanding: 15, 59 | TargetLatency: 10 * time.Millisecond, 60 | }) 61 | 62 | for i := 0; i < 3; i++ { 63 | wg.Add(1) 64 | 65 | priority := 10 * i 66 | go func() { 67 | defer wg.Done() 68 | 69 | inner := sync.WaitGroup{} 70 | 71 | success := int64(0) 72 | error := int64(0) 73 | 74 | for i := 0; i < 1000; i++ { 75 | inner.Add(1) 76 | time.Sleep(msToWait(1000)) 77 | 78 | go func() { 79 | defer inner.Done() 80 | 81 | err := limiter.Acquire(context.Background(), priority) 82 | if err != nil { 83 | atomic.AddInt64(&error, 1) 84 | return 85 | } 86 | defer limiter.Release() 87 | 88 | time.Sleep(msToWait(100)) 89 | atomic.AddInt64(&success, 1) 90 | 91 | }() 92 | } 93 | 94 | inner.Wait() 95 | t.Logf("priority=%d success=%f dropped=%f", priority, (float64(success) / 1000), float64(error)/1000) 96 | 97 | }() 98 | } 99 | 100 | wg.Wait() 101 | 102 | } 103 | 104 | func BenchmarkPLockUnblocked(b *testing.B) { 105 | c := NewPriority(Options{ 106 | MaxPending: 1, 107 | MaxOutstanding: 1, 108 | TargetLatency: 5 * time.Millisecond, 109 | }) 110 | 111 | ctx := context.Background() 112 | b.ResetTimer() 113 | 114 | for i := 0; i < b.N; i++ { 115 | err := c.Acquire(ctx, i) 116 | 117 | if err != nil { 118 | b.Log("Got an error:", err) 119 | return 120 | } 121 | c.Release() 122 | } 123 | b.StopTimer() 124 | } 125 | 126 | func BenchmarkPLockBlocked(b *testing.B) { 127 | const concurrent = 4 128 | 129 | c := NewPriority(Options{ 130 | MaxPending: 1, 131 | MaxOutstanding: concurrent, 132 | TargetLatency: 5 * time.Millisecond, 133 | }) 134 | 135 | ctx := context.Background() 136 | 137 | // Acquire maximum outstanding to avoid fast path 138 | for i := 0; i < concurrent; i++ { 139 | err := c.Acquire(ctx, i) 140 | if err != nil { 141 | b.Error("Got an error:", err) 142 | return 143 | } 144 | } 145 | 146 | b.ResetTimer() 147 | 148 | // Race the release and the acquire in order to benchmark slow path 149 | for i := 0; i < b.N; i++ { 150 | go func() { 151 | c.Release() 152 | 153 | }() 154 | err := c.Acquire(ctx, i) 155 | 156 | if err != nil { 157 | b.Log("Got an error:", err) 158 | return 159 | } 160 | } 161 | } 162 | -------------------------------------------------------------------------------- /cmd/sim/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "math/rand" 9 | "os" 10 | "sync" 11 | "sync/atomic" 12 | "time" 13 | 14 | "github.com/bohde/codel" 15 | "github.com/bohde/codel/stats" 16 | ) 17 | 18 | func msToWait(perSec int64) time.Duration { 19 | ms := rand.ExpFloat64() / (float64(perSec) / 1000) 20 | return time.Duration(ms * float64(time.Millisecond)) 21 | } 22 | 23 | type Simulation struct { 24 | Method string 25 | TimeToRun time.Duration 26 | Deadline time.Duration 27 | InputPerSec int64 28 | OutputPerSec int64 29 | Completed uint64 30 | Rejected uint64 31 | Started int64 32 | Stats stats.Stats 33 | mu sync.Mutex 34 | } 35 | 36 | func (sim *Simulation) Process() { 37 | sim.mu.Lock() 38 | time.Sleep(msToWait(sim.OutputPerSec)) 39 | sim.mu.Unlock() 40 | } 41 | 42 | func (sim *Simulation) String() string { 43 | successPercentage := float64(atomic.LoadUint64(&sim.Completed)) / float64(sim.Started) 44 | rejectedPercentage := float64(atomic.LoadUint64(&sim.Rejected)) / float64(sim.Started) 45 | 46 | return fmt.Sprintf("method=%s duration=%s deadline=%s input=%d output=%d throughput=%.2f completed=%.4f rejected=%.4f p50=%s p95=%s p99=%s ", 47 | sim.Method, sim.TimeToRun, sim.Deadline, 48 | sim.InputPerSec, sim.OutputPerSec, 49 | float64(sim.InputPerSec)*successPercentage, successPercentage, rejectedPercentage, 50 | sim.Stats.Query(0.5), sim.Stats.Query(0.95), sim.Stats.Query(0.99)) 51 | 52 | } 53 | 54 | // Model input & output as random processes with average throughput. 55 | func (sim *Simulation) Run(lock Locker) { 56 | start := time.Now() 57 | 58 | wg := sync.WaitGroup{} 59 | 60 | for { 61 | time.Sleep(msToWait(sim.InputPerSec)) 62 | 63 | if time.Since(start) > sim.TimeToRun { 64 | break 65 | } 66 | 67 | sim.Started++ 68 | wg.Add(1) 69 | 70 | go func() { 71 | ctx, cancel := context.WithTimeout(context.Background(), sim.Deadline) 72 | 73 | timer := sim.Stats.Time() 74 | 75 | err := lock.Acquire(ctx) 76 | cancel() 77 | 78 | if err == nil { 79 | sim.Process() 80 | 81 | timer.Mark() 82 | atomic.AddUint64(&sim.Completed, 1) 83 | lock.Release() 84 | 85 | } else { 86 | atomic.AddUint64(&sim.Rejected, 1) 87 | } 88 | 89 | wg.Done() 90 | }() 91 | 92 | } 93 | } 94 | 95 | func Overloaded(runtime time.Duration, deadline time.Duration, opts codel.Options) []*Simulation { 96 | wg := sync.WaitGroup{} 97 | 98 | runs := []*Simulation{} 99 | 100 | run := func(in, out int64) { 101 | wg.Add(2) 102 | 103 | codelRun := Simulation{ 104 | Method: "codel", 105 | Deadline: deadline, 106 | InputPerSec: in, 107 | OutputPerSec: out, 108 | TimeToRun: runtime, 109 | Stats: stats.New(), 110 | } 111 | 112 | queueRun := Simulation{ 113 | Method: "queue", 114 | Deadline: deadline, 115 | InputPerSec: in, 116 | OutputPerSec: out, 117 | TimeToRun: runtime, 118 | Stats: stats.New(), 119 | } 120 | 121 | runs = append(runs, &codelRun, &queueRun) 122 | 123 | go func() { 124 | codelRun.Run(codel.New(opts)) 125 | wg.Done() 126 | }() 127 | go func() { 128 | queueRun.Run(NewSemaphore(opts)) 129 | wg.Done() 130 | }() 131 | } 132 | 133 | run(1000, 1000) 134 | run(990, 1000) 135 | run(950, 1000) 136 | run(900, 1000) 137 | 138 | run(1000, 900) 139 | run(1000, 750) 140 | run(1000, 500) 141 | run(1000, 250) 142 | run(1000, 100) 143 | 144 | wg.Wait() 145 | return runs 146 | } 147 | 148 | func main() { 149 | log.SetOutput(os.Stdout) 150 | 151 | runtime := flag.Duration("simulation-time", 5*time.Second, "Time to run each simulation") 152 | targetLatency := flag.Duration("target-latency", 5*time.Millisecond, "Target latency") 153 | deadline := flag.Duration("deadline", 1*time.Second, "Hard deadline to remain in the queue") 154 | 155 | flag.Parse() 156 | 157 | opts := codel.Options{ 158 | MaxPending: 1000, 159 | MaxOutstanding: 10, 160 | TargetLatency: *targetLatency, 161 | } 162 | 163 | runs := Overloaded(*runtime, *deadline, opts) 164 | for _, r := range runs { 165 | log.Printf("%s", r.String()) 166 | } 167 | 168 | } 169 | -------------------------------------------------------------------------------- /priority.go: -------------------------------------------------------------------------------- 1 | package codel 2 | 3 | import ( 4 | "context" 5 | "math" 6 | "sync" 7 | "time" 8 | ) 9 | 10 | const maxInt = int((^uint(0)) >> 1) 11 | 12 | // prendezvouz is for returning context to the calling goroutine 13 | type prendezvouz struct { 14 | priority int 15 | index int 16 | enqueuedTime time.Time 17 | errChan chan error 18 | } 19 | 20 | func (r prendezvouz) Drop() { 21 | select { 22 | case r.errChan <- Dropped: 23 | default: 24 | } 25 | } 26 | 27 | func (r prendezvouz) Signal() { 28 | close(r.errChan) 29 | } 30 | 31 | // PLock implements a FIFO lock with concurrency control and priority, based upon the CoDel algorithm (https://queue.acm.org/detail.cfm?id=2209336). 32 | type PLock struct { 33 | mu sync.Mutex 34 | target time.Duration 35 | firstAboveTime time.Time 36 | dropNext time.Time 37 | 38 | droppedCount int64 39 | dropping bool 40 | 41 | waiters priorityQueue 42 | maxPending int64 43 | 44 | outstanding int64 45 | maxOutstanding int64 46 | } 47 | 48 | func NewPriority(opts Options) *PLock { 49 | q := PLock{ 50 | target: opts.TargetLatency, 51 | maxOutstanding: int64(opts.MaxOutstanding), 52 | maxPending: int64(opts.MaxPending), 53 | waiters: newQueue(opts.MaxPending), 54 | } 55 | 56 | return &q 57 | } 58 | 59 | // Acquire a PLock with FIFO ordering, respecting the context. Returns an error it fails to acquire. 60 | func (l *PLock) Acquire(ctx context.Context, priority int) error { 61 | l.mu.Lock() 62 | 63 | // Fast path if we are unblocked. 64 | if l.outstanding < l.maxOutstanding && l.waiters.Len() == 0 { 65 | l.outstanding++ 66 | l.mu.Unlock() 67 | return nil 68 | } 69 | 70 | r := prendezvouz{ 71 | priority: priority, 72 | enqueuedTime: time.Now(), 73 | errChan: make(chan error, 1), 74 | } 75 | 76 | pushed := l.waiters.Push(&r) 77 | 78 | if !pushed { 79 | l.externalDrop() 80 | l.mu.Unlock() 81 | return Dropped 82 | } 83 | 84 | l.mu.Unlock() 85 | 86 | select { 87 | 88 | case err := <-r.errChan: 89 | return err 90 | 91 | case <-ctx.Done(): 92 | err := ctx.Err() 93 | 94 | l.mu.Lock() 95 | defer l.mu.Unlock() 96 | 97 | select { 98 | case err = <-r.errChan: 99 | default: 100 | l.waiters.Remove(&r) 101 | l.externalDrop() 102 | } 103 | 104 | return err 105 | } 106 | } 107 | 108 | // Release a previously acquired lock. 109 | func (l *PLock) Release() { 110 | l.mu.Lock() 111 | l.outstanding-- 112 | if l.outstanding < 0 { 113 | l.mu.Unlock() 114 | panic("lock: bad release") 115 | } 116 | 117 | l.deque() 118 | 119 | l.mu.Unlock() 120 | 121 | } 122 | 123 | // Adjust the time based upon interval / sqrt(droppedCount) 124 | func (l *PLock) controlLaw(t time.Time) time.Time { 125 | return t.Add(time.Duration(float64(interval) / math.Sqrt(float64(l.droppedCount)))) 126 | } 127 | 128 | // Pull a single instance off the queue. This should be 129 | func (l *PLock) doDeque(now time.Time) (r prendezvouz, ok bool, okToDrop bool) { 130 | if l.waiters.Empty() { 131 | return r, false, false 132 | } 133 | 134 | r = l.waiters.Pop() 135 | 136 | sojurnDuration := now.Sub(r.enqueuedTime) 137 | 138 | if sojurnDuration < l.target || l.waiters.Len() == 0 { 139 | l.firstAboveTime = time.Time{} 140 | } else if (l.firstAboveTime == time.Time{}) { 141 | l.firstAboveTime = now.Add(interval) 142 | } else if now.After(l.firstAboveTime) { 143 | okToDrop = true 144 | } 145 | 146 | return r, true, okToDrop 147 | 148 | } 149 | 150 | // Signal that we couldn't write to the queue 151 | func (l *PLock) externalDrop() { 152 | l.dropping = true 153 | l.droppedCount++ 154 | l.dropNext = l.controlLaw(l.dropNext) 155 | } 156 | 157 | // Pull instances off the queue until we no longer drop 158 | func (l *PLock) deque() { 159 | now := time.Now() 160 | 161 | rendezvouz, ok, okToDrop := l.doDeque(now) 162 | 163 | // The queue has no entries, so return 164 | if !ok { 165 | return 166 | } 167 | 168 | if !okToDrop { 169 | l.dropping = false 170 | l.outstanding++ 171 | rendezvouz.Signal() 172 | return 173 | } 174 | 175 | if l.dropping { 176 | for now.After(l.dropNext) && l.dropping { 177 | rendezvouz.Drop() 178 | rendezvouz, ok, okToDrop = l.doDeque(now) 179 | 180 | if !ok { 181 | return 182 | } 183 | 184 | l.droppedCount++ 185 | 186 | if !okToDrop { 187 | l.dropping = false 188 | } else { 189 | l.dropNext = l.controlLaw(l.dropNext) 190 | } 191 | } 192 | } else if now.Sub(l.dropNext) < interval || now.Sub(l.firstAboveTime) >= interval { 193 | rendezvouz.Drop() 194 | rendezvouz, ok, _ = l.doDeque(now) 195 | 196 | if !ok { 197 | return 198 | } 199 | 200 | l.dropping = true 201 | 202 | if l.droppedCount > 2 { 203 | l.droppedCount -= 2 204 | } else { 205 | l.droppedCount = 1 206 | } 207 | 208 | l.dropNext = l.controlLaw(now) 209 | } 210 | 211 | l.outstanding++ 212 | rendezvouz.Signal() 213 | } 214 | -------------------------------------------------------------------------------- /codel.go: -------------------------------------------------------------------------------- 1 | // Package codel implements the Controlled Delay 2 | // (https://queue.acm.org/detail.cfm?id=2209336) algorithm for 3 | // overload detection, providing a mechanism to shed load when 4 | // overloaded. It optimizes for latency while keeping throughput high, 5 | // even when downstream rates dynamically change. 6 | // It keeps latency low when even severely overloaded, by preemptively 7 | // shedding some load when wait latency is long. It is comparable to 8 | // using a queue to handle bursts of load, but improves upon this 9 | // technique by avoiding the latency required to handle all previous 10 | // entries in the queue. 11 | package codel 12 | 13 | import ( 14 | "container/list" 15 | "context" 16 | "errors" 17 | "math" 18 | "sync" 19 | "time" 20 | ) 21 | 22 | // Dropped is the error that will be returned if this token is dropped 23 | var Dropped = errors.New("dropped") 24 | 25 | const ( 26 | interval = 10 * time.Millisecond 27 | ) 28 | 29 | // rendezvouz is for returning context to the calling goroutine 30 | type rendezvouz struct { 31 | enqueuedTime time.Time 32 | errChan chan error 33 | } 34 | 35 | func (r rendezvouz) Drop() { 36 | select { 37 | case r.errChan <- Dropped: 38 | default: 39 | } 40 | } 41 | 42 | func (r rendezvouz) Signal() { 43 | close(r.errChan) 44 | } 45 | 46 | // Options are options to configure a Lock. 47 | type Options struct { 48 | MaxPending int // The maximum number of pending acquires 49 | MaxOutstanding int // The maximum number of concurrent acquires 50 | TargetLatency time.Duration // The target latency to wait for an acquire. Acquires that take longer than this can fail. 51 | } 52 | 53 | // Lock implements a FIFO lock with concurrency control, based upon the CoDel algorithm (https://queue.acm.org/detail.cfm?id=2209336). 54 | type Lock struct { 55 | mu sync.Mutex 56 | target time.Duration 57 | firstAboveTime time.Time 58 | dropNext time.Time 59 | 60 | droppedCount int64 61 | dropping bool 62 | 63 | waiters list.List 64 | maxPending int64 65 | 66 | outstanding int64 67 | maxOutstanding int64 68 | } 69 | 70 | func New(opts Options) *Lock { 71 | q := Lock{ 72 | target: opts.TargetLatency, 73 | maxOutstanding: int64(opts.MaxOutstanding), 74 | maxPending: int64(opts.MaxPending), 75 | } 76 | 77 | return &q 78 | } 79 | 80 | // Acquire a Lock with FIFO ordering, respecting the context. Returns an error it fails to acquire. 81 | func (l *Lock) Acquire(ctx context.Context) error { 82 | l.mu.Lock() 83 | 84 | // Fast path if we are unblocked. 85 | if l.outstanding < l.maxOutstanding && l.waiters.Len() == 0 { 86 | l.outstanding++ 87 | l.mu.Unlock() 88 | return nil 89 | } 90 | 91 | // If our queue is full, drop 92 | if int64(l.waiters.Len()) == l.maxPending { 93 | l.externalDrop() 94 | l.mu.Unlock() 95 | return Dropped 96 | } 97 | 98 | r := rendezvouz{ 99 | enqueuedTime: time.Now(), 100 | errChan: make(chan error), 101 | } 102 | 103 | elem := l.waiters.PushBack(r) 104 | l.mu.Unlock() 105 | 106 | select { 107 | 108 | case err := <-r.errChan: 109 | return err 110 | 111 | case <-ctx.Done(): 112 | err := ctx.Err() 113 | 114 | l.mu.Lock() 115 | 116 | select { 117 | case err = <-r.errChan: 118 | default: 119 | l.waiters.Remove(elem) 120 | l.externalDrop() 121 | } 122 | 123 | l.mu.Unlock() 124 | 125 | return err 126 | } 127 | } 128 | 129 | // Release a previously acquired lock. 130 | func (l *Lock) Release() { 131 | l.mu.Lock() 132 | l.outstanding-- 133 | if l.outstanding < 0 { 134 | l.mu.Unlock() 135 | panic("lock: bad release") 136 | } 137 | 138 | l.deque() 139 | 140 | l.mu.Unlock() 141 | 142 | } 143 | 144 | // Adjust the time based upon interval / sqrt(droppedCount) 145 | func (l *Lock) controlLaw(t time.Time) time.Time { 146 | return t.Add(time.Duration(float64(interval) / math.Sqrt(float64(l.droppedCount)))) 147 | } 148 | 149 | // Pull a single instance off the queue. This should be 150 | func (l *Lock) doDeque(now time.Time) (r rendezvouz, ok bool, okToDrop bool) { 151 | next := l.waiters.Front() 152 | 153 | if next == nil { 154 | return rendezvouz{}, false, false 155 | } 156 | 157 | l.waiters.Remove(next) 158 | 159 | r = next.Value.(rendezvouz) 160 | 161 | sojurnDuration := now.Sub(r.enqueuedTime) 162 | 163 | if sojurnDuration < l.target || l.waiters.Len() == 0 { 164 | l.firstAboveTime = time.Time{} 165 | } else if (l.firstAboveTime == time.Time{}) { 166 | l.firstAboveTime = now.Add(interval) 167 | } else if now.After(l.firstAboveTime) { 168 | okToDrop = true 169 | } 170 | 171 | return r, true, okToDrop 172 | 173 | } 174 | 175 | // Signal that we couldn't write to the queue 176 | func (l *Lock) externalDrop() { 177 | l.dropping = true 178 | l.droppedCount++ 179 | l.dropNext = l.controlLaw(l.dropNext) 180 | } 181 | 182 | // Pull instances off the queue until we no longer drop 183 | func (l *Lock) deque() { 184 | now := time.Now() 185 | 186 | rendezvouz, ok, okToDrop := l.doDeque(now) 187 | 188 | // The queue has no entries, so return 189 | if !ok { 190 | return 191 | } 192 | 193 | if !okToDrop { 194 | l.dropping = false 195 | l.outstanding++ 196 | rendezvouz.Signal() 197 | return 198 | } 199 | 200 | if l.dropping { 201 | for now.After(l.dropNext) && l.dropping { 202 | rendezvouz.Drop() 203 | rendezvouz, ok, okToDrop = l.doDeque(now) 204 | 205 | if !ok { 206 | return 207 | } 208 | 209 | l.droppedCount++ 210 | 211 | if !okToDrop { 212 | l.dropping = false 213 | } else { 214 | l.dropNext = l.controlLaw(l.dropNext) 215 | } 216 | } 217 | } else if now.Sub(l.dropNext) < interval || now.Sub(l.firstAboveTime) >= interval { 218 | rendezvouz.Drop() 219 | rendezvouz, ok, _ = l.doDeque(now) 220 | 221 | if !ok { 222 | return 223 | } 224 | 225 | l.dropping = true 226 | 227 | if l.droppedCount > 2 { 228 | l.droppedCount -= 2 229 | } else { 230 | l.droppedCount = 1 231 | } 232 | 233 | l.dropNext = l.controlLaw(now) 234 | } 235 | 236 | l.outstanding++ 237 | rendezvouz.Signal() 238 | } 239 | --------------------------------------------------------------------------------