├── cmd └── sparkline │ ├── .babelrc │ ├── sparkline.png │ ├── webpack.config.js │ ├── package.json │ ├── main.go │ ├── main.jsx │ └── tpl │ └── index.html ├── buckets.go ├── .gitignore ├── buckets_redisv5_test.go ├── .travis.yml ├── export_test.go ├── LICENSE ├── limiter.go ├── example_test.go ├── counter.go ├── limiter_test.go ├── counter_test.go ├── buckets_redisv5.go ├── README.md └── buckets_redigo.go /cmd/sparkline/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets":[ 3 | "es2015", "react" 4 | ] 5 | } 6 | -------------------------------------------------------------------------------- /cmd/sparkline/sparkline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/abo/rerate/HEAD/cmd/sparkline/sparkline.png -------------------------------------------------------------------------------- /cmd/sparkline/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | module.exports = { 3 | entry: './main.jsx', 4 | output: { 5 | path: path.resolve('assets'), 6 | filename: 'sparkline.js' 7 | }, 8 | module: { 9 | loaders: [ 10 | {test: /\.jsx$/, loader: 'babel-loader', exclude: /node_modules/} 11 | ] 12 | } 13 | } -------------------------------------------------------------------------------- /buckets.go: -------------------------------------------------------------------------------- 1 | package rerate 2 | 3 | import "time" 4 | 5 | // Buckets a set of bucket, each bucket compute and return the number of occurs in itself 6 | type Buckets interface { 7 | Inc(key string, id int64) error 8 | Del(key string, ids ...int64) error 9 | Get(key string, ids ...int64) ([]int64, error) 10 | } 11 | 12 | // BucketsFactory a interface to create Buckets 13 | type BucketsFactory func(size int64, ttl time.Duration) Buckets 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | 27 | .vscode/ 28 | *.log 29 | 30 | cmd/sparkline/node_modules/ 31 | cmd/sparkline/assets/ 32 | -------------------------------------------------------------------------------- /buckets_redisv5_test.go: -------------------------------------------------------------------------------- 1 | package rerate_test 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/abo/rerate" 8 | 9 | redis "gopkg.in/redis.v5" 10 | ) 11 | 12 | func TestCleanup(t *testing.T) { 13 | buckets := rerate.NewRedisV5Buckets(redis.NewClient(&redis.Options{ 14 | Addr: "localhost:6379", 15 | Password: "", 16 | DB: 0, 17 | }))(5, 5*time.Second) 18 | key := "buckets:redisv5:cleanup" 19 | 20 | for index := 0; index < 10; index++ { 21 | buckets.Inc(key, int64(index)) 22 | } 23 | //TODO buckets's len == 5 24 | } 25 | -------------------------------------------------------------------------------- /cmd/sparkline/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "rerate-sparkline", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "webpack.config.js", 6 | "dependencies": { 7 | "react": "^15.4.2", 8 | "react-dom": "^15.4.2", 9 | "react-sparklines": "^1.6.0" 10 | }, 11 | "devDependencies": { 12 | "babel-core": "^6.24.0", 13 | "babel-loader": "^6.4.1", 14 | "babel-preset-es2015": "^6.24.0", 15 | "babel-preset-react": "^6.23.0", 16 | "webpack": "^2.3.2" 17 | }, 18 | "scripts": { 19 | "test": "echo \"Error: no test specified\" && exit 1" 20 | }, 21 | "author": "abo", 22 | "license": "ISC" 23 | } 24 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: go 3 | go: 4 | - 1.5 5 | - 1.6 6 | - tip 7 | 8 | services: 9 | - redis-server 10 | 11 | install: 12 | - go get -v -t ./... 13 | - go get golang.org/x/tools/cmd/cover 14 | - go get github.com/mattn/goveralls 15 | 16 | before_script: 17 | # sleep a bit to allow things to get set up 18 | # - sleep 10 19 | 20 | script: 21 | - go test -v -covermode=count -coverprofile=coverage.out 22 | - $(go env GOPATH | awk 'BEGIN{FS=":"} {print $1}')/bin/goveralls -coverprofile=coverage.out -service=travis-ci -repotoken $COVERALLS_TOKEN 23 | 24 | after_failure: 25 | - tail -n100 ./*.log 26 | -------------------------------------------------------------------------------- /export_test.go: -------------------------------------------------------------------------------- 1 | package rerate 2 | 3 | import "time" 4 | 5 | // func BucketIdsExp(c *Counter, from int) []int { 6 | // return c.bucketIds(from) 7 | // } 8 | 9 | func HashExp(c *Counter, t time.Time) int64 { 10 | return c.hash(t) 11 | } 12 | 13 | func IncAtExp(c *Counter, id string, t time.Time) error { 14 | return c.incAt(id, t) 15 | } 16 | 17 | func HistogramAtExp(c *Counter, id string, t time.Time) ([]int64, error) { 18 | return c.histogramAt(id, t) 19 | } 20 | 21 | func RemainingAtExp(l *Limiter, id string, t time.Time) (int64, error) { 22 | return l.remainingAt(id, t) 23 | } 24 | 25 | func ExceededAtExp(l *Limiter, id string, t time.Time) (bool, error) { 26 | return l.exceededAt(id, t) 27 | } 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /limiter.go: -------------------------------------------------------------------------------- 1 | package rerate 2 | 3 | import "time" 4 | 5 | // Limiter a redis-based ratelimiter 6 | type Limiter struct { 7 | Counter 8 | max int64 9 | } 10 | 11 | // NewLimiter create a new redis-based ratelimiter 12 | // the Limiter limits the rate to max times per period 13 | func NewLimiter(newBuckets BucketsFactory, pfx string, period, interval time.Duration, max int64) *Limiter { 14 | return &Limiter{ 15 | Counter: *NewCounter(newBuckets, pfx, period, interval), 16 | max: max, 17 | } 18 | } 19 | 20 | func (l *Limiter) remainingAt(id string, t time.Time) (int64, error) { 21 | occurs, err := l.countAt(id, t) 22 | if err != nil { 23 | return 0, err 24 | } 25 | return l.max - occurs, nil 26 | } 27 | 28 | // Remaining return the number of requests left for the time window 29 | func (l *Limiter) Remaining(id string) (int64, error) { 30 | return l.remainingAt(id, time.Now()) 31 | } 32 | 33 | func (l *Limiter) exceededAt(id string, t time.Time) (bool, error) { 34 | rem, err := l.Remaining(id) 35 | if err != nil { 36 | return false, err 37 | } 38 | return rem <= 0, nil 39 | } 40 | 41 | // Exceeded is exceeded the rate limit or not 42 | func (l *Limiter) Exceeded(id string) (bool, error) { 43 | return l.exceededAt(id, time.Now()) 44 | } 45 | -------------------------------------------------------------------------------- /cmd/sparkline/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "time" 7 | 8 | redis "gopkg.in/redis.v5" 9 | 10 | "github.com/abo/rerate" 11 | "github.com/gorilla/mux" 12 | ) 13 | 14 | var counter *rerate.Counter 15 | 16 | func init() { 17 | buckets := rerate.NewRedisV5Buckets(redis.NewClient(&redis.Options{ 18 | Addr: "localhost:6379", 19 | Password: "", 20 | DB: 0, 21 | })) 22 | counter = rerate.NewCounter(buckets, "rerate:sparkline", 20*time.Second, 500*time.Millisecond) 23 | } 24 | 25 | func main() { 26 | r := mux.NewRouter() 27 | r.PathPrefix("/assets/").Handler(http.StripPrefix("/assets/", http.FileServer(http.Dir("assets")))) 28 | r.HandleFunc("/histogram/{key}", histogram) 29 | r.HandleFunc("/inc/{key}", inc) 30 | r.PathPrefix("/").Handler(http.FileServer(http.Dir("tpl"))) 31 | 32 | srv := &http.Server{ 33 | Handler: r, 34 | Addr: ":8080", 35 | WriteTimeout: 15 * time.Second, 36 | ReadTimeout: 15 * time.Second, 37 | } 38 | 39 | srv.ListenAndServe() 40 | } 41 | 42 | func inc(w http.ResponseWriter, r *http.Request) { 43 | vars := mux.Vars(r) 44 | counter.Inc(vars["key"]) 45 | } 46 | 47 | func histogram(w http.ResponseWriter, r *http.Request) { 48 | vars := mux.Vars(r) 49 | if hist, err := counter.Histogram(vars["key"]); err != nil { 50 | http.Error(w, err.Error(), http.StatusInternalServerError) 51 | } else if resp, err := json.Marshal(hist); err != nil { 52 | http.Error(w, err.Error(), http.StatusInternalServerError) 53 | } else { 54 | w.Header().Set("Content-Type", "application/json") 55 | w.Write(resp) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /cmd/sparkline/main.jsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | import { Sparklines, SparklinesLine, SparklinesBars } from 'react-sparklines'; 4 | 5 | class Rerate extends React.Component { 6 | constructor(props) { 7 | super(props); 8 | this.state = { histogram:[] }; 9 | } 10 | 11 | componentDidMount() { 12 | this.intervalId = setInterval( () => { 13 | fetch("/histogram/"+this.props.counterId) 14 | .then(response => response.json()) 15 | .then(json => this.setState({histogram:json})); 16 | },this.props.interval); 17 | } 18 | 19 | componentWillUnmount() { 20 | clearInterval(this.intervalId); 21 | } 22 | 23 | render() { 24 | const data = this.state.histogram.reverse(); 25 | return (
{React.Children.map(this.props.children, function(child) { 26 | return React.cloneElement(child, { data }); 27 | })}
); 28 | } 29 | } 30 | 31 | 32 | ReactDOM.render( 33 | 34 | 35 | , document.getElementById("sparkline-0") 36 | ); 37 | ReactDOM.render( 38 | 39 | 40 | , document.getElementById("sparkline-1") 41 | ); 42 | /*ReactDOM.render( 43 | 44 | 45 | , document.getElementById("sparkline-2") 46 | ); 47 | ReactDOM.render( 48 | 49 | 50 | , document.getElementById("sparkline-3") 51 | ); 52 | ReactDOM.render( 53 | 54 | 55 | , document.getElementById("sparkline-4") 56 | ); 57 | ReactDOM.render( 58 | 59 | 60 | , document.getElementById("sparkline-5") 61 | );*/ -------------------------------------------------------------------------------- /example_test.go: -------------------------------------------------------------------------------- 1 | package rerate_test 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/abo/rerate" 8 | "github.com/garyburd/redigo/redis" 9 | ) 10 | 11 | func newRedigoPool(server, password string) *redis.Pool { 12 | return &redis.Pool{ 13 | MaxIdle: 3, 14 | IdleTimeout: 240 * time.Second, 15 | Dial: func() (redis.Conn, error) { 16 | c, err := redis.Dial("tcp", server) 17 | if err != nil { 18 | return nil, err 19 | } 20 | if len(password) == 0 { 21 | return c, err 22 | } 23 | 24 | if _, err := c.Do("AUTH", password); err != nil { 25 | c.Close() 26 | return nil, err 27 | } 28 | return c, err 29 | }, 30 | TestOnBorrow: func(c redis.Conn, t time.Time) error { 31 | _, err := c.Do("PING") 32 | return err 33 | }, 34 | } 35 | } 36 | 37 | func ExampleCounter() { 38 | redigoBuckets := rerate.NewRedigoBuckets(newRedigoPool("localhost:6379", "")) 39 | 40 | key := "pv-home" 41 | // pv count in 5s, try to release per 0.5s 42 | counter := rerate.NewCounter(redigoBuckets, "rr:test:count", 5*time.Second, 500*time.Millisecond) 43 | counter.Reset(key) 44 | 45 | ticker := time.NewTicker(1000 * time.Millisecond) 46 | go func() { 47 | for range ticker.C { 48 | counter.Inc(key) 49 | } 50 | }() 51 | 52 | time.Sleep(4500 * time.Millisecond) 53 | ticker.Stop() 54 | total, _ := counter.Count(key) 55 | his, _ := counter.Histogram(key) 56 | fmt.Println("total:", total, ", histogram:", his) 57 | //Output: total: 4 , histogram: [0 1 0 1 0 1 0 1 0 0] 58 | } 59 | 60 | func ExampleLimiter() { 61 | redigoBuckets := rerate.NewRedigoBuckets(newRedigoPool("localhost:6379", "")) 62 | 63 | key := "pv-dashboard" 64 | // rate limit to 10/2s, release interval 0.2s 65 | limiter := rerate.NewLimiter(redigoBuckets, "rr:test:limit", 2*time.Second, 200*time.Millisecond, 10) 66 | limiter.Reset(key) 67 | 68 | ticker := time.NewTicker(200 * time.Millisecond) 69 | 70 | go func() { 71 | for range ticker.C { 72 | limiter.Inc(key) 73 | if exceed, _ := limiter.Exceeded(key); exceed { 74 | ticker.Stop() 75 | } 76 | } 77 | }() 78 | 79 | time.Sleep(20 * time.Millisecond) 80 | for i := 0; i < 20; i++ { 81 | time.Sleep(200 * time.Millisecond) 82 | 83 | if exceed, _ := limiter.Exceeded(key); exceed { 84 | fmt.Println("exceeded") 85 | } else { 86 | rem, _ := limiter.Remaining(key) 87 | fmt.Println("remaining", rem) 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /counter.go: -------------------------------------------------------------------------------- 1 | package rerate 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | // Counter count total occurs during a period, 9 | // it will store occurs during every time slice interval: (now ~ now - interval), (now - interval ~ now - 2*interval)... 10 | type Counter struct { 11 | pfx string 12 | buckets Buckets 13 | period time.Duration 14 | interval time.Duration 15 | } 16 | 17 | // NewCounter create a new Counter 18 | func NewCounter(newBuckets BucketsFactory, prefix string, period, interval time.Duration) *Counter { 19 | return &Counter{ 20 | buckets: newBuckets(int64(period/interval), period), 21 | pfx: prefix, 22 | period: period, 23 | interval: interval, 24 | } 25 | } 26 | 27 | // hash a time to n buckets(n=c.bkts) 28 | func (c *Counter) hash(t time.Time) int64 { 29 | return t.UnixNano() / int64(c.interval) 30 | } 31 | 32 | func (c *Counter) key(id string) string { 33 | return fmt.Sprintf("%s:%s", c.pfx, id) 34 | } 35 | 36 | func (c *Counter) incAt(id string, t time.Time) error { 37 | bucketID := c.hash(t) 38 | if err := c.buckets.Inc(c.key(id), bucketID); err != nil { 39 | return err 40 | } 41 | return nil 42 | } 43 | 44 | // Inc increment id's occurs with current timestamp, 45 | // the count before period will be cleanup 46 | func (c *Counter) Inc(id string) error { 47 | return c.incAt(id, time.Now()) 48 | } 49 | 50 | func (c *Counter) histogramAt(id string, t time.Time) ([]int64, error) { 51 | from := c.hash(t) 52 | size := int(c.period / c.interval) 53 | bucketIDs := make([]int64, size) 54 | for i := 0; i < size; i++ { 55 | bucketIDs[i] = from - int64(i) 56 | } 57 | 58 | return c.buckets.Get(c.key(id), bucketIDs...) 59 | } 60 | 61 | // Histogram return count histogram in recent period, order by time desc 62 | func (c *Counter) Histogram(id string) ([]int64, error) { 63 | return c.histogramAt(id, time.Now()) 64 | } 65 | 66 | func (c *Counter) countAt(id string, t time.Time) (int64, error) { 67 | h, err := c.histogramAt(id, t) 68 | if err != nil { 69 | return 0, err 70 | } 71 | 72 | total := int64(0) 73 | for _, v := range h { 74 | total += v 75 | } 76 | return total, nil 77 | } 78 | 79 | // Count return total occurs in recent period 80 | func (c *Counter) Count(id string) (int64, error) { 81 | return c.countAt(id, time.Now()) 82 | } 83 | 84 | // Reset cleanup occurs, set it to zero 85 | func (c *Counter) Reset(id string) error { 86 | return c.buckets.Del(c.key(id)) 87 | } 88 | -------------------------------------------------------------------------------- /limiter_test.go: -------------------------------------------------------------------------------- 1 | package rerate_test 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | redis "gopkg.in/redis.v5" 8 | 9 | . "github.com/abo/rerate" 10 | ) 11 | 12 | func TestLimiter(t *testing.T) { 13 | redisBuckets := NewRedisV5Buckets(redis.NewClient(&redis.Options{ 14 | Addr: "localhost:6379", 15 | Password: "", 16 | DB: 0, 17 | })) 18 | limiter := NewLimiter(redisBuckets, "rerate:test:limiter:limiter", time.Minute, time.Second, 20) 19 | k := randkey() 20 | limiter.Reset(k) 21 | 22 | assertExceeded(t, limiter, k, false) 23 | for i := 0; i < 19; i++ { 24 | assertRem(t, limiter, k, int64(20-i)) 25 | limiter.Inc(k) 26 | assertExceeded(t, limiter, k, false) 27 | } 28 | 29 | limiter.Inc(k) 30 | assertExceeded(t, limiter, k, true) 31 | } 32 | 33 | func TestExpire(t *testing.T) { 34 | redisBuckets := NewRedisV5Buckets(redis.NewClient(&redis.Options{ 35 | Addr: "localhost:6379", 36 | Password: "", 37 | DB: 0, 38 | })) 39 | limiter := NewLimiter(redisBuckets, "rerate:test:limiter:expire", 3*time.Second, time.Second, 20) 40 | k := randkey() 41 | limiter.Reset(k) 42 | 43 | limiter.Inc(k) 44 | assertRem(t, limiter, k, 19) 45 | 46 | time.Sleep(time.Second) 47 | limiter.Inc(k) 48 | assertRem(t, limiter, k, 18) 49 | 50 | time.Sleep(2 * time.Second) 51 | assertRem(t, limiter, k, 19) 52 | 53 | time.Sleep(time.Second) 54 | assertRem(t, limiter, k, 20) 55 | } 56 | 57 | //TODO 测试period不是interval的整数倍 58 | 59 | func TestNonOccurs(t *testing.T) { 60 | redisBuckets := NewRedisV5Buckets(redis.NewClient(&redis.Options{ 61 | Addr: "localhost:6379", 62 | Password: "", 63 | DB: 0, 64 | })) 65 | l := NewLimiter(redisBuckets, "rerate:test:limiter:nonoccurs", 3*time.Second, 500*time.Millisecond, 20) 66 | k := randkey() 67 | l.Reset(k) 68 | assertRem(t, l, k, 20) 69 | 70 | for i := 0; i < 6; i++ { 71 | l.Inc(k) 72 | time.Sleep(500 * time.Millisecond) 73 | } 74 | assertRem(t, l, k, 15) 75 | 76 | for i := 0; i < 5; i++ { 77 | time.Sleep(500 * time.Millisecond) 78 | assertRem(t, l, k, int64(15+1+i)) 79 | } 80 | } 81 | 82 | func assertRem(t *testing.T, l *Limiter, k string, expect int64) { 83 | if c, err := l.Remaining(k); err != nil || c != expect { 84 | t.Fatal("expect ", expect, " actual ", c, ", err:", err) 85 | } 86 | } 87 | 88 | func assertExceeded(t *testing.T, l *Limiter, k string, expect bool) { 89 | if exceed, err := l.Exceeded(k); err != nil || exceed != expect { 90 | t.Fatal("expect exceeded:", expect, ",err ", err) 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /cmd/sparkline/tpl/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | rerate's sparkline 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 22 | 23 | 24 |
25 |

Sparkline for rerate

26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 |
rerate:sparkline:0rerate:sparkline:1rerate:sparkline:arerate:sparkline:brerate:sparkline:crerate:sparkline:d
48 |
49 | 50 | 59 | 60 | -------------------------------------------------------------------------------- /counter_test.go: -------------------------------------------------------------------------------- 1 | package rerate_test 2 | 3 | import ( 4 | "math/rand" 5 | "reflect" 6 | "strconv" 7 | "testing" 8 | "time" 9 | 10 | redis "gopkg.in/redis.v5" 11 | 12 | . "github.com/abo/rerate" 13 | ) 14 | 15 | func randkey() string { 16 | return strconv.Itoa(rand.Int()) 17 | } 18 | 19 | func TestHistogram(t *testing.T) { 20 | redisBuckets := NewRedisV5Buckets(redis.NewClient(&redis.Options{ 21 | Addr: "localhost:6379", 22 | Password: "", 23 | DB: 0, 24 | })) 25 | counter := NewCounter(redisBuckets, "rerate:test:counter:count", 4000*time.Millisecond, 400*time.Millisecond) 26 | id := randkey() 27 | counter.Reset(id) 28 | 29 | zero := []int64{0, 0, 0, 0, 0, 0, 0, 0, 0, 0} 30 | if b, err := counter.Histogram(id); err != nil || !reflect.DeepEqual(b, zero) { 31 | t.Fatal("expect all zero without err, actual", b, err) 32 | } 33 | 34 | at := time.Now() 35 | for i := 0; i <= 10; i++ { 36 | for j := 0; j < i; j++ { 37 | IncAtExp(counter, id, at.Add(time.Duration(i)*400*time.Millisecond)) 38 | } 39 | } //[]int64{0,1,2,3,4,5,6,7,8,9,10,0,0,0,0,0,0,0,0,0} 40 | 41 | for i := 0; i < 20; i++ { 42 | for j := len(zero) - 1; j > 0; j-- { 43 | zero[j] = zero[j-1] 44 | } 45 | if i <= 10 { 46 | zero[0] = int64(i) 47 | } else { 48 | zero[0] = 0 49 | } 50 | 51 | assertHist(t, counter, id, at.Add(time.Duration(i)*400*time.Millisecond), zero) 52 | } 53 | } 54 | 55 | func TestCounter(t *testing.T) { 56 | redisBuckets := NewRedisV5Buckets(redis.NewClient(&redis.Options{ 57 | Addr: "localhost:6379", 58 | Password: "", 59 | DB: 0, 60 | })) 61 | counter := NewCounter(redisBuckets, "rerate:test:counter:counter", time.Minute, time.Second) 62 | ip1, ip2 := randkey(), randkey() 63 | 64 | if err := counter.Reset(ip1); err != nil { 65 | t.Fatal("can not reset counter", err) 66 | } 67 | if err := counter.Reset(ip2); err != nil { 68 | t.Fatal("can not reset counter", err) 69 | } 70 | 71 | assertCount(t, counter, ip1, 0) 72 | 73 | for i := 0; i < 10; i++ { 74 | counter.Inc(ip1) 75 | assertCount(t, counter, ip1, int64(i+1)) 76 | assertCount(t, counter, ip2, 0) 77 | } 78 | } 79 | 80 | func assertCount(t *testing.T, c *Counter, k string, expect int64) { 81 | if count, err := c.Count(k); err != nil || count != expect { 82 | t.Fatal("should be ", expect, " without error, actual ", count, err) 83 | } 84 | } 85 | 86 | func assertHist(t *testing.T, c *Counter, k string, from time.Time, expect []int64) { 87 | if b, err := HistogramAtExp(c, k, from); err != nil || !reflect.DeepEqual(b, expect) { 88 | t.Fatal("expect ", expect, " without err, actual", b, err) 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /buckets_redisv5.go: -------------------------------------------------------------------------------- 1 | package rerate 2 | 3 | import ( 4 | "strconv" 5 | "time" 6 | 7 | redis "gopkg.in/redis.v5" 8 | ) 9 | 10 | // RedisV5Buckets is a Buckets using redis.v5 as backend 11 | type RedisV5Buckets struct { 12 | redis *redis.Client 13 | size int64 14 | ttl time.Duration 15 | } 16 | 17 | // NewRedisV5Buckets is a RedisV5Buckets factory 18 | func NewRedisV5Buckets(redis *redis.Client) BucketsFactory { 19 | return func(size int64, ttl time.Duration) Buckets { 20 | return &RedisV5Buckets{ 21 | redis: redis, 22 | size: size, 23 | ttl: ttl, 24 | } 25 | } 26 | } 27 | 28 | func (bs *RedisV5Buckets) cleanup(key string, from int64) { 29 | if l, err := bs.redis.HLen(key).Result(); err != nil || l < bs.size*2 { 30 | return 31 | } 32 | 33 | if ids, err := bs.redis.HKeys(key).Result(); err == nil { 34 | var delIds []int64 35 | for _, s := range ids { 36 | if v, e := strconv.ParseInt(s, 10, 64); e == nil && v <= from-bs.size { 37 | delIds = append(delIds, v) 38 | 39 | } 40 | } 41 | bs.Del(key, delIds...) 42 | } 43 | } 44 | 45 | // Inc increment bucket key:id 's occurs 46 | func (bs *RedisV5Buckets) Inc(key string, id int64) error { 47 | pipe := bs.redis.TxPipeline() 48 | defer pipe.Close() 49 | 50 | count := pipe.HIncrBy(key, strconv.FormatInt(id, 10), 1) 51 | pipe.PExpire(key, bs.ttl) 52 | _, err := pipe.Exec() 53 | if err != nil { 54 | return err 55 | } 56 | 57 | if count.Val() == 1 { // new bucket created 58 | go bs.cleanup(key, id) 59 | } 60 | return nil 61 | } 62 | 63 | // Del delete bucket key:ids, or delete Buckets key when ids is empty. 64 | func (bs *RedisV5Buckets) Del(key string, ids ...int64) error { 65 | if len(ids) == 0 { 66 | _, err := bs.redis.Del(key).Result() 67 | return err 68 | } 69 | 70 | args := make([]string, len(ids)) 71 | for i, v := range ids { 72 | args[i] = strconv.FormatInt(v, 10) 73 | } 74 | _, err := bs.redis.HDel(key, args...).Result() 75 | return err 76 | } 77 | 78 | // Get return bucket key:ids' occurs 79 | func (bs *RedisV5Buckets) Get(key string, ids ...int64) ([]int64, error) { 80 | args := make([]string, len(ids)) 81 | for i, v := range ids { 82 | args[i] = strconv.FormatInt(v, 10) 83 | } 84 | 85 | results, err := bs.redis.HMGet(key, args...).Result() 86 | if err != nil { 87 | return []int64{}, err 88 | } 89 | 90 | vals := make([]int64, len(ids)) 91 | for i, result := range results { 92 | if result == nil { 93 | vals[i] = 0 94 | } else if v, e := strconv.ParseInt(result.(string), 10, 64); e != nil { 95 | vals[i] = 0 96 | } else { 97 | vals[i] = v 98 | } 99 | } 100 | 101 | return vals, nil 102 | } 103 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | rerate 2 | =========== 3 | [![Build Status](https://travis-ci.org/abo/rerate.svg?branch=master)](https://travis-ci.org/abo/rerate) 4 | [![GoDoc](https://godoc.org/github.com/abo/rerate?status.svg)](https://godoc.org/github.com/abo/rerate) 5 | [![Go Report Card](https://goreportcard.com/badge/github.com/abo/rerate)](https://goreportcard.com/report/github.com/abo/rerate) 6 | [![Coverage Status](https://coveralls.io/repos/github/abo/rerate/badge.svg?branch=master)](https://coveralls.io/github/abo/rerate?branch=master) 7 | 8 | rerate is a redis-based ratecounter and ratelimiter 9 | 10 | * Dead simple api 11 | * With redis as backend, multiple rate counters/limiters can work as a cluster 12 | * Count/Limit requests any period, 2 day, 1 hour, 5 minute or 2 second, it's up to you 13 | * Recording requests as a histotram, which can be used to visualize or monitor 14 | * Limit requests from single ip, userid, applicationid, or any other unique identifier 15 | 16 | 17 | Tutorial 18 | -------- 19 | ``` 20 | package main 21 | 22 | import ( 23 | "github.com/abo/rerate" 24 | ) 25 | 26 | ... 27 | 28 | func main() { 29 | // redigo buckets 30 | pool := newRedisPool("localhost:6379", "") 31 | buckets := rerate.NewRedigoBuckets(pool) 32 | 33 | // OR redis buckets 34 | // client := redis.NewClient(&redis.Options{ 35 | // Addr: "localhost:6379", 36 | // Password: "", 37 | // DB: 0, 38 | // }) 39 | // buckets := rerate.NewRedisBuckets(client) 40 | 41 | // Counter 42 | counter := rerate.NewCounter(buckets, "rl:test", 10 * time.Minute, 15 * time.Second) 43 | counter.Inc("click") 44 | c, err := counter.Count("click") 45 | 46 | // Limiter 47 | limiter := rerate.NewLimiter(buckets, "rl:test", 1 * time.Hour, 15 * time.Minute, 100) 48 | limiter.Inc("114.255.86.200") 49 | rem, err := limiter.Remaining("114.255.86.200") 50 | exceed, err := limiter.Exceeded("114.255.86.200") 51 | } 52 | ``` 53 | 54 | 55 | Installation 56 | ------------ 57 | 58 | Install rerate using the "go get" command: 59 | 60 | go get github.com/abo/rerate 61 | 62 | Documentation 63 | ------------- 64 | 65 | - [API Reference](http://godoc.org/github.com/abo/rerate) 66 | - [Wiki](https://github.com/abo/rerate/wiki) 67 | 68 | 69 | Sample - Sparkline 70 | ------------------ 71 | 72 | ![](https://github.com/abo/rerate/raw/master/cmd/sparkline/sparkline.png) 73 | 74 | ``` 75 | cd cmd/sparkline 76 | npm install webpack -g 77 | npm install 78 | webpack && go run main.go 79 | ``` 80 | Open `http://localhost:8080` in Browser, And then move mouse. 81 | 82 | 83 | Contributing 84 | ------------ 85 | WELCOME 86 | 87 | 88 | License 89 | ------- 90 | 91 | rerate is available under the [The MIT License (MIT)](https://opensource.org/licenses/MIT). 92 | -------------------------------------------------------------------------------- /buckets_redigo.go: -------------------------------------------------------------------------------- 1 | package rerate 2 | 3 | import ( 4 | "strconv" 5 | "time" 6 | 7 | "github.com/garyburd/redigo/redis" 8 | ) 9 | 10 | // RedigoBuckets is a Buckets using redigo as backend 11 | type RedigoBuckets struct { 12 | pool *redis.Pool 13 | size int64 14 | ttl time.Duration 15 | } 16 | 17 | // NewRedigoBuckets is a RedigoBuckets factory 18 | func NewRedigoBuckets(redis *redis.Pool) BucketsFactory { 19 | return func(size int64, ttl time.Duration) Buckets { 20 | return &RedigoBuckets{ 21 | pool: redis, 22 | size: size, 23 | ttl: ttl, 24 | } 25 | } 26 | 27 | } 28 | 29 | // cleanup unused bucket(s) 30 | func (bs *RedigoBuckets) cleanup(key string, from int64) { 31 | conn := bs.pool.Get() 32 | defer conn.Close() 33 | 34 | if l, err := redis.Int64(conn.Do("HLEN", key)); err != nil || l < bs.size*2 { 35 | return 36 | } 37 | 38 | if ids, err := redis.Strings(conn.Do("HKEYS", key)); err == nil { 39 | var delIds []int64 40 | for _, s := range ids { 41 | if v, err := strconv.ParseInt(s, 10, 64); err == nil && v <= from-bs.size { 42 | delIds = append(delIds, v) 43 | } 44 | } 45 | bs.Del(key, delIds...) 46 | } 47 | } 48 | 49 | // Inc increment bucket key:id 's occurs 50 | func (bs *RedigoBuckets) Inc(key string, id int64) error { 51 | conn := bs.pool.Get() 52 | defer conn.Close() 53 | 54 | conn.Send("MULTI") 55 | conn.Send("HINCRBY", key, strconv.FormatInt(id, 10), 1) 56 | conn.Send("PEXPIRE", key, int64(bs.ttl/time.Millisecond)) 57 | ret, err := redis.Ints(conn.Do("EXEC")) 58 | if err != nil { 59 | return err 60 | } 61 | 62 | if ret[0] == 1 { // new bucket created 63 | go bs.cleanup(key, id) 64 | } 65 | return nil 66 | } 67 | 68 | // Del delete bucket key:ids, or delete Buckets key when ids is empty. 69 | func (bs *RedigoBuckets) Del(key string, ids ...int64) error { 70 | conn := bs.pool.Get() 71 | defer conn.Close() 72 | 73 | if len(ids) == 0 { 74 | _, err := conn.Do("DEL", key) 75 | return err 76 | } 77 | 78 | args := make([]interface{}, len(ids)+1) 79 | args[0] = key 80 | for i, v := range ids { 81 | args[i+1] = v 82 | } 83 | _, err := conn.Do("HDEL", args...) 84 | return err 85 | } 86 | 87 | // Get return bucket key:ids' occurs 88 | func (bs *RedigoBuckets) Get(key string, ids ...int64) ([]int64, error) { 89 | args := make([]interface{}, len(ids)+1) 90 | args[0] = key 91 | for i, v := range ids { 92 | args[i+1] = v 93 | } 94 | 95 | conn := bs.pool.Get() 96 | defer conn.Close() 97 | 98 | vals, err := redis.Strings(conn.Do("HMGET", args...)) 99 | if err != nil { 100 | return []int64{}, err 101 | } 102 | 103 | ret := make([]int64, len(ids)) 104 | for i, val := range vals { 105 | if v, e := strconv.ParseInt(val, 10, 64); e == nil { 106 | ret[i] = v 107 | } else { 108 | ret[i] = 0 109 | } 110 | } 111 | return ret, nil 112 | } 113 | --------------------------------------------------------------------------------