├── .gitignore ├── README.md ├── constant.go ├── example.go ├── go.mod ├── go.sum ├── sliding_window_assigner.go ├── sliding_window_assigner_test.go ├── sliding_window_operator.go ├── sliding_window_operator_test.go ├── state.go ├── state_backend.go ├── time_window.go ├── tumbling_window_assigner.go ├── tumbling_window_assigner_test.go ├── tumbling_window_operator.go ├── tumbling_window_operator_test.go ├── window_assigner.go ├── window_operator.go ├── window_state.go ├── window_state_test.go ├── windows.go └── windows_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | .idea -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # window 2 | 3 | ## 背景 4 | 5 | 在我们业务开发中，经常遇到类似"用户当天累计消费金额"、"用户最近三天消费金额"等统计的需求。 6 | 7 | 本质上来说，这都是窗口统计(第一种是窗口大小为一天的滚动窗口，第二种是窗口大小为3天，滑动大小为一天的滑动窗口)。 8 | 9 | 通常，我们可以保存所有的消费记录到MySQL，然后通过MySQL的查询，来统计出来。 10 | 11 | 但是，如果数据量非常大，且访问量非常高，MySQL无法支撑起线上业务如此庞大的实时查询，我们该如何实时的统计出用户时间窗口内的值呢？ 12 | 13 | Flink提供了非常丰富的窗口算子，利用Flink提供的API，就可以非常简单的完成窗口计算。 14 | 15 | 但是，很多业务都是非Java业务。另外为了一个简单的统计功能，引入非常复杂的Flink，实现过于重。 16 | 17 | 那么，有没有简洁的方法实现，类似Flink的窗口计算功能呢。 18 | 19 | [Window](https://github.com/nienie/window) 提供了Golang的解决方案。 20 | 21 | ## 特点 22 | 23 | 提供了滑动窗口和滚动窗口算子，让Golang很容易实现类似Flink的窗口统计的逻辑。 24 | 25 | ## 基本概念 26 | 27 | - **Time Window** 28 | 29 | 时间窗口，时间在窗口开始时间和结束时间之内的元素，都会属于这个窗口。时间窗口主要有两种类型：1）**滚动窗口**(**Tumbling Window**)，2）**滑动窗口**(**Sliding Window**)。 30 | 31 | - **Tumbling Window** 32 | 33 | **滚动窗口**，某个时刻只属于一个窗口。例如，1天大小的滚动窗口，时间段[2020-11-11 00:00:00, 2020-11-12 00:00:00)是一个窗口， 34 | 时间段[2020-11-12 00:00:00, 2020-11-13 00:00:00)是另外一个窗口。2020-11-11 10:52:48 只会属于[2020-11-11 00:00:00, 2020-11-12 00:00:00) 这个窗口。 35 | 而不会属于[2020-11-12 00:00:00, 2020-11-13 00:00:00)的窗口。 36 | 37 | - **Sliding Window** 38 | 39 | **滑动窗口**，某个时刻会属于多个窗口。例如，1天大小的滚动窗口，12h滑动一次。时间段[2020-11-10 12:00:00, 2020-11-11 12:00:00)和[2020-11-11 00:00:00, 2020-11-12 00:00:00)都是一个窗口。 40 | 窗口之间有重合，时间2020-11-11 10:44:45，会同时属于这个窗口。 41 | 42 | - **State** 43 | 44 | **状态**，有状态流式计算中的概念，是统计的中间结果也可以是最终的结果。例如，要统计用户充值总金额，State就是用户这个充值的总金额。 45 | 46 | - **StateBackend** 47 | 48 | **StateBackend**，状态存储的抽象。 49 | 50 | ## 使用与示例 51 | 52 | 详见[一天内用户充值金额统计(滚动窗口)](https://github.com/nienie/window/blob/master/tumbling_window_operator_test.go) 53 | 54 | 详见[三天内用户充值金额统计(滑动窗口)](https://github.com/nienie/window/blob/master/sliding_window_operator_test.go) 55 | 56 | 1) 定义State。 57 | 58 | ```golang 59 | //State就是我们要统计的东西。 60 | //ChargeState 用户充值的金额统计结果 61 | type ChargeState struct { 62 | TotalAmount uint64 `json:"total_amount"` 63 | } 64 | 65 | //String ... 66 | func (o *ChargeState) String() string { 67 | data, _ := json.Marshal(o) 68 | return string(data) 69 | } 70 | 71 | var _ State = (*ChargeState)(nil) 72 | ``` 73 | 74 | 2) 定义StateBackend. 75 | 76 | ```golang 77 | //ChargeEvent 用户充值事件 78 | type ChargeEvent struct { 79 | UID int64 `json:"uid"` //用户Uid 80 | Amount int64 `json:"amount"` //充值金额 81 | Ts int64 `json:"ts"` //充值时间戳，单位s 82 | } 83 | 84 | //ChargeStateBackend 充值状态的StateBackend 85 | type ChargeStateBackend struct { 86 | client *redis.Client 87 | } 88 | 89 | //NewChargeState ... 90 | func NewChargeStateBackend(client *redis.Client) *ChargeStateBackend { 91 | return &ChargeStateBackend{ 92 | client: client, 93 | } 94 | } 95 | 96 | //Get 获取充值统计结果 97 | func (o *ChargeStateBackend) Get(ctx context.Context, key string) (State, error) { 98 | count, err := o.client.Get(ctx, key).Uint64() 99 | if err != nil { 100 | return nil, err 101 | } 102 | return &ChargeState{ 103 | TotalAmount: count, 104 | }, nil 105 | } 106 | 107 | //Update 当有新的充值事件发生，更新充值结果 108 | func (o *ChargeStateBackend) Update(ctx context.Context, key string, ev interface{}) (State, error) { 109 | event, ok := ev.(*ChargeEvent) 110 | if !ok { 111 | return nil, fmt.Errorf("invalid event") 112 | } 113 | count, err := o.client.IncrBy(ctx, key, event.Amount).Uint64() 114 | return &ChargeState{ 115 | TotalAmount: count, 116 | }, err 117 | } 118 | 119 | //Expire 给充值结果设置过期时间 120 | func (o *ChargeStateBackend) Expire(ctx context.Context, key string, expireSeconds int64) error { 121 | return o.client.Expire(ctx, key, time.Duration(expireSeconds)*time.Second).Err() 122 | } 123 | 124 | //Del 删除充值结果的State 125 | func (o *ChargeStateBackend) Del(ctx context.Context, key string) error { 126 | return o.client.Del(ctx, key).Err() 127 | } 128 | 129 | ``` 130 | 131 | 3) 定义滚动窗口或者滑动窗口。 132 | 133 | ```golang 134 | var ( 135 | uid int64 //用户ID 136 | windowOperator *TumblingWindowOperator 137 | size time.Duration //窗口大小 138 | offset time.Duration //时间偏移量，跟当前所在时区有关 139 | ) 140 | ctx := context.TODO() 141 | uid = 100000 142 | operatorName := fmt.Sprintf("one-day-charge-%d", uid) //窗口算子名字 143 | size = 24 * time.Hour //窗口大小， 1天，统计一天内，用户uid=100000 充值金额 144 | offset = -8 * time.Hour //偏移量，北京东8区，所以要减去8小时 145 | //一天大小的滚动窗口 146 | windowOperator = NewTumblingWindowOperator(operatorName, size, offset, NewChargeStateBackend(o.redisClient)) 147 | //用户充值的事件 148 | now := time.Now() 149 | events := []*ChargeEvent{ 150 | //昨天第一次充值，99 151 | { 152 | UID: uid, 153 | Amount: 99, 154 | Ts: now.Add(-24 * time.Hour).Unix(), 155 | }, 156 | //昨天第二次充值，199 157 | { 158 | UID: uid, 159 | Amount: 199, 160 | Ts: now.Add(-24 * time.Hour).Unix(), 161 | }, 162 | //今天第一次充值，92 163 | { 164 | UID: uid, 165 | Amount: 92, 166 | Ts: now.Unix(), 167 | }, 168 | //今天第二次充值，180 169 | { 170 | UID: uid, 171 | Amount: 180, 172 | Ts: now.Unix(), 173 | }, 174 | //今天第二次充值，36 175 | { 176 | UID: uid, 177 | Amount: 36, 178 | Ts: now.Unix(), 179 | }, 180 | } 181 | //每个事件都丢进窗口算子处理 182 | for _, event := range events { 183 | windowOperator.Process(ctx, time.Duration(event.Ts)*time.Second, event) 184 | } 185 | //昨天充值金额统计结果 186 | timestamp := now.Add(-24 * time.Hour) 187 | window := windowOperator.GetWindow(ctx, time.Duration(timestamp.UnixNano())) 188 | state, err := windowOperator.GetState(ctx, time.Duration(timestamp.UnixNano())) 189 | o.T().Logf("uid=%d||timestamp=%s||window_size=%s||window_start=%s||window_end=%s||state=%s||err=%v", 190 | uid, timestamp.Format("2006-01-02 15:04:05"), size, 191 | time.Unix(int64(window.GetStart()/time.Second), int64(window.GetStart()%time.Second)).Format("2006-01-02 15:04:05"), 192 | time.Unix(int64(window.GetEnd()/time.Second), int64(window.GetEnd()%time.Second)).Format("2006-01-02 15:04:05"), 193 | state, err) 194 | o.Require().Nil(err) 195 | o.Require().Equal(uint64(298), state.(*ChargeState).TotalAmount) 196 | //清理结果，防止重复执行会失败 197 | windowOperator.GetStateBackend().Del(ctx, window.GetName()) 198 | 199 | //今天充值的金额统计结果 200 | timestamp = now 201 | window = windowOperator.GetWindow(ctx, time.Duration(timestamp.UnixNano())) 202 | state, err = windowOperator.GetState(ctx, time.Duration(timestamp.UnixNano())) 203 | o.T().Logf("uid=%d||timestamp=%s||window_size=%s||window_start=%s||window_end=%s||state=%s||err=%v", 204 | uid, timestamp.Format("2006-01-02 15:04:05"), size, 205 | time.Unix(int64(window.GetStart()/time.Second), int64(window.GetStart()%time.Second)).Format("2006-01-02 15:04:05"), 206 | time.Unix(int64(window.GetEnd()/time.Second), int64(window.GetEnd()%time.Second)).Format("2006-01-02 15:04:05"), 207 | state, err) 208 | o.Require().Nil(err) 209 | o.Require().Equal(uint64(308), state.(*ChargeState).TotalAmount) 210 | //清理结果，防止重复执行会失败 211 | windowOperator.GetStateBackend().Del(ctx, window.GetName()) 212 | //TestTumblingWindowOperator/Test: tumbling_window_operator_test.go:80: uid=100000||timestamp=2020-11-10 11:37:15||window_size=24h0m0s||window_start=2020-11-10 00:00:00||window_end=2020-11-11 00:00:00||state={"TotalAmount":298}||err= 213 | //TestTumblingWindowOperator/Test: tumbling_window_operator_test.go:95: uid=100000||timestamp=2020-11-11 11:37:15||window_size=24h0m0s||window_start=2020-11-11 00:00:00||window_end=2020-11-12 00:00:00||state={"TotalAmount":308}||err= 214 | ``` 215 | 216 | ```golang 217 | var ( 218 | uid int64 //用户ID 219 | windowOperator *SlidingWindowOperator 220 | size time.Duration //窗口大小 221 | slide time.Duration //滑动距离 222 | offset time.Duration //时间偏移量，跟当前所在时区有关 223 | ) 224 | ctx := context.TODO() 225 | uid = 100000 226 | operatorName := fmt.Sprintf("three-day-charge-%d", uid) //窗口算子名字 227 | size = 3 * 24 * time.Hour //窗口大小， 3天，统计3天内，用户uid=100000 充值金额 228 | slide = 24 * time.Hour //滑动距离 229 | offset = -8 * time.Hour //偏移量，北京东8区，所以要减去8小时 230 | //三天时长的滑动窗口，一天滑动一次 231 | windowOperator = NewSlidingWindowOperator(operatorName, size, slide, offset, NewChargeStateBackend(o.redisClient)) 232 | //用户充值的事件 233 | now := time.Now() 234 | events := []*ChargeEvent{ 235 | //4天前充值99 236 | { 237 | UID: uid, 238 | Amount: 99, 239 | Ts: now.Add(-4 * 24 * time.Hour).Unix(), 240 | }, 241 | //三天前充值199 242 | { 243 | UID: uid, 244 | Amount: 199, 245 | Ts: now.Add(-3 * 24 * time.Hour).Unix(), 246 | }, 247 | //两天前充值，92 248 | { 249 | UID: uid, 250 | Amount: 92, 251 | Ts: now.Add(-2 * 24 * time.Hour).Unix(), 252 | }, 253 | //一天前充值180 254 | { 255 | UID: uid, 256 | Amount: 180, 257 | Ts: now.Add(-1 * 24 * time.Hour).Unix(), 258 | }, 259 | //今天充值36 260 | { 261 | UID: uid, 262 | Amount: 36, 263 | Ts: now.Unix(), 264 | }, 265 | } 266 | //每个充值事件都丢进窗口算子进行处理 267 | for _, event := range events { 268 | windowOperator.Process(ctx, time.Duration(event.Ts)*time.Second, event) 269 | } 270 | //昨天开始，3天内的充值金额统计结果 271 | timestamp := now.Add(-24 * time.Hour) 272 | window := windowOperator.GetWindow(ctx, time.Duration(timestamp.UnixNano())) 273 | state, err := windowOperator.GetState(ctx, time.Duration(timestamp.UnixNano())) 274 | o.T().Logf("uid=%d||timestamp=%s||window_size=%s||window_start=%s||window_end=%s||state=%s||err=%v", 275 | uid, timestamp.Format("2006-01-02 15:04:05"), size, 276 | time.Unix(int64(window.GetStart()/time.Second), int64(window.GetStart()%time.Second)).Format("2006-01-02 15:04:05"), 277 | time.Unix(int64(window.GetEnd()/time.Second), int64(window.GetEnd()%time.Second)).Format("2006-01-02 15:04:05"), 278 | state, err) 279 | o.Require().Nil(err) 280 | //471 = 199 + 92 + 180 281 | o.Require().Equal(uint64(471), state.(*ChargeState).TotalAmount) 282 | //清理结果，防止重复执行会失败 283 | windowOperator.GetStateBackend().Del(ctx, window.GetName()) 284 | 285 | //最近3天充值的金额统计结果 286 | timestamp = now 287 | window = windowOperator.GetWindow(ctx, time.Duration(timestamp.UnixNano())) 288 | state, err = windowOperator.GetState(ctx, time.Duration(timestamp.UnixNano())) 289 | o.T().Logf("uid=%d||timestamp=%s||window_size=%s||window_start=%s||window_end=%s||state=%s||err=%v", 290 | uid, timestamp.Format("2006-01-02 15:04:05"), size, 291 | time.Unix(int64(window.GetStart()/time.Second), int64(window.GetStart()%time.Second)).Format("2006-01-02 15:04:05"), 292 | time.Unix(int64(window.GetEnd()/time.Second), int64(window.GetEnd()%time.Second)).Format("2006-01-02 15:04:05"), 293 | state, err) 294 | o.Require().Nil(err) 295 | //308 = 92 + 180 + 36 296 | o.Require().Equal(uint64(308), state.(*ChargeState).TotalAmount) 297 | //清理结果，防止重复执行会失败 298 | windowOperator.GetStateBackend().Del(ctx, window.GetName()) 299 | //TestTumblingWindowOperator/Test: tumbling_window_operator_test.go:80: uid=100000||timestamp=2020-11-10 11:37:15||window_size=24h0m0s||window_start=2020-11-10 00:00:00||window_end=2020-11-11 00:00:00||state={"TotalAmount":298}||err= 300 | //TestTumblingWindowOperator/Test: tumbling_window_operator_test.go:95: uid=100000||timestamp=2020-11-11 11:37:15||window_size=24h0m0s||window_start=2020-11-11 00:00:00||window_end=2020-11-12 00:00:00||state={"TotalAmount":308}||err= 301 | ``` 302 | -------------------------------------------------------------------------------- /constant.go: -------------------------------------------------------------------------------- 1 | package window 2 | 3 | //Type 窗口类型 4 | type Type int 5 | 6 | const ( 7 | //None ... 8 | None Type = iota 9 | 10 | //Tumbling ... 11 | Tumbling 12 | 13 | //Sliding ... 14 | Sliding 15 | ) 16 | -------------------------------------------------------------------------------- /example.go: -------------------------------------------------------------------------------- 1 | package window 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "time" 8 | 9 | "github.com/go-redis/redis/v8" 10 | ) 11 | 12 | //ChargeEvent 用户充值事件 13 | type ChargeEvent struct { 14 | UID int64 `json:"uid"` //用户Uid 15 | Amount int64 `json:"amount"` //充值金额 16 | Ts int64 `json:"ts"` //充值时间戳，单位s 17 | } 18 | 19 | //ChargeState 用户充值的金额统计结果 20 | type ChargeState struct { 21 | TotalAmount uint64 `json:"total_amount"` 22 | } 23 | 24 | //String ... 25 | func (o *ChargeState) String() string { 26 | data, _ := json.Marshal(o) 27 | return string(data) 28 | } 29 | 30 | var _ State = (*ChargeState)(nil) 31 | 32 | //ChargeStateBackend ... 33 | type ChargeStateBackend struct { 34 | client *redis.Client 35 | } 36 | 37 | //NewChargeStateBackend ... 38 | func NewChargeStateBackend(client *redis.Client) *ChargeStateBackend { 39 | return &ChargeStateBackend{ 40 | client: client, 41 | } 42 | } 43 | 44 | //Get 获取窗口的状态 45 | func (o *ChargeStateBackend) Get(ctx context.Context, key string) (State, error) { 46 | count, err := o.client.Get(ctx, key).Uint64() 47 | if err != nil { 48 | return nil, err 49 | } 50 | return &ChargeState{ 51 | TotalAmount: count, 52 | }, nil 53 | } 54 | 55 | //Update 更新窗口的状态 56 | func (o *ChargeStateBackend) Update(ctx context.Context, key string, ev interface{}) (State, error) { 57 | event, ok := ev.(*ChargeEvent) 58 | if !ok { 59 | return nil, fmt.Errorf("invalid event") 60 | } 61 | count, err := o.client.IncrBy(ctx, key, event.Amount).Uint64() 62 | return &ChargeState{ 63 | TotalAmount: count, 64 | }, err 65 | } 66 | 67 | //Expire 给窗口状态设置过期时间 68 | func (o *ChargeStateBackend) Expire(ctx context.Context, key string, expireSeconds int64) error { 69 | return o.client.Expire(ctx, key, time.Duration(expireSeconds)*time.Second).Err() 70 | } 71 | 72 | //Del 删除窗口状态 73 | func (o *ChargeStateBackend) Del(ctx context.Context, key string) error { 74 | return o.client.Del(ctx, key).Err() 75 | } 76 | 77 | var _ StateBackend = (*ChargeStateBackend)(nil) 78 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/nienie/window 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/go-redis/redis/v8 v8.4.0 7 | github.com/stretchr/testify v1.6.1 8 | ) 9 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= 2 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 3 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 4 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 6 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 7 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 8 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= 9 | github.com/go-redis/redis/v8 v8.4.0 h1:J5NCReIgh3QgUJu398hUncxDExN4gMOHI11NVbVicGQ= 10 | github.com/go-redis/redis/v8 v8.4.0/go.mod h1:A1tbYoHSa1fXwN+//ljcCYYJeLmVrwL9hbQN45Jdy0M= 11 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 12 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 13 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 14 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 15 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 16 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 17 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 18 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 19 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 20 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 21 | github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 22 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 23 | github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= 24 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 25 | github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= 26 | github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= 27 | github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= 28 | github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= 29 | github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= 30 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 31 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 32 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 33 | github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= 34 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 35 | go.opentelemetry.io/otel v0.14.0 h1:YFBEfjCk9MTjaytCNSUkp9Q8lF7QJezA06T71FbQxLQ= 36 | go.opentelemetry.io/otel v0.14.0/go.mod h1:vH5xEuwy7Rts0GNtsCW3HYQoZDY+OmBJ6t1bFGGlxgw= 37 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 38 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 39 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 40 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 41 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 42 | golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 43 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 44 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 45 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 46 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 47 | golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 48 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 49 | golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 50 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 51 | golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 52 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 53 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 54 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 55 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 56 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 57 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 58 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 59 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 60 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 61 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 62 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 63 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 64 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 65 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 66 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 67 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 68 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 69 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 70 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 71 | -------------------------------------------------------------------------------- /sliding_window_assigner.go: -------------------------------------------------------------------------------- 1 | package window 2 | 3 | import "time" 4 | 5 | //SlidingWindowAssigner ... 6 | type SlidingWindowAssigner struct { 7 | name string 8 | size time.Duration 9 | offset time.Duration 10 | slide time.Duration 11 | } 12 | 13 | //NewSlidingWindowAssigner ... 14 | func NewSlidingWindowAssigner(name string, size, slide, offset time.Duration) *SlidingWindowAssigner { 15 | return &SlidingWindowAssigner{ 16 | name: name, 17 | size: size, 18 | slide: slide, 19 | offset: offset, 20 | } 21 | } 22 | 23 | //AssignWindows ... 24 | func (o *SlidingWindowAssigner) AssignWindows(timestamp time.Duration) []*TimeWindow { 25 | windows := make([]*TimeWindow, 0, o.size/o.slide) 26 | lastStart := GetWindowStartWithOffset(timestamp, o.offset, o.slide) 27 | for start := lastStart; start > timestamp-o.size; start -= o.slide { 28 | windows = append(windows, NewTimeWindow(o.name, start, start+o.size)) 29 | } 30 | return windows 31 | } 32 | 33 | var _ Assigner = (*SlidingWindowAssigner)(nil) 34 | -------------------------------------------------------------------------------- /sliding_window_assigner_test.go: -------------------------------------------------------------------------------- 1 | package window 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/suite" 8 | ) 9 | 10 | //SlidingWindowAssignerTestSuite ... 11 | type SlidingWindowAssignerTestSuite struct { 12 | suite.Suite 13 | } 14 | 15 | //Test ... 16 | func (o *SlidingWindowAssignerTestSuite) Test() { 17 | now := time.Now() 18 | var ( 19 | window *SlidingWindowAssigner 20 | assignedWindows []*TimeWindow 21 | ) 22 | //7天窗口 23 | window = NewSlidingWindowAssigner("SevenDaySW", 7*24*time.Hour, 24*time.Hour, -8*time.Hour) 24 | assignedWindows = window.AssignWindows(time.Duration(now.UnixNano())) 25 | for _, w := range assignedWindows { 26 | o.T().Logf("name=%s||window_size=%s||now=%s||start=%s||end=%s", w.GetName(), w.GetWindowSize(), now.Format("2006-01-02 15:04:05"), 27 | time.Unix(int64(w.GetStart()/time.Second), int64(w.GetStart()%time.Second)).Format("2006-01-02 15:04:05"), 28 | time.Unix(int64(w.GetEnd()/time.Second), int64(w.GetEnd()%time.Second)).Format("2006-01-02 15:04:05")) 29 | } 30 | //1天的窗口，offset = 0 31 | window = NewSlidingWindowAssigner("OneDaySW", 24*time.Hour, 12*time.Hour, 0) 32 | assignedWindows = window.AssignWindows(time.Duration(now.UnixNano())) 33 | for _, w := range assignedWindows { 34 | o.T().Logf("name=%s||window_size=%s||now=%s||start=%s||end=%s", w.GetName(), w.GetWindowSize(), now.Format("2006-01-02 15:04:05"), 35 | time.Unix(int64(w.GetStart()/time.Second), int64(w.GetStart()%time.Second)).Format("2006-01-02 15:04:05"), 36 | time.Unix(int64(w.GetEnd()/time.Second), int64(w.GetEnd()%time.Second)).Format("2006-01-02 15:04:05")) 37 | } 38 | //1天的窗口，北京东8区，快8小时，offset = -8h 39 | window = NewSlidingWindowAssigner("OneDaySW", 24*time.Hour, 12*time.Hour, -8*time.Hour) 40 | assignedWindows = window.AssignWindows(time.Duration(now.UnixNano())) 41 | for _, w := range assignedWindows { 42 | o.T().Logf("name=%s||window_size=%s||now=%s||start=%s||end=%s", w.GetName(), w.GetWindowSize(), now.Format("2006-01-02 15:04:05"), 43 | time.Unix(int64(w.GetStart()/time.Second), int64(w.GetStart()%time.Second)).Format("2006-01-02 15:04:05"), 44 | time.Unix(int64(w.GetEnd()/time.Second), int64(w.GetEnd()%time.Second)).Format("2006-01-02 15:04:05")) 45 | } 46 | //12小时一个窗口 47 | window = NewSlidingWindowAssigner("TwelveHourSW", 12*time.Hour, 3*time.Hour, -8*time.Hour) 48 | assignedWindows = window.AssignWindows(time.Duration(now.UnixNano())) 49 | for _, w := range assignedWindows { 50 | o.T().Logf("name=%s||window_size=%s||now=%s||start=%s||end=%s", w.GetName(), w.GetWindowSize(), now.Format("2006-01-02 15:04:05"), 51 | time.Unix(int64(w.GetStart()/time.Second), int64(w.GetStart()%time.Second)).Format("2006-01-02 15:04:05"), 52 | time.Unix(int64(w.GetEnd()/time.Second), int64(w.GetEnd()%time.Second)).Format("2006-01-02 15:04:05")) 53 | } 54 | //1小时一个窗口 55 | window = NewSlidingWindowAssigner("OneHourSW", 1*time.Hour, 10*time.Minute, -8*time.Hour) 56 | assignedWindows = window.AssignWindows(time.Duration(now.UnixNano())) 57 | for _, w := range assignedWindows { 58 | o.T().Logf("name=%s||window_size=%s||now=%s||start=%s||end=%s", w.GetName(), w.GetWindowSize(), now.Format("2006-01-02 15:04:05"), 59 | time.Unix(int64(w.GetStart()/time.Second), int64(w.GetStart()%time.Second)).Format("2006-01-02 15:04:05"), 60 | time.Unix(int64(w.GetEnd()/time.Second), int64(w.GetEnd()%time.Second)).Format("2006-01-02 15:04:05")) 61 | } 62 | } 63 | 64 | //TestSlidingWindowAssigner ... 65 | func TestSlidingWindowAssigner(t *testing.T) { 66 | suite.Run(t, new(SlidingWindowAssignerTestSuite)) 67 | } 68 | 69 | //结果： 70 | //TestSlidingWindowAssigner/Test: sliding_window_assigner_test.go:26: name=SevenDaySW:1605024000000-1605628800000||window_size=168h0m0s||now=2020-11-11 10:44:45||start=2020-11-11 00:00:00||end=2020-11-18 00:00:00 71 | //TestSlidingWindowAssigner/Test: sliding_window_assigner_test.go:26: name=SevenDaySW:1604937600000-1605542400000||window_size=168h0m0s||now=2020-11-11 10:44:45||start=2020-11-10 00:00:00||end=2020-11-17 00:00:00 72 | //TestSlidingWindowAssigner/Test: sliding_window_assigner_test.go:26: name=SevenDaySW:1604851200000-1605456000000||window_size=168h0m0s||now=2020-11-11 10:44:45||start=2020-11-09 00:00:00||end=2020-11-16 00:00:00 73 | //TestSlidingWindowAssigner/Test: sliding_window_assigner_test.go:26: name=SevenDaySW:1604764800000-1605369600000||window_size=168h0m0s||now=2020-11-11 10:44:45||start=2020-11-08 00:00:00||end=2020-11-15 00:00:00 74 | //TestSlidingWindowAssigner/Test: sliding_window_assigner_test.go:26: name=SevenDaySW:1604678400000-1605283200000||window_size=168h0m0s||now=2020-11-11 10:44:45||start=2020-11-07 00:00:00||end=2020-11-14 00:00:00 75 | //TestSlidingWindowAssigner/Test: sliding_window_assigner_test.go:26: name=SevenDaySW:1604592000000-1605196800000||window_size=168h0m0s||now=2020-11-11 10:44:45||start=2020-11-06 00:00:00||end=2020-11-13 00:00:00 76 | //TestSlidingWindowAssigner/Test: sliding_window_assigner_test.go:26: name=SevenDaySW:1604505600000-1605110400000||window_size=168h0m0s||now=2020-11-11 10:44:45||start=2020-11-05 00:00:00||end=2020-11-12 00:00:00 77 | //TestSlidingWindowAssigner/Test: sliding_window_assigner_test.go:34: name=OneDaySW:1605052800000-1605139200000||window_size=24h0m0s||now=2020-11-11 10:44:45||start=2020-11-11 08:00:00||end=2020-11-12 08:00:00 78 | //TestSlidingWindowAssigner/Test: sliding_window_assigner_test.go:34: name=OneDaySW:1605009600000-1605096000000||window_size=24h0m0s||now=2020-11-11 10:44:45||start=2020-11-10 20:00:00||end=2020-11-11 20:00:00 79 | //TestSlidingWindowAssigner/Test: sliding_window_assigner_test.go:42: name=OneDaySW:1605024000000-1605110400000||window_size=24h0m0s||now=2020-11-11 10:44:45||start=2020-11-11 00:00:00||end=2020-11-12 00:00:00 80 | //TestSlidingWindowAssigner/Test: sliding_window_assigner_test.go:42: name=OneDaySW:1604980800000-1605067200000||window_size=24h0m0s||now=2020-11-11 10:44:45||start=2020-11-10 12:00:00||end=2020-11-11 12:00:00 81 | //TestSlidingWindowAssigner/Test: sliding_window_assigner_test.go:50: name=TwelveHourSW:1605056400000-1605099600000||window_size=12h0m0s||now=2020-11-11 10:44:45||start=2020-11-11 09:00:00||end=2020-11-11 21:00:00 82 | //TestSlidingWindowAssigner/Test: sliding_window_assigner_test.go:50: name=TwelveHourSW:1605045600000-1605088800000||window_size=12h0m0s||now=2020-11-11 10:44:45||start=2020-11-11 06:00:00||end=2020-11-11 18:00:00 83 | //TestSlidingWindowAssigner/Test: sliding_window_assigner_test.go:50: name=TwelveHourSW:1605034800000-1605078000000||window_size=12h0m0s||now=2020-11-11 10:44:45||start=2020-11-11 03:00:00||end=2020-11-11 15:00:00 84 | //TestSlidingWindowAssigner/Test: sliding_window_assigner_test.go:50: name=TwelveHourSW:1605024000000-1605067200000||window_size=12h0m0s||now=2020-11-11 10:44:45||start=2020-11-11 00:00:00||end=2020-11-11 12:00:00 85 | //TestSlidingWindowAssigner/Test: sliding_window_assigner_test.go:58: name=OneHourSW:1605062400000-1605066000000||window_size=1h0m0s||now=2020-11-11 10:44:45||start=2020-11-11 10:40:00||end=2020-11-11 11:40:00 86 | //TestSlidingWindowAssigner/Test: sliding_window_assigner_test.go:58: name=OneHourSW:1605061800000-1605065400000||window_size=1h0m0s||now=2020-11-11 10:44:45||start=2020-11-11 10:30:00||end=2020-11-11 11:30:00 87 | //TestSlidingWindowAssigner/Test: sliding_window_assigner_test.go:58: name=OneHourSW:1605061200000-1605064800000||window_size=1h0m0s||now=2020-11-11 10:44:45||start=2020-11-11 10:20:00||end=2020-11-11 11:20:00 88 | //TestSlidingWindowAssigner/Test: sliding_window_assigner_test.go:58: name=OneHourSW:1605060600000-1605064200000||window_size=1h0m0s||now=2020-11-11 10:44:45||start=2020-11-11 10:10:00||end=2020-11-11 11:10:00 89 | //TestSlidingWindowAssigner/Test: sliding_window_assigner_test.go:58: name=OneHourSW:1605060000000-1605063600000||window_size=1h0m0s||now=2020-11-11 10:44:45||start=2020-11-11 10:00:00||end=2020-11-11 11:00:00 90 | //TestSlidingWindowAssigner/Test: sliding_window_assigner_test.go:58: name=OneHourSW:1605059400000-1605063000000||window_size=1h0m0s||now=2020-11-11 10:44:45||start=2020-11-11 09:50:00||end=2020-11-11 10:50:00 91 | -------------------------------------------------------------------------------- /sliding_window_operator.go: -------------------------------------------------------------------------------- 1 | package window 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | ) 8 | 9 | //SlidingWindowOperator ... 10 | type SlidingWindowOperator struct { 11 | name string 12 | size time.Duration 13 | slide time.Duration 14 | offset time.Duration 15 | backend StateBackend 16 | assigner *SlidingWindowAssigner 17 | } 18 | 19 | //NewSlidingWindowOperator ... 20 | //size: 窗口大小 21 | //slide: 滑动的时间 22 | //offset: 偏移量，跟时区有关，北京东八区offset = -8h 23 | func NewSlidingWindowOperator(name string, size, slide, offset time.Duration, backend StateBackend) *SlidingWindowOperator { 24 | return &SlidingWindowOperator{ 25 | name: name, 26 | size: size, 27 | slide: slide, 28 | offset: offset, 29 | backend: backend, 30 | assigner: NewSlidingWindowAssigner(name, size, slide, offset), 31 | } 32 | } 33 | 34 | //GetName ... 35 | func (o *SlidingWindowOperator) GetName() string { 36 | return o.name 37 | } 38 | 39 | //GetSize ... 40 | func (o *SlidingWindowOperator) GetSize() time.Duration { 41 | return o.size 42 | } 43 | 44 | //GetOffset ... 45 | func (o *SlidingWindowOperator) GetOffset() time.Duration { 46 | return o.offset 47 | } 48 | 49 | //GetSlide ... 50 | func (o *SlidingWindowOperator) GetSlide() time.Duration { 51 | return o.slide 52 | } 53 | 54 | //GetAssigner ... 55 | func (o *SlidingWindowOperator) GetAssigner() Assigner { 56 | return o.assigner 57 | } 58 | 59 | //GetStateBackend ... 60 | func (o *SlidingWindowOperator) GetStateBackend() StateBackend { 61 | return o.backend 62 | } 63 | 64 | //Process ... 65 | // timestamp unix时间戳时间戳，单位ns 66 | // event 处理的事件 67 | func (o *SlidingWindowOperator) Process(ctx context.Context, timestamp time.Duration, event interface{}) error { 68 | windows := o.assigner.AssignWindows(timestamp) 69 | var ( 70 | gerr error 71 | err error 72 | ) 73 | for _, window := range windows { 74 | windowState := NewTimeWindowState(window, o.backend) 75 | err = windowState.Update(ctx, event) 76 | if err != nil { 77 | gerr = err 78 | continue 79 | } 80 | } 81 | return gerr 82 | } 83 | 84 | //GetWindow 获取某一时刻的时间窗口 85 | func (o *SlidingWindowOperator) GetWindow(ctx context.Context, timestamp time.Duration) *TimeWindow { 86 | windows := o.assigner.AssignWindows(timestamp) 87 | for i := len(windows) - 1; i >= 0; i-- { 88 | if windows[i].GetEnd() > timestamp { 89 | return windows[i] 90 | } 91 | } 92 | return windows[0] 93 | } 94 | 95 | //GetState 获取某一时刻的状态 96 | func (o *SlidingWindowOperator) GetState(ctx context.Context, timestamp time.Duration) (State, error) { 97 | window := o.GetWindow(ctx, timestamp) 98 | windowState := NewTimeWindowState(window, o.backend) 99 | return windowState.Get(ctx) 100 | } 101 | 102 | //GetCurrentState 获取当前时刻的状态 103 | func (o *SlidingWindowOperator) GetCurrentState(ctx context.Context) (State, error) { 104 | return o.GetState(ctx, time.Duration(time.Now().UnixNano())) 105 | } 106 | 107 | //String ... 108 | func (o SlidingWindowOperator) String() string { 109 | return fmt.Sprintf(`SlidingWindowOperator={"type":%v,"name":"%s","size":"%s","slide":"%s","offset":"%s"}`, 110 | Sliding, o.name, o.size, o.slide, o.offset) 111 | } 112 | 113 | var _ Operator = (*SlidingWindowOperator)(nil) 114 | -------------------------------------------------------------------------------- /sliding_window_operator_test.go: -------------------------------------------------------------------------------- 1 | package window 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | "time" 8 | 9 | "github.com/go-redis/redis/v8" 10 | "github.com/stretchr/testify/suite" 11 | ) 12 | 13 | //SlidingWindowOperatorTestSuite .... 14 | type SlidingWindowOperatorTestSuite struct { 15 | suite.Suite 16 | redisClient *redis.Client 17 | } 18 | 19 | //SetupSuite .... 20 | func (o *SlidingWindowOperatorTestSuite) SetupSuite() { 21 | o.redisClient = redis.NewClient(&redis.Options{ 22 | Addr: "localhost:6379", 23 | Password: "", // no password set 24 | DB: 0, // use default DB 25 | }) 26 | err := o.redisClient.Ping(context.TODO()).Err() 27 | o.Require().Nil(err) 28 | } 29 | 30 | func (o *SlidingWindowOperatorTestSuite) Test() { 31 | var ( 32 | uid int64 //用户ID 33 | windowOperator *SlidingWindowOperator 34 | size time.Duration //窗口大小 35 | slide time.Duration //滑动距离 36 | offset time.Duration //时间偏移量，跟当前所在时区有关 37 | ) 38 | ctx := context.TODO() 39 | uid = 100000 40 | operatorName := fmt.Sprintf("three-day-charge-%d", uid) //窗口算子名字 41 | size = 3 * 24 * time.Hour //窗口大小， 3天，统计3天内，用户uid=100000 充值金额 42 | slide = 24 * time.Hour //滑动距离 43 | offset = -8 * time.Hour //偏移量，北京东8区，所以要减去8小时 44 | //三天时长的滑动窗口，一天滑动一次 45 | windowOperator = NewSlidingWindowOperator(operatorName, size, slide, offset, NewChargeStateBackend(o.redisClient)) 46 | //用户充值的事件 47 | now := time.Now() 48 | events := []*ChargeEvent{ 49 | //4天前充值99 50 | { 51 | UID: uid, 52 | Amount: 99, 53 | Ts: now.Add(-4 * 24 * time.Hour).Unix(), 54 | }, 55 | //三天前充值199 56 | { 57 | UID: uid, 58 | Amount: 199, 59 | Ts: now.Add(-3 * 24 * time.Hour).Unix(), 60 | }, 61 | //两天前充值，92 62 | { 63 | UID: uid, 64 | Amount: 92, 65 | Ts: now.Add(-2 * 24 * time.Hour).Unix(), 66 | }, 67 | //一天前充值180 68 | { 69 | UID: uid, 70 | Amount: 180, 71 | Ts: now.Add(-1 * 24 * time.Hour).Unix(), 72 | }, 73 | //今天充值36 74 | { 75 | UID: uid, 76 | Amount: 36, 77 | Ts: now.Unix(), 78 | }, 79 | } 80 | //每个充值事件都丢进窗口算子进行处理 81 | for _, event := range events { 82 | windowOperator.Process(ctx, time.Duration(event.Ts)*time.Second, event) 83 | } 84 | //昨天开始，3天内的充值金额统计结果 85 | timestamp := now.Add(-24 * time.Hour) 86 | window := windowOperator.GetWindow(ctx, time.Duration(timestamp.UnixNano())) 87 | state, err := windowOperator.GetState(ctx, time.Duration(timestamp.UnixNano())) 88 | o.T().Logf("uid=%d||timestamp=%s||window_size=%s||window_start=%s||window_end=%s||state=%s||err=%v", 89 | uid, timestamp.Format("2006-01-02 15:04:05"), size, 90 | time.Unix(int64(window.GetStart()/time.Second), int64(window.GetStart()%time.Second)).Format("2006-01-02 15:04:05"), 91 | time.Unix(int64(window.GetEnd()/time.Second), int64(window.GetEnd()%time.Second)).Format("2006-01-02 15:04:05"), 92 | state, err) 93 | o.Require().Nil(err) 94 | //471 = 199 + 92 + 180 95 | o.Require().Equal(uint64(471), state.(*ChargeState).TotalAmount) 96 | //清理结果，防止重复执行会失败 97 | windowOperator.GetStateBackend().Del(ctx, window.GetName()) 98 | 99 | //最近3天充值的金额统计结果 100 | timestamp = now 101 | window = windowOperator.GetWindow(ctx, time.Duration(timestamp.UnixNano())) 102 | state, err = windowOperator.GetState(ctx, time.Duration(timestamp.UnixNano())) 103 | o.T().Logf("uid=%d||timestamp=%s||window_size=%s||window_start=%s||window_end=%s||state=%s||err=%v", 104 | uid, timestamp.Format("2006-01-02 15:04:05"), size, 105 | time.Unix(int64(window.GetStart()/time.Second), int64(window.GetStart()%time.Second)).Format("2006-01-02 15:04:05"), 106 | time.Unix(int64(window.GetEnd()/time.Second), int64(window.GetEnd()%time.Second)).Format("2006-01-02 15:04:05"), 107 | state, err) 108 | o.Require().Nil(err) 109 | //308 = 92 + 180 + 36 110 | o.Require().Equal(uint64(308), state.(*ChargeState).TotalAmount) 111 | //清理结果，防止重复执行会失败 112 | windowOperator.GetStateBackend().Del(ctx, window.GetName()) 113 | } 114 | 115 | func TestSlidingWindowOperator(t *testing.T) { 116 | suite.Run(t, new(SlidingWindowOperatorTestSuite)) 117 | } 118 | 119 | //结果： 120 | //TestSlidingWindowOperator/Test: sliding_window_operator_test.go:81: uid=100000||timestamp=2020-11-10 12:39:34||window_size=72h0m0s||window_start=2020-11-08 00:00:00||window_end=2020-11-11 00:00:00||state={"TotalAmount":471}||err= 121 | //TestSlidingWindowOperator/Test: sliding_window_operator_test.go:95: uid=100000||timestamp=2020-11-11 12:39:34||window_size=72h0m0s||window_start=2020-11-09 00:00:00||window_end=2020-11-12 00:00:00||state={"TotalAmount":308}||err= 122 | -------------------------------------------------------------------------------- /state.go: -------------------------------------------------------------------------------- 1 | package window 2 | 3 | //State ... 4 | type State interface { 5 | //String ... 6 | String() string 7 | } 8 | 9 | //DummyState 测试用 10 | type DummyState struct { 11 | } 12 | 13 | //String ... 14 | func (o *DummyState) String() string { 15 | return "DummyState" 16 | } 17 | 18 | var _ State = (*DummyState)(nil) 19 | -------------------------------------------------------------------------------- /state_backend.go: -------------------------------------------------------------------------------- 1 | package window 2 | 3 | import "context" 4 | 5 | //StateBackend ... 6 | type StateBackend interface { 7 | 8 | //Update 更新窗口的状态 9 | Update(ctx context.Context, key string, event interface{}) (State, error) 10 | 11 | //Expire 给窗口状态设置过期时间 12 | Expire(ctx context.Context, key string, expireSeconds int64) error 13 | 14 | //Get 获取窗口的状态 15 | Get(ctx context.Context, key string) (State, error) 16 | 17 | //Del 删除窗口的状态 18 | Del(ctx context.Context, key string) error 19 | } 20 | 21 | //DummyStateBackend 测试用 22 | type DummyStateBackend struct { 23 | } 24 | 25 | //Update ... 26 | func (o *DummyStateBackend) Update(ctx context.Context, key string, event interface{}) (State, error) { 27 | return &DummyState{}, nil 28 | } 29 | 30 | //Expire ... 31 | func (o *DummyStateBackend) Expire(ctx context.Context, key string, expireSeconds int64) error { 32 | return nil 33 | } 34 | 35 | //Get ... 36 | func (o *DummyStateBackend) Get(ctx context.Context, key string) (State, error) { 37 | return &DummyState{}, nil 38 | } 39 | 40 | //Del ... 41 | func (o *DummyStateBackend) Del(ctx context.Context, key string) error { 42 | return nil 43 | } 44 | 45 | var _ StateBackend = (*DummyStateBackend)(nil) 46 | -------------------------------------------------------------------------------- /time_window.go: -------------------------------------------------------------------------------- 1 | package window 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | const ( 9 | windowNameFormat = "%s:%d-%d" 10 | ) 11 | 12 | //TimeWindow 事件窗口，精度ms 13 | type TimeWindow struct { 14 | prefix string 15 | start time.Duration //窗口开始时间戳 16 | end time.Duration //窗口结束时间戳 17 | precise time.Duration //窗口精度,ms 18 | name string 19 | } 20 | 21 | //NewTimeWindow ... 22 | func NewTimeWindow(prefix string, start, end time.Duration) *TimeWindow { 23 | return &TimeWindow{ 24 | prefix: prefix, 25 | start: start, 26 | end: end, 27 | precise: time.Millisecond, 28 | name: fmt.Sprintf(windowNameFormat, prefix, start/time.Millisecond, end/time.Millisecond), 29 | } 30 | } 31 | 32 | //String ... 33 | func (o TimeWindow) String() string { 34 | return fmt.Sprintf(`TimeWindow={"name":"%s", prefix":"%s"","start":%d,"end":%d,"size":"%s","precise":"%s"}`, 35 | o.GetName(), o.prefix, o.start, o.end, o.GetWindowSize(), o.precise) 36 | } 37 | 38 | //GetStart 窗口开始时间戳，包括这个时间 39 | func (o *TimeWindow) GetStart() time.Duration { 40 | return o.start 41 | } 42 | 43 | //GetEnd 窗口结束时间戳，不包括这个时间戳 44 | func (o *TimeWindow) GetEnd() time.Duration { 45 | return o.end 46 | } 47 | 48 | //GetPrecise ... 49 | func (o *TimeWindow) GetPrecise() time.Duration { 50 | return o.precise 51 | } 52 | 53 | //GetName ... 54 | func (o *TimeWindow) GetName() string { 55 | return o.name 56 | //return fmt.Sprintf(windowNameFormat, o.prefix, o.start/o.precise, o.end/o.precise) 57 | } 58 | 59 | //GetWindowSize 窗口的大小 60 | func (o *TimeWindow) GetWindowSize() time.Duration { 61 | return o.end - o.start 62 | } 63 | 64 | //MaxTimestamp 窗口最大的时间戳 65 | func (o *TimeWindow) MaxTimestamp() time.Duration { 66 | return o.end - o.precise 67 | } 68 | 69 | //Intersects Returns if this window intersects the given window. 70 | func (o *TimeWindow) Intersects(other *TimeWindow) bool { 71 | return o.start <= other.end && o.end >= other.start 72 | } 73 | 74 | //Cover ... 75 | func (o *TimeWindow) Cover(other *TimeWindow) *TimeWindow { 76 | return NewTimeWindow(o.prefix, time.Duration(minInt64(int64(o.start), int64(other.start))), time.Duration(maxInt64(int64(o.end), int64(other.end)))) 77 | } 78 | 79 | //GetWindowStartWithOffset Method to get the window start for a timestamp 80 | // timestamp epoch to get the window start. 81 | // offset The offset which window start would be shifted by. 82 | // windowSize The size of the generated windows. 83 | func GetWindowStartWithOffset(timestamp, offset, windowSize time.Duration) time.Duration { 84 | return timestamp - (timestamp-offset+windowSize)%windowSize 85 | } 86 | 87 | func minInt64(first, second int64) int64 { 88 | if first <= second { 89 | return first 90 | } 91 | return second 92 | } 93 | 94 | func maxInt64(first, second int64) int64 { 95 | if first >= second { 96 | return first 97 | } 98 | return second 99 | } 100 | -------------------------------------------------------------------------------- /tumbling_window_assigner.go: -------------------------------------------------------------------------------- 1 | package window 2 | 3 | import "time" 4 | 5 | //TumblingWindowAssigner ... 6 | type TumblingWindowAssigner struct { 7 | name string 8 | size time.Duration 9 | offset time.Duration 10 | } 11 | 12 | //NewTumblingWindowAssigner ... 13 | func NewTumblingWindowAssigner(name string, size, offset time.Duration) *TumblingWindowAssigner { 14 | return &TumblingWindowAssigner{ 15 | name: name, 16 | size: size, 17 | offset: offset, 18 | } 19 | } 20 | 21 | //AssignWindows ... 22 | func (o *TumblingWindowAssigner) AssignWindows(timestamp time.Duration) []*TimeWindow { 23 | start := GetWindowStartWithOffset(timestamp, o.offset, o.size) 24 | return []*TimeWindow{NewTimeWindow(o.name, start, start+o.size)} 25 | } 26 | 27 | var _ Assigner = (*TumblingWindowAssigner)(nil) 28 | -------------------------------------------------------------------------------- /tumbling_window_assigner_test.go: -------------------------------------------------------------------------------- 1 | package window 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/suite" 8 | ) 9 | 10 | //TumblingWindowAssignerTestSuite ... 11 | type TumblingWindowAssignerTestSuite struct { 12 | suite.Suite 13 | } 14 | 15 | //Test ... 16 | func (o *TumblingWindowAssignerTestSuite) Test() { 17 | now := time.Now() 18 | var ( 19 | window *TumblingWindowAssigner 20 | assignedWindows []*TimeWindow 21 | ) 22 | //7天窗口 23 | window = NewTumblingWindowAssigner("SevenDayTW", 7*24*time.Hour, -8*time.Hour) 24 | assignedWindows = window.AssignWindows(time.Duration(now.UnixNano())) 25 | for _, w := range assignedWindows { 26 | o.T().Logf("name=%s||window_size=%s||now=%s||start=%s||end=%s", w.GetName(), w.GetWindowSize(), now.Format("2006-01-02 15:04:05"), 27 | time.Unix(int64(w.GetStart()/time.Second), int64(w.GetStart()%time.Second)).Format("2006-01-02 15:04:05"), 28 | time.Unix(int64(w.GetEnd()/time.Second), int64(w.GetEnd()%time.Second)).Format("2006-01-02 15:04:05")) 29 | } 30 | //1天的窗口，offset = 0 31 | window = NewTumblingWindowAssigner("OneDayTW", 24*time.Hour, 0) 32 | assignedWindows = window.AssignWindows(time.Duration(now.UnixNano())) 33 | for _, w := range assignedWindows { 34 | o.T().Logf("name=%s||window_size=%s||now=%s||start=%s||end=%s", w.GetName(), w.GetWindowSize(), now.Format("2006-01-02 15:04:05"), 35 | time.Unix(int64(w.GetStart()/time.Second), int64(w.GetStart()%time.Second)).Format("2006-01-02 15:04:05"), 36 | time.Unix(int64(w.GetEnd()/time.Second), int64(w.GetEnd()%time.Second)).Format("2006-01-02 15:04:05")) 37 | } 38 | //1天的窗口，北京东8区，快8小时，offset = -8h 39 | window = NewTumblingWindowAssigner("OneDayTW", 24*time.Hour, -8*time.Hour) 40 | assignedWindows = window.AssignWindows(time.Duration(now.UnixNano())) 41 | for _, w := range assignedWindows { 42 | o.T().Logf("name=%s||window_size=%s||now=%s||start=%s||end=%s", w.GetName(), w.GetWindowSize(), now.Format("2006-01-02 15:04:05"), 43 | time.Unix(int64(w.GetStart()/time.Second), int64(w.GetStart()%time.Second)).Format("2006-01-02 15:04:05"), 44 | time.Unix(int64(w.GetEnd()/time.Second), int64(w.GetEnd()%time.Second)).Format("2006-01-02 15:04:05")) 45 | } 46 | //1天的窗口，美国西8区时间 47 | window = NewTumblingWindowAssigner("OneDayTW", 24*time.Hour, 8*time.Hour) 48 | assignedWindows = window.AssignWindows(time.Duration(now.UnixNano())) 49 | for _, w := range assignedWindows { 50 | o.T().Logf("name=%s||window_size=%s||now=%s||start=%s||end=%s", w.GetName(), w.GetWindowSize(), 51 | now.In(time.FixedZone("GMT-8", -8*60*60)).Format("2006-01-02 15:04:05"), 52 | time.Unix(int64(w.GetStart()/time.Second), int64(w.GetStart()%time.Second)).In(time.FixedZone("GMT-8", -8*60*60)).Format("2006-01-02 15:04:05"), 53 | time.Unix(int64(w.GetEnd()/time.Second), int64(w.GetEnd()%time.Second)).In(time.FixedZone("GMT-8", -8*60*60)).Format("2006-01-02 15:04:05")) 54 | } 55 | //12小时一个窗口 56 | window = NewTumblingWindowAssigner("TwelveHourTW", 12*time.Hour, -8*time.Hour) 57 | assignedWindows = window.AssignWindows(time.Duration(now.UnixNano())) 58 | for _, w := range assignedWindows { 59 | o.T().Logf("name=%s||window_size=%s||now=%s||start=%s||end=%s", w.GetName(), w.GetWindowSize(), now.Format("2006-01-02 15:04:05"), 60 | time.Unix(int64(w.GetStart()/time.Second), int64(w.GetStart()%time.Second)).Format("2006-01-02 15:04:05"), 61 | time.Unix(int64(w.GetEnd()/time.Second), int64(w.GetEnd()%time.Second)).Format("2006-01-02 15:04:05")) 62 | } 63 | //8小时一个窗口 64 | window = NewTumblingWindowAssigner("EightHourTW", 8*time.Hour, -8*time.Hour) 65 | assignedWindows = window.AssignWindows(time.Duration(now.UnixNano())) 66 | for _, w := range assignedWindows { 67 | o.T().Logf("name=%s||window_size=%s||now=%s||start=%s||end=%s", w.GetName(), w.GetWindowSize(), now.Format("2006-01-02 15:04:05"), 68 | time.Unix(int64(w.GetStart()/time.Second), int64(w.GetStart()%time.Second)).Format("2006-01-02 15:04:05"), 69 | time.Unix(int64(w.GetEnd()/time.Second), int64(w.GetEnd()%time.Second)).Format("2006-01-02 15:04:05")) 70 | } 71 | //1小时一个窗口 72 | window = NewTumblingWindowAssigner("OneHourTW", 1*time.Hour, -8*time.Hour) 73 | assignedWindows = window.AssignWindows(time.Duration(now.UnixNano())) 74 | for _, w := range assignedWindows { 75 | o.T().Logf("name=%s||window_size=%s||now=%s||start=%s||end=%s", w.GetName(), w.GetWindowSize(), now.Format("2006-01-02 15:04:05"), 76 | time.Unix(int64(w.GetStart()/time.Second), int64(w.GetStart()%time.Second)).Format("2006-01-02 15:04:05"), 77 | time.Unix(int64(w.GetEnd()/time.Second), int64(w.GetEnd()%time.Second)).Format("2006-01-02 15:04:05")) 78 | } 79 | } 80 | 81 | //TestTumblingWindowAssigner ... 82 | func TestTumblingWindowAssigner(t *testing.T) { 83 | suite.Run(t, new(TumblingWindowAssignerTestSuite)) 84 | } 85 | 86 | //结果： 87 | //TestTumblingWindowAssigner/Test: tumbling_window_assigner_test.go:26: name=SevenDayTW:1605110400000-1605715200000||window_size=168h0m0s||now=2020-11-13 16:21:59||start=2020-11-12 00:00:00||end=2020-11-19 00:00:00 88 | //TestTumblingWindowAssigner/Test: tumbling_window_assigner_test.go:34: name=OneDayTW:1605225600000-1605312000000||window_size=24h0m0s||now=2020-11-13 16:21:59||start=2020-11-13 08:00:00||end=2020-11-14 08:00:00 89 | //TestTumblingWindowAssigner/Test: tumbling_window_assigner_test.go:42: name=OneDayTW:1605196800000-1605283200000||window_size=24h0m0s||now=2020-11-13 16:21:59||start=2020-11-13 00:00:00||end=2020-11-14 00:00:00 90 | //TestTumblingWindowAssigner/Test: tumbling_window_assigner_test.go:50: name=OneDayTW:1605254400000-1605340800000||window_size=24h0m0s||now=2020-11-13 00:21:59||start=2020-11-13 00:00:00||end=2020-11-14 00:00:00 91 | //TestTumblingWindowAssigner/Test: tumbling_window_assigner_test.go:59: name=TwelveHourTW:1605240000000-1605283200000||window_size=12h0m0s||now=2020-11-13 16:21:59||start=2020-11-13 12:00:00||end=2020-11-14 00:00:00 92 | //TestTumblingWindowAssigner/Test: tumbling_window_assigner_test.go:67: name=EightHourTW:1605254400000-1605283200000||window_size=8h0m0s||now=2020-11-13 16:21:59||start=2020-11-13 16:00:00||end=2020-11-14 00:00:00 93 | //TestTumblingWindowAssigner/Test: tumbling_window_assigner_test.go:75: name=OneHourTW:1605254400000-1605258000000||window_size=1h0m0s||now=2020-11-13 16:21:59||start=2020-11-13 16:00:00||end=2020-11-13 17:00:00 94 | -------------------------------------------------------------------------------- /tumbling_window_operator.go: -------------------------------------------------------------------------------- 1 | package window 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | ) 8 | 9 | //TumblingWindowOperator ... 10 | type TumblingWindowOperator struct { 11 | name string 12 | size time.Duration 13 | offset time.Duration 14 | backend StateBackend 15 | assigner *TumblingWindowAssigner 16 | } 17 | 18 | //NewTumblingWindowOperator ... 19 | //size: 窗口大小 20 | //offset: 偏移量，跟时区有关，北京东八区offset = -8h 21 | func NewTumblingWindowOperator(name string, size, offset time.Duration, backend StateBackend) *TumblingWindowOperator { 22 | return &TumblingWindowOperator{ 23 | name: name, 24 | size: size, 25 | offset: offset, 26 | backend: backend, 27 | assigner: NewTumblingWindowAssigner(name, size, offset), 28 | } 29 | } 30 | 31 | //GetName ... 32 | func (o *TumblingWindowOperator) GetName() string { 33 | return o.name 34 | } 35 | 36 | //GetSize ... 37 | func (o *TumblingWindowOperator) GetSize() time.Duration { 38 | return o.size 39 | } 40 | 41 | //GetOffset ... 42 | func (o *TumblingWindowOperator) GetOffset() time.Duration { 43 | return o.offset 44 | } 45 | 46 | //GetAssigner ... 47 | func (o *TumblingWindowOperator) GetAssigner() Assigner { 48 | return o.assigner 49 | } 50 | 51 | //GetStateBackend ... 52 | func (o *TumblingWindowOperator) GetStateBackend() StateBackend { 53 | return o.backend 54 | } 55 | 56 | //Process ... 57 | // timestamp unix时间戳时间戳，单位ns 58 | // event 处理的事件 59 | func (o *TumblingWindowOperator) Process(ctx context.Context, timestamp time.Duration, event interface{}) error { 60 | windows := o.assigner.AssignWindows(timestamp) 61 | var ( 62 | gerr error 63 | err error 64 | ) 65 | for _, window := range windows { 66 | windowState := NewTimeWindowState(window, o.backend) 67 | err = windowState.Update(ctx, event) 68 | if err != nil { 69 | gerr = err 70 | continue 71 | } 72 | } 73 | return gerr 74 | } 75 | 76 | //GetWindow 获取某一时刻的时间窗口 77 | func (o *TumblingWindowOperator) GetWindow(ctx context.Context, timestamp time.Duration) *TimeWindow { 78 | windows := o.assigner.AssignWindows(timestamp) 79 | return windows[0] 80 | } 81 | 82 | //GetState 获取某一时刻的状态 83 | func (o *TumblingWindowOperator) GetState(ctx context.Context, timestamp time.Duration) (State, error) { 84 | window := o.GetWindow(ctx, timestamp) 85 | windowState := NewTimeWindowState(window, o.backend) 86 | return windowState.Get(ctx) 87 | } 88 | 89 | //GetCurrentState 获取当前时刻的状态 90 | func (o *TumblingWindowOperator) GetCurrentState(ctx context.Context) (State, error) { 91 | return o.GetState(ctx, time.Duration(time.Now().UnixNano())) 92 | } 93 | 94 | //String ... 95 | func (o TumblingWindowOperator) String() string { 96 | return fmt.Sprintf(`TumblingWindowOperator={"type":%v,"name":"%s","size":"%s","offset":"%s"}`, 97 | Tumbling, o.name, o.size, o.offset) 98 | } 99 | 100 | var _ Operator = (*TumblingWindowOperator)(nil) 101 | -------------------------------------------------------------------------------- /tumbling_window_operator_test.go: -------------------------------------------------------------------------------- 1 | package window 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | "time" 8 | 9 | "github.com/go-redis/redis/v8" 10 | "github.com/stretchr/testify/suite" 11 | ) 12 | 13 | //TumblingWindowOperatorTestSuite ... 14 | type TumblingWindowOperatorTestSuite struct { 15 | suite.Suite 16 | redisClient *redis.Client 17 | } 18 | 19 | func (o *TumblingWindowOperatorTestSuite) SetupSuite() { 20 | o.redisClient = redis.NewClient(&redis.Options{ 21 | Addr: "localhost:6379", 22 | Password: "", // no password set 23 | DB: 0, // use default DB 24 | }) 25 | err := o.redisClient.Ping(context.TODO()).Err() 26 | o.Require().Nil(err) 27 | } 28 | 29 | func (o *TumblingWindowOperatorTestSuite) Test() { 30 | var ( 31 | uid int64 //用户ID 32 | windowOperator *TumblingWindowOperator 33 | size time.Duration //窗口大小 34 | offset time.Duration //时间偏移量，跟当前所在时区有关 35 | ) 36 | ctx := context.TODO() 37 | uid = 100000 38 | operatorName := fmt.Sprintf("one-day-charge-%d", uid) //窗口算子名字 39 | size = 24 * time.Hour //窗口大小， 1天，统计一天内，用户uid=100000 充值金额 40 | offset = -8 * time.Hour //偏移量，北京东8区，所以要减去8小时 41 | //一天大小的滚动窗口 42 | windowOperator = NewTumblingWindowOperator(operatorName, size, offset, NewChargeStateBackend(o.redisClient)) 43 | //用户充值的事件 44 | now := time.Now() 45 | events := []*ChargeEvent{ 46 | //昨天第一次充值，99 47 | { 48 | UID: uid, 49 | Amount: 99, 50 | Ts: now.Add(-24 * time.Hour).Unix(), 51 | }, 52 | //昨天第二次充值，199 53 | { 54 | UID: uid, 55 | Amount: 199, 56 | Ts: now.Add(-24 * time.Hour).Unix(), 57 | }, 58 | //今天第一次充值，92 59 | { 60 | UID: uid, 61 | Amount: 92, 62 | Ts: now.Unix(), 63 | }, 64 | //今天第二次充值，180 65 | { 66 | UID: uid, 67 | Amount: 180, 68 | Ts: now.Unix(), 69 | }, 70 | //今天第二次充值，36 71 | { 72 | UID: uid, 73 | Amount: 36, 74 | Ts: now.Unix(), 75 | }, 76 | } 77 | //每个事件都丢进窗口算子处理 78 | for _, event := range events { 79 | windowOperator.Process(ctx, time.Duration(event.Ts)*time.Second, event) 80 | } 81 | //昨天充值金额统计结果 82 | timestamp := now.Add(-24 * time.Hour) 83 | window := windowOperator.GetWindow(ctx, time.Duration(timestamp.UnixNano())) 84 | state, err := windowOperator.GetState(ctx, time.Duration(timestamp.UnixNano())) 85 | o.T().Logf("uid=%d||timestamp=%s||window_size=%s||window_start=%s||window_end=%s||state=%s||err=%v", 86 | uid, timestamp.Format("2006-01-02 15:04:05"), size, 87 | time.Unix(int64(window.GetStart()/time.Second), int64(window.GetStart()%time.Second)).Format("2006-01-02 15:04:05"), 88 | time.Unix(int64(window.GetEnd()/time.Second), int64(window.GetEnd()%time.Second)).Format("2006-01-02 15:04:05"), 89 | state, err) 90 | o.Require().Nil(err) 91 | o.Require().Equal(uint64(298), state.(*ChargeState).TotalAmount) 92 | //清理结果，防止重复执行会失败 93 | windowOperator.GetStateBackend().Del(ctx, window.GetName()) 94 | 95 | //今天充值的金额统计结果 96 | timestamp = now 97 | window = windowOperator.GetWindow(ctx, time.Duration(timestamp.UnixNano())) 98 | state, err = windowOperator.GetState(ctx, time.Duration(timestamp.UnixNano())) 99 | o.T().Logf("uid=%d||timestamp=%s||window_size=%s||window_start=%s||window_end=%s||state=%s||err=%v", 100 | uid, timestamp.Format("2006-01-02 15:04:05"), size, 101 | time.Unix(int64(window.GetStart()/time.Second), int64(window.GetStart()%time.Second)).Format("2006-01-02 15:04:05"), 102 | time.Unix(int64(window.GetEnd()/time.Second), int64(window.GetEnd()%time.Second)).Format("2006-01-02 15:04:05"), 103 | state, err) 104 | o.Require().Nil(err) 105 | o.Require().Equal(uint64(308), state.(*ChargeState).TotalAmount) 106 | //清理结果，防止重复执行会失败 107 | windowOperator.GetStateBackend().Del(ctx, window.GetName()) 108 | } 109 | 110 | //TestTumblingWindowOperator ... 111 | func TestTumblingWindowOperator(t *testing.T) { 112 | suite.Run(t, new(TumblingWindowOperatorTestSuite)) 113 | } 114 | 115 | //结果： 116 | //TestTumblingWindowOperator/Test: tumbling_window_operator_test.go:80: uid=100000||timestamp=2020-11-10 11:37:15||window_size=24h0m0s||window_start=2020-11-10 00:00:00||window_end=2020-11-11 00:00:00||state={"TotalAmount":298}||err= 117 | //TestTumblingWindowOperator/Test: tumbling_window_operator_test.go:95: uid=100000||timestamp=2020-11-11 11:37:15||window_size=24h0m0s||window_start=2020-11-11 00:00:00||window_end=2020-11-12 00:00:00||state={"TotalAmount":308}||err= 118 | -------------------------------------------------------------------------------- /window_assigner.go: -------------------------------------------------------------------------------- 1 | package window 2 | 3 | import "time" 4 | 5 | //Assigner ... 6 | type Assigner interface { 7 | 8 | //AssignWindows ... 9 | AssignWindows(timestamp time.Duration) []*TimeWindow 10 | } 11 | -------------------------------------------------------------------------------- /window_operator.go: -------------------------------------------------------------------------------- 1 | package window 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | //Operator ... 9 | type Operator interface { 10 | 11 | //Process 窗口统计处理 12 | //timestamp unix的时间戳，单位ns 13 | //event: 事件 14 | Process(ctx context.Context, timestamp time.Duration, event interface{}) error 15 | 16 | //GetState 获取某一时刻窗口状态 17 | //timestamp unix时间戳，单位ns 18 | GetState(ctx context.Context, timestamp time.Duration) (State, error) 19 | 20 | //GetCurrentState 获取当前时刻的窗口状态 21 | GetCurrentState(ctx context.Context) (State, error) 22 | 23 | //GetWindow 获取某一时刻的时间窗口 24 | GetWindow(ctx context.Context, timestamp time.Duration) *TimeWindow 25 | } 26 | -------------------------------------------------------------------------------- /window_state.go: -------------------------------------------------------------------------------- 1 | package window 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | //TimeWindowState ... 9 | type TimeWindowState struct { 10 | window *TimeWindow 11 | backend StateBackend 12 | } 13 | 14 | //NewTimeWindowState ... 15 | func NewTimeWindowState(window *TimeWindow, backend StateBackend) *TimeWindowState { 16 | return &TimeWindowState{ 17 | window: window, 18 | backend: backend, 19 | } 20 | } 21 | 22 | //Get ... 23 | func (o *TimeWindowState) Get(ctx context.Context) (State, error) { 24 | return o.backend.Get(ctx, o.window.GetName()) 25 | } 26 | 27 | //Update ... 28 | func (o *TimeWindowState) Update(ctx context.Context, event interface{}) error { 29 | _, err := o.backend.Update(ctx, o.window.GetName(), event) 30 | if err != nil { 31 | return err 32 | } 33 | return o.backend.Expire(ctx, o.window.GetName(), int64(o.window.GetWindowSize()/time.Second)) 34 | } 35 | 36 | //Del ... 37 | func (o *TimeWindowState) Del(ctx context.Context) error { 38 | return o.backend.Del(ctx, o.window.GetName()) 39 | } 40 | -------------------------------------------------------------------------------- /window_state_test.go: -------------------------------------------------------------------------------- 1 | package window 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/suite" 9 | ) 10 | 11 | //WindowStateTestSuite ... 12 | type WindowStateTestSuite struct { 13 | suite.Suite 14 | } 15 | 16 | func (o *WindowStateTestSuite) Test() { 17 | window := NewTimeWindow("window-state-test", time.Duration(time.Now().UnixNano()), time.Duration(time.Now().Add(24*time.Hour).UnixNano())) 18 | windowState := NewTimeWindowState(window, new(DummyStateBackend)) 19 | ctx := context.TODO() 20 | err := windowState.Update(ctx, nil) 21 | o.Require().Nil(err) 22 | state, err := windowState.Get(ctx) 23 | o.Require().Nil(err) 24 | o.T().Logf("state=%s", state) 25 | err = windowState.Del(ctx) 26 | o.Require().Nil(err) 27 | } 28 | 29 | //TestWindowState ... 30 | func TestWindowState(t *testing.T) { 31 | suite.Run(t, new(WindowStateTestSuite)) 32 | } 33 | -------------------------------------------------------------------------------- /windows.go: -------------------------------------------------------------------------------- 1 | package window 2 | 3 | import "sort" 4 | 5 | //Windows ... 6 | type Windows []*TimeWindow 7 | 8 | //NewWindows ... 9 | func NewWindows() Windows { 10 | return make([]*TimeWindow, 0) 11 | } 12 | 13 | //Len ... 14 | func (o Windows) Len() int { 15 | return len(o) 16 | } 17 | 18 | //Swap ... 19 | func (o Windows) Swap(i, j int) { 20 | o[i], o[j] = o[j], o[i] 21 | } 22 | 23 | //Less ... 24 | func (o Windows) Less(i, j int) bool { 25 | return o[i].GetStart() <= o[j].GetStart() 26 | } 27 | 28 | //Sort ... 29 | func (o *Windows) Sort() { 30 | sort.Stable(o) 31 | } 32 | 33 | //Add ... 34 | func (o *Windows) Add(w *TimeWindow) bool { 35 | for _, tw := range *o { 36 | //w已经在了，直接返回 37 | if tw.GetName() == w.GetName() { 38 | return false 39 | } 40 | } 41 | //添加到windows中 42 | *o = append(*o, w) 43 | //从小到大排列 44 | o.Sort() 45 | return true 46 | } 47 | 48 | //Peek ... 49 | func (o Windows) Peek() *TimeWindow { 50 | if len(o) == 0 { 51 | return nil 52 | } 53 | return o[0] 54 | } 55 | 56 | //PopFront ... 57 | func (o *Windows) PopFront() *TimeWindow { 58 | if len(*o) == 0 { 59 | return nil 60 | } 61 | tw := (*o)[0] 62 | *o = (*o)[1:] 63 | return tw 64 | } 65 | -------------------------------------------------------------------------------- /windows_test.go: -------------------------------------------------------------------------------- 1 | package window 2 | 3 | import ( 4 | "github.com/stretchr/testify/suite" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | type WindowsTestSuite struct { 10 | suite.Suite 11 | } 12 | 13 | func (o *WindowsTestSuite) TestAll() { 14 | var ( 15 | w *TimeWindow 16 | added bool 17 | ) 18 | windows := NewWindows() 19 | prefix := "test-windows" 20 | w1 := NewTimeWindow(prefix, 1*time.Second, 5*time.Second) 21 | w2 := NewTimeWindow(prefix, 3*time.Second, 7*time.Second) 22 | w3 := NewTimeWindow(prefix, 2*time.Second, 6*time.Second) 23 | 24 | added = windows.Add(w1) 25 | o.T().Logf("w=%s||added=%v", w1, added) 26 | o.Require().True(added) 27 | o.Require().Len(windows, 1) 28 | 29 | added = windows.Add(w1) 30 | o.T().Logf("w=%s||added=%v", w1, added) 31 | o.Require().False(added) 32 | o.Require().Len(windows, 1) 33 | 34 | added = windows.Add(w2) 35 | o.T().Logf("w=%s||added=%v", w2, added) 36 | o.Require().True(added) 37 | o.Require().Len(windows, 2) 38 | w = windows.Peek() 39 | o.Require().Equal(w1, w) 40 | 41 | added = windows.Add(w3) 42 | o.T().Logf("w=%s||added=%v", w3, added) 43 | o.Require().True(added) 44 | o.Require().Len(windows, 3) 45 | w = windows.Peek() 46 | o.Require().Equal(w1, w) 47 | 48 | //从小到大 pop 49 | w = windows.PopFront() 50 | o.Require().Equal(w1, w) 51 | w = windows.PopFront() 52 | o.Require().Equal(w3, w) 53 | w = windows.PopFront() 54 | o.Require().Equal(w2, w) 55 | } 56 | 57 | func TestWindows(t *testing.T) { 58 | suite.Run(t, new(WindowsTestSuite)) 59 | } 60 | --------------------------------------------------------------------------------