├── .gitignore ├── E0B82999-82BB-4963-BC2D-FDEAF42230FC.jpeg ├── README.md ├── bandwidth.png ├── common ├── Errors.go └── Protocol.go ├── cpu.png ├── gateway ├── Bucket.go ├── Config.go ├── ConnMgr.go ├── Merger.go ├── Room.go ├── Service.go ├── Stats.go ├── WSConnection.go ├── WSHandler.go ├── WSServer.go ├── cli │ ├── client.html │ ├── default.key │ ├── default.pem │ ├── gateway.json │ └── main.go └── test │ ├── client.go │ └── stats.go ├── go.mod ├── go.sum ├── logic ├── Config.go ├── GateConn.go ├── GateConnMgr.go ├── Service.go ├── Stats.go └── cli │ ├── logic.json │ └── main.go └── xingqiu.png /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | -------------------------------------------------------------------------------- /E0B82999-82BB-4963-BC2D-FDEAF42230FC.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/owenliang/go-push/b5b6a2e1cf9172762800f263198b65058a99cbe9/E0B82999-82BB-4963-BC2D-FDEAF42230FC.jpeg -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-push 2 | 3 | golang实现的、可扩展的通用消息推送原型。 4 | 5 | # 安装 6 | 7 | 已升级到golang1.13，基于gomod管理依赖。 8 | 9 | * 下载go-push 10 | 11 | ``` 12 | go get github.com/owenliang/go-push 13 | ``` 14 | 15 | * 安装依赖 16 | 17 | ``` 18 | export GOPROXY=goproxy.io 19 | go mod download 20 | ``` 21 | 22 | * 编译gateway服务 23 | 24 | ``` 25 | cd gateway/cli && go build && cd - 26 | ``` 27 | 28 | * 编译logic服务 29 | 30 | ``` 31 | cd logic/cli && go build && cd - 32 | ``` 33 | 34 | # 架构 35 | 36 | * gateway: 长连接网关 37 | * 海量长连接按BUCKET打散, 减小推送遍历的锁粒度 38 | * 按广播/房间粒度的消息前置合并, 减少编码CPU损耗, 减少系统网络调用, 巨幅提升吞吐 39 | * logic: 逻辑服务器 40 | * 本身无状态, 负责将推送消息分发到所有gateway节点 41 | * 对调用方暴露HTTP/1接口, 方便业务对接 42 | * 采用HTTP/2长连接RPC向gateway集群分发消息 43 | 44 | # 潜在问题 45 | 46 | * 推送主要瓶颈是gateway层而不是内部通讯, 所以gateway和logic之间仍旧采用了小包通讯(对网卡有PPS压力), 同时logic为业务提供了批量推送接口来缓解特殊需求. 47 | 48 | # 压测 49 | 50 | ## 环境 51 | 52 | * 16 vcore 53 | * client, logic, gateway deployed together 54 | 55 | ## 带宽 56 | 57 | ![bandwidth](https://github.com/owenliang/go-push/blob/master/bandwidth.png?raw=true) 58 | 59 | ## CPU占用 60 | 61 | ![cpu usage](https://github.com/owenliang/go-push/blob/master/cpu.png?raw=true) 62 | 63 | # logic的推送API 64 | 65 | * 全员广播 66 | 67 | ``` 68 | curl http://localhost:7799/push/all -d 'items=[{"msg": "hi"},{"msg": "bye"}]' 69 | ``` 70 | 71 | * 房间广播 72 | 73 | ``` 74 | curl http://localhost:7799/push/room -d 'room=default&items=[{"msg": "hi"},{"msg": "bye"}]' 75 | ``` 76 | 77 | ## gateway的websocekt协议 78 | 79 | * PING(客户端->服务端) 80 | 81 | ``` 82 | {"type": "PING", "data": {}} 83 | ``` 84 | 85 | * PONG(服务端->客户端) 86 | 87 | ``` 88 | {"type": "PONG", "data": {}} 89 | ``` 90 | 91 | * JOIN(客户端->服务端) 92 | 93 | ``` 94 | {"type": "JOIN", "data": {"room": "fengtimo"}} 95 | ``` 96 | 97 | * LEAVE(客户端->服务端) 98 | 99 | ``` 100 | {"type": "LEAVE", "data": {"room": "fengtimo"}} 101 | ``` 102 | 103 | * PUSH(服务端->客户端) 104 | 105 | ``` 106 | {"type": "PUSH", "data": {"items": [{"name": "go-push"}, {"age": "1"}]}} 107 | ``` 108 | 109 | # 加入社群 110 | 111 | 我的博客： 112 | 113 | [鱼儿的博客](https://yuerblog.cc) 114 | 115 | Go微信群： 116 | 117 | ![wechat](https://github.com/owenliang/go-push/blob/master/E0B82999-82BB-4963-BC2D-FDEAF42230FC.jpeg?raw=true) 118 | 119 | 知识星球（独家知识）： 120 | ![xingqiu](https://github.com/owenliang/go-push/blob/master/xingqiu.png?raw=true) -------------------------------------------------------------------------------- /bandwidth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/owenliang/go-push/b5b6a2e1cf9172762800f263198b65058a99cbe9/bandwidth.png -------------------------------------------------------------------------------- /common/Errors.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "errors" 4 | 5 | var ( 6 | ERR_CONNECTION_LOSS = errors.New("ERR_CONNECTION_LOSS") 7 | 8 | ERR_SEND_MESSAGE_FULL = errors.New("ERR_SEND_MESSAGE_FULL") 9 | 10 | ERR_JOIN_ROOM_TWICE = errors.New("ERR_JOIN_ROOM_TWICE") 11 | 12 | ERR_NOT_IN_ROOM = errors.New("ERR_NOT_IN_ROOM") 13 | 14 | ERR_ROOM_ID_INVALID = errors.New("ERR_ROOM_ID_INVALID") 15 | 16 | ERR_DISPATCH_CHANNEL_FULL = errors.New("ERR_DISPATCH_CHANNEL_FULL") 17 | 18 | ERR_MERGE_CHANNEL_FULL = errors.New("ERR_MERGE_CHANNEL_FULL") 19 | 20 | ERR_CERT_INVALID = errors.New("ERR_CERT_INVALID") 21 | 22 | ERR_LOGIC_DISPATCH_CHANNEL_FULL = errors.New("ERR_LOGIC_DISPATCH_CHANNEL_FULL") 23 | ) 24 | -------------------------------------------------------------------------------- /common/Protocol.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/gorilla/websocket" 6 | ) 7 | 8 | // 推送类型 9 | const ( 10 | PUSH_TYPE_ROOM = 1 // 推送房间 11 | PUSH_TYPE_ALL = 2 // 推送在线 12 | ) 13 | 14 | // websocket的Message对象 15 | type WSMessage struct { 16 | MsgType int 17 | MsgData []byte 18 | } 19 | 20 | // 业务消息的固定格式(type+data) 21 | type BizMessage struct { 22 | Type string `json:"type"` // type消息类型: PING, PONG, JOIN, LEAVE, PUSH 23 | Data json.RawMessage `json:"data"` // data数据字段 24 | } 25 | 26 | // Data数据类型 27 | 28 | // PUSH 29 | type BizPushData struct { 30 | Items []*json.RawMessage `json:"items"` 31 | } 32 | 33 | // PING 34 | type BizPingData struct {} 35 | 36 | // PONG 37 | type BizPongData struct {} 38 | 39 | // JOIN 40 | type BizJoinData struct { 41 | Room string `json:"room"` 42 | } 43 | 44 | // LEAVE 45 | type BizLeaveData struct { 46 | Room string `json:"room"` 47 | } 48 | 49 | func BuildWSMessage(msgType int, msgData []byte) (wsMessage *WSMessage) { 50 | return &WSMessage{ 51 | MsgType: msgType, 52 | MsgData: msgData, 53 | } 54 | } 55 | 56 | func EncodeWSMessage(bizMessage *BizMessage) (wsMessage *WSMessage, err error){ 57 | var ( 58 | buf []byte 59 | ) 60 | if buf, err = json.Marshal(*bizMessage); err != nil { 61 | return 62 | } 63 | wsMessage = &WSMessage{websocket.TextMessage, buf} 64 | return 65 | } 66 | 67 | // 解析{"type": "PING", "data": {...}}的包 68 | func DecodeBizMessage(buf []byte) (bizMessage *BizMessage, err error) { 69 | var ( 70 | bizMsgObj BizMessage 71 | ) 72 | 73 | if err = json.Unmarshal(buf, &bizMsgObj); err != nil { 74 | return 75 | } 76 | 77 | bizMessage = &bizMsgObj 78 | return 79 | } 80 | -------------------------------------------------------------------------------- /cpu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/owenliang/go-push/b5b6a2e1cf9172762800f263198b65058a99cbe9/cpu.png -------------------------------------------------------------------------------- /gateway/Bucket.go: -------------------------------------------------------------------------------- 1 | package gateway 2 | 3 | import ( 4 | "sync" 5 | "github.com/owenliang/go-push/common" 6 | ) 7 | 8 | type Bucket struct { 9 | rwMutex sync.RWMutex 10 | index int // 我是第几个桶 11 | id2Conn map[uint64]*WSConnection // 连接列表(key=连接唯一ID) 12 | rooms map[string]*Room // 房间列表 13 | } 14 | 15 | func InitBucket(bucketIdx int) (bucket *Bucket) { 16 | bucket = &Bucket{ 17 | index: bucketIdx, 18 | id2Conn: make(map[uint64]*WSConnection), 19 | rooms: make(map[string]*Room), 20 | } 21 | return 22 | } 23 | 24 | func (bucket *Bucket) AddConn(wsConn *WSConnection) { 25 | bucket.rwMutex.Lock() 26 | defer bucket.rwMutex.Unlock() 27 | 28 | bucket.id2Conn[wsConn.connId] = wsConn 29 | } 30 | 31 | func (bucket *Bucket) DelConn(wsConn *WSConnection) { 32 | bucket.rwMutex.Lock() 33 | defer bucket.rwMutex.Unlock() 34 | 35 | delete(bucket.id2Conn, wsConn.connId) 36 | } 37 | 38 | func (bucket *Bucket) JoinRoom(roomId string, wsConn *WSConnection) (err error) { 39 | var ( 40 | existed bool 41 | room *Room 42 | ) 43 | bucket.rwMutex.Lock() 44 | defer bucket.rwMutex.Unlock() 45 | 46 | // 找到房间 47 | if room, existed = bucket.rooms[roomId]; !existed { 48 | room = InitRoom(roomId) 49 | bucket.rooms[roomId] = room 50 | RoomCount_INCR() 51 | } 52 | // 加入房间 53 | err = room.Join(wsConn) 54 | return 55 | } 56 | 57 | func (bucket *Bucket) LeaveRoom(roomId string, wsConn *WSConnection) (err error) { 58 | var ( 59 | existed bool 60 | room *Room 61 | ) 62 | bucket.rwMutex.Lock() 63 | defer bucket.rwMutex.Unlock() 64 | 65 | // 找到房间 66 | if room, existed = bucket.rooms[roomId]; !existed { 67 | err = common.ERR_NOT_IN_ROOM 68 | return 69 | } 70 | 71 | err = room.Leave(wsConn) 72 | 73 | // 房间为空, 则删除 74 | if room.Count() == 0 { 75 | delete(bucket.rooms, roomId) 76 | RoomCount_DESC() 77 | } 78 | return 79 | } 80 | 81 | // 推送给Bucket内所有用户 82 | func (bucket *Bucket) PushAll(wsMsg *common.WSMessage) { 83 | var ( 84 | wsConn *WSConnection 85 | ) 86 | 87 | // 锁Bucket 88 | bucket.rwMutex.RLock() 89 | defer bucket.rwMutex.RUnlock() 90 | 91 | // 全量非阻塞推送 92 | for _, wsConn = range bucket.id2Conn { 93 | wsConn.SendMessage(wsMsg) 94 | } 95 | } 96 | 97 | // 推送给某个房间的所有用户 98 | func (bucket *Bucket) PushRoom(roomId string, wsMsg *common.WSMessage) { 99 | var ( 100 | room *Room 101 | existed bool 102 | ) 103 | 104 | // 锁Bucket 105 | bucket.rwMutex.RLock() 106 | room, existed = bucket.rooms[roomId] 107 | bucket.rwMutex.RUnlock() 108 | 109 | // 房间不存在 110 | if !existed { 111 | return 112 | } 113 | 114 | // 向房间做推送 115 | room.Push(wsMsg) 116 | } 117 | -------------------------------------------------------------------------------- /gateway/Config.go: -------------------------------------------------------------------------------- 1 | package gateway 2 | 3 | import ( 4 | "io/ioutil" 5 | "encoding/json" 6 | ) 7 | 8 | // 程序配置 9 | type Config struct { 10 | WsPort int `json:"wsPort"` 11 | WsReadTimeout int `json:"wsReadTimeout"` 12 | WsWriteTimeout int `json:"wsWriteTimeout"` 13 | WsInChannelSize int `json:"wsInChannelSize"` 14 | WsOutChannelSize int `json:"wsOutChannelSize"` 15 | WsHeartbeatInterval int `json:"wsHeartbeatInterval"` 16 | MaxMergerDelay int `json:"maxMergerDelay"` 17 | MaxMergerBatchSize int `json:"maxMergerBatchSize"` 18 | MergerWorkerCount int `json:"mergerWorkerCount"` 19 | MergerChannelSize int `json:"mergerChannelSize"` 20 | ServicePort int `json:"servicePort"` 21 | ServiceReadTimeout int `json:"serviceReadTimeout"` 22 | ServiceWriteTimeout int `json:"serviceWriteTimeout"` 23 | ServerPem string `json:"serverPem"` 24 | ServerKey string `json:"serverKey"` 25 | BucketCount int `json:"bucketCount"` 26 | BucketWorkerCount int `json:"bucketWorkerCount"` 27 | MaxJoinRoom int`json:"maxJoinRoom"` 28 | DispatchChannelSize int `json:"dispatchChannelSize"` 29 | DispatchWorkerCount int `json:"dispatchWorkerCount"` 30 | BucketJobChannelSize int `json:"bucketJobChannelSize"` 31 | BucketJobWorkerCount int `json:"bucketJobWorkerCount"` 32 | } 33 | 34 | var ( 35 | G_config *Config 36 | ) 37 | 38 | func InitConfig(filename string) (err error) { 39 | var ( 40 | content []byte 41 | conf Config 42 | ) 43 | 44 | if content, err = ioutil.ReadFile(filename); err != nil { 45 | return 46 | } 47 | 48 | if err = json.Unmarshal(content, &conf); err != nil { 49 | return 50 | } 51 | 52 | G_config = &conf 53 | return 54 | } -------------------------------------------------------------------------------- /gateway/ConnMgr.go: -------------------------------------------------------------------------------- 1 | package gateway 2 | 3 | import "github.com/owenliang/go-push/common" 4 | 5 | // 推送任务 6 | type PushJob struct { 7 | pushType int // 推送类型 8 | roomId string // 房间ID 9 | // union { 10 | bizMsg *common.BizMessage // 未序列化的业务消息 11 | wsMsg *common.WSMessage // 已序列化的业务消息 12 | // } 13 | } 14 | 15 | // 连接管理器 16 | type ConnMgr struct { 17 | buckets []*Bucket 18 | jobChan []chan*PushJob // 每个Bucket对应一个Job Queue 19 | 20 | dispatchChan chan *PushJob // 待分发消息队列 21 | } 22 | 23 | var ( 24 | G_connMgr *ConnMgr 25 | ) 26 | 27 | // 消息分发到Bucket 28 | func (connMgr *ConnMgr)dispatchWorkerMain(dispatchWorkerIdx int) { 29 | var ( 30 | bucketIdx int 31 | pushJob *PushJob 32 | err error 33 | ) 34 | for { 35 | select { 36 | case pushJob = <- connMgr.dispatchChan: 37 | DispatchPending_DESC() 38 | 39 | // 序列化 40 | if pushJob.wsMsg, err = common.EncodeWSMessage(pushJob.bizMsg); err != nil { 41 | continue 42 | } 43 | // 分发给所有Bucket, 若Bucket拥塞则等待 44 | for bucketIdx, _ = range connMgr.buckets { 45 | PushJobPending_INCR() 46 | connMgr.jobChan[bucketIdx] <- pushJob 47 | } 48 | } 49 | } 50 | } 51 | 52 | // Job负责消息广播给客户端 53 | func (connMgr *ConnMgr)jobWorkerMain(jobWorkerIdx int, bucketIdx int) { 54 | var ( 55 | bucket = connMgr.buckets[bucketIdx] 56 | pushJob *PushJob 57 | ) 58 | 59 | for { 60 | select { 61 | case pushJob = <-connMgr.jobChan[bucketIdx]: // 从Bucket的job queue取出一个任务 62 | PushJobPending_DESC() 63 | if pushJob.pushType == common.PUSH_TYPE_ALL { 64 | bucket.PushAll(pushJob.wsMsg) 65 | } else if pushJob.pushType == common.PUSH_TYPE_ROOM { 66 | bucket.PushRoom(pushJob.roomId, pushJob.wsMsg) 67 | } 68 | } 69 | } 70 | } 71 | 72 | /** 73 | 以下是API 74 | */ 75 | 76 | func InitConnMgr() (err error) { 77 | var ( 78 | bucketIdx int 79 | jobWorkerIdx int 80 | dispatchWorkerIdx int 81 | connMgr *ConnMgr 82 | ) 83 | 84 | connMgr = &ConnMgr{ 85 | buckets: make([]*Bucket, G_config.BucketCount), 86 | jobChan: make([]chan*PushJob, G_config.BucketCount), 87 | dispatchChan: make(chan*PushJob, G_config.DispatchChannelSize), 88 | } 89 | for bucketIdx, _ = range connMgr.buckets { 90 | connMgr.buckets[bucketIdx] = InitBucket(bucketIdx) // 初始化Bucket 91 | connMgr.jobChan[bucketIdx] = make(chan*PushJob, G_config.BucketJobChannelSize) // Bucket的Job队列 92 | // Bucket的Job worker 93 | for jobWorkerIdx = 0; jobWorkerIdx < G_config.BucketJobWorkerCount; jobWorkerIdx++ { 94 | go connMgr.jobWorkerMain(jobWorkerIdx, bucketIdx) 95 | } 96 | } 97 | // 初始化分发协程, 用于将消息扇出给各个Bucket 98 | for dispatchWorkerIdx = 0; dispatchWorkerIdx < G_config.DispatchWorkerCount; dispatchWorkerIdx++ { 99 | go connMgr.dispatchWorkerMain(dispatchWorkerIdx) 100 | } 101 | 102 | G_connMgr = connMgr 103 | return 104 | } 105 | 106 | func (connMgr *ConnMgr) GetBucket(wsConnection *WSConnection) (bucket *Bucket) { 107 | bucket = connMgr.buckets[wsConnection.connId % uint64(len(connMgr.buckets))] 108 | return 109 | } 110 | 111 | func (connMgr *ConnMgr) AddConn(wsConnection *WSConnection) { 112 | var ( 113 | bucket *Bucket 114 | ) 115 | 116 | bucket = connMgr.GetBucket(wsConnection) 117 | bucket.AddConn(wsConnection) 118 | 119 | OnlineConnections_INCR() 120 | } 121 | 122 | func (connMgr *ConnMgr) DelConn(wsConnection *WSConnection) { 123 | var ( 124 | bucket *Bucket 125 | ) 126 | 127 | bucket = connMgr.GetBucket(wsConnection) 128 | bucket.DelConn(wsConnection) 129 | 130 | OnlineConnections_DESC() 131 | } 132 | 133 | func (connMgr *ConnMgr) JoinRoom(roomId string, wsConn *WSConnection) (err error) { 134 | var ( 135 | bucket *Bucket 136 | ) 137 | 138 | bucket = connMgr.GetBucket(wsConn) 139 | err = bucket.JoinRoom(roomId, wsConn) 140 | return 141 | } 142 | 143 | func (connMgr *ConnMgr) LeaveRoom(roomId string, wsConn *WSConnection) (err error) { 144 | var ( 145 | bucket *Bucket 146 | ) 147 | 148 | bucket = connMgr.GetBucket(wsConn) 149 | err = bucket.LeaveRoom(roomId, wsConn) 150 | return 151 | } 152 | 153 | // 向所有在线用户发送消息 154 | func (connMgr *ConnMgr) PushAll(bizMsg *common.BizMessage) (err error) { 155 | var ( 156 | pushJob *PushJob 157 | ) 158 | 159 | pushJob = &PushJob{ 160 | pushType: common.PUSH_TYPE_ALL, 161 | bizMsg: bizMsg, 162 | } 163 | 164 | select { 165 | case connMgr.dispatchChan <- pushJob: 166 | DispatchPending_INCR() 167 | default: 168 | err = common.ERR_DISPATCH_CHANNEL_FULL 169 | DispatchFail_INCR() 170 | } 171 | return 172 | } 173 | 174 | // 向指定房间发送消息 175 | func (connMgr *ConnMgr) PushRoom(roomId string, bizMsg *common.BizMessage) (err error) { 176 | var ( 177 | pushJob *PushJob 178 | ) 179 | 180 | pushJob = &PushJob{ 181 | pushType: common.PUSH_TYPE_ROOM, 182 | bizMsg: bizMsg, 183 | roomId: roomId, 184 | } 185 | 186 | select { 187 | case connMgr.dispatchChan <- pushJob: 188 | DispatchPending_INCR() 189 | default: 190 | err = common.ERR_DISPATCH_CHANNEL_FULL 191 | DispatchFail_INCR() 192 | } 193 | return 194 | } -------------------------------------------------------------------------------- /gateway/Merger.go: -------------------------------------------------------------------------------- 1 | package gateway 2 | 3 | import ( 4 | "encoding/json" 5 | "time" 6 | "github.com/owenliang/go-push/common" 7 | ) 8 | 9 | type PushBatch struct { 10 | items []*json.RawMessage 11 | commitTimer *time.Timer 12 | 13 | // union { 14 | room string // 按room合并 15 | // } 16 | } 17 | 18 | type PushContext struct { 19 | msg *json.RawMessage 20 | 21 | // union { 22 | room string // 按room合并 23 | // } 24 | } 25 | 26 | type MergeWorker struct { 27 | mergeType int // 合并类型: 广播, room, uid... 28 | 29 | contextChan chan*PushContext 30 | timeoutChan chan*PushBatch 31 | 32 | // union { 33 | room2Batch map[string]*PushBatch // room合并 34 | allBatch *PushBatch // 广播合并 35 | // } 36 | } 37 | 38 | // 广播消息、房间消息的合并 39 | type Merger struct { 40 | roomWorkers []*MergeWorker // 房间合并 41 | broadcastWorker *MergeWorker // 广播合并 42 | } 43 | 44 | var ( 45 | G_merger *Merger 46 | ) 47 | 48 | func (worker *MergeWorker) autoCommit(batch *PushBatch) func() { 49 | return func() { 50 | worker.timeoutChan <- batch 51 | } 52 | } 53 | 54 | func (worker *MergeWorker) commitBatch(batch *PushBatch) (err error) { 55 | var ( 56 | bizPushData *common.BizPushData 57 | bizMessage *common.BizMessage 58 | buf []byte 59 | ) 60 | 61 | bizPushData = &common.BizPushData{ 62 | Items: batch.items, 63 | } 64 | if buf, err = json.Marshal(*bizPushData); err != nil { 65 | return 66 | } 67 | 68 | bizMessage = &common.BizMessage{ 69 | Type: "PUSH", 70 | Data: json.RawMessage(buf), 71 | } 72 | 73 | // 打包发送 74 | if worker.mergeType == common.PUSH_TYPE_ROOM { 75 | delete(worker.room2Batch, batch.room) 76 | err = G_connMgr.PushRoom(batch.room, bizMessage) 77 | } else if worker.mergeType == common.PUSH_TYPE_ALL { 78 | worker.allBatch = nil 79 | err = G_connMgr.PushAll(bizMessage) 80 | } 81 | return 82 | } 83 | 84 | func (worker *MergeWorker) mergeWorkerMain() { 85 | var ( 86 | context *PushContext 87 | batch *PushBatch 88 | timeoutBatch *PushBatch 89 | existed bool 90 | isCreated bool 91 | err error 92 | ) 93 | for { 94 | select { 95 | case context = <- worker.contextChan: 96 | MergerPending_DESC() 97 | 98 | isCreated = false 99 | // 按房间合并 100 | if worker.mergeType == common.PUSH_TYPE_ROOM { 101 | if batch, existed = worker.room2Batch[context.room]; !existed { 102 | batch = &PushBatch{room: context.room} 103 | worker.room2Batch[context.room] = batch 104 | isCreated = true 105 | } 106 | } else if worker.mergeType == common.PUSH_TYPE_ALL { // 广播合并 107 | batch = worker.allBatch 108 | if batch == nil { 109 | batch = &PushBatch{} 110 | worker.allBatch = batch 111 | isCreated = true 112 | } 113 | } 114 | 115 | // 合并消息 116 | batch.items = append(batch.items, context.msg) 117 | 118 | // 新建批次, 启动超时自动提交 119 | if isCreated { 120 | batch.commitTimer = time.AfterFunc(time.Duration(G_config.MaxMergerDelay) * time.Millisecond, worker.autoCommit(batch)) 121 | } 122 | 123 | // 批次未满, 继续等待下次提交 124 | if len(batch.items) < G_config.MaxMergerBatchSize { 125 | continue 126 | } 127 | 128 | // 批次已满, 取消超时自动提交 129 | batch.commitTimer.Stop() 130 | case timeoutBatch = <- worker.timeoutChan: 131 | if worker.mergeType == common.PUSH_TYPE_ROOM { 132 | // 定时器触发时, 批次已被提交 133 | if batch, existed = worker.room2Batch[timeoutBatch.room]; !existed { 134 | continue 135 | } 136 | 137 | // 定时器触发时, 前一个批次已提交, 下一个批次已建立 138 | if batch != timeoutBatch { 139 | continue 140 | } 141 | } else if worker.mergeType == common.PUSH_TYPE_ALL { 142 | batch = worker.allBatch 143 | // 定时器触发时, 批次已被提交 144 | if timeoutBatch != batch { 145 | continue 146 | } 147 | } 148 | } 149 | // 提交批次 150 | err = worker.commitBatch(batch) 151 | 152 | // 打点统计 153 | if worker.mergeType == common.PUSH_TYPE_ALL { 154 | MergerAllTotal_INCR(int64(len(batch.items))) 155 | if err != nil { 156 | MergerAllFail_INCR(int64(len(batch.items))) 157 | } 158 | } else if worker.mergeType == common.PUSH_TYPE_ROOM { 159 | MergerRoomTotal_INCR(int64(len(batch.items))) 160 | if err != nil { 161 | MergerRoomFail_INCR(int64(len(batch.items))) 162 | } 163 | } 164 | } 165 | } 166 | 167 | func initMergeWorker(mergeType int) (worker *MergeWorker) { 168 | worker = &MergeWorker{ 169 | mergeType: mergeType, 170 | room2Batch: make(map[string]*PushBatch), 171 | contextChan: make(chan*PushContext, G_config.MergerChannelSize), 172 | timeoutChan: make(chan*PushBatch, G_config.MergerChannelSize), 173 | } 174 | go worker.mergeWorkerMain() 175 | return 176 | } 177 | 178 | func (worker *MergeWorker) pushRoom(room string, msg *json.RawMessage) (err error) { 179 | var ( 180 | context *PushContext 181 | ) 182 | context = &PushContext{ 183 | room: room, 184 | msg: msg, 185 | } 186 | select { 187 | case worker.contextChan <- context: 188 | MergerPending_INCR() 189 | default: 190 | err = common.ERR_MERGE_CHANNEL_FULL 191 | } 192 | return 193 | } 194 | 195 | func (worker *MergeWorker) pushAll(msg *json.RawMessage) (err error) { 196 | var ( 197 | context *PushContext 198 | ) 199 | context = &PushContext{ 200 | msg: msg, 201 | } 202 | select { 203 | case worker.contextChan <- context: 204 | MergerPending_INCR() 205 | default: 206 | err = common.ERR_MERGE_CHANNEL_FULL 207 | } 208 | return 209 | } 210 | 211 | /** 212 | API 213 | */ 214 | 215 | func InitMerger() (err error) { 216 | var ( 217 | workerIdx int 218 | merger *Merger 219 | ) 220 | 221 | merger = &Merger{ 222 | roomWorkers: make([]*MergeWorker, G_config.MergerWorkerCount), 223 | } 224 | for workerIdx = 0; workerIdx < G_config.MergerWorkerCount; workerIdx++ { 225 | merger.roomWorkers[workerIdx] = initMergeWorker(common.PUSH_TYPE_ROOM) 226 | } 227 | merger.broadcastWorker = initMergeWorker(common.PUSH_TYPE_ALL) 228 | 229 | G_merger = merger 230 | return 231 | } 232 | 233 | // 广播合并推送 234 | func (merger *Merger) PushAll(msg *json.RawMessage) (err error) { 235 | return merger.broadcastWorker.pushAll(msg) 236 | } 237 | 238 | // 房间合并推送 239 | func (merger *Merger) PushRoom(room string, msg *json.RawMessage) (err error) { 240 | // 计算room hash到某个worker 241 | var ( 242 | workerIdx uint32= 0 243 | ch byte 244 | ) 245 | for _, ch = range []byte(room) { 246 | workerIdx = (workerIdx + uint32(ch) * 33) % uint32(G_config.MergerWorkerCount) 247 | } 248 | return merger.roomWorkers[workerIdx].pushRoom(room, msg) 249 | } -------------------------------------------------------------------------------- /gateway/Room.go: -------------------------------------------------------------------------------- 1 | package gateway 2 | 3 | import ( 4 | "sync" 5 | "github.com/owenliang/go-push/common" 6 | ) 7 | 8 | // 房间 9 | type Room struct { 10 | rwMutex sync.RWMutex 11 | roomId string 12 | id2Conn map[uint64]*WSConnection 13 | } 14 | 15 | func InitRoom(roomId string) (room *Room) { 16 | room = &Room { 17 | roomId: roomId, 18 | id2Conn: make(map[uint64]*WSConnection), 19 | } 20 | return 21 | } 22 | 23 | func (room *Room) Join(wsConn *WSConnection) (err error) { 24 | var ( 25 | existed bool 26 | ) 27 | 28 | room.rwMutex.Lock() 29 | defer room.rwMutex.Unlock() 30 | 31 | if _, existed = room.id2Conn[wsConn.connId]; existed { 32 | err = common.ERR_JOIN_ROOM_TWICE 33 | return 34 | } 35 | 36 | room.id2Conn[wsConn.connId] = wsConn 37 | return 38 | } 39 | 40 | func (room *Room) Leave(wsConn* WSConnection) (err error) { 41 | var ( 42 | existed bool 43 | ) 44 | 45 | room.rwMutex.Lock() 46 | defer room.rwMutex.Unlock() 47 | 48 | if _, existed = room.id2Conn[wsConn.connId]; !existed { 49 | err = common.ERR_NOT_IN_ROOM 50 | return 51 | } 52 | 53 | delete(room.id2Conn, wsConn.connId) 54 | return 55 | } 56 | 57 | func (room *Room) Count() int { 58 | room.rwMutex.RLock() 59 | defer room.rwMutex.RUnlock() 60 | 61 | return len(room.id2Conn) 62 | } 63 | 64 | func (room *Room) Push(wsMsg *common.WSMessage) { 65 | var ( 66 | wsConn *WSConnection 67 | ) 68 | room.rwMutex.RLock() 69 | defer room.rwMutex.RUnlock() 70 | 71 | for _, wsConn = range room.id2Conn { 72 | wsConn.SendMessage(wsMsg) 73 | } 74 | } -------------------------------------------------------------------------------- /gateway/Service.go: -------------------------------------------------------------------------------- 1 | package gateway 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | "net" 7 | "strconv" 8 | "encoding/json" 9 | "crypto/tls" 10 | "github.com/owenliang/go-push/common" 11 | ) 12 | 13 | type Service struct { 14 | server *http.Server 15 | } 16 | 17 | var ( 18 | G_service *Service 19 | ) 20 | 21 | // 全量推送POST msg={} 22 | func handlePushAll(resp http.ResponseWriter, req *http.Request) { 23 | var ( 24 | err error 25 | items string 26 | msgArr []json.RawMessage 27 | msgIdx int 28 | ) 29 | if err = req.ParseForm(); err != nil { 30 | return 31 | } 32 | 33 | items = req.PostForm.Get("items") 34 | if err = json.Unmarshal([]byte(items), &msgArr); err != nil { 35 | return 36 | } 37 | 38 | for msgIdx, _ = range msgArr { 39 | G_merger.PushAll(&msgArr[msgIdx]) 40 | } 41 | } 42 | 43 | // 房间推送POST room=xxx&msg 44 | func handlePushRoom(resp http.ResponseWriter, req *http.Request) { 45 | var ( 46 | err error 47 | room string 48 | items string 49 | msgArr []json.RawMessage 50 | msgIdx int 51 | ) 52 | if err = req.ParseForm(); err != nil { 53 | return 54 | } 55 | 56 | room = req.PostForm.Get("room") 57 | items = req.PostForm.Get("items") 58 | 59 | if err = json.Unmarshal([]byte(items), &msgArr); err != nil { 60 | return 61 | } 62 | 63 | for msgIdx, _ = range msgArr { 64 | G_merger.PushRoom(room, &msgArr[msgIdx]) 65 | } 66 | } 67 | 68 | // 统计 69 | func handleStats(resp http.ResponseWriter, req *http.Request) { 70 | var ( 71 | data []byte 72 | err error 73 | ) 74 | 75 | if data, err = G_stats.Dump(); err != nil { 76 | return 77 | } 78 | 79 | resp.Write(data) 80 | } 81 | 82 | func InitService() (err error) { 83 | var ( 84 | mux *http.ServeMux 85 | server *http.Server 86 | listener net.Listener 87 | ) 88 | 89 | // 路由 90 | mux = http.NewServeMux() 91 | mux.HandleFunc("/push/all", handlePushAll) 92 | mux.HandleFunc("/push/room", handlePushRoom) 93 | mux.HandleFunc("/stats", handleStats) 94 | 95 | // TLS证书解析验证 96 | if _, err = tls.LoadX509KeyPair(G_config.ServerPem, G_config.ServerKey); err != nil { 97 | return common.ERR_CERT_INVALID 98 | } 99 | 100 | // HTTP/2 TLS服务 101 | server = &http.Server{ 102 | ReadTimeout: time.Duration(G_config.ServiceReadTimeout) * time.Millisecond, 103 | WriteTimeout: time.Duration(G_config.ServiceWriteTimeout) * time.Millisecond, 104 | Handler: mux, 105 | } 106 | 107 | // 监听端口 108 | if listener, err = net.Listen("tcp", ":" + strconv.Itoa(G_config.ServicePort)); err != nil { 109 | return 110 | } 111 | 112 | // 赋值全局变量 113 | G_service = &Service{ 114 | server: server, 115 | } 116 | 117 | // 拉起服务 118 | go server.ServeTLS(listener, G_config.ServerPem, G_config.ServerKey) 119 | 120 | return 121 | } -------------------------------------------------------------------------------- /gateway/Stats.go: -------------------------------------------------------------------------------- 1 | package gateway 2 | 3 | import ( 4 | "sync/atomic" 5 | "encoding/json" 6 | ) 7 | 8 | type Stats struct { 9 | // 反馈在线长连接的数量 10 | OnlineConnections int64 `json:"onlineConnections"` 11 | 12 | // 反馈客户端的推送压力 13 | SendMessageTotal int64 `json:"sendMessageTotal"` 14 | SendMessageFail int64 `json:"sendMessageFail"` 15 | 16 | // 反馈ConnMgr消息分发模块的压力 17 | DispatchPending int64 `json:"dispatchPending"` 18 | PushJobPending int64 `json:"pushJobPending"` 19 | DispatchFail int64 `json:"dispatchFail"` 20 | 21 | // 返回出在线的房间总数, 有利于分析内存上涨的原因 22 | RoomCount int64 `json:"roomCount"` 23 | 24 | // Merger模块处理队列, 反馈出消息合并的压力情况 25 | MergerPending int64 `json:"mergerPending"` 26 | 27 | // Merger模块合并发送的消息总数与失败总数 28 | MergerRoomTotal int64 `json:"mergerRoomTotal"` 29 | MergerAllTotal int64 `json:"mergerAllTotal"` 30 | MergerRoomFail int64 `json:"mergerRoomFail"` 31 | MergerAllFail int64 `json:"mergerAllFail"` 32 | } 33 | 34 | var ( 35 | G_stats *Stats 36 | ) 37 | 38 | func InitStats() (err error) { 39 | G_stats = &Stats{} 40 | return 41 | } 42 | 43 | func DispatchPending_INCR() { 44 | atomic.AddInt64(&G_stats.DispatchPending, 1) 45 | } 46 | 47 | func DispatchPending_DESC() { 48 | atomic.AddInt64(&G_stats.DispatchPending, -1) 49 | } 50 | 51 | func PushJobPending_INCR() { 52 | atomic.AddInt64(&G_stats.PushJobPending, 1) 53 | } 54 | 55 | func PushJobPending_DESC() { 56 | atomic.AddInt64(&G_stats.PushJobPending, -1) 57 | } 58 | 59 | func OnlineConnections_INCR() { 60 | atomic.AddInt64(&G_stats.OnlineConnections, 1) 61 | } 62 | 63 | func OnlineConnections_DESC() { 64 | atomic.AddInt64(&G_stats.OnlineConnections, -1) 65 | } 66 | 67 | func RoomCount_INCR() { 68 | atomic.AddInt64(&G_stats.RoomCount, 1) 69 | } 70 | 71 | func RoomCount_DESC() { 72 | atomic.AddInt64(&G_stats.RoomCount, -1) 73 | } 74 | 75 | func MergerPending_INCR() { 76 | atomic.AddInt64(&G_stats.MergerPending, 1) 77 | } 78 | 79 | func MergerPending_DESC() { 80 | atomic.AddInt64(&G_stats.MergerPending, -1) 81 | } 82 | 83 | func MergerRoomTotal_INCR(batchSize int64) { 84 | atomic.AddInt64(&G_stats.MergerRoomTotal, batchSize) 85 | } 86 | 87 | func MergerAllTotal_INCR(batchSize int64) { 88 | atomic.AddInt64(&G_stats.MergerAllTotal, batchSize) 89 | } 90 | 91 | func MergerRoomFail_INCR(batchSize int64) { 92 | atomic.AddInt64(&G_stats.MergerRoomFail, batchSize) 93 | } 94 | 95 | func MergerAllFail_INCR(batchSize int64) { 96 | atomic.AddInt64(&G_stats.MergerAllFail, batchSize) 97 | } 98 | 99 | func DispatchFail_INCR() { 100 | atomic.AddInt64(&G_stats.DispatchFail, 1) 101 | } 102 | 103 | func SendMessageFail_INCR() { 104 | atomic.AddInt64(&G_stats.SendMessageFail, 1) 105 | } 106 | 107 | func SendMessageTotal_INCR() { 108 | atomic.AddInt64(&G_stats.SendMessageTotal, 1) 109 | } 110 | 111 | func (stats *Stats) Dump() (data []byte, err error){ 112 | return json.Marshal(G_stats) 113 | } -------------------------------------------------------------------------------- /gateway/WSConnection.go: -------------------------------------------------------------------------------- 1 | package gateway 2 | 3 | import ( 4 | "github.com/gorilla/websocket" 5 | "sync" 6 | "time" 7 | "github.com/owenliang/go-push/common" 8 | ) 9 | 10 | type WSConnection struct { 11 | mutex sync.Mutex 12 | connId uint64 13 | wsSocket *websocket.Conn 14 | inChan chan*common.WSMessage 15 | outChan chan*common.WSMessage 16 | closeChan chan byte 17 | isClosed bool 18 | lastHeartbeatTime time.Time // 最近一次心跳时间 19 | rooms map[string]bool // 加入了哪些房间 20 | } 21 | 22 | // 读websocket 23 | func (wsConnection *WSConnection) readLoop() { 24 | var ( 25 | msgType int 26 | msgData []byte 27 | message *common.WSMessage 28 | err error 29 | ) 30 | for { 31 | if msgType, msgData, err = wsConnection.wsSocket.ReadMessage(); err != nil { 32 | goto ERR 33 | } 34 | 35 | message = common.BuildWSMessage(msgType, msgData) 36 | 37 | select { 38 | case wsConnection.inChan <- message: 39 | case <- wsConnection.closeChan: 40 | goto CLOSED 41 | } 42 | } 43 | 44 | ERR: 45 | wsConnection.Close() 46 | CLOSED: 47 | } 48 | 49 | // 写websocket 50 | func (wsConnection *WSConnection) writeLoop() { 51 | var ( 52 | message *common.WSMessage 53 | err error 54 | ) 55 | for { 56 | select { 57 | case message = <- wsConnection.outChan: 58 | if err = wsConnection.wsSocket.WriteMessage(message.MsgType, message.MsgData); err != nil { 59 | goto ERR 60 | } 61 | case <- wsConnection.closeChan: 62 | goto CLOSED 63 | } 64 | } 65 | ERR: 66 | wsConnection.Close() 67 | CLOSED: 68 | } 69 | 70 | /** 71 | 以下是API 72 | */ 73 | 74 | func InitWSConnection(connId uint64, wsSocket *websocket.Conn) (wsConnection *WSConnection) { 75 | wsConnection = &WSConnection{ 76 | wsSocket: wsSocket, 77 | connId: connId, 78 | inChan: make(chan *common.WSMessage, G_config.WsInChannelSize), 79 | outChan: make(chan *common.WSMessage, G_config.WsOutChannelSize), 80 | closeChan: make(chan byte), 81 | lastHeartbeatTime: time.Now(), 82 | rooms: make(map[string]bool), 83 | } 84 | 85 | go wsConnection.readLoop() 86 | go wsConnection.writeLoop() 87 | 88 | return 89 | } 90 | 91 | // 发送消息 92 | func (wsConnection *WSConnection) SendMessage(message *common.WSMessage) (err error) { 93 | select { 94 | case wsConnection.outChan <- message: 95 | SendMessageTotal_INCR() 96 | case <- wsConnection.closeChan: 97 | err = common.ERR_CONNECTION_LOSS 98 | default: // 写操作不会阻塞, 因为channel已经预留给websocket一定的缓冲空间 99 | err = common.ERR_SEND_MESSAGE_FULL 100 | SendMessageFail_INCR() 101 | } 102 | return 103 | } 104 | 105 | // 读取消息 106 | func (wsConnection *WSConnection) ReadMessage() (message *common.WSMessage, err error) { 107 | select { 108 | case message = <- wsConnection.inChan: 109 | case <- wsConnection.closeChan: 110 | err = common.ERR_CONNECTION_LOSS 111 | } 112 | return 113 | } 114 | 115 | // 关闭连接 116 | func (wsConnection *WSConnection) Close() { 117 | wsConnection.wsSocket.Close() 118 | 119 | wsConnection.mutex.Lock() 120 | defer wsConnection.mutex.Unlock() 121 | 122 | if !wsConnection.isClosed { 123 | wsConnection.isClosed = true 124 | close(wsConnection.closeChan) 125 | } 126 | } 127 | 128 | // 检查心跳（不需要太频繁） 129 | func (wsConnection *WSConnection) IsAlive() bool { 130 | var ( 131 | now = time.Now() 132 | ) 133 | 134 | wsConnection.mutex.Lock() 135 | defer wsConnection.mutex.Unlock() 136 | 137 | // 连接已关闭 或者 太久没有心跳 138 | if wsConnection.isClosed || now.Sub(wsConnection.lastHeartbeatTime) > time.Duration(G_config.WsHeartbeatInterval) * time.Second { 139 | return false 140 | } 141 | return true 142 | } 143 | 144 | // 更新心跳 145 | func (WSConnection *WSConnection) KeepAlive() { 146 | var ( 147 | now = time.Now() 148 | ) 149 | 150 | WSConnection.mutex.Lock() 151 | defer WSConnection.mutex.Unlock() 152 | 153 | WSConnection.lastHeartbeatTime = now 154 | } -------------------------------------------------------------------------------- /gateway/WSHandler.go: -------------------------------------------------------------------------------- 1 | package gateway 2 | 3 | import ( 4 | "time" 5 | "github.com/gorilla/websocket" 6 | "encoding/json" 7 | "github.com/owenliang/go-push/common" 8 | ) 9 | 10 | // 每隔1秒, 检查一次连接是否健康 11 | func (wsConnection *WSConnection) heartbeatChecker() { 12 | var ( 13 | timer *time.Timer 14 | ) 15 | timer = time.NewTimer(time.Duration(G_config.WsHeartbeatInterval) * time.Second) 16 | for { 17 | select { 18 | case <- timer.C: 19 | if !wsConnection.IsAlive() { 20 | wsConnection.Close() 21 | goto EXIT 22 | } 23 | timer.Reset(time.Duration(G_config.WsHeartbeatInterval) * time.Second) 24 | case <- wsConnection.closeChan: 25 | timer.Stop() 26 | goto EXIT 27 | } 28 | } 29 | 30 | EXIT: 31 | // 确保连接被关闭 32 | } 33 | 34 | // 处理PING请求 35 | func (wsConnection *WSConnection) handlePing(bizReq *common.BizMessage) (bizResp *common.BizMessage, err error) { 36 | var ( 37 | buf []byte 38 | ) 39 | 40 | wsConnection.KeepAlive() 41 | 42 | if buf, err = json.Marshal(common.BizPongData{}); err != nil { 43 | return 44 | } 45 | bizResp = &common.BizMessage{ 46 | Type: "PONG", 47 | Data: json.RawMessage(buf), 48 | } 49 | return 50 | } 51 | 52 | // 处理JOIN请求 53 | func (wsConnection *WSConnection) handleJoin(bizReq *common.BizMessage) (bizResp *common.BizMessage, err error) { 54 | var ( 55 | bizJoinData *common.BizJoinData 56 | existed bool 57 | ) 58 | bizJoinData = &common.BizJoinData{} 59 | if err = json.Unmarshal(bizReq.Data, bizJoinData); err != nil { 60 | return 61 | } 62 | if len(bizJoinData.Room) == 0 { 63 | err = common.ERR_ROOM_ID_INVALID 64 | return 65 | } 66 | if len(wsConnection.rooms) >= G_config.MaxJoinRoom { 67 | // 超过了房间数量限制, 忽略这个请求 68 | return 69 | } 70 | // 已加入过 71 | if _, existed = wsConnection.rooms[bizJoinData.Room]; existed { 72 | // 忽略掉这个请求 73 | return 74 | } 75 | // 建立房间 -> 连接的关系 76 | if err = G_connMgr.JoinRoom(bizJoinData.Room, wsConnection); err != nil { 77 | return 78 | } 79 | // 建立连接 -> 房间的关系 80 | wsConnection.rooms[bizJoinData.Room] = true 81 | return 82 | } 83 | 84 | // 处理LEAVE请求 85 | func (wsConnection *WSConnection) handleLeave(bizReq *common.BizMessage) (bizResp *common.BizMessage, err error) { 86 | var ( 87 | bizLeaveData *common.BizLeaveData 88 | existed bool 89 | ) 90 | bizLeaveData = &common.BizLeaveData{} 91 | if err = json.Unmarshal(bizReq.Data, bizLeaveData); err != nil { 92 | return 93 | } 94 | if len(bizLeaveData.Room) == 0 { 95 | err = common.ERR_ROOM_ID_INVALID 96 | return 97 | } 98 | // 未加入过 99 | if _, existed = wsConnection.rooms[bizLeaveData.Room]; !existed { 100 | // 忽略掉这个请求 101 | return 102 | } 103 | // 删除房间 -> 连接的关系 104 | if err = G_connMgr.LeaveRoom(bizLeaveData.Room, wsConnection); err != nil { 105 | return 106 | } 107 | // 删除连接 -> 房间的关系 108 | delete(wsConnection.rooms, bizLeaveData.Room) 109 | return 110 | } 111 | 112 | func (wsConnection *WSConnection) leaveAll() { 113 | var ( 114 | roomId string 115 | ) 116 | // 从所有房间中退出 117 | for roomId, _ = range wsConnection.rooms { 118 | G_connMgr.LeaveRoom(roomId, wsConnection) 119 | delete(wsConnection.rooms, roomId) 120 | } 121 | } 122 | 123 | // 处理websocket请求 124 | func (wsConnection *WSConnection) WSHandle() { 125 | var ( 126 | message *common.WSMessage 127 | bizReq *common.BizMessage 128 | bizResp *common.BizMessage 129 | err error 130 | buf []byte 131 | ) 132 | 133 | // 连接加入管理器, 可以推送端查找到 134 | G_connMgr.AddConn(wsConnection) 135 | 136 | // 心跳检测线程 137 | go wsConnection.heartbeatChecker() 138 | 139 | // 请求处理协程 140 | for { 141 | if message, err = wsConnection.ReadMessage(); err != nil { 142 | goto ERR 143 | } 144 | 145 | // 只处理文本消息 146 | if message.MsgType != websocket.TextMessage { 147 | continue 148 | } 149 | 150 | // 解析消息体 151 | if bizReq, err = common.DecodeBizMessage(message.MsgData); err != nil { 152 | goto ERR 153 | } 154 | 155 | bizResp = nil 156 | 157 | // 1,收到PING则响应PONG: {"type": "PING"}, {"type": "PONG"} 158 | // 2,收到JOIN则加入ROOM: {"type": "JOIN", "data": {"room": "chrome-plugin"}} 159 | // 3,收到LEAVE则离开ROOM: {"type": "LEAVE", "data": {"room": "chrome-plugin"}} 160 | 161 | // 请求串行处理 162 | switch bizReq.Type { 163 | case "PING": 164 | if bizResp, err = wsConnection.handlePing(bizReq); err != nil { 165 | goto ERR 166 | } 167 | case "JOIN": 168 | if bizResp, err = wsConnection.handleJoin(bizReq); err != nil { 169 | goto ERR 170 | } 171 | case "LEAVE": 172 | if bizResp, err = wsConnection.handleLeave(bizReq); err != nil { 173 | goto ERR 174 | } 175 | } 176 | 177 | if bizResp != nil { 178 | if buf, err = json.Marshal(*bizResp); err != nil { 179 | goto ERR 180 | } 181 | // socket缓冲区写满不是致命错误 182 | if err = wsConnection.SendMessage(&common.WSMessage{websocket.TextMessage, buf}); err != nil { 183 | if err != common.ERR_SEND_MESSAGE_FULL { 184 | goto ERR 185 | } else { 186 | err = nil 187 | } 188 | } 189 | } 190 | } 191 | 192 | ERR: 193 | // 确保连接关闭 194 | wsConnection.Close() 195 | 196 | // 离开所有房间 197 | wsConnection.leaveAll() 198 | 199 | // 从连接池中移除 200 | G_connMgr.DelConn(wsConnection) 201 | return 202 | } -------------------------------------------------------------------------------- /gateway/WSServer.go: -------------------------------------------------------------------------------- 1 | package gateway 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | "net" 7 | "strconv" 8 | "github.com/gorilla/websocket" 9 | "sync/atomic" 10 | ) 11 | 12 | // WebSocket服务端 13 | type WSServer struct { 14 | server *http.Server 15 | curConnId uint64 16 | } 17 | 18 | var ( 19 | G_wsServer *WSServer 20 | 21 | wsUpgrader = websocket.Upgrader{ 22 | // 允许所有CORS跨域请求 23 | CheckOrigin: func(r *http.Request) bool { 24 | return true 25 | }, 26 | } 27 | ) 28 | 29 | func handleConnect(resp http.ResponseWriter, req *http.Request) { 30 | var ( 31 | err error 32 | wsSocket *websocket.Conn 33 | connId uint64 34 | wsConn *WSConnection 35 | ) 36 | 37 | // WebSocket握手 38 | if wsSocket, err = wsUpgrader.Upgrade(resp, req, nil); err != nil { 39 | return 40 | } 41 | 42 | // 连接唯一标识 43 | connId = atomic.AddUint64(&G_wsServer.curConnId, 1) 44 | 45 | // 初始化WebSocket的读写协程 46 | wsConn = InitWSConnection(connId, wsSocket) 47 | 48 | // 开始处理websocket消息 49 | wsConn.WSHandle() 50 | } 51 | 52 | func InitWSServer() (err error) { 53 | var ( 54 | mux *http.ServeMux 55 | server *http.Server 56 | listener net.Listener 57 | ) 58 | 59 | // 路由 60 | mux = http.NewServeMux() 61 | mux.HandleFunc("/connect", handleConnect) 62 | 63 | // HTTP服务 64 | server = &http.Server{ 65 | ReadTimeout: time.Duration(G_config.WsReadTimeout) * time.Millisecond, 66 | WriteTimeout: time.Duration(G_config.WsWriteTimeout) * time.Millisecond, 67 | Handler: mux, 68 | } 69 | 70 | // 监听端口 71 | if listener, err = net.Listen("tcp", ":" + strconv.Itoa(G_config.WsPort)); err != nil { 72 | return 73 | } 74 | 75 | // 赋值全局变量 76 | G_wsServer = &WSServer{ 77 | server: server, 78 | curConnId: uint64(time.Now().Unix()), 79 | } 80 | 81 | // 拉起服务 82 | go server.Serve(listener) 83 | 84 | return 85 | } -------------------------------------------------------------------------------- /gateway/cli/client.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 63 | 64 | 65 | 66 |
67 |

Click "Open" to create a connection to the server, 68 | "Send" to send a message to the server and "Close" to close the connection. 69 | You can change the message and send multiple times. 70 |

71 |
72 | 73 | 74 | 75 | 76 | 77 |
78 | 		79 |
80 |
81 | 82 | -------------------------------------------------------------------------------- /gateway/cli/default.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpAIBAAKCAQEAp9gCQKUTG5dpXMyukl9jDTQs80YFe6fO8QT7qpBqTYbjamp1 3 | yFMG8ePI+jrdq3K5iTl2cZM+94MbNz+vA8dYOW0O+djeIgLpujKmCaii0tzQjhgW 4 | JDmwZZGcirrIWPLsZQDN278leoGZD/w3cbwaYhBgTx9rNpStRqQJJUg3095u7TNq 5 | JqVjfAOQTMjlABJnEZY94Tz75wtds0Gln6Xx680OVTwwCSo++hyU9da7jh9xRMcZ 6 | gq5dyNQ5rzKvablzNQyO6E91CzppeHfDctfzT32PfnrIV6y4dg8+H98nwDk5UXJJ 7 | VMo3KfM1AIQLP9WvywmhiwmbNG5cc3NLMzNpLQIDAQABAoIBAFn29n4f/TX02ozb 8 | SVc7uaQCK3XaOmYldE7MFPk/nzse6hbIKYnOtxJAviEiHF8hh0F+g3YtLrsMkzHx 9 | +KVV3HwrcLOLAsXIOe+L5mOW+G993GvNjVCte1d1zSqvI2RKEVuyKqV2t2gKvzhK 10 | QI29/YZCsNy4Qodm+dm7YwuQwhvVZ1wiBlneFft9IDh1RTgGy2nD1UY2rnyzp6Tc 11 | Wvcd4CW1Xc1JhI4g4ejav2FjK0YAwEyraWkrm1SLSkCPJIIJoyRkPXgFLqPoopQd 12 | m1Y6VatIyURAA+1ovw4uBOdXouPd873gBP/cedS5oyMNtrm7rsBGnalOcegkpw8U 13 | aAf4lgECgYEA3ztQ1UhEZHLdnyT94hwgWAOErUcYyO+P61/FhZTI5wasdzOTBw/r 14 | +xJsBZuMEsT6XLgUl6IsIJknI9weMbquFKoMvfCS1aNP9L+dIAcwBwiPpTE8nwMS 15 | IZUpkPnbf1nC2rrHiL4sPDThiD0nTIt3UpubvoeThKsm1aENYNziTL0CgYEAwHtO 16 | IP7+wpDoo/uAotR3bTUE66dJ4rhEq5LyEcqq96Eql7H1/MiydYGv6r0X50ajQZOq 17 | l6XwGo3mlDF1lWdY/RPz97T2d5QOFsOyWzdCoCc5jo60oL/12zcWZMH+r5qivUzF 18 | qpwWL02Ld4AEsoYPXM5+ZqhsYvqve7erxDwTrTECgYEAv5aR9qtSf3+SM/80orYP 19 | EFrcqTcGz5XDyHhm9xHOJ3Gz3Y53Fq2Uk+Sor1tjqcxDMAaRLd7yznuDeyR1Cas1 20 | suiLjQ0HiLHkcqNtwEpK7w5q8pVCeRrSmd4CUboPp8orET0S+Yp2PqoIEryhmPFt 21 | 1IlW7vw/ILMf3mOeLs8ErmUCgYAzue9JFr3H88FRMCllmMtvYaws2AwXDYYGxbqf 22 | 5WMEoR8dHQoKILU0tuFbp+1gja8Z1GEn18Qqnq+0a4Y53Egh2tbZXpxNjlAq9fRc 23 | ZwFUoLXrBZatDGk5vBgcg7W540iQrq0AKGod1C0CtDiO8U/3lNaLJ//YqZ23Fajn 24 | V23CQQKBgQCdUWiotxNRrT0GI+am8k/wOnKkMNFRSybrT5o78AxsjP8MD9KoZ1px 25 | a1/CcdT/8InvGHrdZhRMvKoL0ps2pHwQWa5kTDyF6wqXxL6qH8jIiHwxPkG64AjT 26 | RMMJ+ay9BAo5Azkkhwox1RtAxXHEHGa2yGgFPC6WHUGRHdmtVHIefg== 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /gateway/cli/default.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDIDCCAggCCQDqiYzSuPI9hzANBgkqhkiG9w0BAQsFADBSMQswCQYDVQQGEwJj 3 | bjERMA8GA1UECAwIc2hhbmRvbmcxEDAOBgNVBAcMB3FpbmdkYW8xDjAMBgNVBAoM 4 | BXNtemRtMQ4wDAYDVQQLDAVzbXpkbTAeFw0xODA2MjUwMjQ4NDRaFw0yODA2MjIw 5 | MjQ4NDRaMFIxCzAJBgNVBAYTAmNuMREwDwYDVQQIDAhzaGFuZG9uZzEQMA4GA1UE 6 | BwwHcWluZ2RhbzEOMAwGA1UECgwFc216ZG0xDjAMBgNVBAsMBXNtemRtMIIBIjAN 7 | BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAp9gCQKUTG5dpXMyukl9jDTQs80YF 8 | e6fO8QT7qpBqTYbjamp1yFMG8ePI+jrdq3K5iTl2cZM+94MbNz+vA8dYOW0O+dje 9 | IgLpujKmCaii0tzQjhgWJDmwZZGcirrIWPLsZQDN278leoGZD/w3cbwaYhBgTx9r 10 | NpStRqQJJUg3095u7TNqJqVjfAOQTMjlABJnEZY94Tz75wtds0Gln6Xx680OVTww 11 | CSo++hyU9da7jh9xRMcZgq5dyNQ5rzKvablzNQyO6E91CzppeHfDctfzT32PfnrI 12 | V6y4dg8+H98nwDk5UXJJVMo3KfM1AIQLP9WvywmhiwmbNG5cc3NLMzNpLQIDAQAB 13 | MA0GCSqGSIb3DQEBCwUAA4IBAQB3YhrhFUXMvo0UUjacKsveyfnLgKmGBfsX1c+Z 14 | KZCY9mmIQLNfrPKC0gYUoCV1izLR8ihl6WoloppdWH0mzQ/ogW6ZRgiTbyEM2sOD 15 | TUd050EkykN8nBweVEqbcq9+j07oNm8gr0VmCGulS1lJxJfDzlhcepUcKmUxLIDq 16 | tajMQnsBKpE8hkRSbxmHhKmddWpL//gjxzVAWuZyP6cqMt3vGhS/cuPmWmvN3PQW 17 | c0kfovm9kODlFqYxun5EeJXkHFyvf/FwLqtElqe3A7VNjGET2i8Bay51qR0cjpN5 18 | EMyUDdpwgWJwO/zZ5yKch4oiqEThfdgLm0WESyRRuOdu0p4s 19 | -----END CERTIFICATE----- 20 | -------------------------------------------------------------------------------- /gateway/cli/gateway.json: -------------------------------------------------------------------------------- 1 | { 2 | "websocket监听端口": "建议nginx做代理转发", 3 | "wsPort": 7777, 4 | 5 | "websocket HTTP握手读超时": "单位毫秒", 6 | "wsReadTimeout": 2000, 7 | 8 | "websocket HTTP握手写超时": "单位毫秒", 9 | "wsWriteTimeout": 2000, 10 | 11 | "websocket读队列长度": "一般不需要修改", 12 | "wsInChannelSize": 1000, 13 | 14 | "WebSocket写队列长度": "一般不需要修改", 15 | "wsOutChannelSize": 1000, 16 | 17 | "WebSocket心跳检查间隔": "单位秒, 超过时间没有收到心跳, 服务端将主动断开链接", 18 | "wsHeartbeatInterval": 60, 19 | 20 | "合并推送的最大延迟时间": "单位毫秒, 在抵达maxPushBatchSize之前超时则发送", 21 | "maxMergerDelay": 1000, 22 | 23 | "合并最多消息条数": "消息推送频次越高, 应该使用更大的合并批次, 得到更高的吞吐收益", 24 | "maxMergerBatchSize": 100, 25 | 26 | "消息合并协程的数量": "消息合并与json编码耗费CPU, 注意一个房间的消息只会由同一个协程处理.", 27 | "MergerWorkerCount": 32, 28 | 29 | "消息合并队列的容量": "每个房间消息合并线程有一个队列, 推送量超过队列将被丢弃", 30 | "mergerChannelSize": 1000, 31 | 32 | "内部通讯HTTP2端口": "严禁该端口暴露到外网", 33 | "servicePort": 7788, 34 | 35 | "内部通讯HTTP2读超时": "单位毫秒", 36 | "serviceReadTimeout": 2000, 37 | 38 | "内部通讯HTTP2写超时": "单位毫秒", 39 | "serviceWriteTimeout": 2000, 40 | 41 | "内部通讯HTTP2 TLS证书": "私有证书,默认有效期10年", 42 | "serverPem": "./default.pem", 43 | 44 | "内部通讯HTTP2 TLS密钥": "与证书配对", 45 | "serverKey": "./default.key", 46 | 47 | "连接分桶的数量": "桶越多, 推送的锁粒度越小, 推送并发度越高", 48 | "bucketCount": 512, 49 | 50 | "每个桶的处理协程数量": "影响同一时刻可以有多少个不同消息被分发出去", 51 | "bucketWorkerCount": 32, 52 | 53 | "每个连接最多加入房间数量": "目前房间ID没有校验, 所以先做简单的数量控制", 54 | "maxJoinRoom": 5, 55 | 56 | "待分发队列的长度": "分发队列缓冲所有待推送的消息, 等待被分发到Bucket", 57 | "dispatchChannelSize": 100000, 58 | 59 | "分发协程的数量": "分发协程用于将待推送消息扇出给各个Bucket", 60 | "dispatchWorkerCount": 32, 61 | 62 | "Bucket工作队列长度": "每个Bucket的分发任务放在一个独立队列中", 63 | "bucketJobChannelSize": 1000, 64 | 65 | "Bucket发送协程的数量": "每个Bucket有多个协程并发的推送消息", 66 | "bucketJobWorkerCount": 32 67 | } -------------------------------------------------------------------------------- /gateway/cli/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/owenliang/go-push/gateway" 5 | "fmt" 6 | "os" 7 | "flag" 8 | "runtime" 9 | "time" 10 | ) 11 | 12 | var ( 13 | confFile string // 配置文件路径 14 | ) 15 | 16 | func initArgs() { 17 | flag.StringVar(&confFile, "config", "./gateway.json", "where gateway.json is.") 18 | flag.Parse() 19 | } 20 | 21 | func initEnv() { 22 | runtime.GOMAXPROCS(runtime.NumCPU()) 23 | } 24 | 25 | func main() { 26 | var ( 27 | err error 28 | ) 29 | 30 | // 初始化环境 31 | initArgs() 32 | initEnv() 33 | 34 | // 加载配置 35 | if err = gateway.InitConfig(confFile); err != nil { 36 | goto ERR 37 | } 38 | 39 | // 统计 40 | if err = gateway.InitStats(); err != nil { 41 | goto ERR 42 | } 43 | 44 | // 初始化连接管理器 45 | if err = gateway.InitConnMgr(); err != nil { 46 | goto ERR 47 | } 48 | 49 | // 初始化websocket服务器 50 | if err = gateway.InitWSServer(); err != nil { 51 | goto ERR 52 | } 53 | 54 | // 初始化merger合并层 55 | if err = gateway.InitMerger(); err != nil { 56 | goto ERR 57 | } 58 | 59 | // 初始化service接口 60 | if err = gateway.InitService(); err != nil { 61 | goto ERR 62 | } 63 | 64 | for { 65 | time.Sleep(1 * time.Second) 66 | } 67 | 68 | os.Exit(0) 69 | 70 | ERR: 71 | fmt.Fprintln(os.Stderr, err) 72 | os.Exit(-1) 73 | } 74 | -------------------------------------------------------------------------------- /gateway/test/client.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Gorilla WebSocket Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build ignore 6 | 7 | package main 8 | 9 | import ( 10 | "flag" 11 | "log" 12 | "net/url" 13 | "github.com/gorilla/websocket" 14 | "time" 15 | ) 16 | 17 | var addr = flag.String("addr", "localhost:7777", "http service address") 18 | 19 | func loop() { 20 | for { 21 | u := url.URL{Scheme: "ws", Host: *addr, Path: "/connect"} 22 | c, _, err := websocket.DefaultDialer.Dial(u.String(), nil) 23 | if err != nil { 24 | continue 25 | } 26 | // 循环读消息 27 | for { 28 | _, _, err := c.ReadMessage() 29 | if err != nil { 30 | // log.Println("read:", err) 31 | break 32 | } 33 | // log.Printf("recv: %s", message) 34 | } 35 | c.Close() 36 | } 37 | } 38 | 39 | func main() { 40 | flag.Parse() 41 | log.SetFlags(0) 42 | 43 | for i := 0; i < 10000; i++ { 44 | go loop() 45 | } 46 | 47 | for { 48 | time.Sleep(1 * time.Second) 49 | } 50 | } -------------------------------------------------------------------------------- /gateway/test/stats.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | "fmt" 6 | "os" 7 | "io/ioutil" 8 | "crypto/tls" 9 | ) 10 | 11 | func main() { 12 | var ( 13 | resp *http.Response 14 | err error 15 | buf []byte 16 | client *http.Client 17 | ) 18 | 19 | client = &http.Client{ 20 | Transport: &http.Transport{ 21 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true,}, // 不校验服务端证书 22 | }, 23 | } 24 | 25 | if resp, err = client.Get("https://localhost:7788/stats"); err != nil { 26 | goto ERR 27 | } 28 | 29 | defer resp.Body.Close() 30 | 31 | if buf, err = ioutil.ReadAll(resp.Body); err != nil { 32 | goto ERR 33 | } 34 | 35 | fmt.Println("返回值:", string(buf)) 36 | return 37 | 38 | ERR: 39 | fmt.Println(err) 40 | os.Exit(-1) 41 | return 42 | } -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/owenliang/go-push 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/gorilla/websocket v1.4.1-0.20181206070239-95ba29eb981b 7 | golang.org/x/net v0.0.0-20191108225301-c7154b74f18f 8 | ) 9 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/gorilla/websocket v1.4.1-0.20181206070239-95ba29eb981b h1:iJ/E8JhWXX4SFiO262uaUow6La0iNf/50wQ1fdPNEDQ= 2 | github.com/gorilla/websocket v1.4.1-0.20181206070239-95ba29eb981b/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= 3 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 4 | golang.org/x/net v0.0.0-20191108225301-c7154b74f18f h1:nughwhJbEPZ8KHzITXxn59RmA0bhS7vDoe8gQi6Od4M= 5 | golang.org/x/net v0.0.0-20191108225301-c7154b74f18f/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 6 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 7 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 8 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 9 | -------------------------------------------------------------------------------- /logic/Config.go: -------------------------------------------------------------------------------- 1 | package logic 2 | 3 | import ( 4 | "io/ioutil" 5 | "encoding/json" 6 | ) 7 | 8 | type GatewayConfig struct { 9 | Hostname string `json:"hostname"` 10 | Port int `json:"port"` 11 | } 12 | 13 | // 程序配置 14 | type Config struct { 15 | ServicePort int `json:"servicePort"` 16 | ServiceReadTimeout int `json:"serviceReadTimeout"` 17 | ServiceWriteTimeout int `json:"serviceWriteTimeout"` 18 | GatewayList []GatewayConfig`json:"gatewayList"` 19 | GatewayMaxConnection int `json:"gatewayMaxConnection"` 20 | GatewayTimeout int `json:"gatewayTimeout"` 21 | GatewayIdleTimeout int `json:"gatewayIdleTimeout"` 22 | GatewayDispatchWorkerCount int `json:"gatewayDispatchWorkerCount"` 23 | GatewayDispatchChannelSize int `json:"gatewayDispatchChannelSize"` 24 | GatewayMaxPendingCount int `json:"gatewayMaxPendingCount"` 25 | GatewayPushRetry int `json:"gatewayPushRetry"` 26 | } 27 | 28 | var ( 29 | G_config *Config 30 | ) 31 | 32 | func InitConfig(filename string) (err error) { 33 | var ( 34 | content []byte 35 | conf Config 36 | ) 37 | 38 | if content, err = ioutil.ReadFile(filename); err != nil { 39 | return 40 | } 41 | 42 | if err = json.Unmarshal(content, &conf); err != nil { 43 | return 44 | } 45 | 46 | G_config = &conf 47 | return 48 | } -------------------------------------------------------------------------------- /logic/GateConn.go: -------------------------------------------------------------------------------- 1 | package logic 2 | 3 | import ( 4 | "net/http" 5 | "crypto/tls" 6 | "time" 7 | "net/url" 8 | "strconv" 9 | "golang.org/x/net/http2" 10 | ) 11 | 12 | // 与网关之间的通讯 13 | type GateConn struct { 14 | schema string 15 | client *http.Client // 内置长连接+并发连接数 16 | } 17 | 18 | func InitGateConn(gatewayConfig *GatewayConfig) (gateConn *GateConn, err error) { 19 | var ( 20 | transport *http.Transport 21 | ) 22 | 23 | gateConn = &GateConn{ 24 | schema: "https://" + gatewayConfig.Hostname + ":" + strconv.Itoa(gatewayConfig.Port), 25 | } 26 | 27 | transport = &http.Transport{ 28 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true,}, // 不校验服务端证书 29 | MaxIdleConns: G_config.GatewayMaxConnection, 30 | MaxIdleConnsPerHost: G_config.GatewayMaxConnection, 31 | IdleConnTimeout: time.Duration(G_config.GatewayIdleTimeout) * time.Second, // 连接空闲超时 32 | } 33 | // 启动HTTP/2协议 34 | http2.ConfigureTransport(transport) 35 | 36 | // HTTP/2 客户端 37 | gateConn.client = &http.Client{ 38 | Transport: transport, 39 | Timeout: time.Duration(G_config.GatewayTimeout) * time.Millisecond, // 请求超时 40 | } 41 | return 42 | } 43 | 44 | // 出于性能考虑, 消息数组在此前已经编码成json 45 | func (gateConn *GateConn) PushAll(itemsJson []byte) (err error) { 46 | var ( 47 | apiUrl string 48 | form url.Values 49 | resp *http.Response 50 | retry int 51 | ) 52 | 53 | apiUrl = gateConn.schema + "/push/all" 54 | 55 | form = url.Values{} 56 | form.Set("items", string(itemsJson)) 57 | 58 | for retry = 0; retry < G_config.GatewayPushRetry; retry++ { 59 | if resp, err = gateConn.client.PostForm(apiUrl, form); err != nil { 60 | PushFail_INCR() 61 | continue 62 | } 63 | resp.Body.Close() 64 | break 65 | } 66 | return 67 | } 68 | 69 | // 出于性能考虑, 消息数组在此前已经编码成json 70 | func (gateConn *GateConn) PushRoom(room string, itemsJson []byte) (err error) { 71 | var ( 72 | apiUrl string 73 | form url.Values 74 | resp *http.Response 75 | retry int 76 | ) 77 | 78 | apiUrl = gateConn.schema + "/push/room" 79 | 80 | form = url.Values{} 81 | form.Set("room", room) 82 | form.Set("items", string(itemsJson)) 83 | 84 | for retry = 0; retry < G_config.GatewayPushRetry; retry++ { 85 | if resp, err = gateConn.client.PostForm(apiUrl, form); err != nil { 86 | PushFail_INCR() 87 | continue 88 | } 89 | resp.Body.Close() 90 | break 91 | } 92 | return 93 | } -------------------------------------------------------------------------------- /logic/GateConnMgr.go: -------------------------------------------------------------------------------- 1 | package logic 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/owenliang/go-push/common" 6 | ) 7 | 8 | type PushJob struct { 9 | pushType int // 推送类型 10 | roomId string // 房间ID 11 | items []json.RawMessage // 要推送的消息数组 12 | } 13 | 14 | type GateConnMgr struct { 15 | gateConns []*GateConn // 到所有gateway的连接数组 16 | pendingChan []chan byte // gateway的并发请求控制 17 | dispatchChan chan*PushJob // 待分发的推送 18 | } 19 | 20 | var ( 21 | G_gateConnMgr *GateConnMgr 22 | ) 23 | 24 | // 推送给一个gateway 25 | func (gateConnMgr *GateConnMgr) doPush(gatewayIdx int, pushJob *PushJob, itemsJson []byte) { 26 | if pushJob.pushType == common.PUSH_TYPE_ALL { 27 | gateConnMgr.gateConns[gatewayIdx].PushAll(itemsJson) 28 | } else if pushJob.pushType == common.PUSH_TYPE_ROOM { 29 | gateConnMgr.gateConns[gatewayIdx].PushRoom(pushJob.roomId, itemsJson) 30 | } 31 | 32 | // 释放名额 33 | <- gateConnMgr.pendingChan[gatewayIdx] 34 | } 35 | 36 | // 消息分发协程 37 | func (gateConnMgr* GateConnMgr) dispatchWorkerMain(dispatchWorkerIdx int) { 38 | var ( 39 | pushJob *PushJob 40 | gatewayIdx int 41 | itemsJson []byte 42 | err error 43 | ) 44 | for { 45 | select { 46 | case pushJob = <- gateConnMgr.dispatchChan: 47 | // 序列化 48 | if itemsJson, err = json.Marshal(pushJob.items); err != nil { 49 | continue 50 | } 51 | // 分发到所有gateway 52 | for gatewayIdx = 0; gatewayIdx < len(gateConnMgr.gateConns); gatewayIdx++ { 53 | select { 54 | case gateConnMgr.pendingChan[gatewayIdx] <- 1: // 并发控制 55 | go gateConnMgr.doPush(gatewayIdx, pushJob, itemsJson) 56 | default: // 并发已满, 直接丢弃 57 | } 58 | } 59 | } 60 | } 61 | } 62 | 63 | func InitGateConnMgr() (err error) { 64 | var ( 65 | gatewayIdx int 66 | dispatchWorkerIdx int 67 | gatewayConfig GatewayConfig 68 | gateConnMgr *GateConnMgr 69 | ) 70 | 71 | gateConnMgr = &GateConnMgr{ 72 | gateConns: make([]*GateConn, len(G_config.GatewayList)), 73 | pendingChan: make([]chan byte, len(G_config.GatewayList)), 74 | dispatchChan: make(chan*PushJob, G_config.GatewayDispatchChannelSize), 75 | } 76 | 77 | for gatewayIdx, gatewayConfig = range G_config.GatewayList { 78 | if gateConnMgr.gateConns[gatewayIdx], err = InitGateConn(&gatewayConfig); err != nil { 79 | return 80 | } 81 | gateConnMgr.pendingChan[gatewayIdx] = make(chan byte, G_config.GatewayMaxPendingCount) 82 | } 83 | 84 | for dispatchWorkerIdx = 0; dispatchWorkerIdx < G_config.GatewayDispatchWorkerCount; dispatchWorkerIdx++ { 85 | go gateConnMgr.dispatchWorkerMain(dispatchWorkerIdx) 86 | } 87 | 88 | G_gateConnMgr = gateConnMgr 89 | return 90 | } 91 | 92 | func (gateConnMgr *GateConnMgr) PushAll(items []json.RawMessage) (err error) { 93 | var ( 94 | pushJob *PushJob 95 | ) 96 | 97 | pushJob = &PushJob{ 98 | pushType: common.PUSH_TYPE_ALL, 99 | items: items, 100 | } 101 | 102 | select { 103 | case gateConnMgr.dispatchChan <- pushJob: 104 | DispatchTotal_INCR(int64(len(items))) 105 | default: 106 | DispatchFail_INCR(int64(len(items))) 107 | err = common.ERR_LOGIC_DISPATCH_CHANNEL_FULL 108 | } 109 | return 110 | } 111 | 112 | func (gateConnMgr *GateConnMgr) PushRoom(roomId string, items []json.RawMessage) (err error) { 113 | var ( 114 | pushJob *PushJob 115 | ) 116 | 117 | pushJob = &PushJob{ 118 | pushType: common.PUSH_TYPE_ROOM, 119 | roomId: roomId, 120 | items: items, 121 | } 122 | 123 | select { 124 | case gateConnMgr.dispatchChan <- pushJob: 125 | DispatchTotal_INCR(int64(len(items))) 126 | default: 127 | DispatchFail_INCR(int64(len(items))) 128 | err = common.ERR_LOGIC_DISPATCH_CHANNEL_FULL 129 | } 130 | return 131 | } -------------------------------------------------------------------------------- /logic/Service.go: -------------------------------------------------------------------------------- 1 | package logic 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | "net" 7 | "strconv" 8 | "encoding/json" 9 | ) 10 | 11 | type Service struct { 12 | server *http.Server 13 | } 14 | 15 | var ( 16 | G_service *Service 17 | ) 18 | 19 | // 全量推送POST msg={} 20 | func handlePushAll(resp http.ResponseWriter, req *http.Request) { 21 | var ( 22 | err error 23 | items string 24 | msgArr []json.RawMessage 25 | ) 26 | if err = req.ParseForm(); err != nil { 27 | return 28 | } 29 | 30 | items = req.PostForm.Get("items") 31 | if err = json.Unmarshal([]byte(items), &msgArr); err != nil { 32 | return 33 | } 34 | 35 | G_gateConnMgr.PushAll(msgArr) 36 | } 37 | 38 | // 房间推送POST room=xxx&msg 39 | func handlePushRoom(resp http.ResponseWriter, req *http.Request) { 40 | var ( 41 | err error 42 | room string 43 | items string 44 | msgArr []json.RawMessage 45 | ) 46 | if err = req.ParseForm(); err != nil { 47 | return 48 | } 49 | 50 | room = req.PostForm.Get("room") 51 | items = req.PostForm.Get("items") 52 | 53 | if err = json.Unmarshal([]byte(items), &msgArr); err != nil { 54 | return 55 | } 56 | 57 | G_gateConnMgr.PushRoom(room, msgArr) 58 | } 59 | 60 | // 处理统计 61 | func handleStats(resp http.ResponseWriter, req *http.Request) { 62 | var ( 63 | data []byte 64 | err error 65 | ) 66 | 67 | if data, err = G_stats.Dump(); err != nil { 68 | return 69 | } 70 | 71 | resp.Write(data) 72 | } 73 | 74 | func InitService() (err error) { 75 | var ( 76 | mux *http.ServeMux 77 | server *http.Server 78 | listener net.Listener 79 | ) 80 | 81 | // 路由 82 | mux = http.NewServeMux() 83 | mux.HandleFunc("/push/all", handlePushAll) 84 | mux.HandleFunc("/push/room", handlePushRoom) 85 | mux.HandleFunc("/stats", handleStats) 86 | 87 | // HTTP/1服务 88 | server = &http.Server{ 89 | ReadTimeout: time.Duration(G_config.ServiceReadTimeout) * time.Millisecond, 90 | WriteTimeout: time.Duration(G_config.ServiceWriteTimeout) * time.Millisecond, 91 | Handler: mux, 92 | } 93 | 94 | // 监听端口 95 | if listener, err = net.Listen("tcp", ":" + strconv.Itoa(G_config.ServicePort)); err != nil { 96 | return 97 | } 98 | 99 | // 赋值全局变量 100 | G_service = &Service{ 101 | server: server, 102 | } 103 | 104 | // 拉起服务 105 | go server.Serve(listener) 106 | 107 | return 108 | } -------------------------------------------------------------------------------- /logic/Stats.go: -------------------------------------------------------------------------------- 1 | package logic 2 | 3 | import ( 4 | "sync/atomic" 5 | "encoding/json" 6 | ) 7 | 8 | type Stats struct { 9 | // 分发总消息数 10 | DispatchTotal int64 `json:"DispatchTotal"` 11 | // 分发丢弃消息数 12 | DispatchFail int64 `json:"DispatchFail"` 13 | // 推送失败次数 14 | PushFail int64 `json:"PushFail"` 15 | } 16 | 17 | var ( 18 | G_stats *Stats 19 | ) 20 | 21 | func InitStats() (err error) { 22 | G_stats = &Stats{} 23 | return 24 | } 25 | 26 | func DispatchTotal_INCR(batchSize int64) { 27 | atomic.AddInt64(&G_stats.DispatchTotal, batchSize) 28 | } 29 | 30 | func DispatchFail_INCR(batchSize int64) { 31 | atomic.AddInt64(&G_stats.DispatchFail, batchSize) 32 | } 33 | 34 | func PushFail_INCR() { 35 | atomic.AddInt64(&G_stats.PushFail, 1) 36 | } 37 | 38 | func (stats *Stats) Dump() (data []byte, err error){ 39 | return json.Marshal(G_stats) 40 | } -------------------------------------------------------------------------------- /logic/cli/logic.json: -------------------------------------------------------------------------------- 1 | { 2 | "HTTP/1服务端口": "接收业务方调用", 3 | "servicePort": 7799, 4 | 5 | "接口读超时": "单位毫秒", 6 | "serviceReadTimeout": 2000, 7 | 8 | "接口写超时": "单位毫秒", 9 | "serviceWriteTimeout": 2000, 10 | 11 | "网关列表": "推送将分发给所有网关", 12 | "gatewayList": [ 13 | { 14 | "hostname": "localhost", 15 | "port": 7788 16 | } 17 | ], 18 | 19 | "每个网关的最多并发连接数": "建议与gateway的CPU核数相等, 提升内部通讯吞吐", 20 | "gatewayMaxConnection": 32, 21 | 22 | "网关单个请求的超时时间": "单位是毫秒", 23 | "gatewayTimeout": 3000, 24 | 25 | "网关连接的空闲关闭时间": "单位是秒", 26 | "gatewayIdleTimeout": 60, 27 | 28 | "向各个网关分发消息的协程数量": "CPU密集型, 与CPU个数相当即可", 29 | "gatewayDispatchWorkerCount": 32, 30 | 31 | "待分发消息队列长度": "分发本身很快, 队列不需要太大", 32 | "gatewayDispatchChannelSize": 100000, 33 | 34 | "每个网关的最大拥塞推送数": "当消息拥塞时, 后续发往该网关的消息将被丢弃", 35 | "gatewayMaxPendingCount": 200000, 36 | 37 | "每条推送的最大重试次数": "超过重试次数后, 消息将被丢弃", 38 | "gatewayPushRetry": 3 39 | } -------------------------------------------------------------------------------- /logic/cli/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "flag" 7 | "runtime" 8 | "time" 9 | "github.com/owenliang/go-push/logic" 10 | ) 11 | 12 | var ( 13 | confFile string // 配置文件路径 14 | ) 15 | 16 | func initArgs() { 17 | flag.StringVar(&confFile, "config", "./logic.json", "where logic.json is.") 18 | flag.Parse() 19 | } 20 | 21 | func initEnv() { 22 | runtime.GOMAXPROCS(runtime.NumCPU()) 23 | } 24 | 25 | func main() { 26 | var ( 27 | err error 28 | ) 29 | 30 | // 初始化环境 31 | initArgs() 32 | initEnv() 33 | 34 | if err = logic.InitConfig(confFile); err != nil { 35 | goto ERR 36 | } 37 | 38 | if err = logic.InitStats(); err != nil { 39 | goto ERR 40 | } 41 | 42 | if err = logic.InitGateConnMgr(); err != nil { 43 | goto ERR 44 | } 45 | 46 | if err = logic.InitService(); err != nil { 47 | goto ERR 48 | } 49 | 50 | for { 51 | time.Sleep(1 * time.Second) 52 | } 53 | 54 | os.Exit(0) 55 | 56 | ERR: 57 | fmt.Fprintln(os.Stderr, err) 58 | os.Exit(-1) 59 | } 60 | -------------------------------------------------------------------------------- /xingqiu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/owenliang/go-push/b5b6a2e1cf9172762800f263198b65058a99cbe9/xingqiu.png --------------------------------------------------------------------------------