├── .gitignore ├── LICENSE ├── README.md ├── config.go ├── config ├── env.example └── queues.example.yml ├── glide.lock ├── glide.yaml ├── main.go ├── message.go ├── restart.sh ├── test-tools ├── send.go └── server.go └── util.go /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/go 3 | 4 | ### Go ### 5 | # Binaries for programs and plugins 6 | *.exe 7 | *.dll 8 | *.so 9 | *.dylib 10 | *.swp 11 | 12 | # Test binary, build with `go test -c` 13 | *.test 14 | 15 | # Output of the go coverage tool, specifically when used with LiteIDE 16 | *.out 17 | 18 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 19 | .glide/ 20 | 21 | run/* 22 | log/ 23 | vendor/ 24 | 25 | # End of https://www.gitignore.io/api/go 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 fishtrip 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # Feature 特性 3 | 4 | * 高性能。在 macbook pro 15 上测试，每个队列的处理能力可以轻松达到 3000 message/second 以上，多个队列也可以做到线性的增加性能，整体应用达到几万每秒很轻松。同时，得益于 golang 的协程设计，如果下游出现了慢调用，那么也不会影响并发。 5 | * 优雅关闭。通过对信号的监听，整个程序可以在不丢消息的情况下优雅关闭，利于配置更改和程序重启。这个在生产环境非常重要。 6 | * 自动重连。当 RabbitMQ 服务无法连接的时候，应用可以自动重连。 7 | 8 | # Usage 使用 9 | 10 | ## Build 11 | go build -o watchman 12 | 13 | ## Usage 14 | 15 | ``` 16 | Usage of ./watchman: 17 | -c string 18 | config file path (default "config/queues.example.yml") 19 | -log string 20 | logging file, default STDOUT 21 | -pidfile string 22 | If specified, write pid to file. 23 | ``` 24 | 25 | 优雅关闭： 26 | ``` 27 | # kill -QUIT $PID 28 | ``` 29 | 30 | # config 配置 31 | ## ENV 文件 32 | 使用前需要先加载 env，样例见 config/env.example.yml，主要是 RabbitMQ 的配置。 33 | 34 | ## 队列配置文件 35 | 样例见 config/queues.example.yml, 主要是注明消息队列的配置以及回调地址和参数。 36 | 37 | # Author 作者 38 | 大鱼（fishtrip.cn) 是国内领先的全球民宿平台，给用户提供全球特色的民宿、客栈、别墅等非标准住宿。目前公司的技术栈以 ruby 和 Java 为主，部分中间件采用 golang 编写。 39 | 40 | # Licence 许可 41 | MIT Licence 42 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "log" 7 | "strings" 8 | 9 | "github.com/streadway/amqp" 10 | yaml "gopkg.in/yaml.v2" 11 | ) 12 | 13 | type ProjectsConfig struct { 14 | Projects []ProjectConfig `yaml:"projects"` 15 | } 16 | type ProjectConfig struct { 17 | Name string `yaml:"name"` 18 | QueuesDefaultConfig QueuesDefaultConfig `yaml:"queues_default"` 19 | Queues []QueueConfig `yaml:"queues"` 20 | } 21 | type QueuesDefaultConfig struct { 22 | NotifyBase string `yaml:"notify_base"` 23 | NotifyTimeout int `yaml:"notify_timeout"` 24 | RetryTimes int `yaml:"retry_times"` 25 | RetryDuration int `yaml:"retry_duration"` 26 | BindingExchange string `yaml:"binding_exchange"` 27 | } 28 | type QueueConfig struct { 29 | QueueName string `yaml:"queue_name"` 30 | RoutingKey []string `yaml:"routing_key"` 31 | NotifyPath string `yaml:"notify_path"` 32 | NotifyTimeout int `yaml:"notify_timeout"` 33 | RetryTimes int `yaml:"retry_times"` 34 | RetryDuration int `yaml:"retry_duration"` 35 | BindingExchange string `yaml:"binding_exchange"` 36 | 37 | project *ProjectConfig 38 | } 39 | 40 | func (qc QueueConfig) WorkerQueueName() string { 41 | return qc.QueueName 42 | } 43 | func (qc QueueConfig) RetryQueueName() string { 44 | return fmt.Sprintf("%s-retry", qc.QueueName) 45 | } 46 | func (qc QueueConfig) ErrorQueueName() string { 47 | return fmt.Sprintf("%s-error", qc.QueueName) 48 | } 49 | func (qc QueueConfig) RetryExchangeName() string { 50 | return fmt.Sprintf("%s-retry", qc.QueueName) 51 | } 52 | func (qc QueueConfig) RequeueExchangeName() string { 53 | return fmt.Sprintf("%s-retry-requeue", qc.QueueName) 54 | } 55 | func (qc QueueConfig) ErrorExchangeName() string { 56 | return fmt.Sprintf("%s-error", qc.QueueName) 57 | } 58 | func (qc QueueConfig) WorkerExchangeName() string { 59 | if qc.BindingExchange == "" { 60 | return qc.project.QueuesDefaultConfig.BindingExchange 61 | } 62 | return qc.BindingExchange 63 | } 64 | 65 | func (qc QueueConfig) NotifyUrl() string { 66 | if strings.HasPrefix(qc.NotifyPath, "http://") || strings.HasPrefix(qc.NotifyPath, "https://") { 67 | return qc.NotifyPath 68 | } 69 | return fmt.Sprintf("%s%s", qc.project.QueuesDefaultConfig.NotifyBase, qc.NotifyPath) 70 | } 71 | 72 | func (qc QueueConfig) NotifyTimeoutWithDefault() int { 73 | if qc.NotifyTimeout == 0 { 74 | return qc.project.QueuesDefaultConfig.NotifyTimeout 75 | } 76 | return qc.NotifyTimeout 77 | } 78 | 79 | func (qc QueueConfig) RetryTimesWithDefault() int { 80 | if qc.RetryTimes == 0 { 81 | return qc.project.QueuesDefaultConfig.RetryTimes 82 | } 83 | return qc.RetryTimes 84 | } 85 | 86 | func (qc QueueConfig) RetryDurationWithDefault() int { 87 | if qc.RetryDuration == 0 { 88 | return qc.project.QueuesDefaultConfig.RetryDuration 89 | } 90 | return qc.RetryDuration 91 | } 92 | 93 | func (qc QueueConfig) DeclareExchange(channel *amqp.Channel) { 94 | exchanges := []string{ 95 | qc.WorkerExchangeName(), 96 | qc.RetryExchangeName(), 97 | qc.ErrorExchangeName(), 98 | qc.RequeueExchangeName(), 99 | } 100 | 101 | for _, e := range exchanges { 102 | log.Printf("declaring exchange: %s\n", e) 103 | 104 | err := channel.ExchangeDeclare(e, "topic", true, false, false, false, nil) 105 | PanicOnError(err) 106 | } 107 | } 108 | 109 | func (qc QueueConfig) DeclareQueue(channel *amqp.Channel) { 110 | var err error 111 | 112 | // 定义重试队列 113 | log.Printf("declaring retry queue: %s\n", qc.RetryQueueName()) 114 | retryQueueOptions := map[string]interface{}{ 115 | "x-dead-letter-exchange": qc.RequeueExchangeName(), 116 | "x-message-ttl": int32(qc.RetryDurationWithDefault() * 1000), 117 | } 118 | 119 | _, err = channel.QueueDeclare(qc.RetryQueueName(), true, false, false, false, retryQueueOptions) 120 | PanicOnError(err) 121 | err = channel.QueueBind(qc.RetryQueueName(), "#", qc.RetryExchangeName(), false, nil) 122 | PanicOnError(err) 123 | 124 | // 定义错误队列 125 | log.Printf("declaring error queue: %s\n", qc.ErrorQueueName()) 126 | 127 | _, err = channel.QueueDeclare(qc.ErrorQueueName(), true, false, false, false, nil) 128 | PanicOnError(err) 129 | err = channel.QueueBind(qc.ErrorQueueName(), "#", qc.ErrorExchangeName(), false, nil) 130 | PanicOnError(err) 131 | 132 | // 定义工作队列 133 | log.Printf("declaring worker queue: %s\n", qc.WorkerQueueName()) 134 | 135 | workerQueueOptions := map[string]interface{}{ 136 | "x-dead-letter-exchange": qc.RetryExchangeName(), 137 | } 138 | _, err = channel.QueueDeclare(qc.WorkerQueueName(), true, false, false, false, workerQueueOptions) 139 | PanicOnError(err) 140 | 141 | for _, key := range qc.RoutingKey { 142 | err = channel.QueueBind(qc.WorkerQueueName(), key, qc.WorkerExchangeName(), false, nil) 143 | PanicOnError(err) 144 | } 145 | 146 | // 最后，绑定工作队列 和 requeue Exchange 147 | err = channel.QueueBind(qc.WorkerQueueName(), "#", qc.RequeueExchangeName(), false, nil) 148 | PanicOnError(err) 149 | } 150 | 151 | func loadQueuesConfig(configFileName string, allQueues []*QueueConfig) []*QueueConfig { 152 | configFile, err := ioutil.ReadFile(configFileName) 153 | PanicOnError(err) 154 | 155 | projectsConfig := ProjectsConfig{} 156 | err = yaml.Unmarshal(configFile, &projectsConfig) 157 | PanicOnError(err) 158 | log.Printf("find config: %v", projectsConfig) 159 | 160 | projects := projectsConfig.Projects 161 | for i, project := range projects { 162 | log.Printf("find project: %s", project.Name) 163 | 164 | queues := projects[i].Queues 165 | for j, queue := range queues { 166 | log.Printf("find queue: %v", queue) 167 | 168 | queues[j].project = &projects[i] 169 | allQueues = append(allQueues, &queues[j]) 170 | } 171 | } 172 | 173 | return allQueues 174 | } 175 | -------------------------------------------------------------------------------- /config/env.example: -------------------------------------------------------------------------------- 1 | export AMQP_URL="amqp://guest:guest@localhost:5672" 2 | -------------------------------------------------------------------------------- /config/queues.example.yml: -------------------------------------------------------------------------------- 1 | projects: 2 | - name: finance 3 | queues_default: 4 | notify_base: "http://localhost:8080" 5 | notify_timeout: 5 6 | retry_times: 40 7 | retry_duration: 300 8 | binding_exchange: fishtrip 9 | queues: 10 | - queue_name: "processor" 11 | notify_path: "/test.html" 12 | routing_key: 13 | - "order.state.paid" 14 | - queue_name: "wakasms.orders" 15 | notify_path: "/test.html" 16 | routing_key: 17 | - "order.state.*" 18 | -------------------------------------------------------------------------------- /glide.lock: -------------------------------------------------------------------------------- 1 | hash: 62485fe6290eedab08db02b0935bf41a55268502369de208a4e104a34fdf78ae 2 | updated: 2017-07-06T01:09:04.171177079+08:00 3 | imports: 4 | - name: github.com/facebookgo/atomicfile 5 | version: 2de1f203e7d5e386a6833233882782932729f27e 6 | - name: github.com/facebookgo/pidfile 7 | version: f242e2999868dcd267a2b86e49ce1f9cf9e15b16 8 | - name: github.com/google/uuid 9 | version: 064e2069ce9c359c118179501254f67d7d37ba24 10 | - name: github.com/satori/go.uuid 11 | version: 879c5887cd475cd7864858769793b2ceb0d44feb 12 | - name: github.com/streadway/amqp 13 | version: 27859d32540aebd2e5befa52dc59ae8e6a0132b6 14 | - name: gopkg.in/yaml.v2 15 | version: cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b 16 | testImports: [] 17 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: fishtrip.cn/watchman 2 | import: 3 | - package: github.com/streadway/amqp 4 | - package: gopkg.in/yaml.v2 5 | - package: github.com/satori/go.uuid 6 | version: ^1.1.0 7 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "log" 7 | "os" 8 | "os/signal" 9 | "runtime" 10 | "sync" 11 | "syscall" 12 | "time" 13 | 14 | "github.com/facebookgo/pidfile" 15 | uuid "github.com/satori/go.uuid" 16 | "github.com/streadway/amqp" 17 | ) 18 | 19 | const ( 20 | // chan 21 | ChannelBufferLength = 100 22 | 23 | //worker number 24 | ReceiverNum = 5 25 | AckerNum = 10 26 | ResenderNum = 5 27 | 28 | // http tune 29 | HttpMaxIdleConns = 500 // default 100 in net/http 30 | HttpMaxIdleConnsPerHost = 500 // default 2 in net/http 31 | HttpIdleConnTimeout = 30 // default 90 in net/http 32 | ) 33 | 34 | func receiveMessage(queues []*QueueConfig, done <-chan struct{}) <-chan Message { 35 | out := make(chan Message, ChannelBufferLength) 36 | var wg sync.WaitGroup 37 | 38 | receiver := func(qc QueueConfig) { 39 | defer wg.Done() 40 | 41 | RECONNECT: 42 | for { 43 | _, channel, err := setupChannel() 44 | if err != nil { 45 | PanicOnError(err) 46 | } 47 | 48 | msgs, err := channel.Consume( 49 | qc.WorkerQueueName(), // queue 50 | "", // consumer 51 | false, // auto-ack 52 | false, // exclusive 53 | false, // no-local 54 | false, // no-wait 55 | nil, // args 56 | ) 57 | PanicOnError(err) 58 | 59 | for { 60 | select { 61 | case msg, ok := <-msgs: 62 | if !ok { 63 | log.Printf("receiver: channel is closed, maybe lost connection") 64 | time.Sleep(5 * time.Second) 65 | continue RECONNECT 66 | } 67 | msg.MessageId = fmt.Sprintf("%s", uuid.NewV4()) 68 | message := Message{qc, &msg, 0} 69 | out <- message 70 | 71 | message.Printf("receiver: received msg") 72 | case <-done: 73 | log.Printf("receiver: received a done signal") 74 | return 75 | } 76 | } 77 | } 78 | } 79 | 80 | for _, queue := range queues { 81 | wg.Add(ReceiverNum) 82 | for i := 0; i < ReceiverNum; i++ { 83 | go receiver(*queue) 84 | } 85 | } 86 | 87 | go func() { 88 | wg.Wait() 89 | log.Printf("all receiver is done, closing channel") 90 | close(out) 91 | }() 92 | 93 | return out 94 | } 95 | 96 | func workMessage(in <-chan Message) <-chan Message { 97 | var wg sync.WaitGroup 98 | out := make(chan Message, ChannelBufferLength) 99 | client := newHttpClient(HttpMaxIdleConns, HttpMaxIdleConnsPerHost, HttpIdleConnTimeout) 100 | 101 | worker := func(m Message, o chan<- Message) { 102 | m.Printf("worker: received a msg, body: %s", string(m.amqpDelivery.Body)) 103 | 104 | defer wg.Done() 105 | m.Notify(client) 106 | o <- m 107 | } 108 | 109 | wg.Add(1) 110 | go func() { 111 | defer wg.Done() 112 | 113 | for message := range in { 114 | wg.Add(1) 115 | go worker(message, out) 116 | } 117 | }() 118 | 119 | go func() { 120 | wg.Wait() 121 | log.Printf("all worker is done, closing channel") 122 | close(out) 123 | }() 124 | 125 | return out 126 | } 127 | 128 | func ackMessage(in <-chan Message) <-chan Message { 129 | out := make(chan Message) 130 | var wg sync.WaitGroup 131 | 132 | acker := func() { 133 | defer wg.Done() 134 | 135 | for m := range in { 136 | m.Printf("acker: received a msg") 137 | 138 | if m.IsNotifySuccess() { 139 | m.Ack() 140 | } else if m.IsMaxRetry() { 141 | m.Republish(out) 142 | } else { 143 | m.Reject() 144 | } 145 | } 146 | } 147 | 148 | for i := 0; i < AckerNum; i++ { 149 | wg.Add(1) 150 | go acker() 151 | } 152 | 153 | go func() { 154 | wg.Wait() 155 | log.Printf("all acker is done, close out") 156 | close(out) 157 | }() 158 | 159 | return out 160 | } 161 | 162 | func resendMessage(in <-chan Message) <-chan Message { 163 | out := make(chan Message) 164 | 165 | var wg sync.WaitGroup 166 | 167 | resender := func() { 168 | defer wg.Done() 169 | 170 | RECONNECT: 171 | for { 172 | conn, channel, err := setupChannel() 173 | if err != nil { 174 | PanicOnError(err) 175 | } 176 | 177 | for m := range in { 178 | err := m.CloneAndPublish(channel) 179 | if err == amqp.ErrClosed { 180 | time.Sleep(5 * time.Second) 181 | continue RECONNECT 182 | } 183 | } 184 | 185 | // normally quit , we quit too 186 | conn.Close() 187 | break 188 | } 189 | } 190 | 191 | for i := 0; i < ResenderNum; i++ { 192 | wg.Add(1) 193 | go resender() 194 | } 195 | 196 | go func() { 197 | wg.Wait() 198 | log.Printf("all resender is done, close out") 199 | close(out) 200 | }() 201 | 202 | return out 203 | } 204 | 205 | func handleSignal(done chan<- struct{}) { 206 | chan_sigs := make(chan os.Signal, 1) 207 | signal.Notify(chan_sigs, syscall.SIGQUIT) 208 | 209 | go func() { 210 | sig := <-chan_sigs 211 | 212 | if sig != nil { 213 | log.Printf("received a signal %v, close done channel", sig) 214 | close(done) 215 | } 216 | }() 217 | } 218 | 219 | func main() { 220 | // parse command line args 221 | configFileName := flag.String("c", "config/queues.example.yml", "config file path") 222 | logFileName := flag.String("log", "", "logging file, default STDOUT") 223 | flag.Parse() 224 | 225 | // write pid file 226 | pidfile.Write() 227 | 228 | // set loger 229 | if *logFileName != "" { 230 | f, err := os.OpenFile(*logFileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) 231 | PanicOnError(err) 232 | defer f.Close() 233 | 234 | log.SetOutput(f) 235 | } 236 | 237 | // read yaml config 238 | var allQueues []*QueueConfig 239 | allQueues = loadQueuesConfig(*configFileName, allQueues) 240 | 241 | // create queues 242 | _, channel, err := setupChannel() 243 | if err != nil { 244 | PanicOnError(err) 245 | } 246 | for _, queue := range allQueues { 247 | log.Printf("allQueues: queue config: %v", queue) 248 | queue.DeclareExchange(channel) 249 | queue.DeclareQueue(channel) 250 | } 251 | 252 | // register signal 253 | done := make(chan struct{}, 1) 254 | handleSignal(done) 255 | 256 | // change gorouting config 257 | log.Printf("set gorouting to the number of logical CPU: %d", runtime.NumCPU()) 258 | runtime.GOMAXPROCS(runtime.NumCPU()) 259 | 260 | // main logic 261 | <-resendMessage(ackMessage(workMessage(receiveMessage(allQueues, done)))) 262 | 263 | log.Printf("exiting program") 264 | } 265 | -------------------------------------------------------------------------------- /message.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "time" 7 | 8 | "github.com/streadway/amqp" 9 | ) 10 | 11 | type NotifyResponse int 12 | 13 | const ( 14 | NotifySuccess = 1 15 | NotifyFailure = 0 16 | ) 17 | 18 | type Message struct { 19 | queueConfig QueueConfig 20 | amqpDelivery *amqp.Delivery // message read from rabbitmq 21 | notifyResponse NotifyResponse // notify result from callback url 22 | } 23 | 24 | func (m Message) CurrentMessageRetries() int { 25 | msg := m.amqpDelivery 26 | 27 | xDeathArray, ok := msg.Headers["x-death"].([]interface{}) 28 | if !ok { 29 | m.Printf("x-death array case fail") 30 | return 0 31 | } 32 | 33 | if len(xDeathArray) <= 0 { 34 | return 0 35 | } 36 | 37 | for _, h := range xDeathArray { 38 | xDeathItem := h.(amqp.Table) 39 | 40 | if xDeathItem["reason"] == "rejected" { 41 | return int(xDeathItem["count"].(int64)) 42 | } 43 | } 44 | 45 | return 0 46 | } 47 | 48 | func (m *Message) Notify(client *http.Client) *Message { 49 | qc := m.queueConfig 50 | msg := m.amqpDelivery 51 | 52 | client.Timeout = time.Duration(qc.NotifyTimeoutWithDefault()) * time.Second 53 | statusCode := notifyUrl(client, qc.NotifyUrl(), msg.Body) 54 | 55 | m.Printf("notify url %s, result: %d", qc.NotifyUrl(), statusCode) 56 | 57 | if statusCode == 200 || statusCode == 201 { 58 | m.notifyResponse = NotifySuccess 59 | } else { 60 | m.notifyResponse = NotifyFailure 61 | } 62 | 63 | return m 64 | } 65 | 66 | func (m Message) IsMaxRetry() bool { 67 | retries := m.CurrentMessageRetries() 68 | maxRetries := m.queueConfig.RetryTimesWithDefault() 69 | return retries >= maxRetries 70 | } 71 | 72 | func (m Message) IsNotifySuccess() bool { 73 | return m.notifyResponse == NotifySuccess 74 | } 75 | 76 | func (m Message) Ack() error { 77 | m.Printf("acker: ack message") 78 | err := m.amqpDelivery.Ack(false) 79 | LogOnError(err) 80 | return err 81 | } 82 | 83 | func (m Message) Reject() error { 84 | m.Printf("acker: reject message") 85 | err := m.amqpDelivery.Reject(false) 86 | LogOnError(err) 87 | return err 88 | } 89 | 90 | func (m Message) Republish(out chan<- Message) error { 91 | m.Printf("acker: ERROR republish message") 92 | out <- m 93 | err := m.amqpDelivery.Ack(false) 94 | LogOnError(err) 95 | return err 96 | } 97 | 98 | func (m Message) CloneAndPublish(channel *amqp.Channel) error { 99 | msg := m.amqpDelivery 100 | qc := m.queueConfig 101 | 102 | errMsg := cloneToPublishMsg(msg) 103 | err := channel.Publish(qc.ErrorExchangeName(), msg.RoutingKey, false, false, *errMsg) 104 | LogOnError(err) 105 | return err 106 | } 107 | 108 | func (m Message) Printf(v ...interface{}) { 109 | msg := m.amqpDelivery 110 | 111 | vv := []interface{}{} 112 | vv = append(vv, msg.MessageId, msg.RoutingKey) 113 | vv = append(vv, v[1:]...) 114 | 115 | log.Printf("[%s] [%s] "+v[0].(string), vv...) 116 | } 117 | -------------------------------------------------------------------------------- /restart.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source config/env 4 | 5 | echo "kill old process ..." 6 | kill -QUIT `cat run/watchmen.pid` 7 | 8 | echo "sleep 5 second ..." 9 | sleep 5 10 | 11 | echo "start new process ..." 12 | nohup ./watchmen -c config/queues.yml -log log/watchmen.log -pidfile run/watchmen.pid & 13 | 14 | exit 0 15 | -------------------------------------------------------------------------------- /test-tools/send.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "time" 7 | 8 | "github.com/streadway/amqp" 9 | ) 10 | 11 | func main() { 12 | url := os.Getenv("AMQP_URL") 13 | 14 | conn, err := amqp.Dial(url) 15 | if err != nil { 16 | log.Fatalf("connection.open: %s", err) 17 | } 18 | 19 | // This waits for a server acknowledgment which means the sockets will have 20 | // flushed all outbound publishings prior to returning. It's important to 21 | // block on Close to not lose any publishings. 22 | defer conn.Close() 23 | 24 | c, err := conn.Channel() 25 | if err != nil { 26 | log.Fatalf("channel.open: %s", err) 27 | } 28 | 29 | log.Printf("%s", time.Now()) 30 | 31 | for i := 0; i < 100000; i++ { 32 | msg := amqp.Publishing{ 33 | DeliveryMode: amqp.Persistent, 34 | Timestamp: time.Now(), 35 | ContentType: "application/json", 36 | Body: []byte("{\"hello\": \"world\"}"), 37 | } 38 | 39 | // This is not a mandatory delivery, so it will be dropped if there are no 40 | // queues bound to the logs exchange. 41 | err = c.Publish("fishtrip", "order.state.paid", false, false, msg) 42 | if err != nil { 43 | // Since publish is asynchronous this can happen if the network connection 44 | // is reset or if the server has run out of resources. 45 | log.Fatalf("basic.publish: %v", err) 46 | } 47 | } 48 | log.Printf("%s", time.Now()) 49 | } 50 | -------------------------------------------------------------------------------- /test-tools/server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "html" 7 | "log" 8 | "net/http" 9 | ) 10 | 11 | func main() { 12 | http.HandleFunc("/test.html", func(w http.ResponseWriter, r *http.Request) { 13 | var t interface{} 14 | 15 | fmt.Printf("Request, %v\n", r) 16 | 17 | decoder := json.NewDecoder(r.Body) 18 | err := decoder.Decode(&t) 19 | 20 | if err != nil { 21 | fmt.Printf("decode error, %s\n", err) 22 | w.WriteHeader(http.StatusBadRequest) 23 | return 24 | } 25 | 26 | //n := rand.Intn(10) 27 | //fmt.Printf("Prepare to Sleep %d seconds\n", n) 28 | //time.Sleep(time.Duration(n) * time.Second) 29 | 30 | fmt.Printf("Request Body, %v\n", t) 31 | fmt.Fprintf(w, "Hello, %q", html.EscapeString(r.URL.Path)) 32 | }) 33 | 34 | log.Fatal(http.ListenAndServe(":8080", nil)) 35 | } 36 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "log" 9 | "net/http" 10 | "os" 11 | "time" 12 | 13 | "github.com/streadway/amqp" 14 | ) 15 | 16 | func LogOnError(err error) { 17 | if err != nil { 18 | fmt.Printf("ERROR - %s\n", err) 19 | } 20 | } 21 | 22 | func PanicOnError(err error) { 23 | if err != nil { 24 | panic(err) 25 | } 26 | } 27 | 28 | func setupChannel() (*amqp.Connection, *amqp.Channel, error) { 29 | url := os.Getenv("AMQP_URL") 30 | 31 | conn, err := amqp.Dial(url) 32 | if err != nil { 33 | LogOnError(err) 34 | return nil, nil, err 35 | } 36 | 37 | channel, err := conn.Channel() 38 | if err != nil { 39 | LogOnError(err) 40 | return nil, nil, err 41 | } 42 | 43 | err = channel.Qos(1, 0, false) 44 | if err != nil { 45 | LogOnError(err) 46 | return nil, nil, err 47 | } 48 | 49 | log.Printf("setup channel success!") 50 | 51 | return conn, channel, nil 52 | } 53 | 54 | func cloneToPublishMsg(msg *amqp.Delivery) *amqp.Publishing { 55 | newMsg := amqp.Publishing{ 56 | Headers: msg.Headers, 57 | 58 | ContentType: msg.ContentType, 59 | ContentEncoding: msg.ContentEncoding, 60 | DeliveryMode: msg.DeliveryMode, 61 | Priority: msg.Priority, 62 | CorrelationId: msg.CorrelationId, 63 | ReplyTo: msg.ReplyTo, 64 | Expiration: msg.Expiration, 65 | MessageId: msg.MessageId, 66 | Timestamp: msg.Timestamp, 67 | Type: msg.Type, 68 | UserId: msg.UserId, 69 | AppId: msg.AppId, 70 | 71 | Body: msg.Body, 72 | } 73 | 74 | return &newMsg 75 | } 76 | 77 | func newHttpClient(maxIdleConns, maxIdleConnsPerHost, idleConnTimeout int) *http.Client { 78 | tr := &http.Transport{ 79 | MaxIdleConns: maxIdleConns, 80 | MaxIdleConnsPerHost: maxIdleConnsPerHost, 81 | IdleConnTimeout: time.Duration(idleConnTimeout) * time.Second, 82 | } 83 | 84 | client := &http.Client{ 85 | Transport: tr, 86 | } 87 | 88 | return client 89 | } 90 | 91 | func notifyUrl(client *http.Client, url string, body []byte) int { 92 | req, err := http.NewRequest("POST", url, bytes.NewReader(body)) 93 | if err != nil { 94 | log.Printf("notify url create req fail: %s", err) 95 | return 0 96 | } 97 | 98 | req.Header.Set("Content-Type", "application/json") 99 | response, err := client.Do(req) 100 | 101 | if err != nil { 102 | log.Printf("notify url %s fail: %s", url, err) 103 | return 0 104 | } 105 | defer response.Body.Close() 106 | 107 | io.Copy(ioutil.Discard, response.Body) 108 | 109 | return response.StatusCode 110 | } 111 | --------------------------------------------------------------------------------