├── .gitignore ├── nats ├── receiver │ └── receiver.go └── sender │ └── sender.go ├── model └── model.go ├── redis ├── receiver │ └── receiver.go └── sender │ └── sender.go ├── faker └── faker.go ├── nats-streaming-BETA ├── sender │ └── sender.go └── receiver │ └── receiver.go └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.gitignore.io/api/go 2 | # Edit at https://www.gitignore.io/?templates=go 3 | 4 | ### Go ### 5 | # Binaries for programs and plugins 6 | *.exe 7 | *.exe~ 8 | *.dll 9 | *.so 10 | *.dylib 11 | 12 | # Test binary, built with `go test -c` 13 | *.test 14 | 15 | # Output of the go coverage tool, specifically when used with LiteIDE 16 | *.out 17 | 18 | # Dependency directories (remove the comment below to include it) 19 | # vendor/ 20 | 21 | ### Go Patch ### 22 | /vendor/ 23 | /Godeps/ 24 | 25 | # End of https://www.gitignore.io/api/go 26 | 27 | json/* -------------------------------------------------------------------------------- /nats/receiver/receiver.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "redis-vs-nats/model" 7 | "sync" 8 | 9 | "github.com/nats-io/nats.go" 10 | ) 11 | 12 | var wg sync.WaitGroup // 1 13 | var client *nats.Conn 14 | var err error 15 | 16 | func init() { 17 | client, err = nats.Connect("nats://0.0.0.0:4222") 18 | checkErr(err) 19 | } 20 | 21 | func main() { 22 | wg.Add(1) 23 | go worker() 24 | wg.Wait() 25 | } 26 | 27 | func worker() { 28 | // defer wg.Done() 29 | 30 | client.Subscribe("message", func(m *nats.Msg) { 31 | message := model.Message{} 32 | json.Unmarshal([]byte(m.Data), &message) 33 | }) 34 | 35 | } 36 | 37 | func checkErr(err error) { 38 | if err != nil { 39 | log.Println(err) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /model/model.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type Message struct { 8 | ID uint 9 | CreatedAt time.Time 10 | UpdatedAt time.Time 11 | DeletedAt time.Time 12 | 13 | Name string `faker:"name"` 14 | Summary string `faker:"sentence"` 15 | Paragraph string `faker:"paragraph"` 16 | 17 | IPV4 string `faker:"ipv4"` 18 | IPV6 string `faker:"ipv6"` 19 | MAC string `faker:"mac_address"` 20 | 21 | Latitude float32 `faker:"lat"` 22 | Longitude float32 `faker:"long"` 23 | 24 | FirstName string `faker:"first_name"` 25 | LastName string `faker:"last_name"` 26 | Email string `faker:"email"` 27 | PhoneNumber string `faker:"phone_number"` 28 | CreditCardNumber string `faker:"cc_number"` 29 | CreditCardType string `faker:"cc_type"` 30 | 31 | Criticality int `faker:"boundary_start=1, boundary_end=5"` 32 | Status bool 33 | } 34 | -------------------------------------------------------------------------------- /redis/receiver/receiver.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "redis-vs-nats/model" 6 | "sync" 7 | 8 | "github.com/go-redis/redis" 9 | ) 10 | 11 | var wg sync.WaitGroup // 1 12 | var client *redis.Client 13 | 14 | func init() { 15 | client = redis.NewClient(&redis.Options{ 16 | Addr: "127.0.0.1:6379", 17 | Password: "", 18 | DB: 1, 19 | }) 20 | } 21 | 22 | func main() { 23 | wg.Add(1) 24 | go worker() 25 | wg.Wait() 26 | } 27 | 28 | func worker() { 29 | defer wg.Done() 30 | 31 | pubsub := client.Subscribe("message") 32 | defer pubsub.Close() 33 | 34 | // Wait for confirmation that subscription is created before publishing anything. 35 | _, err := pubsub.Receive() 36 | if err != nil { 37 | panic(err) 38 | } 39 | 40 | channel := pubsub.ChannelSize(1000000) 41 | 42 | message := model.Message{} 43 | for packet := range channel { 44 | json.Unmarshal([]byte(packet.Payload), &message) 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /faker/faker.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "redis-vs-nats/model" 9 | 10 | "github.com/bxcodec/faker" 11 | ) 12 | 13 | func main() { 14 | 15 | datasets := map[string]int{ 16 | "1k.json": 1000, 17 | "10k.json": 10000, 18 | "100k.json": 100000, 19 | } 20 | 21 | for filename, value := range datasets { 22 | message := model.Message{} 23 | messages := []model.Message{} 24 | 25 | // Added +1 to start with 1 not 0 26 | for i := 1; i < value+1; i++ { 27 | faker.FakeData(&message) 28 | message.ID = uint(i) 29 | messages = append(messages, message) 30 | } 31 | 32 | f, err := os.Create("../json/" + filename) 33 | if err != nil { 34 | fmt.Println(err) 35 | } 36 | defer f.Close() 37 | 38 | json, _ := json.MarshalIndent(messages, "", " ") 39 | err = ioutil.WriteFile("../json/"+filename, json, 0644) 40 | if err != nil { 41 | fmt.Println(err) 42 | } 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /nats/sender/sender.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "os" 7 | "redis-vs-nats/model" 8 | "time" 9 | 10 | "github.com/nats-io/nats.go" 11 | ) 12 | 13 | var filename string = "../../json/100k.json" 14 | var client *nats.Conn 15 | var err error 16 | 17 | func init() { 18 | client, err = nats.Connect(nats.DefaultURL) 19 | checkErr(err) 20 | } 21 | 22 | func main() { 23 | defer client.Close() 24 | 25 | start := time.Now() 26 | 27 | jsonFile, err := os.Open(filename) 28 | checkErr(err) 29 | 30 | decoder := json.NewDecoder(jsonFile) 31 | 32 | // Read opening file 33 | _, err = decoder.Token() 34 | checkErr(err) 35 | 36 | var message model.Message 37 | for decoder.More() { 38 | err := decoder.Decode(&message) 39 | checkErr(err) 40 | 41 | messageJSON, err := json.Marshal(message) 42 | checkErr(err) 43 | 44 | AddNats(messageJSON) 45 | } 46 | 47 | // Close the file 48 | _, err = decoder.Token() 49 | checkErr(err) 50 | 51 | elapsed := time.Since(start) 52 | log.Println("Nats Sender took %s", elapsed) 53 | 54 | } 55 | 56 | func AddNats(data []byte) { 57 | err = client.Publish("message", data) 58 | checkErr(err) 59 | } 60 | 61 | func checkErr(err error) { 62 | if err != nil { 63 | log.Println(err) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /redis/sender/sender.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "os" 7 | "redis-vs-nats/model" 8 | "time" 9 | 10 | "github.com/go-redis/redis" 11 | ) 12 | 13 | var filename string = "../../json/100k.json" 14 | var client *redis.Client 15 | var err error 16 | 17 | func init() { 18 | client = redis.NewClient(&redis.Options{ 19 | Addr: "0.0.0.0:6379", 20 | Password: "", // no password set 21 | DB: 1, // use default DB 22 | }) 23 | } 24 | 25 | func main() { 26 | 27 | start := time.Now() 28 | 29 | jsonFile, err := os.Open(filename) 30 | checkErr(err) 31 | 32 | decoder := json.NewDecoder(jsonFile) 33 | 34 | // Read opening file 35 | _, err = decoder.Token() 36 | checkErr(err) 37 | 38 | var message model.Message 39 | for decoder.More() { 40 | err := decoder.Decode(&message) 41 | checkErr(err) 42 | 43 | messageJSON, err := json.Marshal(message) 44 | checkErr(err) 45 | 46 | AddRedis(messageJSON) 47 | } 48 | 49 | // Close the file 50 | _, err = decoder.Token() 51 | checkErr(err) 52 | 53 | elapsed := time.Since(start) 54 | log.Println("Redis Sender took %s", elapsed) 55 | 56 | } 57 | 58 | /*AddRedis publish incident data to insight channel*/ 59 | func AddRedis(data []byte) { 60 | _, err = client.Do("PUBLISH", "message", string(data)).Result() 61 | if err != nil { 62 | log.Println(err) 63 | } 64 | } 65 | 66 | func checkErr(err error) { 67 | if err != nil { 68 | log.Println(err) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /nats-streaming-BETA/sender/sender.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "os" 7 | "redis-vs-nats/model" 8 | "time" 9 | 10 | "github.com/nats-io/nats.go" 11 | "github.com/nats-io/stan.go" 12 | ) 13 | 14 | var filename string = "../../json/100bin.json" 15 | var client *nats.Conn 16 | var sc stan.Conn 17 | var err error 18 | 19 | func init() { 20 | client, err = nats.Connect(nats.DefaultURL) 21 | checkErr(err) 22 | 23 | sc, err = stan.Connect("test-cluster", "pubID", stan.NatsConn(client)) 24 | checkErr(err) 25 | } 26 | 27 | func main() { 28 | defer client.Close() 29 | defer sc.Close() 30 | 31 | start := time.Now() 32 | 33 | jsonFile, err := os.Open(filename) 34 | checkErr(err) 35 | 36 | decoder := json.NewDecoder(jsonFile) 37 | 38 | // Read opening file 39 | _, err = decoder.Token() 40 | checkErr(err) 41 | 42 | var message model.Message 43 | for decoder.More() { 44 | err := decoder.Decode(&message) 45 | checkErr(err) 46 | 47 | messageJSON, err := json.Marshal(message) 48 | checkErr(err) 49 | 50 | AddNats(messageJSON) 51 | } 52 | 53 | // Close the file 54 | _, err = decoder.Token() 55 | checkErr(err) 56 | 57 | elapsed := time.Since(start) 58 | log.Println("Asset Nats Parser took %s", elapsed) 59 | 60 | } 61 | 62 | func AddNats(data []byte) { 63 | err = sc.Publish("message", data) 64 | checkErr(err) 65 | } 66 | 67 | func checkErr(err error) { 68 | if err != nil { 69 | log.Println(err) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /nats-streaming-BETA/receiver/receiver.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "sync" 7 | 8 | "github.com/nats-io/nats.go" 9 | "github.com/nats-io/stan.go" 10 | ) 11 | 12 | var wg sync.WaitGroup // 1 13 | var nc *nats.Conn 14 | var sc stan.Conn 15 | var err error 16 | 17 | // var json = jsoniter.ConfigCompatibleWithStandardLibrary 18 | 19 | func init() { 20 | 21 | // sc, err = stan.Connect("test-cluster", "pubID", stan.NatsConn(client)) 22 | sc, err = stan.Connect("test-cluster", "subID") 23 | checkErr(err) 24 | 25 | } 26 | 27 | func main() { 28 | // Simple Synchronous Publisher 29 | sc.Publish("message", []byte("Hello World")) // does not return until an ack has been received from NATS Streaming 30 | 31 | // Simple Async Subscriber 32 | sub, err := sc.Subscribe("message", func(m *stan.Msg) { 33 | fmt.Printf("Received a message: %s\n", string(m.Data)) 34 | }) 35 | checkErr(err) 36 | 37 | // Unsubscribe 38 | sub.Unsubscribe() 39 | 40 | // Close connection 41 | sc.Close() 42 | } 43 | 44 | func DBWorker() { 45 | // defer wg.Done() 46 | 47 | // Simple Synchronous Publisher 48 | sc.Publish("message", []byte("Hello World")) // does not return until an ack has been received from NATS Streaming 49 | 50 | // Simple Async Subscriber 51 | sub, _ := sc.Subscribe("message", func(m *stan.Msg) { 52 | fmt.Printf("Received a message: %s\n", string(m.Data)) 53 | }) 54 | 55 | // Unsubscribe 56 | sub.Unsubscribe() 57 | 58 | // Close connection 59 | sc.Close() 60 | 61 | // sub, err := sc.Subscribe("message", func(m *stan.Msg) { 62 | // fmt.Println("geldi") 63 | // message := model.Message{} 64 | // json.Unmarshal([]byte(m.Data), &message) 65 | // fmt.Println(message) 66 | // }) 67 | // checkErr(err) 68 | 69 | // // Unsubscribe 70 | // sub.Unsubscribe() 71 | 72 | // // Close connection 73 | // sc.Close() 74 | 75 | } 76 | 77 | func checkErr(err error) { 78 | if err != nil { 79 | log.Println(err) 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NATS vs REDIS 2 | This repo is prepared to test the performance of nats and redis in pubsub messaging. 3 | 4 | ## PREQUISITES 5 | Redis server - https://redis.io/ 6 | Nats server - https://nats.io/download/nats-io/nats-server/ 7 | NATS streaming server - https://nats.io/download/nats-io/nats-streaming-server/ 8 | 9 | ## FOLDERS 10 | ```sh 11 | ➜ redis-vs-nats git:(master) ls -lah 12 | total 16 13 | drwxr-xr-x 3 yakuter staff 96B Apr 10 10:47 faker 14 | drwxr-xr-x 5 yakuter staff 160B Apr 10 10:47 json 15 | drwxr-xr-x 3 yakuter staff 96B Apr 7 08:46 model 16 | drwxr-xr-x 4 yakuter staff 128B Apr 10 10:50 nats 17 | drwxr-xr-x 4 yakuter staff 128B Apr 10 10:52 nats-streaming-BETA 18 | drwxr-xr-x 4 yakuter staff 128B Apr 10 10:50 redis 19 | ``` 20 | ### faker 21 | With faker, the user can generate json files with fake messages. Usage: 22 | ```go 23 | // In faker folder 24 | go run faker.go 25 | ``` 26 | ### json 27 | JSON files generated with faker are located in this folder. Default files and sizes are listed below: 28 | ```shell 29 | ➜ json git:(master) ✗ ls -lah 30 | total 244896 31 | drwxr-xr-x 5 yakuter staff 160B Apr 10 10:47 . 32 | drwxr-xr-x 11 yakuter staff 352B Apr 10 10:53 .. 33 | -rw-r--r-- 1 yakuter staff 108M Apr 10 10:47 100k.json 34 | -rw-r--r-- 1 yakuter staff 11M Apr 10 10:47 10k.json 35 | -rw-r--r-- 1 yakuter staff 1.1M Apr 10 10:47 1k.json 36 | ``` 37 | ### model 38 | This folder contains the struct for message. The Message struct has faker tags to generate meaningful dummy content like IP address etc. 39 | 40 | ### redis, nats and nats-streaming 41 | These folders have same structure. They all have a sender and a receiver. The test steps are below. The order is important! 42 | 43 | 1. Start the relevant server according to the test (Redis, Nats or Nats Streaming Server). 44 | 2. Go into receiver folder of the test and run receiver with `go run receiver.go` 45 | 3. Go into sender folder of the test and run sender with `go run sender.go` 46 | 47 | ## MONITORING 48 | ### Nats 49 | NATS has a great tool for monioring called `nats-top` which is a `top`-like tool for monitoring NATS servers. 50 | https://github.com/nats-io/nats-top 51 | 52 | ```sh 53 | $ nats-top 54 | 55 | NATS server version 0.7.3 (uptime: 3m34s) 56 | Server: 57 | Load: CPU: 58.3% Memory: 8.6M Slow Consumers: 0 58 | In: Msgs: 568.7K Bytes: 1.7M Msgs/Sec: 13129.0 Bytes/Sec: 38.5K 59 | Out: Msgs: 1.6M Bytes: 4.7M Msgs/Sec: 131290.9 Bytes/Sec: 384.6K 60 | 61 | Connections: 10 62 | HOST CID NAME SUBS PENDING MSGS_TO MSGS_FROM BYTES_TO BYTES_FROM LANG VERSION UPTIME LAST ACTIVITY 63 | 127.0.0.1:57487 13 example 1 12.0K 161.6K 0 484.7K 0 go 1.1.7 17s 2016-02-09 00:13:24.753062715 -0800 PST 64 | 127.0.0.1:57488 14 example 1 11.9K 161.6K 0 484.7K 0 go 1.1.7 17s 2016-02-09 00:13:24.753040168 -0800 PST 65 | 127.0.0.1:57489 15 example 1 12.1K 161.6K 0 484.7K 0 go 1.1.7 17s 2016-02-09 00:13:24.753069442 -0800 PST 66 | 127.0.0.1:57490 16 example 1 12.0K 161.6K 0 484.7K 0 go 1.1.7 17s 2016-02-09 00:13:24.753057413 -0800 PST 67 | ``` 68 | 69 | ### Redis 70 | Unfortunately, couldn't find a useful tool like nats-top. If you know any, I would like to hear. 71 | 72 | ## MY RESULTS 73 | 74 | **File**: 100k.json | 100.000 JSON messages | 108mb 75 | **NATS**: 6.207s 76 | **REDIS**: 8.212s 77 | 78 | **File**: 10k.json | 10.000 JSON messages | 11mb 79 | **NATS**: 623.52ms 80 | **REDIS**: 785.28ms 81 | 82 | --------------------------------------------------------------------------------