├── .gitignore └── patterns-nats-streaming ├── 1-ephemeral.go ├── 2-manual-ack.go ├── 3-durable.go ├── 4-exactly-once.go └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.sw? 3 | -------------------------------------------------------------------------------- /patterns-nats-streaming/1-ephemeral.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "log" 7 | "math/rand" 8 | "sync" 9 | "time" 10 | 11 | stan "github.com/nats-io/go-nats-streaming" 12 | ) 13 | 14 | func logCloser(c io.Closer) { 15 | if err := c.Close(); err != nil { 16 | log.Printf("close error: %s", err) 17 | } 18 | } 19 | 20 | func main() { 21 | if err := run(); err != nil { 22 | log.Fatal(err) 23 | } 24 | } 25 | 26 | func run() error { 27 | conn, err := stan.Connect( 28 | "test-cluster", 29 | "test-client", 30 | stan.NatsURL("nats://localhost:4222"), 31 | ) 32 | if err != nil { 33 | return err 34 | } 35 | defer logCloser(conn) 36 | 37 | wg := &sync.WaitGroup{} 38 | 39 | sub, err := conn.Subscribe("counter", func(msg *stan.Msg) { 40 | // Print the value and whether it was redelivered. 41 | fmt.Printf("seq = %d [redelivered = %v]\n", msg.Sequence, msg.Redelivered) 42 | 43 | // Add jitter.. 44 | time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond) 45 | 46 | // Mark it is done. 47 | wg.Done() 48 | }) 49 | if err != nil { 50 | return err 51 | } 52 | defer logCloser(sub) 53 | 54 | // Publish up to 10. 55 | for i := 0; i < 10; i++ { 56 | wg.Add(1) 57 | 58 | err := conn.Publish("counter", nil) 59 | if err != nil { 60 | return err 61 | } 62 | } 63 | 64 | // Wait until all messages have been processed. 65 | wg.Wait() 66 | 67 | return nil 68 | } 69 | -------------------------------------------------------------------------------- /patterns-nats-streaming/2-manual-ack.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "log" 7 | "math/rand" 8 | "sync" 9 | "time" 10 | 11 | stan "github.com/nats-io/go-nats-streaming" 12 | ) 13 | 14 | func logCloser(c io.Closer) { 15 | if err := c.Close(); err != nil { 16 | log.Printf("close error: %s", err) 17 | } 18 | } 19 | 20 | func main() { 21 | if err := run(); err != nil { 22 | log.Fatal(err) 23 | } 24 | } 25 | 26 | func run() error { 27 | conn, err := stan.Connect( 28 | "test-cluster", 29 | "test-client", 30 | stan.NatsURL("nats://localhost:4222"), 31 | ) 32 | if err != nil { 33 | return err 34 | } 35 | defer logCloser(conn) 36 | 37 | wg := &sync.WaitGroup{} 38 | 39 | var sub stan.Subscription 40 | 41 | sub, err = conn.Subscribe("counter", func(msg *stan.Msg) { 42 | // Add jitter.. 43 | time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond) 44 | 45 | // Simulate failed acks to show redelivery behavior. 80% of the time 46 | // the ack will "succeed" 47 | if rand.Float32() > 0.2 { 48 | if err := msg.Ack(); err != nil { 49 | log.Printf("failed to ack") 50 | sub.Close() 51 | return 52 | } 53 | 54 | fmt.Printf("seq = %d [redelivered = %v, acked = true]\n", msg.Sequence, msg.Redelivered) 55 | wg.Done() 56 | } else { 57 | fmt.Printf("seq = %d [redelivered = %v, acked = false]\n", msg.Sequence, msg.Redelivered) 58 | } 59 | }, stan.SetManualAckMode(), stan.AckWait(time.Second)) 60 | if err != nil { 61 | return err 62 | } 63 | defer logCloser(sub) 64 | 65 | // Publish up to 10. 66 | for i := 0; i < 10; i++ { 67 | wg.Add(1) 68 | 69 | err := conn.Publish("counter", nil) 70 | if err != nil { 71 | return err 72 | } 73 | } 74 | 75 | // Wait until all messages have been processed. 76 | wg.Wait() 77 | 78 | return nil 79 | } 80 | -------------------------------------------------------------------------------- /patterns-nats-streaming/3-durable.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "log" 7 | "math/rand" 8 | "sync" 9 | "time" 10 | 11 | stan "github.com/nats-io/go-nats-streaming" 12 | ) 13 | 14 | func logCloser(c io.Closer) { 15 | if err := c.Close(); err != nil { 16 | log.Printf("close error: %s", err) 17 | } 18 | } 19 | 20 | func startSubscriber(conn stan.Conn, wg *sync.WaitGroup, n int, d chan<- struct{}) { 21 | var ( 22 | i int 23 | err error 24 | sub stan.Subscription 25 | ) 26 | 27 | sub, err = conn.Subscribe("counter", func(msg *stan.Msg) { 28 | // Print the value and whether it was redelivered. 29 | fmt.Printf("seq = %d [redelivered = %v]\n", msg.Sequence, msg.Redelivered) 30 | 31 | // Add jitter.. 32 | time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond) 33 | 34 | // Mark it is done. 35 | wg.Done() 36 | 37 | i++ 38 | 39 | msg.Ack() 40 | 41 | if i == n { 42 | sub.Close() 43 | d <- struct{}{} 44 | } 45 | }, stan.DurableName("i-will-remember"), stan.MaxInflight(1), stan.SetManualAckMode()) 46 | if err != nil { 47 | log.Print(err) 48 | } 49 | } 50 | 51 | func main() { 52 | if err := run(); err != nil { 53 | log.Fatal(err) 54 | } 55 | } 56 | 57 | func run() error { 58 | conn, err := stan.Connect( 59 | "test-cluster", 60 | "test-client", 61 | stan.NatsURL("nats://localhost:4222"), 62 | ) 63 | if err != nil { 64 | return err 65 | } 66 | defer logCloser(conn) 67 | 68 | wg := &sync.WaitGroup{} 69 | 70 | go func() { 71 | done := make(chan struct{}) 72 | startSubscriber(conn, wg, 5, done) 73 | <-done 74 | log.Print("subscriber disconnected..") 75 | log.Print("reconnecting..") 76 | startSubscriber(conn, wg, 5, done) 77 | }() 78 | 79 | // Publish up to 10. 80 | for i := 0; i < 10; i++ { 81 | wg.Add(1) 82 | 83 | err := conn.Publish("counter", nil) 84 | if err != nil { 85 | return err 86 | } 87 | } 88 | 89 | // Wait until all messages have been processed. 90 | wg.Wait() 91 | 92 | return nil 93 | } 94 | -------------------------------------------------------------------------------- /patterns-nats-streaming/4-exactly-once.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "log" 7 | "math/rand" 8 | "sync" 9 | "sync/atomic" 10 | "time" 11 | 12 | stan "github.com/nats-io/go-nats-streaming" 13 | ) 14 | 15 | func logCloser(c io.Closer) { 16 | if err := c.Close(); err != nil { 17 | log.Printf("close error: %s", err) 18 | } 19 | } 20 | 21 | func main() { 22 | if err := run(); err != nil { 23 | log.Fatal(err) 24 | } 25 | } 26 | 27 | func run() error { 28 | conn, err := stan.Connect( 29 | "test-cluster", 30 | "test-client", 31 | stan.NatsURL("nats://localhost:4222"), 32 | ) 33 | if err != nil { 34 | return err 35 | } 36 | defer logCloser(conn) 37 | 38 | wg := &sync.WaitGroup{} 39 | 40 | var lastProcessed uint64 41 | var i int 42 | 43 | sub, err := conn.Subscribe("counter", func(msg *stan.Msg) { 44 | var processed bool 45 | 46 | if msg.Sequence > lastProcessed { 47 | processed = true 48 | atomic.SwapUint64(&lastProcessed, msg.Sequence) 49 | } 50 | 51 | // Add jitter.. 52 | time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond) 53 | 54 | i++ 55 | 56 | var acked bool 57 | if i <= 5 { 58 | msg.Ack() 59 | // Mark it is done. 60 | wg.Done() 61 | acked = true 62 | } else if i == 9 { 63 | i = -5 64 | } 65 | 66 | // Print the value and whether it was redelivered. 67 | fmt.Printf("seq = %d [redelivered = %v, acked = %v, processed = %v]\n", msg.Sequence, msg.Redelivered, acked, processed) 68 | 69 | }, stan.SetManualAckMode(), stan.AckWait(time.Second)) 70 | if err != nil { 71 | return err 72 | } 73 | defer logCloser(sub) 74 | 75 | // Publish up to 10. 76 | for i := 0; i < 10; i++ { 77 | wg.Add(1) 78 | 79 | err := conn.Publish("counter", nil) 80 | if err != nil { 81 | return err 82 | } 83 | } 84 | 85 | // Wait until all messages have been processed. 86 | wg.Wait() 87 | 88 | return nil 89 | } 90 | -------------------------------------------------------------------------------- /patterns-nats-streaming/README.md: -------------------------------------------------------------------------------- 1 | # Use cases for persistent logs with NATS Streaming 2 | 3 | These are fully working examples of the patterns described in the post [Use cases for persistent logs with NATS Streaming](). 4 | 5 | To run, [Go must be installed](https://golang.org/dl/). [Download NATS streaming](https://github.com/nats-io/nats-streaming-server/releases) and run it: 6 | 7 | ``` 8 | $ ./nats-streaming-server \ 9 | --cluster_id test-cluster \ 10 | --max_msgs 0 \ 11 | --max_bytes 0 \ 12 | --max_age 0s \ 13 | ``` 14 | 15 | Each section belows links to the script and show example output (so you don't have to run them). If you choose to, just use `go run`, e.g. `go run 1-ephemeral.go`. 16 | 17 | ## [ephemeral](./1-ephemeral.go) 18 | 19 | This example shows the standard subscriber. 20 | 21 | *Note your sequence numbers may be different for all examples. They are the sequence number of the message within the log, so if you see the same sequence number twice, they are the same message.* 22 | 23 | ``` 24 | seq = 11 [redelivered = false] 25 | seq = 12 [redelivered = false] 26 | seq = 13 [redelivered = false] 27 | seq = 14 [redelivered = false] 28 | seq = 15 [redelivered = false] 29 | seq = 16 [redelivered = false] 30 | seq = 17 [redelivered = false] 31 | seq = 18 [redelivered = false] 32 | seq = 19 [redelivered = false] 33 | seq = 20 [redelivered = false] 34 | ``` 35 | 36 | Running it again you will see the same thing but for the next 10 messages published. 37 | 38 | ## [manual ack](./2-manual-ack.go) 39 | 40 | This example simulates failing to ack and getting a redelivery from the server. Message 34 is not acked, the remaining queued messages were processed, then 34 was redelivered a couple times until it was successful. 41 | 42 | ``` 43 | seq = 31 [redelivered = false, acked = true] 44 | seq = 32 [redelivered = false, acked = true] 45 | seq = 33 [redelivered = false, acked = true] 46 | seq = 34 [redelivered = false, acked = false] 47 | seq = 35 [redelivered = false, acked = true] 48 | seq = 36 [redelivered = false, acked = true] 49 | seq = 37 [redelivered = false, acked = true] 50 | seq = 38 [redelivered = false, acked = true] 51 | seq = 39 [redelivered = false, acked = true] 52 | seq = 40 [redelivered = false, acked = true] 53 | seq = 34 [redelivered = true, acked = false] 54 | seq = 34 [redelivered = true, acked = false] 55 | seq = 34 [redelivered = true, acked = true] 56 | ``` 57 | 58 | ## [durable](./3-durable.go) 59 | 60 | With this example, a subscriber is *disconnected* after 5 messages, reconnects and processes the remaining messages. 61 | 62 | ``` 63 | seq = 21 [redelivered = false] 64 | seq = 22 [redelivered = false] 65 | seq = 23 [redelivered = false] 66 | seq = 24 [redelivered = false] 67 | seq = 25 [redelivered = false] 68 | 2017/09/10 16:04:04 subscriber disconnected.. 69 | 2017/09/10 16:04:04 reconnecting.. 70 | seq = 26 [redelivered = true] 71 | seq = 27 [redelivered = false] 72 | seq = 28 [redelivered = false] 73 | seq = 29 [redelivered = false] 74 | seq = 30 [redelivered = false] 75 | ``` 76 | 77 | ## [exactly once](./4-exactly-once.go) 78 | 79 | This example emulates a failed ACK, but when the a subscription receives the same msg, it does not reprocess it. It only acknowledges it. 80 | 81 | ``` 82 | seq = 81 [redelivered = false, acked = true, processed = true] 83 | seq = 82 [redelivered = false, acked = true, processed = true] 84 | seq = 83 [redelivered = false, acked = true, processed = true] 85 | seq = 84 [redelivered = false, acked = true, processed = true] 86 | seq = 85 [redelivered = false, acked = true, processed = true] 87 | seq = 86 [redelivered = false, acked = false, processed = true] 88 | seq = 87 [redelivered = false, acked = false, processed = true] 89 | seq = 88 [redelivered = false, acked = false, processed = true] 90 | seq = 89 [redelivered = false, acked = false, processed = true] 91 | seq = 90 [redelivered = false, acked = true, processed = true] 92 | seq = 86 [redelivered = true, acked = true, processed = false] 93 | seq = 87 [redelivered = true, acked = true, processed = false] 94 | seq = 88 [redelivered = true, acked = true, processed = false] 95 | seq = 89 [redelivered = true, acked = true, processed = false] 96 | ``` 97 | --------------------------------------------------------------------------------