├── LICENSE ├── README.md ├── examples ├── ex1.go ├── ex2.go └── ex3.go └── milkthisbuffer.go /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Leonid Titov, CS 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # milkthisbuffer 2 | Mutex-free/Lock-Free Thread-Safe Buffer in Go 3 | 4 | ## For Shell automation in Golang: 5 | 6 | - #### See the output of long-running Shell command in real time; 7 | - #### Process that output programmatically by your code, in real time; 8 | - #### Pipe that output to another Shell command; 9 | - #### Execute shell commands really easily, like in the Bash; 10 | 11 | ## Mutex/Lock-Free Thread-Safe Buffer 12 | 13 | The same as bytes.Buffer, but thread-safe. 14 | 15 | Wraps (implements) the io.Reader and io.Writer with a channel. 16 | 17 | Can also be used as a "pipe" between goroutines. 18 | 19 | Etymology: 20 | 1. Mutex/Lock-Free Thread-Safe Buffer 21 | 1. MxLkF_ThS_Buffer 22 | 1. MilkThisBuffer 23 | 24 | ## What's the use of it 25 | 26 | It's an absolutely useful tool for a shell automation. Basically, the standard lib, specifically the os/exec, allows for quite convenient shell commands execution out-of-the-box, nothing else is needed.... Except one thing. That thing is the ability to see the output of long-running command in real time, and/or process that output programmatically, and/or pipe that output to another command. All these cases are covered by this simple pipe-buffer. 27 | 28 | You can: 29 | 30 | - See the output of long-running Shell command in real time; 31 | - Process that output programmatically by your code, in real time; 32 | - Pipe that output to another Shell command; 33 | 34 | ## Example usage 35 | 36 | package main 37 | 38 | import ( 39 | "github.com/latitov/milkthisbuffer" 40 | "log" 41 | "os" 42 | "os/exec" 43 | ) 44 | 45 | // Ctrl-D to end the input (Ctrl-Z on Windows) 46 | 47 | func main() { 48 | mtb1 := milkthisbuffer.New(500) 49 | go mtb1.StdoutAsync() 50 | 51 | cmd := exec.Command("tr", "a-z", "A-Z") 52 | cmd.Stdin = os.Stdin 53 | cmd.Stdout = mtb1 54 | cmd.Stderr = mtb1 55 | 56 | err := cmd.Run() 57 | if err != nil { 58 | log.Fatal(err) 59 | } 60 | } 61 | 62 | ## One more example, piping two commands in a chain: 63 | 64 | package main 65 | 66 | import ( 67 | "github.com/latitov/milkthisbuffer" 68 | "os/exec" 69 | "time" 70 | ) 71 | 72 | // Ctrl-D to end the input (Ctrl-Z on Windows) 73 | 74 | func main() { 75 | pipe1 := milkthisbuffer.New(500) 76 | pipe2 := milkthisbuffer.New(500) 77 | 78 | go pipe2.StdoutAsync() 79 | 80 | go func() { 81 | cmd := exec.Command("tr", "a-z", "A-Z") 82 | cmd.Stdin = pipe1 83 | cmd.Stdout = pipe2 84 | cmd.Stderr = pipe2 85 | cmd.Run() 86 | }() 87 | 88 | cmd := exec.Command("echo", "hey-hi-hello, 1-2-3, repeat") 89 | cmd.Stdout = pipe1 90 | cmd.Stderr = pipe1 91 | cmd.Run() 92 | pipe1.Close() // signal the EOF 93 | 94 | time.Sleep(1 * time.Second) 95 | } 96 | 97 | The output will be: 98 | 99 | HEY-HI-HELLO, 1-2-3, REPEAT 100 | 101 | Basically, you are free to build any graph out of pipes (this buffer as a pipe), and any commands. This opens the road to use the Go language for a Shell automation (which is faster than Bash or Python or Perl). 102 | 103 | ## And one more thing )) 104 | 105 | There is this standard thing: https://golang.org/pkg/io/#Pipe 106 | 107 | The example above can be written in 100% standard Go, without any third-party libs, this way: 108 | 109 | package main 110 | 111 | import ( 112 | "fmt" 113 | "io" 114 | "os/exec" 115 | "time" 116 | ) 117 | 118 | // Ctrl-D to end the input (Ctrl-Z on Windows) 119 | 120 | func main() { 121 | pipe1_reader, pipe1_writer := io.Pipe() 122 | pipe2_reader, pipe2_writer := io.Pipe() 123 | 124 | go StdoutAsync(pipe2_reader) 125 | 126 | go func() { 127 | cmd := exec.Command("tr", "a-z", "A-Z") 128 | cmd.Stdin = pipe1_reader 129 | cmd.Stdout = pipe2_writer 130 | cmd.Stderr = pipe2_writer 131 | cmd.Run() 132 | }() 133 | 134 | cmd := exec.Command("echo", "hey-hi-hello, 1-2-3, repeat") 135 | cmd.Stdout = pipe1_writer 136 | cmd.Stderr = pipe1_writer 137 | cmd.Run() 138 | pipe1_writer.Close() // signal the EOF 139 | 140 | time.Sleep(1 * time.Second) 141 | } 142 | 143 | func StdoutAsync(b io.Reader) { 144 | buf := make([]byte, 100) 145 | for { 146 | n, err := b.Read(buf) 147 | if err != nil { 148 | fmt.Printf("StdoutAsync: %v\n", err) 149 | break 150 | } 151 | fmt.Printf("%v", string(buf[:n])) 152 | } 153 | } 154 | 155 | And the output will be, identically: 156 | 157 | HEY-HI-HELLO, 1-2-3, REPEAT 158 | 159 | It's up to you how to proceed. I myself am in doubt, if I wasted my time. Did I? 160 | 161 | ## THE BONUS 162 | 163 | Actually, it's more than that. I also added a wrapper for the code above, so now you can write the example this short: 164 | 165 | package main 166 | 167 | import ( 168 | "github.com/latitov/milkthisbuffer" 169 | "time" 170 | ) 171 | 172 | // Ctrl-D to end the input (Ctrl-Z on Windows) 173 | 174 | func main() { 175 | pipe1 := milkthisbuffer.New(500) 176 | pipe2 := milkthisbuffer.New(500) 177 | 178 | go pipe2.StdoutAsync() 179 | 180 | co1 := &milkthisbuffer.CommandObject{ 181 | Stdout: pipe1, 182 | Stderr: pipe1, 183 | } 184 | co2 := &milkthisbuffer.CommandObject{ 185 | Stdin: pipe1, 186 | Stdout: pipe2, 187 | Stderr: pipe2, 188 | } 189 | 190 | go func() { 191 | co2.Execf("tr", "a-z", "A-Z") 192 | }() 193 | 194 | co1.Execf("echo", "hey-hi-hello, 1-2-3, repeat") 195 | 196 | pipe1.Close() // signal the EOF 197 | 198 | time.Sleep(1 * time.Second) 199 | } 200 | One more example, without piping, illustrating the ease of calling many commands in sequence: 201 | 202 | package main 203 | 204 | import ( 205 | "github.com/latitov/milkthisbuffer" 206 | "time" 207 | ) 208 | 209 | // Ctrl-D to end the input (Ctrl-Z on Windows) 210 | 211 | func main() { 212 | pipe1 := milkthisbuffer.New(500) 213 | 214 | go pipe1.StdoutAsync() 215 | 216 | co1 := &milkthisbuffer.CommandObject{ 217 | Stdout: pipe1, 218 | Stderr: pipe1, 219 | } 220 | 221 | co1.Execf("echo", "hey-hi-hello, 1-2-3, repeat") 222 | co1.Execf("echo", "hey-hi-hello, 1-2-3-5, repeat") 223 | co1.Execf("echo", "hey-hi-hello, 1-2-3-5-7, repeat") 224 | co1.Execf("echo", "hey-hi-hello, 1-2-3-5-7-9, repeat") 225 | 226 | co1.Execf("git", "add", "-A") 227 | co1.Execf("git", "commit", "-m", "'.'") 228 | 229 | pipe1.Close() // signal the EOF 230 | 231 | time.Sleep(1 * time.Second) 232 | } 233 | 234 | To do, maybe also: 235 | 236 | - A wrapper with Context; 237 | - A wrapper that return the error; 238 | 239 | Very useful. 240 | -------------------------------------------------------------------------------- /examples/ex1.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/latitov/milkthisbuffer" 5 | "os/exec" 6 | "log" 7 | "time" 8 | ) 9 | 10 | // Ctrl-D to end the input (Ctrl-Z on Windows) 11 | 12 | func main() { 13 | pipe1 := milkthisbuffer.New(500) 14 | pipe2 := milkthisbuffer.New(500) 15 | 16 | go pipe2.StdoutAsync() 17 | 18 | var err error 19 | 20 | go func() { 21 | cmd := exec.Command("tr", "a-z", "A-Z") 22 | cmd.Stdin = pipe1 23 | cmd.Stdout = pipe2 24 | cmd.Stderr = pipe2 25 | cmd.Run() 26 | if err != nil { 27 | log.Fatal(err) 28 | } 29 | }() 30 | 31 | cmd := exec.Command("echo", "hey-hi-hello, 1-2-3, repeat") 32 | cmd.Stdout = pipe1 33 | cmd.Stderr = pipe1 34 | cmd.Run() 35 | if err != nil { 36 | log.Fatal(err) 37 | } 38 | 39 | pipe1.Close() // signal the EOF 40 | 41 | time.Sleep(1 * time.Second) 42 | } 43 | -------------------------------------------------------------------------------- /examples/ex2.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/latitov/milkthisbuffer" 5 | "time" 6 | ) 7 | 8 | // Ctrl-D to end the input (Ctrl-Z on Windows) 9 | 10 | func main() { 11 | pipe1 := milkthisbuffer.New(500) 12 | pipe2 := milkthisbuffer.New(500) 13 | 14 | go pipe2.StdoutAsync() 15 | 16 | co1 := &milkthisbuffer.CommandObject{ 17 | Stdout: pipe1, 18 | Stderr: pipe1, 19 | } 20 | co2 := &milkthisbuffer.CommandObject{ 21 | Stdin: pipe1, 22 | Stdout: pipe2, 23 | Stderr: pipe2, 24 | } 25 | 26 | go func() { 27 | co2.Execf("tr", "a-z", "A-Z") 28 | }() 29 | 30 | co1.Execf("echo", "hey-hi-hello, 1-2-3, repeat") 31 | 32 | pipe1.Close() // signal the EOF 33 | 34 | time.Sleep(1 * time.Second) 35 | } 36 | -------------------------------------------------------------------------------- /examples/ex3.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/latitov/milkthisbuffer" 5 | "time" 6 | ) 7 | 8 | // Ctrl-D to end the input (Ctrl-Z on Windows) 9 | 10 | func main() { 11 | pipe1 := milkthisbuffer.New(500) 12 | 13 | go pipe1.StdoutAsync() 14 | 15 | co1 := &milkthisbuffer.CommandObject{ 16 | Stdout: pipe1, 17 | Stderr: pipe1, 18 | } 19 | 20 | co1.Execf("echo", "hey-hi-hello, 1-2-3, repeat") 21 | co1.Execf("echo", "hey-hi-hello, 1-2-3-5, repeat") 22 | co1.Execf("echo", "hey-hi-hello, 1-2-3-5-7, repeat") 23 | co1.Execf("echo", "hey-hi-hello, 1-2-3-5-7-9, repeat") 24 | 25 | co1.Execf("git", "add", "-A") 26 | co1.Execf("git", "commit", "-m", "'.'") 27 | 28 | pipe1.Close() // signal the EOF 29 | 30 | time.Sleep(1 * time.Second) 31 | } 32 | -------------------------------------------------------------------------------- /milkthisbuffer.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020 Leonid Titov. All rights reserved. 2 | // MIT licence. 3 | // Version 2020-12-23 4 | 5 | package milkthisbuffer 6 | 7 | // Mutex/Lock-Free Thread-Safe Buffer 8 | 9 | // The same as bytes.Buffer, but thread-safe. 10 | 11 | // Wraps (implements) the io.Reader and io.Writer with a channel. 12 | 13 | // Can be used as a "pipe" between goroutines. 14 | 15 | // Etymology: 16 | // Mutex/Lock-Free Thread-Safe Buffer 17 | // MxLkF_ThS_Buffer 18 | // MilkThisBuffer 19 | 20 | import ( 21 | // keep this sorted, please 22 | "fmt" 23 | "io" 24 | "log" 25 | "os/exec" 26 | ) 27 | 28 | type MilkThisBuffer struct { 29 | ch chan byte 30 | } 31 | 32 | func New(len int) (b *MilkThisBuffer) { 33 | b = &MilkThisBuffer{ 34 | ch: make(chan byte, len), 35 | } 36 | return 37 | } 38 | 39 | // blocking behaviour 40 | func (b *MilkThisBuffer) Write(p []byte) (n int, err error) { 41 | 42 | // here's why: 43 | // https://stackoverflow.com/a/34899098/11729048 44 | defer func() { 45 | if err2 := recover(); err2 != nil { 46 | err = io.ErrClosedPipe 47 | } 48 | }() 49 | 50 | for n = 0; n < len(p); n++ { 51 | b.ch <- p[n] 52 | } 53 | return 54 | } 55 | 56 | // blocking behaviour until at least one byte read, then behaves non-blockingly 57 | func (b *MilkThisBuffer) Read(p []byte) (n int, err error) { 58 | L: 59 | for n = 0; n < len(p); { 60 | if n == 0 { 61 | b1, ok := <-b.ch 62 | if !ok { 63 | err = io.EOF 64 | break L 65 | } 66 | p[n] = b1 67 | n++ 68 | } else { 69 | select { 70 | case b1, ok := <-b.ch: 71 | if !ok { 72 | err = io.EOF 73 | break L 74 | } 75 | p[n] = b1 76 | n++ 77 | default: 78 | break L 79 | } 80 | } 81 | } 82 | return 83 | } 84 | 85 | func (b *MilkThisBuffer) Close() error { 86 | close(b.ch) 87 | return nil 88 | } 89 | 90 | func (b *MilkThisBuffer) StdoutAsync() { 91 | buf := make([]byte, 100) 92 | for { 93 | n, err := b.Read(buf) 94 | if err != nil { 95 | fmt.Printf("milkthisbuffer.StdoutAsync: %v\n", err) 96 | break 97 | } 98 | fmt.Printf("%v", string(buf[:n])) 99 | } 100 | } 101 | 102 | type CommandObject struct { 103 | Env []string 104 | Dir string 105 | Stdin io.Reader 106 | Stdout io.Writer 107 | Stderr io.Writer 108 | } 109 | 110 | func (co *CommandObject) Execf(name string, arg ...string) (err error) { 111 | 112 | cmd := exec.Command(name, arg...) 113 | 114 | cmd.Env = co.Env 115 | cmd.Dir = co.Dir 116 | cmd.Stdin = co.Stdin 117 | cmd.Stdout = co.Stdout 118 | cmd.Stderr = co.Stderr 119 | 120 | err = cmd.Run() 121 | if err != nil { 122 | log.Println(err) 123 | } 124 | return 125 | } 126 | --------------------------------------------------------------------------------