├── LICENSE.md ├── README.md ├── SECURITY.md ├── go.mod ├── go.sum ├── pkt_line.go ├── pkt_line_test.go ├── reader.go ├── reader_test.go ├── tools.go ├── writer.go └── writer_test.go /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2014- GitHub, Inc. and Git LFS contributors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | Note that Git LFS uses components from other Go modules (included in `vendor/`) 24 | which are under different licenses. See those LICENSE files for details. 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Git pkt-line Toolkit 2 | 3 | This package is a Go language toolkit for reading and writing files using the 4 | Git pkt-line format used in various Git operations. 5 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | Please see 2 | [SECURITY.md](https://github.com/git-lfs/git-lfs/blob/main/SECURITY.md) 3 | in the main Git LFS repository for information on how to report security 4 | vulnerabilities in this package. 5 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/git-lfs/pktline 2 | 3 | require ( 4 | github.com/davecgh/go-spew v1.1.1 // indirect 5 | github.com/pmezard/go-difflib v1.0.0 // indirect 6 | github.com/stretchr/testify v1.2.2 7 | ) 8 | 9 | go 1.12 10 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 2 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 4 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 5 | github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= 6 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 7 | -------------------------------------------------------------------------------- /pkt_line.go: -------------------------------------------------------------------------------- 1 | package pktline 2 | 3 | import ( 4 | "bufio" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "strconv" 10 | "strings" 11 | ) 12 | 13 | const ( 14 | // MaxPacketLength is the maximum total (header+payload) length 15 | // encode-able within one packet using Git's pkt-line protocol. 16 | MaxPacketLength = 65516 17 | ) 18 | 19 | type Pktline struct { 20 | r *bufio.Reader 21 | w *bufio.Writer 22 | } 23 | 24 | func NewPktline(r io.Reader, w io.Writer) *Pktline { 25 | return &Pktline{ 26 | r: bufio.NewReader(r), 27 | w: bufio.NewWriter(w), 28 | } 29 | } 30 | 31 | // ReadPacket reads a single packet entirely and returns the data encoded within 32 | // it. Errors can occur in several cases, as described below. 33 | // 34 | // 1) If no data was present in the reader, and no more data could be read (the 35 | // pipe was closed, etc) than an io.EOF will be returned. 36 | // 2) If there was some data to be read, but the pipe or reader was closed 37 | // before an entire packet (or header) could be ingested, an 38 | // io.ErrShortBuffer error will be returned. 39 | // 3) If there was a valid header, but no body associated with the packet, an 40 | // "Invalid packet length." error will be returned. 41 | // 4) If the data in the header could not be parsed as a hexadecimal length in 42 | // the Git Pktline format, the parse error will be returned. 43 | // 44 | // If none of the above cases fit the state of the data on the wire, the packet 45 | // is returned along with a nil error. 46 | func (p *Pktline) ReadPacket() ([]byte, error) { 47 | slice, _, err := p.ReadPacketWithLength() 48 | return slice, err 49 | } 50 | 51 | // ReadPacketWithLength is exactly like ReadPacket, but on success, it also 52 | // returns the packet length header value. This is useful to distinguish 53 | // between flush and delim packets, which will return 0 and 1 respectively. For 54 | // data packets, the length will be four more than the number of bytes in the 55 | // slice. 56 | func (p *Pktline) ReadPacketWithLength() ([]byte, int, error) { 57 | var pktLenHex [4]byte 58 | if n, err := io.ReadFull(p.r, pktLenHex[:]); err != nil { 59 | return nil, 0, err 60 | } else if n != 4 { 61 | return nil, 0, io.ErrShortBuffer 62 | } 63 | 64 | pktLen, err := strconv.ParseInt(string(pktLenHex[:]), 16, 0) 65 | if err != nil { 66 | return nil, 0, err 67 | } 68 | 69 | // 0: flush packet, 1: delim packet 70 | if pktLen == 0 || pktLen == 1 { 71 | return nil, int(pktLen), nil 72 | } 73 | if pktLen < 4 { 74 | return nil, int(pktLen), errors.New("Invalid packet length.") 75 | } 76 | 77 | payload, err := ioutil.ReadAll(io.LimitReader(p.r, pktLen-4)) 78 | return payload, int(pktLen), err 79 | } 80 | 81 | // ReadPacketText follows identical semantics to the `readPacket()` function, 82 | // but additionally removes the trailing `\n` LF from the end of the packet, if 83 | // present. 84 | func (p *Pktline) ReadPacketText() (string, error) { 85 | data, err := p.ReadPacket() 86 | return strings.TrimSuffix(string(data), "\n"), err 87 | } 88 | 89 | // ReadPacketTextWithLength follows identical semantics to the 90 | // `ReadPacketWithLength()` function, but additionally removes the trailing `\n` 91 | // LF from the end of the packet, if present. The length field is not modified. 92 | func (p *Pktline) ReadPacketTextWithLength() (string, int, error) { 93 | data, pktLen, err := p.ReadPacketWithLength() 94 | return strings.TrimSuffix(string(data), "\n"), pktLen, err 95 | } 96 | 97 | // ReadPacketList reads as many packets as possible using the `readPacketText` 98 | // function before encountering a flush packet. It returns a slice of all the 99 | // packets it read, or an error if one was encountered. 100 | func (p *Pktline) ReadPacketList() ([]string, error) { 101 | var list []string 102 | for { 103 | data, pktLen, err := p.ReadPacketTextWithLength() 104 | if err != nil { 105 | return nil, err 106 | } 107 | 108 | if pktLen == 0 { 109 | break 110 | } 111 | 112 | list = append(list, data) 113 | } 114 | 115 | return list, nil 116 | } 117 | 118 | // WritePacket writes the given data in "data" to the underlying data stream 119 | // using Git's `pkt-line` format. 120 | // 121 | // If the data was longer than MaxPacketLength, an error will be returned. If 122 | // there was any error encountered while writing any component of the packet 123 | // (hdr, payload), it will be returned. 124 | // 125 | // NB: writePacket does _not_ flush the underlying buffered writer. See instead: 126 | // `writeFlush()`. 127 | func (p *Pktline) WritePacket(data []byte) error { 128 | if len(data) > MaxPacketLength { 129 | return errors.New("Packet length exceeds maximal length") 130 | } 131 | 132 | if _, err := p.w.WriteString(fmt.Sprintf("%04x", len(data)+4)); err != nil { 133 | return err 134 | } 135 | 136 | if _, err := p.w.Write(data); err != nil { 137 | return err 138 | } 139 | 140 | return nil 141 | } 142 | 143 | // WriteDelim writes the separating "delim" packet and then flushes the 144 | // underlying buffered writer. 145 | // 146 | // If any error was encountered along the way, it will be returned immediately 147 | func (p *Pktline) WriteDelim() error { 148 | if _, err := p.w.WriteString(fmt.Sprintf("%04x", 1)); err != nil { 149 | return err 150 | } 151 | 152 | if err := p.w.Flush(); err != nil { 153 | return err 154 | } 155 | 156 | return nil 157 | } 158 | 159 | // WriteFlush writes the terminating "flush" packet and then flushes the 160 | // underlying buffered writer. 161 | // 162 | // If any error was encountered along the way, it will be returned immediately 163 | func (p *Pktline) WriteFlush() error { 164 | if _, err := p.w.WriteString(fmt.Sprintf("%04x", 0)); err != nil { 165 | return err 166 | } 167 | 168 | if err := p.w.Flush(); err != nil { 169 | return err 170 | } 171 | 172 | return nil 173 | } 174 | 175 | // WritePacketText follows the same semantics as `writePacket`, but appends a 176 | // trailing "\n" LF character to the end of the data. 177 | func (p *Pktline) WritePacketText(data string) error { 178 | return p.WritePacket([]byte(data + "\n")) 179 | } 180 | 181 | // WritePacketList writes a slice of strings using the semantics of 182 | // and then writes a terminating flush sequence afterwords. 183 | // 184 | // If any error was encountered, it will be returned immediately. 185 | func (p *Pktline) WritePacketList(list []string) error { 186 | for _, i := range list { 187 | if err := p.WritePacketText(i); err != nil { 188 | return err 189 | } 190 | } 191 | 192 | return p.WriteFlush() 193 | } 194 | -------------------------------------------------------------------------------- /pkt_line_test.go: -------------------------------------------------------------------------------- 1 | package pktline 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | type PacketReadTestCase struct { 13 | In []byte 14 | 15 | Length int 16 | Payload []byte 17 | Err string 18 | } 19 | 20 | func (c *PacketReadTestCase) Assert(t *testing.T) { 21 | buf := bytes.NewReader(c.In) 22 | rw := NewPktline(buf, nil) 23 | 24 | pkt, err := rw.ReadPacket() 25 | 26 | if len(c.Payload) > 0 { 27 | assert.Equal(t, c.Payload, pkt) 28 | } else if c.Payload == nil { 29 | assert.Nil(t, pkt) 30 | } else { 31 | assert.Empty(t, pkt) 32 | } 33 | 34 | if len(c.Err) > 0 { 35 | require.NotNil(t, err) 36 | assert.Equal(t, c.Err, err.Error()) 37 | } else { 38 | assert.Nil(t, err) 39 | } 40 | 41 | buf = bytes.NewReader(c.In) 42 | rw = NewPktline(buf, nil) 43 | 44 | pkt, plen, err := rw.ReadPacketWithLength() 45 | 46 | if len(c.Payload) > 0 { 47 | assert.Equal(t, c.Payload, pkt) 48 | } else if c.Payload == nil { 49 | assert.Nil(t, pkt) 50 | } else { 51 | assert.Empty(t, pkt) 52 | } 53 | 54 | if len(c.Err) > 0 { 55 | require.NotNil(t, err) 56 | assert.Equal(t, c.Err, err.Error()) 57 | } else { 58 | assert.Equal(t, c.Length, plen) 59 | assert.Nil(t, err) 60 | } 61 | } 62 | 63 | func TestPktLineReadsWholePackets(t *testing.T) { 64 | tc := &PacketReadTestCase{ 65 | In: []byte{ 66 | 0x30, 0x30, 0x30, 0x38, // 0008 (hex. length) 67 | 0x1, 0x2, 0x3, 0x4, // payload 68 | }, 69 | Length: 8, 70 | Payload: []byte{0x1, 0x2, 0x3, 0x4}, 71 | } 72 | 73 | tc.Assert(t) 74 | } 75 | 76 | func TestPktLineNoPacket(t *testing.T) { 77 | tc := &PacketReadTestCase{ 78 | In: []byte{}, 79 | Err: io.EOF.Error(), 80 | } 81 | 82 | tc.Assert(t) 83 | } 84 | 85 | func TestPktLineInvalidPacket(t *testing.T) { 86 | tc := &PacketReadTestCase{ 87 | In: []byte{ 88 | 0x30, 0x30, 0x30, 0x33, 89 | }, 90 | 91 | Err: "Invalid packet length.", 92 | } 93 | 94 | tc.Assert(t) 95 | 96 | } 97 | 98 | func TestPktLineEmptyPacket(t *testing.T) { 99 | tc := &PacketReadTestCase{ 100 | In: []byte{ 101 | 0x30, 0x30, 0x30, 0x34, 102 | }, 103 | Length: 4, 104 | Payload: []byte{}, 105 | } 106 | 107 | tc.Assert(t) 108 | } 109 | 110 | func TestPktLineFlushPacket(t *testing.T) { 111 | tc := &PacketReadTestCase{ 112 | In: []byte{0x30, 0x30, 0x30, 0x30}, // Flush packet 113 | 114 | Payload: nil, 115 | Length: 0, 116 | Err: "", 117 | } 118 | 119 | tc.Assert(t) 120 | } 121 | 122 | func TestPktLineDelimPacket(t *testing.T) { 123 | tc := &PacketReadTestCase{ 124 | In: []byte{0x30, 0x30, 0x30, 0x31}, // Delim packet 125 | 126 | Payload: nil, 127 | Length: 1, 128 | Err: "", 129 | } 130 | 131 | tc.Assert(t) 132 | } 133 | 134 | func TestPktLineDiscardsPacketsWithUnparseableLength(t *testing.T) { 135 | tc := &PacketReadTestCase{ 136 | In: []byte{ 137 | 0xff, 0xff, 0xff, 0xff, // ÿÿÿÿ (invalid hex. length) 138 | // No body 139 | }, 140 | Err: "strconv.ParseInt: parsing \"\\xff\\xff\\xff\\xff\": invalid syntax", 141 | } 142 | 143 | tc.Assert(t) 144 | } 145 | 146 | func TestPktLineReadsTextWithNewline(t *testing.T) { 147 | rw := NewPktline(bytes.NewReader([]byte{ 148 | 0x30, 0x30, 0x30, 0x39, // 0009 (hex. length) 149 | 0x61, 0x62, 0x63, 0x64, 0xa, 150 | // Empty body 151 | }), nil) 152 | 153 | str, err := rw.ReadPacketText() 154 | 155 | assert.Nil(t, err) 156 | assert.Equal(t, "abcd", str) 157 | } 158 | 159 | func TestPktLineReadsTextWithLengthWithNewline(t *testing.T) { 160 | rw := NewPktline(bytes.NewReader([]byte{ 161 | 0x30, 0x30, 0x30, 0x39, // 0009 (hex. length) 162 | 0x61, 0x62, 0x63, 0x64, 0xa, 163 | // Empty body 164 | }), nil) 165 | 166 | str, plen, err := rw.ReadPacketTextWithLength() 167 | 168 | assert.Nil(t, err) 169 | assert.Equal(t, "abcd", str) 170 | assert.Equal(t, plen, 9) 171 | } 172 | 173 | func TestPktLineReadsTextWithoutNewline(t *testing.T) { 174 | rw := NewPktline(bytes.NewReader([]byte{ 175 | 0x30, 0x30, 0x30, 0x38, // 0009 (hex. length) 176 | 0x61, 0x62, 0x63, 0x64, 177 | }), nil) 178 | 179 | str, err := rw.ReadPacketText() 180 | 181 | assert.Nil(t, err) 182 | assert.Equal(t, "abcd", str) 183 | } 184 | 185 | func TestPktLineReadsTextWithLengthWithoutNewline(t *testing.T) { 186 | rw := NewPktline(bytes.NewReader([]byte{ 187 | 0x30, 0x30, 0x30, 0x38, // 0009 (hex. length) 188 | 0x61, 0x62, 0x63, 0x64, 189 | }), nil) 190 | 191 | str, plen, err := rw.ReadPacketTextWithLength() 192 | 193 | assert.Nil(t, err) 194 | assert.Equal(t, "abcd", str) 195 | assert.Equal(t, plen, 8) 196 | } 197 | 198 | func TestPktLineReadsTextWithEmpty(t *testing.T) { 199 | rw := NewPktline(bytes.NewReader([]byte{ 200 | 0x30, 0x30, 0x30, 0x34, // 0004 (hex. length) 201 | // No body 202 | }), nil) 203 | 204 | str, err := rw.ReadPacketText() 205 | 206 | assert.Nil(t, err) 207 | assert.Equal(t, "", str) 208 | } 209 | 210 | func TestPktLineReadsTextWithLengthWithEmpty(t *testing.T) { 211 | rw := NewPktline(bytes.NewReader([]byte{ 212 | 0x30, 0x30, 0x30, 0x34, // 0004 (hex. length) 213 | // No body 214 | }), nil) 215 | 216 | str, plen, err := rw.ReadPacketTextWithLength() 217 | 218 | assert.Nil(t, err) 219 | assert.Equal(t, plen, 4) 220 | assert.Equal(t, "", str) 221 | } 222 | 223 | func TestPktLineReadsTextWithErr(t *testing.T) { 224 | rw := NewPktline(bytes.NewReader([]byte{ 225 | 0x30, 0x30, 0x30, 0x33, // 0003 (invalid) 226 | }), nil) 227 | 228 | str, err := rw.ReadPacketText() 229 | 230 | require.NotNil(t, err) 231 | assert.Equal(t, "Invalid packet length.", err.Error()) 232 | assert.Equal(t, "", str) 233 | } 234 | 235 | func TestPktLineReadsTextWithLengthWithErr(t *testing.T) { 236 | rw := NewPktline(bytes.NewReader([]byte{ 237 | 0x30, 0x30, 0x30, 0x33, // 0003 (invalid) 238 | }), nil) 239 | 240 | str, plen, err := rw.ReadPacketTextWithLength() 241 | 242 | require.NotNil(t, err) 243 | assert.Equal(t, "Invalid packet length.", err.Error()) 244 | assert.Equal(t, plen, 3) 245 | assert.Equal(t, "", str) 246 | } 247 | 248 | func TestPktLineAppendsPacketLists(t *testing.T) { 249 | rw := NewPktline(bytes.NewReader([]byte{ 250 | 0x30, 0x30, 0x30, 0x38, // 0009 (hex. length) 251 | 0x61, 0x62, 0x63, 0x64, // "abcd" 252 | 253 | 0x30, 0x30, 0x30, 0x38, // 0008 (hex. length) 254 | 0x65, 0x66, 0x67, 0x68, // "efgh" 255 | 256 | 0x30, 0x30, 0x30, 0x30, // 0000 (hex. length) 257 | }), nil) 258 | 259 | str, err := rw.ReadPacketList() 260 | 261 | assert.Nil(t, err) 262 | assert.Equal(t, []string{"abcd", "efgh"}, str) 263 | } 264 | 265 | func TestPktLineAppendsPacketListsAndReturnsErrs(t *testing.T) { 266 | rw := NewPktline(bytes.NewReader([]byte{ 267 | 0x30, 0x30, 0x30, 0x38, // 0009 (hex. length) 268 | 0x61, 0x62, 0x63, 0x64, // "abcd" 269 | 270 | 0x30, 0x30, 0x30, 0x33, // 0003 (hex. length) 271 | // No body 272 | }), nil) 273 | 274 | str, err := rw.ReadPacketList() 275 | 276 | require.NotNil(t, err) 277 | assert.Equal(t, "Invalid packet length.", err.Error()) 278 | assert.Empty(t, str) 279 | } 280 | 281 | func TestPktLineWritesPackets(t *testing.T) { 282 | var buf bytes.Buffer 283 | 284 | rw := NewPktline(nil, &buf) 285 | require.Nil(t, rw.WritePacket([]byte{ 286 | 0x1, 0x2, 0x3, 0x4, 287 | })) 288 | require.Nil(t, rw.WriteDelim()) 289 | require.Nil(t, rw.WriteFlush()) 290 | 291 | assert.Equal(t, []byte{ 292 | 0x30, 0x30, 0x30, 0x38, // 0008 (hex. length) 293 | 0x1, 0x2, 0x3, 0x4, // payload 294 | 0x30, 0x30, 0x30, 0x31, // 0001 (delim packet) 295 | 0x30, 0x30, 0x30, 0x30, // 0000 (flush packet) 296 | }, buf.Bytes()) 297 | } 298 | 299 | func TestPktLineWritesPacketsEqualToMaxLength(t *testing.T) { 300 | var buf bytes.Buffer 301 | 302 | rw := NewPktline(nil, &buf) 303 | err := rw.WritePacket(make([]byte, MaxPacketLength)) 304 | 305 | assert.Nil(t, err) 306 | assert.Equal(t, 4+MaxPacketLength, len(buf.Bytes())) 307 | } 308 | 309 | func TestPktLineDoesNotWritePacketsExceedingMaxLength(t *testing.T) { 310 | var buf bytes.Buffer 311 | 312 | rw := NewPktline(nil, &buf) 313 | err := rw.WritePacket(make([]byte, MaxPacketLength+1)) 314 | 315 | require.NotNil(t, err) 316 | assert.Equal(t, "Packet length exceeds maximal length", err.Error()) 317 | assert.Empty(t, buf.Bytes()) 318 | } 319 | 320 | func TestPktLineWritesPacketText(t *testing.T) { 321 | var buf bytes.Buffer 322 | 323 | rw := NewPktline(nil, &buf) 324 | 325 | require.Nil(t, rw.WritePacketText("abcd")) 326 | require.Nil(t, rw.WriteFlush()) 327 | 328 | assert.Equal(t, []byte{ 329 | 0x30, 0x30, 0x30, 0x39, // 0009 (hex. length) 330 | 0x61, 0x62, 0x63, 0x64, 0xa, // "abcd\n" (payload) 331 | 0x30, 0x30, 0x30, 0x30, // 0000 (flush packet) 332 | }, buf.Bytes()) 333 | } 334 | 335 | func TestPktLineWritesPacketLists(t *testing.T) { 336 | var buf bytes.Buffer 337 | 338 | rw := NewPktline(nil, &buf) 339 | err := rw.WritePacketList([]string{"foo", "bar"}) 340 | 341 | assert.Nil(t, err) 342 | assert.Equal(t, []byte{ 343 | 0x30, 0x30, 0x30, 0x38, // 0008 (hex. length) 344 | 0x66, 0x6f, 0x6f, 0xa, // "foo\n" (payload) 345 | 346 | 0x30, 0x30, 0x30, 0x38, // 0008 (hex. length) 347 | 0x62, 0x61, 0x72, 0xa, // "bar\n" (payload) 348 | 349 | 0x30, 0x30, 0x30, 0x30, // 0000 (hex. length) 350 | }, buf.Bytes()) 351 | } 352 | -------------------------------------------------------------------------------- /reader.go: -------------------------------------------------------------------------------- 1 | package pktline 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | type PktlineReader struct { 8 | pl *Pktline 9 | 10 | buf []byte 11 | 12 | eof bool 13 | } 14 | 15 | // NewPktlineReader returns a new *PktlineReader, which will read from the 16 | // underlying data stream "r". The internal buffer is initialized with the given 17 | // capacity, "c". 18 | // 19 | // If "r" is already a `*PktlineReader`, it will be returned as-is. 20 | func NewPktlineReader(r io.Reader, c int) *PktlineReader { 21 | if pr, ok := r.(*PktlineReader); ok { 22 | return pr 23 | } 24 | 25 | return &PktlineReader{ 26 | buf: make([]byte, 0, c), 27 | pl: NewPktline(r, nil), 28 | } 29 | } 30 | 31 | // NewPktlineReaderFromPktline returns a new *PktlineReader based on the 32 | // underlying *Pktline object. The internal buffer is initialized with the 33 | // given capacity, "c". 34 | func NewPktlineReaderFromPktline(pl *Pktline, c int) *PktlineReader { 35 | return &PktlineReader{ 36 | buf: make([]byte, 0, c), 37 | pl: pl, 38 | } 39 | } 40 | 41 | func (r *PktlineReader) Read(p []byte) (int, error) { 42 | var n int 43 | 44 | if r.eof { 45 | return 0, io.EOF 46 | } 47 | 48 | if len(r.buf) > 0 { 49 | // If there is data in the buffer, shift as much out of it and 50 | // into the given "p" as we can. 51 | n = minInt(len(p), len(r.buf)) 52 | 53 | copy(p, r.buf[:n]) 54 | r.buf = r.buf[n:] 55 | } 56 | 57 | // Loop and grab as many packets as we can in a given "run", until we 58 | // have either, a) overfilled the given buffer "p", or we have started 59 | // to internally buffer in "r.buf". 60 | for len(r.buf) == 0 { 61 | chunk, err := r.pl.ReadPacket() 62 | if err != nil { 63 | return n, err 64 | } 65 | 66 | if len(chunk) == 0 { 67 | // If we got an empty chunk, then we know that we have 68 | // reached the end of processing for this particular 69 | // packet, so let's terminate. 70 | 71 | r.eof = true 72 | return n, io.EOF 73 | } 74 | 75 | // Figure out how much of the packet we can read into "p". 76 | nn := minInt(len(chunk), len(p[n:])) 77 | 78 | // Move that amount into "p", from where we left off. 79 | copy(p[n:], chunk[:nn]) 80 | // And move the rest into the buffer. 81 | r.buf = append(r.buf, chunk[nn:]...) 82 | 83 | // Mark that we have read "nn" bytes into "p" 84 | n += nn 85 | } 86 | 87 | return n, nil 88 | } 89 | 90 | // Reset causes the reader to reset the end-of-file indicator and continue 91 | // reading packets from the underlying reader. 92 | func (r *PktlineReader) Reset() { 93 | r.eof = false 94 | } 95 | -------------------------------------------------------------------------------- /reader_test.go: -------------------------------------------------------------------------------- 1 | package pktline 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "io/ioutil" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | // writePackets 14 | func writePacket(t *testing.T, w io.Writer, datas ...[]byte) { 15 | pl := NewPktline(nil, w) 16 | 17 | for _, data := range datas { 18 | require.Nil(t, pl.WritePacket(data)) 19 | } 20 | 21 | require.Nil(t, pl.WriteFlush()) 22 | } 23 | 24 | func TestPktlineReaderReadsSinglePacketsInOneCall(t *testing.T) { 25 | var buf bytes.Buffer 26 | 27 | writePacket(t, &buf, []byte("asdf")) 28 | 29 | pr := NewPktlineReader(&buf, 10) 30 | 31 | data, err := ioutil.ReadAll(pr) 32 | 33 | assert.Nil(t, err) 34 | assert.Equal(t, []byte("asdf"), data) 35 | } 36 | 37 | func TestPktlineReaderReadsManyPacketsInOneCall(t *testing.T) { 38 | var buf bytes.Buffer 39 | 40 | writePacket(t, &buf, []byte("first\n"), []byte("second")) 41 | 42 | pr := NewPktlineReaderFromPktline(NewPktline(&buf, nil), 10) 43 | 44 | data, err := ioutil.ReadAll(pr) 45 | 46 | assert.Nil(t, err) 47 | assert.Equal(t, []byte("first\nsecond"), data) 48 | } 49 | 50 | func TestPktlineReaderReadsSinglePacketsInMultipleCallsWithUnevenBuffering(t *testing.T) { 51 | var buf bytes.Buffer 52 | 53 | writePacket(t, &buf, []byte("asdf")) 54 | 55 | pr := NewPktlineReaderFromPktline(NewPktline(&buf, nil), 10) 56 | 57 | var p1 [3]byte 58 | var p2 [1]byte 59 | 60 | n1, e1 := pr.Read(p1[:]) 61 | assert.Equal(t, 3, n1) 62 | assert.Equal(t, []byte("asd"), p1[:]) 63 | assert.Nil(t, e1) 64 | 65 | n2, e2 := pr.Read(p2[:]) 66 | assert.Equal(t, 1, n2) 67 | assert.Equal(t, []byte("f"), p2[:]) 68 | assert.Equal(t, io.EOF, e2) 69 | } 70 | 71 | func TestPktlineReaderReadsManyPacketsInMultipleCallsWithUnevenBuffering(t *testing.T) { 72 | var buf bytes.Buffer 73 | 74 | writePacket(t, &buf, []byte("first"), []byte("second")) 75 | 76 | pr := NewPktlineReaderFromPktline(NewPktline(&buf, nil), 10) 77 | 78 | var p1 [4]byte 79 | var p2 [7]byte 80 | var p3 []byte 81 | 82 | n1, e1 := pr.Read(p1[:]) 83 | assert.Equal(t, 4, n1) 84 | assert.Equal(t, []byte("firs"), p1[:]) 85 | assert.Nil(t, e1) 86 | 87 | n2, e2 := pr.Read(p2[:]) 88 | assert.Equal(t, 7, n2) 89 | assert.Equal(t, []byte("tsecond"), p2[:]) 90 | assert.Equal(t, io.EOF, e2) 91 | 92 | n3, e3 := pr.Read(p3[:]) 93 | assert.Equal(t, 0, n3) 94 | assert.Empty(t, p3) 95 | assert.Equal(t, io.EOF, e3) 96 | } 97 | 98 | func TestPktlineReaderReadsSinglePacketsInMultipleCallsWithEvenBuffering(t *testing.T) { 99 | var buf bytes.Buffer 100 | 101 | writePacket(t, &buf, []byte("firstother")) 102 | 103 | pr := NewPktlineReaderFromPktline(NewPktline(&buf, nil), 10) 104 | 105 | var p1 [5]byte 106 | var p2 [5]byte 107 | 108 | n1, e1 := pr.Read(p1[:]) 109 | assert.Equal(t, 5, n1) 110 | assert.Equal(t, []byte("first"), p1[:]) 111 | assert.Nil(t, e1) 112 | 113 | n2, e2 := pr.Read(p2[:]) 114 | assert.Equal(t, 5, n2) 115 | assert.Equal(t, []byte("other"), p2[:]) 116 | assert.Equal(t, io.EOF, e2) 117 | } 118 | 119 | func TestPktlineReaderReadsManyPacketsInMultipleCallsWithEvenBuffering(t *testing.T) { 120 | var buf bytes.Buffer 121 | 122 | writePacket(t, &buf, []byte("first"), []byte("other")) 123 | 124 | pr := NewPktlineReaderFromPktline(NewPktline(&buf, nil), 10) 125 | 126 | var p1 [5]byte 127 | var p2 [5]byte 128 | var p3 []byte 129 | 130 | n1, e1 := pr.Read(p1[:]) 131 | assert.Equal(t, 5, n1) 132 | assert.Equal(t, []byte("first"), p1[:]) 133 | assert.Nil(t, e1) 134 | 135 | n2, e2 := pr.Read(p2[:]) 136 | assert.Equal(t, 5, n2) 137 | assert.Equal(t, []byte("other"), p2[:]) 138 | assert.Equal(t, io.EOF, e2) 139 | 140 | n3, e3 := pr.Read(p3) 141 | assert.Equal(t, 0, n3) 142 | assert.Equal(t, io.EOF, e3) 143 | } 144 | 145 | func TestPktlineReaderReadEOFAfterEOF(t *testing.T) { 146 | var buf bytes.Buffer 147 | 148 | writePacket(t, &buf, []byte("asdf")) 149 | writePacket(t, &buf, []byte("abcd")) 150 | 151 | pr := NewPktlineReader(&buf, 10) 152 | 153 | var p1 [5]byte 154 | 155 | n, err := pr.Read(p1[:]) 156 | assert.Equal(t, err, io.EOF) 157 | assert.Equal(t, 4, n) 158 | assert.Equal(t, []byte("asdf"), p1[0:n]) 159 | 160 | n, err = pr.Read(p1[:]) 161 | assert.Equal(t, err, io.EOF) 162 | assert.Equal(t, 0, n) 163 | 164 | pr.Reset() 165 | 166 | n, err = pr.Read(p1[:]) 167 | assert.Equal(t, err, io.EOF) 168 | assert.Equal(t, 4, n) 169 | assert.Equal(t, []byte("abcd"), p1[0:n]) 170 | 171 | n, err = pr.Read(p1[:]) 172 | assert.Equal(t, err, io.EOF) 173 | assert.Equal(t, 0, n) 174 | } 175 | -------------------------------------------------------------------------------- /tools.go: -------------------------------------------------------------------------------- 1 | package pktline 2 | 3 | func minInt(a, b int) int { 4 | if a < b { 5 | return a 6 | } 7 | return b 8 | } 9 | -------------------------------------------------------------------------------- /writer.go: -------------------------------------------------------------------------------- 1 | package pktline 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | // PktlineWriter is an implementation of `io.Writer` which writes data buffers 8 | // "p" to an underlying pkt-line stream for use with the Git pkt-line format. 9 | type PktlineWriter struct { 10 | // buf is an internal buffer used to store data until enough has been 11 | // collected to write a full packet, or the buffer was instructed to 12 | // flush. 13 | buf []byte 14 | // pl is the place where packets get written. 15 | pl *Pktline 16 | } 17 | 18 | var _ io.Writer = new(PktlineWriter) 19 | 20 | // NewPktlineWriter returns a new *PktlineWriter, which will write to the 21 | // underlying data stream "w". The internal buffer is initialized with the given 22 | // capacity, "c". 23 | // 24 | // If "w" is already a `*PktlineWriter`, it will be returned as-is. 25 | func NewPktlineWriter(w io.Writer, c int) *PktlineWriter { 26 | if pw, ok := w.(*PktlineWriter); ok { 27 | return pw 28 | } 29 | 30 | return &PktlineWriter{ 31 | buf: make([]byte, 0, c), 32 | pl: NewPktline(nil, w), 33 | } 34 | } 35 | 36 | // NewPktlineWriterFromPktline returns a new *PktlineWriter based on the 37 | // underlying *Pktline object. The internal buffer is initialized with the 38 | // given capacity, "c". 39 | func NewPktlineWriterFromPktline(pl *Pktline, c int) *PktlineWriter { 40 | return &PktlineWriter{ 41 | buf: make([]byte, 0, c), 42 | pl: pl, 43 | } 44 | } 45 | 46 | // Write implements the io.Writer interface's `Write` method by providing a 47 | // packet-based backend to the given buffer "p". 48 | // 49 | // As many bytes are removed from "p" as possible and stored in an internal 50 | // buffer until the amount of data in the internal buffer is enough to write a 51 | // single packet. Once the internal buffer is full, a packet is written to the 52 | // underlying stream of data, and the process repeats. 53 | // 54 | // When the caller has no more data to write in the given chunk of packets, a 55 | // subsequent call to `Flush()` SHOULD be made in order to signify that the 56 | // current pkt sequence has terminated, and a new one can begin. 57 | // 58 | // Write returns the number of bytes in "p" accepted into the writer, which 59 | // _MAY_ be written to the underlying protocol stream, or may be written into 60 | // the internal buffer. 61 | // 62 | // If any error was encountered while either buffering or writing, that 63 | // error is returned, along with the number of bytes written to the underlying 64 | // protocol stream, as described above. 65 | func (w *PktlineWriter) Write(p []byte) (int, error) { 66 | var n int 67 | 68 | for len(p[n:]) > 0 { 69 | // While there is still data left to process in "p", grab as 70 | // much of it as we can while not allowing the internal buffer 71 | // to exceed the MaxPacketLength const. 72 | m := minInt(len(p[n:]), MaxPacketLength-len(w.buf)) 73 | 74 | // Append on all of the data that we could into the internal 75 | // buffer. 76 | w.buf = append(w.buf, p[n:n+m]...) 77 | 78 | n += m 79 | 80 | if len(w.buf) == MaxPacketLength { 81 | // If we were able to grab an entire packet's worth of 82 | // data, flush the buffer. 83 | 84 | if _, err := w.flush(); err != nil { 85 | return n, err 86 | } 87 | 88 | } 89 | } 90 | 91 | return n, nil 92 | } 93 | 94 | // Flush empties the internal buffer used to store data temporarily and then 95 | // writes the pkt-line's FLUSH packet, to signal that it is done writing this 96 | // chunk of data. 97 | func (w *PktlineWriter) Flush() error { 98 | if w == nil { 99 | return nil 100 | } 101 | 102 | if _, err := w.flush(); err != nil { 103 | return err 104 | } 105 | 106 | if err := w.pl.WriteFlush(); err != nil { 107 | return err 108 | } 109 | 110 | return nil 111 | } 112 | 113 | // flush writes any data in the internal buffer out to the underlying protocol 114 | // stream. If the amount of data in the internal buffer exceeds the 115 | // MaxPacketLength, the data will be written in multiple packets to accommodate. 116 | // 117 | // flush returns the number of bytes written to the underlying packet stream, 118 | // and any error that it encountered along the way. 119 | func (w *PktlineWriter) flush() (int, error) { 120 | var n int 121 | 122 | for len(w.buf) > 0 { 123 | if err := w.pl.WritePacket(w.buf); err != nil { 124 | return 0, err 125 | } 126 | 127 | m := minInt(len(w.buf), MaxPacketLength) 128 | 129 | w.buf = w.buf[m:] 130 | 131 | n = n + m 132 | } 133 | 134 | return n, nil 135 | } 136 | -------------------------------------------------------------------------------- /writer_test.go: -------------------------------------------------------------------------------- 1 | package pktline 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestPktlineWriterWritesPacketsShorterThanMaxPacketSize(t *testing.T) { 11 | var buf bytes.Buffer 12 | 13 | w := NewPktlineWriterFromPktline(NewPktline(nil, &buf), 0) 14 | assertWriterWrite(t, w, []byte("Hello, world!"), 13) 15 | assertWriterWrite(t, w, nil, 0) 16 | 17 | pl := NewPktline(&buf, nil) 18 | assertPacketRead(t, pl, []byte("Hello, world!")) 19 | assertPacketRead(t, pl, nil) 20 | } 21 | 22 | func TestPktlineWriterWritesPacketsEqualToMaxPacketLength(t *testing.T) { 23 | big := make([]byte, MaxPacketLength) 24 | for i, _ := range big { 25 | big[i] = 1 26 | } 27 | 28 | // Make a copy so that we can drain the data inside of it 29 | p := make([]byte, MaxPacketLength) 30 | copy(p, big) 31 | 32 | var buf bytes.Buffer 33 | 34 | w := NewPktlineWriter(&buf, 0) 35 | assertWriterWrite(t, w, p, len(big)) 36 | assertWriterWrite(t, w, nil, 0) 37 | 38 | pl := NewPktline(&buf, nil) 39 | assertPacketRead(t, pl, big) 40 | assertPacketRead(t, pl, nil) 41 | } 42 | 43 | func TestPktlineWriterWritesMultiplePacketsLessThanMaxPacketLength(t *testing.T) { 44 | var buf bytes.Buffer 45 | 46 | w := NewPktlineWriter(&buf, 0) 47 | assertWriterWrite(t, w, []byte("first\n"), len("first\n")) 48 | assertWriterWrite(t, w, []byte("second"), len("second")) 49 | assertWriterWrite(t, w, nil, 0) 50 | 51 | pl := NewPktline(&buf, nil) 52 | assertPacketRead(t, pl, []byte("first\nsecond")) 53 | assertPacketRead(t, pl, nil) 54 | } 55 | 56 | func TestPktlineWriterWritesMultiplePacketsGreaterThanMaxPacketLength(t *testing.T) { 57 | var buf bytes.Buffer 58 | 59 | b1 := make([]byte, MaxPacketLength*3/4) 60 | for i, _ := range b1 { 61 | b1[i] = 1 62 | } 63 | 64 | b2 := make([]byte, MaxPacketLength*3/4) 65 | for i, _ := range b2 { 66 | b2[i] = 2 67 | } 68 | 69 | w := NewPktlineWriter(&buf, 0) 70 | assertWriterWrite(t, w, b1, len(b1)) 71 | assertWriterWrite(t, w, b2, len(b2)) 72 | assertWriterWrite(t, w, nil, 0) 73 | 74 | // offs is how far into b2 we needed to buffer before writing an entire 75 | // packet 76 | offs := MaxPacketLength - len(b1) 77 | 78 | pl := NewPktline(&buf, nil) 79 | assertPacketRead(t, pl, append(b1, b2[:offs]...)) 80 | assertPacketRead(t, pl, b2[offs:]) 81 | assertPacketRead(t, pl, nil) 82 | } 83 | 84 | func TestPktlineWriterAllowsFlushesOnNil(t *testing.T) { 85 | assert.NoError(t, (*PktlineWriter)(nil).Flush()) 86 | } 87 | 88 | func TestPktlineWriterDoesntWrapItself(t *testing.T) { 89 | itself := &PktlineWriter{} 90 | nw := NewPktlineWriter(itself, 0) 91 | 92 | assert.Equal(t, itself, nw) 93 | } 94 | 95 | func assertWriterWrite(t *testing.T, w *PktlineWriter, p []byte, plen int) { 96 | var n int 97 | var err error 98 | 99 | if p == nil { 100 | err = w.Flush() 101 | } else { 102 | n, err = w.Write(p) 103 | } 104 | 105 | assert.Nil(t, err) 106 | assert.Equal(t, plen, n) 107 | } 108 | 109 | func assertPacketRead(t *testing.T, pl *Pktline, expected []byte) { 110 | got, err := pl.ReadPacket() 111 | 112 | assert.Nil(t, err) 113 | assert.Equal(t, expected, got) 114 | } 115 | --------------------------------------------------------------------------------