├── .travis.yml ├── LICENSE ├── README.md ├── xopen.go └── xopen_test.go /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.4 5 | - 1.5 6 | - 1.6 7 | - tip 8 | 9 | before_install: 10 | - go get github.com/axw/gocov/gocov 11 | - go get github.com/mattn/goveralls 12 | - if ! go get code.google.com/p/go.tools/cmd/cover; then go get golang.org/x/tools/cmd/cover; fi 13 | script: 14 | - $HOME/gopath/bin/goveralls -service=travis-ci 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Brent Pedersen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![GoDoc](https://godoc.org/github.com/brentp/xopen?status.png)](https://godoc.org/github.com/brentp/xopen) 2 | [![Build Status](https://travis-ci.org/brentp/xopen.svg)](https://travis-ci.org/brentp/xopen) 3 | [![Coverage Status](https://coveralls.io/repos/brentp/xopen/badge.svg?branch=master)](https://coveralls.io/r/brentp/xopen?branch=master) 4 | 5 | # xopen 6 | -- 7 | import "github.com/brentp/xopen" 8 | 9 | xopen makes it easy to get buffered (possibly gzipped) readers and writers. and 10 | close all of the associated files. Ropen opens a file for reading. Wopen opens a 11 | file for writing. Both will use gzip when appropriate and will use buffered IO. 12 | 13 | ## Usage 14 | 15 | Here's how to get a buffered reader: 16 | ```go 17 | // gzipped 18 | rdr, err := xopen.Ropen("some.gz") 19 | // normal 20 | rdr, err := xopen.Ropen("some.txt") 21 | // stdin (possibly gzipped) 22 | rdr, err := xopen.Ropen("-") 23 | // https:// 24 | rdr, err := xopen.Ropen("http://example.com/some-file.txt") 25 | // Cmd 26 | rdr, err := xopen.Ropen("|ls -lh somefile.gz") 27 | // User directory: 28 | rdr, err := xopen.Ropen("~/brentp/somefile") 29 | 30 | ``` 31 | Get a buffered writer with `xopen.Wopen`. 32 | Get a temp file with `xopen.Wopen("tmp:prefix")` 33 | 34 | 35 | #### func CheckBytes 36 | 37 | ```go 38 | func CheckBytes(b *bufio.Reader, buf []byte) (bool, error) 39 | ``` 40 | CheckBytes peeks at a buffered stream and checks if the first read bytes match. 41 | 42 | #### func IsGzip 43 | 44 | ```go 45 | func IsGzip(b *bufio.Reader) (bool, error) 46 | ``` 47 | IsGzip returns true buffered Reader has the gzip magic. 48 | 49 | #### func IsStdin 50 | 51 | ```go 52 | func IsStdin() bool 53 | ``` 54 | IsStdin checks if we are getting data from stdin. 55 | 56 | #### func XReader 57 | 58 | ```go 59 | func XReader(f string) (io.Reader, error) 60 | ``` 61 | XReader returns a reader from a url string or a file. 62 | 63 | 64 | #### type Reader 65 | 66 | ```go 67 | type Reader struct { 68 | *bufio.Reader 69 | } 70 | ``` 71 | 72 | Reader is returned by Ropen 73 | 74 | #### func Buf 75 | 76 | ```go 77 | func Buf(r io.Reader) *Reader 78 | ``` 79 | Return a buffered reader from an io.Reader If f == "-", then it will attempt to 80 | read from os.Stdin. If the file is gzipped, it will be read as such. 81 | 82 | #### func Ropen 83 | 84 | ```go 85 | func Ropen(f string) (*Reader, error) 86 | ``` 87 | Ropen opens a buffered reader. 88 | 89 | #### func (*Reader) Close 90 | 91 | ```go 92 | func (r *Reader) Close() error 93 | ``` 94 | Close the associated files. 95 | 96 | #### type Writer 97 | 98 | ```go 99 | type Writer struct { 100 | *bufio.Writer 101 | } 102 | ``` 103 | 104 | Writer is returned by Wopen 105 | 106 | #### func Wopen 107 | 108 | ```go 109 | func Wopen(f string) (*Writer, error) 110 | ``` 111 | Wopen opens a buffered writer. If f == "-", then stdout will be used. If f 112 | endswith ".gz", then the output will be gzipped. 113 | If f startswith "tmp:" then a tempfile will be created with a prefix of the string following ":" 114 | 115 | 116 | #### func (*Writer) Name 117 | 118 | ```go 119 | func (w *Writer) Name() string 120 | ``` 121 | The path to the underlying file handle. 122 | 123 | #### func (*Writer) Close 124 | 125 | ```go 126 | func (w *Writer) Close() error 127 | ``` 128 | Close the associated files. 129 | 130 | #### func (*Writer) Flush 131 | 132 | ```go 133 | func (w *Writer) Flush() 134 | ``` 135 | Flush the writer. 136 | -------------------------------------------------------------------------------- /xopen.go: -------------------------------------------------------------------------------- 1 | // Package xopen makes it easy to get buffered readers and writers. 2 | // Ropen opens a (possibly gzipped) file/process/http site for buffered reading. 3 | // Wopen opens a (possibly gzipped) file for buffered writing. 4 | // Both will use gzip when appropriate and will user buffered IO. 5 | package xopen 6 | 7 | import ( 8 | "bufio" 9 | "errors" 10 | "fmt" 11 | "io" 12 | "io/ioutil" 13 | "log" 14 | "net/http" 15 | "os" 16 | "os/exec" 17 | "os/user" 18 | "strings" 19 | 20 | //gzip "github.com/klauspost/pgzip" 21 | //"github.com/klauspost/compress/gzip" 22 | 23 | "compress/gzip" 24 | ) 25 | 26 | // IsGzip returns true buffered Reader has the gzip magic. 27 | func IsGzip(b *bufio.Reader) (bool, error) { 28 | return CheckBytes(b, []byte{0x1f, 0x8b}) 29 | } 30 | 31 | // IsStdin checks if we are getting data from stdin. 32 | func IsStdin() bool { 33 | // http://stackoverflow.com/a/26567513 34 | stat, err := os.Stdin.Stat() 35 | if err != nil { 36 | return false 37 | } 38 | return (stat.Mode() & os.ModeCharDevice) == 0 39 | } 40 | 41 | // ExpandUser expands ~/path and ~otheruser/path appropriately. 42 | func ExpandUser(path string) (string, error) { 43 | if len(path) == 0 || path[0] != '~' { 44 | return path, nil 45 | } 46 | var u *user.User 47 | var err error 48 | if len(path) == 1 || path[1] == '/' { 49 | u, err = user.Current() 50 | } else { 51 | name := strings.Split(path[1:], "/")[0] 52 | u, err = user.Lookup(name) 53 | } 54 | if err != nil { 55 | return "", err 56 | } 57 | home := u.HomeDir 58 | path = home + "/" + path[1:] 59 | return path, nil 60 | } 61 | 62 | // Exists checks if a local file exits 63 | func Exists(path string) bool { 64 | path, perr := ExpandUser(path) 65 | if perr != nil { 66 | return false 67 | } 68 | _, err := os.Stat(path) 69 | return err == nil 70 | } 71 | 72 | // CheckBytes peeks at a buffered stream and checks if the first read bytes match. 73 | func CheckBytes(b *bufio.Reader, buf []byte) (bool, error) { 74 | 75 | m, err := b.Peek(len(buf)) 76 | if err != nil { 77 | return false, err 78 | } 79 | for i := range buf { 80 | if m[i] != buf[i] { 81 | return false, nil 82 | } 83 | } 84 | return true, nil 85 | } 86 | 87 | // Reader is returned by Ropen 88 | type Reader struct { 89 | *bufio.Reader 90 | rdr io.Reader 91 | gz io.ReadCloser 92 | } 93 | 94 | // Close the associated files. 95 | func (r *Reader) Close() error { 96 | if r.gz != nil { 97 | r.gz.Close() 98 | } 99 | if c, ok := r.rdr.(io.ReadCloser); ok { 100 | c.Close() 101 | } 102 | return nil 103 | } 104 | 105 | // Writer is returned by Wopen 106 | type Writer struct { 107 | *bufio.Writer 108 | wtr *os.File 109 | gz *gzip.Writer 110 | } 111 | 112 | // Name returns the path to the underlying file. 113 | func (w *Writer) Name() string { 114 | return w.wtr.Name() 115 | } 116 | 117 | // Close the associated files. 118 | func (w *Writer) Close() error { 119 | w.Flush() 120 | if w.gz != nil { 121 | w.gz.Close() 122 | } 123 | w.wtr.Close() 124 | return nil 125 | } 126 | 127 | // Flush the writer. 128 | func (w *Writer) Flush() { 129 | w.Writer.Flush() 130 | if w.gz != nil { 131 | w.gz.Flush() 132 | } 133 | } 134 | 135 | var pageSize = os.Getpagesize() * 2 136 | 137 | // Buf returns a buffered reader from an io.Reader. 138 | // If f == "-", then it will attempt to read from os.Stdin. 139 | // If the file is gzipped, it will be read as such. 140 | func Buf(r io.Reader) *Reader { 141 | b := bufio.NewReaderSize(r, pageSize) 142 | var rdr io.ReadCloser 143 | if is, err := IsGzip(b); err != nil && err != io.EOF { 144 | log.Fatal(err) 145 | } else if is { 146 | rdr, err = gzip.NewReader(b) 147 | if err != nil { 148 | log.Fatal(err) 149 | } 150 | b = bufio.NewReaderSize(rdr, pageSize) 151 | } 152 | return &Reader{b, r, rdr} 153 | } 154 | 155 | // XReader returns a reader from a url string or a file. 156 | func XReader(f string) (io.Reader, error) { 157 | if strings.HasPrefix(f, "http://") || strings.HasPrefix(f, "https://") { 158 | var rsp *http.Response 159 | rsp, err := http.Get(f) 160 | if err != nil { 161 | return nil, err 162 | } 163 | if rsp.StatusCode != 200 { 164 | return nil, fmt.Errorf("http error downloading %s. status: %s", f, rsp.Status) 165 | } 166 | rdr := rsp.Body 167 | return rdr, nil 168 | } 169 | f, err := ExpandUser(f) 170 | if err != nil { 171 | return nil, err 172 | } 173 | return os.Open(f) 174 | } 175 | 176 | // Ropen opens a buffered reader. 177 | func Ropen(f string) (*Reader, error) { 178 | var err error 179 | var rdr io.Reader 180 | if f == "" { 181 | return nil, errors.New("sent empty file-name to xopen.Ropen") 182 | } 183 | if f == "-" { 184 | if !IsStdin() { 185 | return nil, errors.New("warning: stdin not detected") 186 | } 187 | b := Buf(os.Stdin) 188 | return b, nil 189 | } else if f[0] == '|' { 190 | // TODO: use csv to handle quoted file names. 191 | cmdStrs := strings.Split(f[1:], " ") 192 | var cmd *exec.Cmd 193 | if len(cmdStrs) == 2 { 194 | cmd = exec.Command(cmdStrs[0], cmdStrs[1:]...) 195 | } else { 196 | cmd = exec.Command(cmdStrs[0]) 197 | } 198 | rdr, err = cmd.StdoutPipe() 199 | if err != nil { 200 | return nil, err 201 | } 202 | err = cmd.Start() 203 | if err != nil { 204 | return nil, err 205 | } 206 | } else { 207 | rdr, err = XReader(f) 208 | } 209 | if err != nil { 210 | return nil, err 211 | } 212 | b := Buf(rdr) 213 | return b, nil 214 | } 215 | 216 | // Wopen opens a buffered reader. 217 | // If f == "-", then stdout will be used. 218 | // If f endswith ".gz", then the output will be gzipped. 219 | // If f startswith "tmp:" then a tmpfile will be created with the prefix being the value after tmp:. e.g. tmp:fx.gz will create a gzip writer for /tmp/fx${random}.gz 220 | func Wopen(f string) (*Writer, error) { 221 | var wtr *os.File 222 | var err error 223 | if f == "-" { 224 | wtr = os.Stdout 225 | } else if strings.HasPrefix(f, "tmp:") { 226 | prefix := "" 227 | if len(f) > 4 { 228 | prefix = strings.TrimSuffix(strings.Split(f, ":")[1], ".gz") 229 | } 230 | wtr, err = ioutil.TempFile("", prefix) 231 | } else { 232 | wtr, err = os.Create(f) 233 | if err != nil { 234 | return nil, err 235 | } 236 | } 237 | if !strings.HasSuffix(f, ".gz") { 238 | return &Writer{bufio.NewWriterSize(wtr, pageSize), wtr, nil}, nil 239 | } 240 | gz := gzip.NewWriter(wtr) 241 | w, err := &Writer{bufio.NewWriterSize(gz, pageSize), wtr, gz}, nil 242 | return w, err 243 | } 244 | -------------------------------------------------------------------------------- /xopen_test.go: -------------------------------------------------------------------------------- 1 | package xopen 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "compress/gzip" 7 | "fmt" 8 | "io" 9 | "os" 10 | "strings" 11 | "testing" 12 | 13 | . "gopkg.in/check.v1" 14 | ) 15 | 16 | func Test(t *testing.T) { TestingT(t) } 17 | 18 | type XopenTest struct{} 19 | 20 | var _ = Suite(&XopenTest{}) 21 | 22 | func gzFromString(s string) string { 23 | var c bytes.Buffer 24 | gz := gzip.NewWriter(&c) 25 | gz.Write([]byte(s)) 26 | return c.String() 27 | } 28 | 29 | var gzTests = []struct { 30 | isGz bool 31 | data string 32 | }{ 33 | {false, "asdf"}, 34 | {true, gzFromString("asdf")}, 35 | } 36 | 37 | func (s *XopenTest) TestIsGzip(c *C) { 38 | for _, t := range gzTests { 39 | isGz, err := IsGzip(bufio.NewReader(strings.NewReader(t.data))) 40 | c.Assert(err, IsNil) 41 | c.Assert(t.isGz, Equals, isGz) 42 | } 43 | } 44 | 45 | func (s *XopenTest) TestIsStdin(c *C) { 46 | r := IsStdin() 47 | c.Assert(r, Equals, false) 48 | } 49 | 50 | func (s *XopenTest) TestRopen(c *C) { 51 | rdr, err := Ropen("-") 52 | c.Assert(err, ErrorMatches, ".* stdin not detected") 53 | c.Assert(rdr, IsNil) 54 | } 55 | 56 | func (s *XopenTest) TestWopen(c *C) { 57 | for _, f := range []string{"t.gz", "t"} { 58 | testString := "ASDF1234" 59 | wtr, err := Wopen(f) 60 | c.Assert(err, IsNil) 61 | _, err = os.Stat(f) 62 | c.Assert(err, IsNil) 63 | c.Assert(wtr.wtr, NotNil) 64 | fmt.Fprintf(wtr, testString) 65 | wtr.Close() 66 | 67 | rdr, err := Ropen(f) 68 | c.Assert(err, IsNil) 69 | 70 | str, err := rdr.ReadString(99) 71 | c.Assert(str, Equals, testString) 72 | c.Assert(err, Equals, io.EOF) 73 | str, err = rdr.ReadString(99) 74 | c.Assert(str, Equals, "") 75 | c.Assert(err, Equals, io.EOF) 76 | 77 | rdr.Close() 78 | os.Remove(f) 79 | } 80 | } 81 | 82 | func (s *XopenTest) TestWopenTmp(c *C) { 83 | wtr, err := Wopen("tmp:x.gz") 84 | c.Assert(err, IsNil) 85 | c.Assert(wtr.wtr, NotNil) 86 | fmt.Fprintf(wtr, "xx") 87 | c.Assert(wtr.Close(), IsNil) 88 | _, err = os.Stat(wtr.Name()) 89 | c.Assert(err, IsNil) 90 | 91 | os.Remove(wtr.Name()) 92 | 93 | } 94 | 95 | var httpTests = []struct { 96 | url string 97 | expectError bool 98 | }{ 99 | {"https://raw.githubusercontent.com/brentp/xopen/master/README.md", false}, 100 | {"http://raw.githubusercontent.com/brentp/xopen/master/README.md", false}, 101 | {"http://raw.githubusercontent.com/brentp/xopen/master/BAD.md", true}, 102 | } 103 | 104 | func (s *XopenTest) TestReadHttp(c *C) { 105 | for _, t := range httpTests { 106 | rdr, err := Ropen(t.url) 107 | if !t.expectError { 108 | c.Assert(err, IsNil) 109 | v, err := rdr.ReadString(byte('\n')) 110 | c.Assert(err, IsNil) 111 | c.Assert(len(v), Not(Equals), 0) 112 | } else { 113 | c.Assert(err, ErrorMatches, ".* 404 Not Found") 114 | } 115 | } 116 | } 117 | 118 | func (s *XopenTest) TestReadProcess(c *C) { 119 | for _, cmd := range []string{"|ls -lh", "|ls", "|ls -lh xopen_test.go"} { 120 | rdr, err := Ropen(cmd) 121 | c.Assert(err, IsNil) 122 | b := make([]byte, 1000) 123 | _, err = rdr.Read(b) 124 | if err != io.EOF { 125 | c.Assert(err, IsNil) 126 | } 127 | lines := strings.Split(string(b), "\n") 128 | has := false 129 | for _, line := range lines { 130 | if strings.Contains(line, "xopen_test.go") { 131 | has = true 132 | } 133 | } 134 | c.Assert(has, Equals, true) 135 | } 136 | } 137 | 138 | func (s *XopenTest) TestOpenStdout(c *C) { 139 | w, err := Wopen("-") 140 | c.Assert(err, IsNil) 141 | c.Assert(w.wtr, Equals, os.Stdout) 142 | } 143 | 144 | func (s *XopenTest) TestOpenBadFile(c *C) { 145 | r, err := Ropen("XXXXXXXXXXXXXXXXXXXXXXX") 146 | c.Assert(r, IsNil) 147 | c.Assert(err, ErrorMatches, ".* no such file .*") 148 | } 149 | 150 | func (s *XopenTest) TestWOpenBadFile(c *C) { 151 | w, err := Wopen("XX/XXX/XXX/XXX/XXX/XXXXXXXXX") 152 | c.Assert(w, IsNil) 153 | c.Assert(err, ErrorMatches, ".* no such file .*") 154 | } 155 | 156 | func (s *XopenTest) TestExists(c *C) { 157 | c.Assert(Exists("xopen.go"), Equals, true) 158 | c.Assert(Exists("____xx"), Equals, false) 159 | } 160 | 161 | func (s *XopenTest) TestUser(c *C) { 162 | c.Assert(Exists("~"), Equals, true) 163 | } 164 | 165 | func (s *XopenTest) TestExpand(c *C) { 166 | _, err := ExpandUser("~baduser66") 167 | c.Assert(err, Not(IsNil)) 168 | } 169 | --------------------------------------------------------------------------------