├── .gitignore ├── .travis.yml ├── AUTHORS ├── LICENSE ├── README.md ├── bytereplacer ├── bytereplacer.go └── bytereplacer_test.go ├── cloud ├── cloudlaunch │ └── cloudlaunch.go └── google │ ├── gceutil │ └── gceutil.go │ └── gcsutil │ └── storage.go ├── ctxutil └── ctxutil.go ├── errorutil └── highlight.go ├── fault └── fault.go ├── go.mod ├── go.sum ├── go4test └── cloudlaunch │ └── serve_on_cloud.go ├── jsonconfig ├── eval.go ├── jsonconfig.go ├── jsonconfig_test.go └── testdata │ ├── boolenv.json │ ├── include1.json │ ├── include1bis.json │ ├── include2.json │ ├── listexpand.json │ ├── loop1.json │ └── loop2.json ├── legal ├── legal.go └── legal_test.go ├── lock ├── .gitignore ├── lock.go ├── lock_appengine.go ├── lock_plan9.go ├── lock_sigzero.go ├── lock_test.go ├── lock_unix.go └── lock_windows.go ├── media └── heif │ ├── bmff │ └── bmff.go │ ├── dumpheif │ └── dumpheif.go │ ├── heif.go │ ├── heif_test.go │ └── testdata │ ├── park.heic │ └── rotate.heic ├── must └── must.go ├── net └── throttle │ └── throttle.go ├── oauthutil └── oauth.go ├── osutil ├── exec_plan9.go ├── exec_procfs.go ├── exec_solaris_amd64.go ├── exec_sysctl.go ├── exec_test.go ├── exec_windows.go └── osutil.go ├── readerutil ├── bufreaderat.go ├── bufreaderat_test.go ├── countingreader.go ├── fakeseeker.go ├── fakeseeker_test.go ├── multireaderat.go ├── multireaderat_test.go ├── readersize.go ├── readersize_test.go ├── readerutil.go ├── readerutil_test.go └── singlereader │ ├── opener.go │ └── opener_test.go ├── reflectutil ├── asm_b.s ├── asm_b_14.s ├── asm_jmp.s ├── asm_jmp_14.s ├── reflectutil.go ├── reflectutil_test.go ├── swapper.go ├── swapper_safe.go ├── swapper_std.go ├── swapper_test.go ├── swapper_unsafe.go ├── swapper_unsafe_14.go └── swapper_unsafe_15.go ├── rollsum ├── rollsum.go └── rollsum_test.go ├── sort ├── example_interface_test.go ├── example_keys_test.go ├── example_multi_test.go ├── example_slice_test.go ├── example_test.go ├── example_wrapper_test.go ├── export_test.go ├── genzfunc.go ├── search.go ├── search_test.go ├── sort.go ├── sort_test.go └── zfuncversion.go ├── strutil ├── intern.go ├── strconv.go ├── strutil.go └── strutil_test.go ├── syncutil ├── gate.go ├── group.go ├── once.go ├── once_test.go ├── sem.go ├── sem_test.go ├── singleflight │ ├── singleflight.go │ └── singleflight_test.go ├── syncdebug │ ├── syncdebug.go │ └── syncdebug_test.go └── syncutil.go ├── testing └── functest │ ├── functest.go │ └── functest_test.go ├── types ├── types.go └── types_test.go ├── wkfs ├── gcs │ ├── gcs.go │ └── gcs_test.go └── wkfs.go ├── writerutil ├── writerutil.go └── writerutil_test.go └── xdgdir ├── example_test.go ├── xdgdir.go └── xdgdir_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | arch: 3 | - amd64 4 | - ppc64le 5 | 6 | go: 7 | - "1.13.x" 8 | - tip 9 | 10 | go_import_path: go4.org 11 | 12 | before_install: 13 | - go mod tidy 14 | - git diff --exit-code go.mod 15 | - git diff --exit-code go.sum 16 | - go mod download 17 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # This is the official list of go4 authors for copyright purposes. 2 | # This is distinct from the CONTRIBUTORS file, which is the list of 3 | # people who have contributed, even if they don't own the copyright on 4 | # their work. 5 | 6 | Mathieu Lonjaret 7 | Daniel Theophanes 8 | Google 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go4 2 | 3 | [![travis badge](https://travis-ci.org/go4org/go4.svg?branch=master)](https://travis-ci.org/go4org/go4 "Travis CI") 4 | 5 | [go4.org](http://go4.org) is a collection of packages for 6 | Go programmers. 7 | 8 | They started out living in [Perkeep](https://perkeep.org)'s repo 9 | and elsewhere but they have nothing to do with Perkeep, so we're 10 | moving them here. 11 | 12 | ## Details 13 | 14 | * **single repo**. go4 is a single repo. That means things can be 15 | changed and rearranged globally atomically with ease and 16 | confidence. 17 | 18 | * **no backwards compatibility**. go4 makes no backwards compatibility 19 | promises. If you want to use go4, vendor it. And next time you 20 | update your vendor tree, update to the latest API if things in go4 21 | changed. The plan is to eventually provide tools to make this 22 | easier. 23 | 24 | * **forward progress** because we have no backwards compatibility, 25 | it's always okay to change things to make things better. That also 26 | means the bar for contributions is lower. We don't have to get the 27 | API 100% correct in the first commit. 28 | 29 | * **no Go version policy** go4 packages are usually built and tested 30 | with the latest Go stable version. However, go4 has no overarching 31 | version policy; each package can declare its own set of supported 32 | Go versions. 33 | 34 | * **code review** contributions must be code-reviewed. We're trying 35 | out Gerrithub, to see if we can find a mix of Github Pull Requests 36 | and Gerrit that works well for many people. We'll see. 37 | 38 | * **CLA compliant** contributors must agree to the Google CLA (the 39 | same as Go itself). This ensures we can move things into Go as 40 | necessary in the future. It also makes lawyers at various 41 | companies happy. The CLA is **not** a copyright *assignment*; you 42 | retain the copyright on your work. The CLA just says that your 43 | work is open source and you have permission to open source it. See 44 | https://golang.org/doc/contribute.html#cla 45 | 46 | * **docs, tests, portability** all code should be documented in the 47 | normal Go style, have tests, and be portable to different 48 | operating systems and architectures. We'll try to get builders in 49 | place to help run the tests on different OS/arches. For now we 50 | have Travis at least. 51 | 52 | ## Contact 53 | 54 | For any question, or communication when a Github issue is not appropriate, 55 | please contact the [Perkeep mailing 56 | list](https://groups.google.com/forum/#!forum/perkeep). 57 | 58 | -------------------------------------------------------------------------------- /cloud/google/gceutil/gceutil.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 The Perkeep Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package gceutil provides utility functions to help with instances on 18 | // Google Compute Engine. 19 | package gceutil // import "go4.org/cloud/google/gceutil" 20 | 21 | import ( 22 | "encoding/json" 23 | "errors" 24 | "net/http" 25 | "strings" 26 | "time" 27 | 28 | "google.golang.org/api/compute/v1" 29 | ) 30 | 31 | // CoreOSImageURL returns the URL of the latest stable CoreOS image for running 32 | // on Google Compute Engine. 33 | func CoreOSImageURL(cl *http.Client) (string, error) { 34 | return osImageURL(cl, false) 35 | } 36 | 37 | // COSImageURL returns the URL of the latest stable Container-Optimized OS image 38 | // for running on Google Compute Engine. 39 | func COSImageURL(cl *http.Client) (string, error) { 40 | return osImageURL(cl, true) 41 | } 42 | 43 | func osImageURL(cl *http.Client, cos bool) (string, error) { 44 | project := "coreos-cloud" 45 | if cos { 46 | project = "cos-cloud" 47 | } 48 | resp, err := cl.Get("https://www.googleapis.com/compute/v1/projects/" + project + "/global/images") 49 | if err != nil { 50 | return "", err 51 | } 52 | defer resp.Body.Close() 53 | 54 | type osImage struct { 55 | SelfLink string 56 | CreationTimestamp time.Time 57 | Name string 58 | } 59 | 60 | type osImageList struct { 61 | Items []osImage 62 | } 63 | 64 | imageList := &osImageList{} 65 | if err := json.NewDecoder(resp.Body).Decode(imageList); err != nil { 66 | return "", err 67 | } 68 | if imageList == nil || len(imageList.Items) == 0 { 69 | return "", errors.New("no images list in response") 70 | } 71 | 72 | imageURL := "" 73 | var max time.Time // latest stable image creation time 74 | imgPrefix := "coreos-stable" 75 | if cos { 76 | imgPrefix = "cos-stable" 77 | } 78 | for _, v := range imageList.Items { 79 | if !strings.HasPrefix(v.Name, imgPrefix) { 80 | continue 81 | } 82 | if v.CreationTimestamp.After(max) { 83 | max = v.CreationTimestamp 84 | imageURL = v.SelfLink 85 | } 86 | } 87 | if imageURL == "" { 88 | if cos { 89 | return "", errors.New("no stable Container-Optimized OS image found") 90 | } 91 | return "", errors.New("no stable coreOS image found") 92 | } 93 | return imageURL, nil 94 | } 95 | 96 | // InstanceGroupAndManager contains both an InstanceGroup and 97 | // its InstanceGroupManager, if any. 98 | type InstanceGroupAndManager struct { 99 | Group *compute.InstanceGroup 100 | 101 | // Manager is the manager of the Group. It may be nil. 102 | Manager *compute.InstanceGroupManager 103 | } 104 | 105 | // InstanceGroups returns all the instance groups in a project's zone, along 106 | // with their associated InstanceGroupManagers. 107 | // The returned map is keyed by the instance group identifier URL. 108 | func InstanceGroups(svc *compute.Service, proj, zone string) (map[string]InstanceGroupAndManager, error) { 109 | managerList, err := svc.InstanceGroupManagers.List(proj, zone).Do() 110 | if err != nil { 111 | return nil, err 112 | } 113 | if managerList.NextPageToken != "" { 114 | return nil, errors.New("too many managers; pagination not supported") 115 | } 116 | managedBy := make(map[string]*compute.InstanceGroupManager) // instance group URL -> its manager 117 | for _, it := range managerList.Items { 118 | managedBy[it.InstanceGroup] = it 119 | } 120 | groupList, err := svc.InstanceGroups.List(proj, zone).Do() 121 | if err != nil { 122 | return nil, err 123 | } 124 | if groupList.NextPageToken != "" { 125 | return nil, errors.New("too many instance groups; pagination not supported") 126 | } 127 | ret := make(map[string]InstanceGroupAndManager) 128 | for _, it := range groupList.Items { 129 | ret[it.SelfLink] = InstanceGroupAndManager{it, managedBy[it.SelfLink]} 130 | } 131 | return ret, nil 132 | } 133 | -------------------------------------------------------------------------------- /cloud/google/gcsutil/storage.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 The Go4 Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package gcsutil provides tools for accessing Google Cloud Storage until they can be 18 | // completely replaced by cloud.google.com/go/storage. 19 | package gcsutil // import "go4.org/cloud/google/gcsutil" 20 | 21 | import ( 22 | "encoding/xml" 23 | "errors" 24 | "fmt" 25 | "io" 26 | "net/http" 27 | "net/url" 28 | "os" 29 | "strings" 30 | 31 | "cloud.google.com/go/storage" 32 | "go4.org/ctxutil" 33 | "golang.org/x/net/context" 34 | ) 35 | 36 | const gsAccessURL = "https://storage.googleapis.com" 37 | 38 | // An Object holds the name of an object (its bucket and key) within 39 | // Google Cloud Storage. 40 | type Object struct { 41 | Bucket string 42 | Key string 43 | } 44 | 45 | func (o *Object) valid() error { 46 | if o == nil { 47 | return errors.New("invalid nil Object") 48 | } 49 | if o.Bucket == "" { 50 | return errors.New("missing required Bucket field in Object") 51 | } 52 | if o.Key == "" { 53 | return errors.New("missing required Key field in Object") 54 | } 55 | return nil 56 | } 57 | 58 | // A SizedObject holds the bucket, key, and size of an object. 59 | type SizedObject struct { 60 | Object 61 | Size int64 62 | } 63 | 64 | func (o *Object) String() string { 65 | if o == nil { 66 | return "" 67 | } 68 | return fmt.Sprintf("%v/%v", o.Bucket, o.Key) 69 | } 70 | 71 | func (so SizedObject) String() string { 72 | return fmt.Sprintf("%v/%v (%vB)", so.Bucket, so.Key, so.Size) 73 | } 74 | 75 | // Makes a simple body-less google storage request 76 | func simpleRequest(method, url_ string) (*http.Request, error) { 77 | req, err := http.NewRequest(method, url_, nil) 78 | if err != nil { 79 | return nil, err 80 | } 81 | req.Header.Set("x-goog-api-version", "2") 82 | return req, err 83 | } 84 | 85 | // ErrInvalidRange is used when the server has returned http.StatusRequestedRangeNotSatisfiable. 86 | var ErrInvalidRange = errors.New("gcsutil: requested range not satisfiable") 87 | 88 | // GetPartialObject fetches part of a Google Cloud Storage object. 89 | // This function relies on the ctx ctxutil.HTTPClient value being set to an OAuth2 90 | // authorized and authenticated HTTP client. 91 | // If length is negative, the rest of the object is returned. 92 | // It returns ErrInvalidRange if the server replies with http.StatusRequestedRangeNotSatisfiable. 93 | // The caller must call Close on the returned value. 94 | func GetPartialObject(ctx context.Context, obj Object, offset, length int64) (io.ReadCloser, error) { 95 | if offset < 0 { 96 | return nil, errors.New("invalid negative offset") 97 | } 98 | if err := obj.valid(); err != nil { 99 | return nil, err 100 | } 101 | 102 | req, err := simpleRequest("GET", gsAccessURL+"/"+obj.Bucket+"/"+obj.Key) 103 | if err != nil { 104 | return nil, err 105 | } 106 | if length >= 0 { 107 | req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)) 108 | } else { 109 | req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset)) 110 | } 111 | req.Cancel = ctx.Done() 112 | res, err := ctxutil.Client(ctx).Do(req) 113 | if err != nil { 114 | return nil, fmt.Errorf("GET (offset=%d, length=%d) failed: %v\n", offset, length, err) 115 | } 116 | if res.StatusCode == http.StatusNotFound { 117 | res.Body.Close() 118 | return nil, os.ErrNotExist 119 | } 120 | if !(res.StatusCode == http.StatusPartialContent || (offset == 0 && res.StatusCode == http.StatusOK)) { 121 | res.Body.Close() 122 | if res.StatusCode == http.StatusRequestedRangeNotSatisfiable { 123 | return nil, ErrInvalidRange 124 | } 125 | return nil, fmt.Errorf("GET (offset=%d, length=%d) got failed status: %v\n", offset, length, res.Status) 126 | } 127 | 128 | return res.Body, nil 129 | } 130 | 131 | // EnumerateObjects lists the objects in a bucket. 132 | // This function relies on the ctx oauth2.HTTPClient value being set to an OAuth2 133 | // authorized and authenticated HTTP client. 134 | // If after is non-empty, listing will begin with lexically greater object names. 135 | // If limit is non-zero, the length of the list will be limited to that number. 136 | func EnumerateObjects(ctx context.Context, bucket, after string, limit int) ([]*storage.ObjectAttrs, error) { 137 | // Build url, with query params 138 | var params []string 139 | if after != "" { 140 | params = append(params, "marker="+url.QueryEscape(after)) 141 | } 142 | if limit > 0 { 143 | params = append(params, fmt.Sprintf("max-keys=%v", limit)) 144 | } 145 | query := "" 146 | if len(params) > 0 { 147 | query = "?" + strings.Join(params, "&") 148 | } 149 | 150 | req, err := simpleRequest("GET", gsAccessURL+"/"+bucket+"/"+query) 151 | if err != nil { 152 | return nil, err 153 | } 154 | req.Cancel = ctx.Done() 155 | res, err := ctxutil.Client(ctx).Do(req) 156 | if err != nil { 157 | return nil, err 158 | } 159 | defer res.Body.Close() 160 | if res.StatusCode != http.StatusOK { 161 | return nil, fmt.Errorf("gcsutil: bad enumerate response code: %v", res.Status) 162 | } 163 | 164 | var xres struct { 165 | Contents []SizedObject 166 | } 167 | if err = xml.NewDecoder(res.Body).Decode(&xres); err != nil { 168 | return nil, err 169 | } 170 | 171 | objAttrs := make([]*storage.ObjectAttrs, len(xres.Contents)) 172 | for k, o := range xres.Contents { 173 | objAttrs[k] = &storage.ObjectAttrs{ 174 | Name: o.Key, 175 | Size: o.Size, 176 | } 177 | } 178 | 179 | return objAttrs, nil 180 | } 181 | -------------------------------------------------------------------------------- /ctxutil/ctxutil.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 The Go4 Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package ctxutil contains golang.org/x/net/context related utilities. 18 | package ctxutil // import "go4.org/ctxutil" 19 | 20 | import ( 21 | "net/http" 22 | 23 | "golang.org/x/net/context" 24 | "golang.org/x/oauth2" 25 | ) 26 | 27 | // HTTPClient is the context key to use with golang.org/x/net/context's WithValue function 28 | // to associate an *http.Client value with a context. 29 | // 30 | // We use the same value as the oauth2 package (which first introduced this key) rather 31 | // than creating a new one and forcing users to possibly set two. 32 | var HTTPClient = oauth2.HTTPClient 33 | 34 | // Client returns the HTTP client to use for the provided context. 35 | // If ctx is non-nil and has an associated HTTP client, that client is returned. 36 | // Otherwise, http.DefaultClient is returned. 37 | func Client(ctx context.Context) *http.Client { 38 | if ctx != nil { 39 | if hc, ok := ctx.Value(HTTPClient).(*http.Client); ok { 40 | return hc 41 | } 42 | } 43 | return http.DefaultClient 44 | } 45 | -------------------------------------------------------------------------------- /errorutil/highlight.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2011 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package errorutil helps make better error messages. 18 | package errorutil // import "go4.org/errorutil" 19 | 20 | import ( 21 | "bufio" 22 | "bytes" 23 | "fmt" 24 | "io" 25 | "strings" 26 | ) 27 | 28 | // HighlightBytePosition takes a reader and the location in bytes of a parse 29 | // error (for instance, from json.SyntaxError.Offset) and returns the line, column, 30 | // and pretty-printed context around the error with an arrow indicating the exact 31 | // position of the syntax error. 32 | func HighlightBytePosition(f io.Reader, pos int64) (line, col int, highlight string) { 33 | line = 1 34 | br := bufio.NewReader(f) 35 | lastLine := "" 36 | thisLine := new(bytes.Buffer) 37 | for n := int64(0); n < pos; n++ { 38 | b, err := br.ReadByte() 39 | if err != nil { 40 | break 41 | } 42 | if b == '\n' { 43 | lastLine = thisLine.String() 44 | thisLine.Reset() 45 | line++ 46 | col = 1 47 | } else { 48 | col++ 49 | thisLine.WriteByte(b) 50 | } 51 | } 52 | if line > 1 { 53 | highlight += fmt.Sprintf("%5d: %s\n", line-1, lastLine) 54 | } 55 | highlight += fmt.Sprintf("%5d: %s\n", line, thisLine.String()) 56 | highlight += fmt.Sprintf("%s^\n", strings.Repeat(" ", col+5)) 57 | return 58 | } 59 | -------------------------------------------------------------------------------- /fault/fault.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014 The Go4 Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package fault handles fault injection for testing. 18 | package fault // import "go4.org/fault" 19 | 20 | import ( 21 | "errors" 22 | "math/rand" 23 | "os" 24 | "strconv" 25 | "strings" 26 | ) 27 | 28 | var fakeErr = errors.New("fake injected error for testing") 29 | 30 | // An Injector reports whether fake errors should be returned. 31 | type Injector struct { 32 | failPercent int 33 | } 34 | 35 | // NewInjector returns a new fault injector with the given name. The 36 | // environment variable "FAULT_" + capital(name) + "_FAIL_PERCENT" 37 | // controls the percentage of requests that fail. If undefined or 38 | // zero, no requests fail. 39 | func NewInjector(name string) *Injector { 40 | var failPercent, _ = strconv.Atoi(os.Getenv("FAULT_" + strings.ToUpper(name) + "_FAIL_PERCENT")) 41 | return &Injector{ 42 | failPercent: failPercent, 43 | } 44 | } 45 | 46 | // ShouldFail reports whether a fake error should be returned. 47 | func (in *Injector) ShouldFail() bool { 48 | return in.failPercent > 0 && in.failPercent > rand.Intn(100) 49 | } 50 | 51 | // FailErr checks ShouldFail and, if true, assigns a fake error to err 52 | // and returns true. 53 | func (in *Injector) FailErr(err *error) bool { 54 | if !in.ShouldFail() { 55 | return false 56 | } 57 | *err = fakeErr 58 | return true 59 | } 60 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module go4.org 2 | 3 | go 1.13 4 | 5 | require ( 6 | cloud.google.com/go v0.53.0 7 | cloud.google.com/go/storage v1.5.0 8 | github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd 9 | golang.org/x/net v0.7.0 10 | golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d 11 | golang.org/x/sys v0.5.0 12 | google.golang.org/api v0.17.0 13 | ) 14 | -------------------------------------------------------------------------------- /go4test/cloudlaunch/serve_on_cloud.go: -------------------------------------------------------------------------------- 1 | // +build ignore 2 | 3 | /* 4 | Copyright 2016 The Go4 Authors. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | // The serve_on_cloud program deploys an HTTP server on Google Compute Engine, 20 | // serving from Google Cloud Storage. Its purpose is to help testing 21 | // go4.org/cloud/cloudlaunch and go4.org/wkfs/gcs. 22 | package main 23 | 24 | import ( 25 | "flag" 26 | "fmt" 27 | "log" 28 | "net/http" 29 | "os" 30 | "path" 31 | "regexp" 32 | "time" 33 | 34 | "go4.org/cloud/cloudlaunch" 35 | "go4.org/wkfs" 36 | _ "go4.org/wkfs/gcs" 37 | 38 | "cloud.google.com/go/compute/metadata" 39 | storageapi "google.golang.org/api/storage/v1" 40 | compute "google.golang.org/api/compute/v1" 41 | ) 42 | 43 | var httpAddr = flag.String("http", ":80", "HTTP address") 44 | 45 | var gcsBucket string 46 | 47 | func serveHTTP(w http.ResponseWriter, r *http.Request) { 48 | rc, err := wkfs.Open(path.Join("/gcs", gcsBucket, r.URL.Path)) 49 | if err != nil { 50 | http.Error(w, fmt.Sprintf("could not open %v: %v", r.URL.Path, err), 500) 51 | return 52 | } 53 | defer rc.Close() 54 | http.ServeContent(w, r, r.URL.Path, time.Now(), rc) 55 | } 56 | 57 | func main() { 58 | if !metadata.OnGCE() { 59 | bucket := os.Getenv("GCSBUCKET") 60 | if bucket == "" { 61 | log.Fatal("You need to set the GCSBUCKET env var to specify the Google Cloud Storage bucket to serve from.") 62 | } 63 | projectID := os.Getenv("GCEPROJECTID") 64 | if projectID == "" { 65 | log.Fatal("You need to set the GCEPROJECTID env var to specify the Google Cloud project where the instance will run.") 66 | } 67 | (&cloudlaunch.Config{ 68 | Name: "serveoncloud", 69 | BinaryBucket: bucket, 70 | GCEProjectID: projectID, 71 | Scopes: []string{ 72 | storageapi.DevstorageFullControlScope, 73 | compute.ComputeScope, 74 | }, 75 | }).MaybeDeploy() 76 | return 77 | } 78 | 79 | flag.Parse() 80 | 81 | storageURLRxp := regexp.MustCompile(`https://storage.googleapis.com/(.+?)/serveoncloud.*`) 82 | cloudConfig, err := metadata.InstanceAttributeValue("user-data") 83 | if err != nil || cloudConfig == "" { 84 | log.Fatalf("could not get cloud config from metadata: %v", err) 85 | } 86 | m := storageURLRxp.FindStringSubmatch(cloudConfig) 87 | if len(m) < 2 { 88 | log.Fatal("storage URL not found in cloud config") 89 | } 90 | gcsBucket = m[1] 91 | 92 | http.HandleFunc("/", serveHTTP) 93 | 94 | log.Fatal(http.ListenAndServe(*httpAddr, nil)) 95 | } 96 | -------------------------------------------------------------------------------- /jsonconfig/jsonconfig_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2011 The go4 Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package jsonconfig 18 | 19 | import ( 20 | "os" 21 | "reflect" 22 | "strings" 23 | "testing" 24 | ) 25 | 26 | func testIncludes(configFile string, t *testing.T) { 27 | var c ConfigParser 28 | c.IncludeDirs = []string{"testdata"} 29 | obj, err := c.ReadFile(configFile) 30 | if err != nil { 31 | t.Fatal(err) 32 | } 33 | two := obj.RequiredObject("two") 34 | if err := obj.Validate(); err != nil { 35 | t.Error(err) 36 | } 37 | if g, e := two.RequiredString("key"), "value"; g != e { 38 | t.Errorf("sub object key = %q; want %q", g, e) 39 | } 40 | } 41 | 42 | func TestIncludesCWD(t *testing.T) { 43 | testIncludes("testdata/include1.json", t) 44 | } 45 | 46 | func TestIncludesIncludeDirs(t *testing.T) { 47 | testIncludes("testdata/include1bis.json", t) 48 | } 49 | 50 | func TestIncludeLoop(t *testing.T) { 51 | _, err := ReadFile("testdata/loop1.json") 52 | if err == nil { 53 | t.Fatal("expected an error about import cycles.") 54 | } 55 | if !strings.Contains(err.Error(), "include cycle detected") { 56 | t.Fatalf("expected an error about import cycles; got: %v", err) 57 | } 58 | } 59 | 60 | func TestBoolEnvs(t *testing.T) { 61 | os.Setenv("TEST_EMPTY", "") 62 | os.Setenv("TEST_TRUE", "true") 63 | os.Setenv("TEST_ONE", "1") 64 | os.Setenv("TEST_ZERO", "0") 65 | os.Setenv("TEST_FALSE", "false") 66 | obj, err := ReadFile("testdata/boolenv.json") 67 | if err != nil { 68 | t.Fatal(err) 69 | } 70 | if str := obj.RequiredString("emptystr"); str != "" { 71 | t.Errorf("str = %q, want empty", str) 72 | } 73 | tests := []struct { 74 | key string 75 | want bool 76 | }{ 77 | {"def_false", false}, 78 | {"def_true", true}, 79 | {"set_true_def_false", true}, 80 | {"set_false_def_true", false}, 81 | {"lit_true", true}, 82 | {"lit_false", false}, 83 | {"one", true}, 84 | {"zero", false}, 85 | } 86 | for _, tt := range tests { 87 | if v := obj.RequiredBool(tt.key); v != tt.want { 88 | t.Errorf("key %q = %v; want %v", tt.key, v, tt.want) 89 | } 90 | } 91 | if err := obj.Validate(); err != nil { 92 | t.Error(err) 93 | } 94 | } 95 | 96 | func TestListExpansion(t *testing.T) { 97 | os.Setenv("TEST_BAR", "bar") 98 | obj, err := ReadFile("testdata/listexpand.json") 99 | if err != nil { 100 | t.Fatal(err) 101 | } 102 | s := obj.RequiredString("str") 103 | l := obj.RequiredList("list") 104 | if err := obj.Validate(); err != nil { 105 | t.Error(err) 106 | } 107 | want := []string{"foo", "bar"} 108 | if !reflect.DeepEqual(l, want) { 109 | t.Errorf("got = %#v\nwant = %#v", l, want) 110 | } 111 | if s != "bar" { 112 | t.Errorf("str = %q, want %q", s, "bar") 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /jsonconfig/testdata/boolenv.json: -------------------------------------------------------------------------------- 1 | { 2 | "emptystr": ["_env", "${TEST_EMPTY}", ""], 3 | "def_false": ["_env", "${TEST_EMPTY}", false], 4 | "def_true": ["_env", "${TEST_EMPTY}", true], 5 | "set_true_def_false": ["_env", "${TEST_TRUE}", false], 6 | "set_false_def_true": ["_env", "${TEST_FALSE}", true], 7 | "one": ["_env", "${TEST_ONE}"], 8 | "zero": ["_env", "${TEST_ZERO}"], 9 | "lit_true": true, 10 | "lit_false": false 11 | } 12 | -------------------------------------------------------------------------------- /jsonconfig/testdata/include1.json: -------------------------------------------------------------------------------- 1 | { 2 | "two": ["_fileobj", "testdata/include2.json"] 3 | } 4 | -------------------------------------------------------------------------------- /jsonconfig/testdata/include1bis.json: -------------------------------------------------------------------------------- 1 | { 2 | "two": ["_fileobj", "include2.json"] 3 | } 4 | -------------------------------------------------------------------------------- /jsonconfig/testdata/include2.json: -------------------------------------------------------------------------------- 1 | { 2 | "key": "value" 3 | } 4 | -------------------------------------------------------------------------------- /jsonconfig/testdata/listexpand.json: -------------------------------------------------------------------------------- 1 | { 2 | "list": ["foo", ["_env", "${TEST_BAR}"]], 3 | "str": ["_env", "${TEST_BAR}"] 4 | } 5 | -------------------------------------------------------------------------------- /jsonconfig/testdata/loop1.json: -------------------------------------------------------------------------------- 1 | { 2 | "obj": ["_fileobj", "testdata/loop2.json"] 3 | } 4 | -------------------------------------------------------------------------------- /jsonconfig/testdata/loop2.json: -------------------------------------------------------------------------------- 1 | { 2 | "obj": ["_fileobj", "testdata/loop1.json"] 3 | } 4 | -------------------------------------------------------------------------------- /legal/legal.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014 The Go4 Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package legal provides in-process storage for compiled-in licenses. 18 | package legal // import "go4.org/legal" 19 | 20 | var licenses []string 21 | 22 | // RegisterLicense stores the license text. 23 | // It doesn't check whether the text was already present. 24 | func RegisterLicense(text string) { 25 | licenses = append(licenses, text) 26 | return 27 | } 28 | 29 | // Licenses returns a slice of the licenses. 30 | func Licenses() []string { 31 | return licenses 32 | } 33 | -------------------------------------------------------------------------------- /legal/legal_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014 The Go4 Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package legal 18 | 19 | import ( 20 | "testing" 21 | ) 22 | 23 | func TestRegisterLicense(t *testing.T) { 24 | initial := len(licenses) 25 | RegisterLicense("dummy") 26 | if initial+1 != len(licenses) { 27 | t.Fatal("didn't add a license") 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /lock/.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | -------------------------------------------------------------------------------- /lock/lock.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2013 The Go Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package lock is a file locking library. 18 | package lock // import "go4.org/lock" 19 | 20 | import ( 21 | "encoding/json" 22 | "fmt" 23 | "io" 24 | "os" 25 | "path/filepath" 26 | "sync" 27 | ) 28 | 29 | // Lock locks the given file, creating the file if necessary. If the 30 | // file already exists, it must have zero size or an error is returned. 31 | // The lock is an exclusive lock (a write lock), but locked files 32 | // should neither be read from nor written to. Such files should have 33 | // zero size and only exist to co-ordinate ownership across processes. 34 | // 35 | // A nil Closer is returned if an error occurred. Otherwise, close that 36 | // Closer to release the lock. 37 | // 38 | // On Linux, FreeBSD and OSX, a lock has the same semantics as fcntl(2)'s 39 | // advisory locks. In particular, closing any other file descriptor for the 40 | // same file will release the lock prematurely. 41 | // 42 | // Attempting to lock a file that is already locked by the current process 43 | // has undefined behavior. 44 | // 45 | // On other operating systems, lock will fallback to using the presence and 46 | // content of a file named name + '.lock' to implement locking behavior. 47 | func Lock(name string) (io.Closer, error) { 48 | abs, err := filepath.Abs(name) 49 | if err != nil { 50 | return nil, err 51 | } 52 | lockmu.Lock() 53 | defer lockmu.Unlock() 54 | if locked[abs] { 55 | return nil, fmt.Errorf("file %q already locked", abs) 56 | } 57 | 58 | c, err := lockFn(abs) 59 | if err != nil { 60 | return nil, fmt.Errorf("cannot acquire lock: %v", err) 61 | } 62 | locked[abs] = true 63 | return c, nil 64 | } 65 | 66 | var lockFn = lockPortable 67 | 68 | // lockPortable is a portable version not using fcntl. Doesn't handle crashes as gracefully, 69 | // since it can leave stale lock files. 70 | func lockPortable(name string) (io.Closer, error) { 71 | fi, err := os.Stat(name) 72 | if err == nil && fi.Size() > 0 { 73 | st := portableLockStatus(name) 74 | switch st { 75 | case statusLocked: 76 | return nil, fmt.Errorf("file %q already locked", name) 77 | case statusStale: 78 | os.Remove(name) 79 | case statusInvalid: 80 | return nil, fmt.Errorf("can't Lock file %q: has invalid contents", name) 81 | } 82 | } 83 | f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_EXCL, 0666) 84 | if err != nil { 85 | return nil, fmt.Errorf("failed to create lock file %s %v", name, err) 86 | } 87 | if err := json.NewEncoder(f).Encode(&pidLockMeta{OwnerPID: os.Getpid()}); err != nil { 88 | return nil, fmt.Errorf("cannot write owner pid: %v", err) 89 | } 90 | return &unlocker{ 91 | f: f, 92 | abs: name, 93 | portable: true, 94 | }, nil 95 | } 96 | 97 | type lockStatus int 98 | 99 | const ( 100 | statusInvalid lockStatus = iota 101 | statusLocked 102 | statusUnlocked 103 | statusStale 104 | ) 105 | 106 | type pidLockMeta struct { 107 | OwnerPID int 108 | } 109 | 110 | func portableLockStatus(path string) lockStatus { 111 | f, err := os.Open(path) 112 | if err != nil { 113 | return statusUnlocked 114 | } 115 | defer f.Close() 116 | var meta pidLockMeta 117 | if json.NewDecoder(f).Decode(&meta) != nil { 118 | return statusInvalid 119 | } 120 | if meta.OwnerPID == 0 { 121 | return statusInvalid 122 | } 123 | p, err := os.FindProcess(meta.OwnerPID) 124 | if err != nil { 125 | // e.g. on Windows 126 | return statusStale 127 | } 128 | // On unix, os.FindProcess always is true, so we have to send 129 | // it a signal to see if it's alive. 130 | if signalZero != nil { 131 | if p.Signal(signalZero) != nil { 132 | return statusStale 133 | } 134 | } 135 | return statusLocked 136 | } 137 | 138 | var signalZero os.Signal // nil or set by lock_sigzero.go 139 | 140 | var ( 141 | lockmu sync.Mutex 142 | locked = map[string]bool{} // abs path -> true 143 | ) 144 | 145 | type unlocker struct { 146 | portable bool 147 | f *os.File 148 | abs string 149 | // once guards the close method call. 150 | once sync.Once 151 | // err holds the error returned by Close. 152 | err error 153 | } 154 | 155 | func (u *unlocker) Close() error { 156 | u.once.Do(u.close) 157 | return u.err 158 | } 159 | 160 | func (u *unlocker) close() { 161 | lockmu.Lock() 162 | defer lockmu.Unlock() 163 | delete(locked, u.abs) 164 | 165 | if u.portable { 166 | // In the portable lock implementation, it's 167 | // important to close before removing because 168 | // Windows won't allow us to remove an open 169 | // file. 170 | if err := u.f.Close(); err != nil { 171 | u.err = err 172 | } 173 | if err := os.Remove(u.abs); err != nil { 174 | // Note that if both Close and Remove fail, 175 | // we care more about the latter than the former 176 | // so we'll return that error. 177 | u.err = err 178 | } 179 | return 180 | } 181 | // In other implementatioons, it's nice for us to clean up. 182 | // If we do do this, though, it needs to be before the 183 | // u.f.Close below. 184 | os.Remove(u.abs) 185 | u.err = u.f.Close() 186 | } 187 | -------------------------------------------------------------------------------- /lock/lock_appengine.go: -------------------------------------------------------------------------------- 1 | // +build appengine 2 | 3 | /* 4 | Copyright 2013 The Go Authors 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | package lock 20 | 21 | import ( 22 | "errors" 23 | "io" 24 | ) 25 | 26 | func init() { 27 | lockFn = lockAppEngine 28 | } 29 | 30 | func lockAppEngine(name string) (io.Closer, error) { 31 | return nil, errors.New("Lock not available on App Engine") 32 | } 33 | -------------------------------------------------------------------------------- /lock/lock_plan9.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2013 The Go Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package lock 18 | 19 | import ( 20 | "fmt" 21 | "io" 22 | "os" 23 | ) 24 | 25 | func init() { 26 | lockFn = lockPlan9 27 | } 28 | 29 | func lockPlan9(name string) (io.Closer, error) { 30 | fi, err := os.Stat(name) 31 | if err == nil && fi.Size() > 0 { 32 | return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name) 33 | } 34 | 35 | f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE, os.ModeExclusive|0644) 36 | if err != nil { 37 | return nil, fmt.Errorf("Lock Create of %s failed: %v", name, err) 38 | } 39 | 40 | return &unlocker{f: f, abs: name}, nil 41 | } 42 | -------------------------------------------------------------------------------- /lock/lock_sigzero.go: -------------------------------------------------------------------------------- 1 | // +build !appengine 2 | // +build linux darwin freebsd openbsd netbsd dragonfly 3 | 4 | /* 5 | Copyright 2013 The Go Authors 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | */ 19 | 20 | package lock 21 | 22 | import "syscall" 23 | 24 | func init() { 25 | signalZero = syscall.Signal(0) 26 | } 27 | -------------------------------------------------------------------------------- /lock/lock_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2013 The Go Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package lock 18 | 19 | import ( 20 | "bufio" 21 | "errors" 22 | "fmt" 23 | "io" 24 | "io/ioutil" 25 | "os" 26 | "os/exec" 27 | "path/filepath" 28 | "strconv" 29 | "testing" 30 | ) 31 | 32 | func TestLock(t *testing.T) { 33 | testLock(t, false) 34 | } 35 | 36 | func TestLockPortable(t *testing.T) { 37 | testLock(t, true) 38 | } 39 | 40 | func TestLockInChild(t *testing.T) { 41 | f := os.Getenv("TEST_LOCK_FILE") 42 | if f == "" { 43 | // not child 44 | return 45 | } 46 | lock := Lock 47 | if v, _ := strconv.ParseBool(os.Getenv("TEST_LOCK_PORTABLE")); v { 48 | lock = lockPortable 49 | } 50 | 51 | var lk io.Closer 52 | for scan := bufio.NewScanner(os.Stdin); scan.Scan(); { 53 | var err error 54 | switch scan.Text() { 55 | case "lock": 56 | lk, err = lock(f) 57 | case "unlock": 58 | err = lk.Close() 59 | lk = nil 60 | case "exit": 61 | // Simulate a crash, or at least not unlocking the lock. 62 | os.Exit(0) 63 | default: 64 | err = fmt.Errorf("unexpected child command %q", scan.Text()) 65 | } 66 | if err != nil { 67 | fmt.Println(err) 68 | } else { 69 | fmt.Println("") 70 | } 71 | } 72 | } 73 | 74 | func testLock(t *testing.T, portable bool) { 75 | lock := Lock 76 | if portable { 77 | lock = lockPortable 78 | } 79 | t.Logf("test lock, portable %v", portable) 80 | 81 | td, err := ioutil.TempDir("", "") 82 | if err != nil { 83 | t.Fatal(err) 84 | } 85 | defer os.RemoveAll(td) 86 | 87 | path := filepath.Join(td, "foo.lock") 88 | 89 | proc := newChildProc(t, path, portable) 90 | defer proc.kill() 91 | 92 | t.Logf("First lock in child") 93 | if err := proc.do("lock"); err != nil { 94 | t.Fatalf("first lock in child process: %v", err) 95 | } 96 | 97 | t.Logf("Crash child") 98 | if err := proc.do("exit"); err != nil { 99 | t.Fatalf("crash in child process: %v", err) 100 | } 101 | 102 | proc = newChildProc(t, path, portable) 103 | defer proc.kill() 104 | 105 | t.Logf("Locking+unlocking in child...") 106 | if err := proc.do("lock"); err != nil { 107 | t.Fatalf("lock in child process after crashing child: %v", err) 108 | } 109 | if err := proc.do("unlock"); err != nil { 110 | t.Fatalf("lock in child process after crashing child: %v", err) 111 | } 112 | 113 | t.Logf("Locking in parent...") 114 | lk1, err := lock(path) 115 | if err != nil { 116 | t.Fatal(err) 117 | } 118 | 119 | t.Logf("Again in parent...") 120 | _, err = lock(path) 121 | if err == nil { 122 | t.Fatal("expected second lock to fail") 123 | } 124 | 125 | t.Logf("Locking in child...") 126 | if err := proc.do("lock"); err == nil { 127 | t.Fatalf("expected lock in child process to fail") 128 | } 129 | 130 | t.Logf("Unlocking lock in parent") 131 | if err := lk1.Close(); err != nil { 132 | t.Fatal(err) 133 | } 134 | 135 | t.Logf("Trying lock again in child...") 136 | if err := proc.do("lock"); err != nil { 137 | t.Fatal(err) 138 | } 139 | if err := proc.do("unlock"); err != nil { 140 | t.Fatal(err) 141 | } 142 | 143 | lk3, err := lock(path) 144 | if err != nil { 145 | t.Fatal(err) 146 | } 147 | lk3.Close() 148 | } 149 | 150 | type childLockCmd struct { 151 | op string 152 | reply chan<- error 153 | } 154 | 155 | type childProc struct { 156 | proc *os.Process 157 | c chan childLockCmd 158 | } 159 | 160 | func (c *childProc) kill() { 161 | c.proc.Kill() 162 | } 163 | 164 | func (c *childProc) do(op string) error { 165 | reply := make(chan error) 166 | c.c <- childLockCmd{ 167 | op: op, 168 | reply: reply, 169 | } 170 | return <-reply 171 | } 172 | 173 | func newChildProc(t *testing.T, path string, portable bool) *childProc { 174 | cmd := exec.Command(os.Args[0], "-test.run=LockInChild$") 175 | cmd.Env = []string{"TEST_LOCK_FILE=" + path} 176 | toChild, err := cmd.StdinPipe() 177 | if err != nil { 178 | t.Fatalf("cannot make pipe: %v", err) 179 | } 180 | fromChild, err := cmd.StdoutPipe() 181 | if err != nil { 182 | t.Fatalf("cannot make pipe: %v", err) 183 | } 184 | cmd.Stderr = os.Stderr 185 | if portable { 186 | cmd.Env = append(cmd.Env, "TEST_LOCK_PORTABLE=1") 187 | } 188 | if err := cmd.Start(); err != nil { 189 | t.Fatalf("cannot start child: %v", err) 190 | } 191 | cmdChan := make(chan childLockCmd) 192 | go func() { 193 | defer fromChild.Close() 194 | defer toChild.Close() 195 | inScan := bufio.NewScanner(fromChild) 196 | for c := range cmdChan { 197 | fmt.Fprintln(toChild, c.op) 198 | ok := inScan.Scan() 199 | if c.op == "exit" { 200 | if ok { 201 | c.reply <- errors.New("child did not exit") 202 | } else { 203 | cmd.Wait() 204 | c.reply <- nil 205 | } 206 | break 207 | } 208 | if !ok { 209 | panic("child exited early") 210 | } 211 | if errText := inScan.Text(); errText != "" { 212 | c.reply <- errors.New(errText) 213 | } else { 214 | c.reply <- nil 215 | } 216 | } 217 | }() 218 | return &childProc{ 219 | c: cmdChan, 220 | proc: cmd.Process, 221 | } 222 | } 223 | -------------------------------------------------------------------------------- /lock/lock_unix.go: -------------------------------------------------------------------------------- 1 | // +build linux darwin freebsd openbsd netbsd dragonfly solaris 2 | // +build !appengine 3 | 4 | /* 5 | Copyright 2013 The Go Authors 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | */ 19 | 20 | package lock 21 | 22 | import ( 23 | "fmt" 24 | "io" 25 | "os" 26 | 27 | "golang.org/x/sys/unix" 28 | ) 29 | 30 | func init() { 31 | lockFn = lockFcntl 32 | } 33 | 34 | func lockFcntl(name string) (io.Closer, error) { 35 | fi, err := os.Stat(name) 36 | if err == nil && fi.Size() > 0 { 37 | return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name) 38 | } 39 | 40 | f, err := os.Create(name) 41 | if err != nil { 42 | return nil, fmt.Errorf("Lock Create of %s failed: %v", name, err) 43 | } 44 | 45 | err = unix.FcntlFlock(f.Fd(), unix.F_SETLK, &unix.Flock_t{ 46 | Type: unix.F_WRLCK, 47 | Whence: int16(os.SEEK_SET), 48 | Start: 0, 49 | Len: 0, // 0 means to lock the entire file. 50 | Pid: 0, // only used by F_GETLK 51 | }) 52 | 53 | if err != nil { 54 | f.Close() 55 | return nil, fmt.Errorf("Lock FcntlFlock of %s failed: %v", name, err) 56 | } 57 | return &unlocker{f: f, abs: name}, nil 58 | } 59 | -------------------------------------------------------------------------------- /lock/lock_windows.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2013 The Go Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package lock 18 | 19 | import ( 20 | "fmt" 21 | "io" 22 | "os" 23 | "sync" 24 | 25 | "golang.org/x/sys/windows" 26 | ) 27 | 28 | func init() { 29 | lockFn = lockWindows 30 | } 31 | 32 | type winUnlocker struct { 33 | h windows.Handle 34 | abs string 35 | // err holds the error returned by Close. 36 | err error 37 | // once guards the close method call. 38 | once sync.Once 39 | } 40 | 41 | func (u *winUnlocker) Close() error { 42 | u.once.Do(u.close) 43 | return u.err 44 | } 45 | 46 | func (u *winUnlocker) close() { 47 | lockmu.Lock() 48 | defer lockmu.Unlock() 49 | delete(locked, u.abs) 50 | 51 | u.err = windows.CloseHandle(u.h) 52 | } 53 | 54 | func lockWindows(name string) (io.Closer, error) { 55 | fi, err := os.Stat(name) 56 | if err == nil && fi.Size() > 0 { 57 | return nil, fmt.Errorf("can't lock file %q: %s", name, "has non-zero size") 58 | } 59 | 60 | handle, err := winCreateEphemeral(name) 61 | if err != nil { 62 | return nil, fmt.Errorf("creation of lock %s failed: %v", name, err) 63 | } 64 | 65 | return &winUnlocker{h: handle, abs: name}, nil 66 | } 67 | 68 | func winCreateEphemeral(name string) (windows.Handle, error) { 69 | const ( 70 | FILE_ATTRIBUTE_TEMPORARY = 0x100 71 | FILE_FLAG_DELETE_ON_CLOSE = 0x04000000 72 | ) 73 | handle, err := windows.CreateFile(windows.StringToUTF16Ptr(name), 0, 0, nil, windows.OPEN_ALWAYS, FILE_ATTRIBUTE_TEMPORARY|FILE_FLAG_DELETE_ON_CLOSE, 0) 74 | if err != nil { 75 | return 0, err 76 | } 77 | return handle, nil 78 | } 79 | -------------------------------------------------------------------------------- /media/heif/dumpheif/dumpheif.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The go4 Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // The dumpheif program dumps the structure and metadata of a HEIF file. 18 | // 19 | // It exists purely for debugging the go4.org/media/heif and 20 | // go4.org/media/heif/bmff packages; it makes no backwards 21 | // compatibility promises. 22 | package main 23 | 24 | import ( 25 | "bytes" 26 | "flag" 27 | "fmt" 28 | "io" 29 | "io/ioutil" 30 | "log" 31 | "os" 32 | "strings" 33 | 34 | "github.com/rwcarlsen/goexif/exif" 35 | "github.com/rwcarlsen/goexif/tiff" 36 | 37 | "go4.org/media/heif" 38 | "go4.org/media/heif/bmff" 39 | ) 40 | 41 | var ( 42 | exifItemID uint16 43 | exifLoc bmff.ItemLocationBoxEntry 44 | ) 45 | 46 | func main() { 47 | flag.Parse() 48 | if flag.NArg() != 1 { 49 | fmt.Fprintf(os.Stderr, "usage: dumpheif \n") 50 | os.Exit(1) 51 | } 52 | f, err := os.Open(flag.Arg(0)) 53 | if err != nil { 54 | log.Fatal(err) 55 | } 56 | defer f.Close() 57 | 58 | hf := heif.Open(f) 59 | 60 | it, err := hf.PrimaryItem() 61 | if err != nil { 62 | log.Fatalf("PrimaryItem: %v", err) 63 | } 64 | fmt.Printf("primary item: %v\n", it.ID) 65 | 66 | width, height, ok := it.SpatialExtents() 67 | if ok { 68 | fmt.Printf("spatial extents: %d x %d\n", width, height) 69 | } 70 | fmt.Printf("properties:\n") 71 | for _, prop := range it.Properties { 72 | fmt.Printf("\t%q: %#v\n", prop.Type(), prop) 73 | } 74 | if len(it.Properties) == 0 { 75 | fmt.Printf("\t(no properties)\n") 76 | } 77 | 78 | if ex, err := hf.EXIF(); err == nil { 79 | fmt.Printf("EXIF dump:\n") 80 | ex, err := exif.Decode(bytes.NewReader(ex)) 81 | if err != nil { 82 | log.Fatalf("EXIF decode: %v", err) 83 | } 84 | ex.Walk(exifWalkFunc(func(name exif.FieldName, tag *tiff.Tag) error { 85 | fmt.Printf("\t%v = %v\n", name, tag) 86 | return nil 87 | })) 88 | fmt.Printf("\n") 89 | } 90 | 91 | fmt.Printf("BMFF boxes:\n") 92 | r := bmff.NewReader(f) 93 | for { 94 | box, err := r.ReadBox() 95 | if err == io.EOF { 96 | break 97 | } 98 | if err != nil { 99 | log.Fatalf("ReadBox: %v", err) 100 | } 101 | dumpBox(box, 0) 102 | } 103 | 104 | } 105 | 106 | type exifWalkFunc func(exif.FieldName, *tiff.Tag) error 107 | 108 | func (f exifWalkFunc) Walk(name exif.FieldName, tag *tiff.Tag) error { 109 | return f(name, tag) 110 | } 111 | 112 | func dumpBox(box bmff.Box, depth int) { 113 | indent := strings.Repeat(" ", depth) 114 | fmt.Printf("%sBox: type %q, size %v\n", indent, box.Type(), box.Size()) 115 | 116 | box2, err := box.Parse() 117 | if err == bmff.ErrUnknownBox { 118 | slurp, err := ioutil.ReadAll(box.Body()) 119 | if err != nil { 120 | log.Fatalf("%sreading body: %v", indent, err) 121 | } 122 | if len(slurp) < 5000 { 123 | fmt.Printf("%s- contents: %q\n", indent, slurp) 124 | } else { 125 | fmt.Printf("%s- contents: (... %d bytes, starting with %q ...)\n", indent, len(slurp), slurp[:100]) 126 | } 127 | return 128 | } 129 | if err != nil { 130 | slurp, _ := ioutil.ReadAll(box.Body()) 131 | log.Fatalf("Parse box type %q: %v; slurp: %q", box.Type(), err, slurp) 132 | } 133 | 134 | switch v := box2.(type) { 135 | case *bmff.FileTypeBox, *bmff.HandlerBox, *bmff.PrimaryItemBox: 136 | fmt.Printf("%s- %T: %+v\n", indent, v, v) 137 | case *bmff.MetaBox: 138 | fmt.Printf("%s- %T, %d children:\n", indent, v, len(v.Children)) 139 | for _, child := range v.Children { 140 | dumpBox(child, depth+1) 141 | } 142 | case *bmff.ItemInfoBox: 143 | //slurp, _ := ioutil.ReadAll(box.Body()) 144 | //fmt.Printf("%s- %T raw: %q\n", indent, v, slurp) 145 | fmt.Printf("%s- %T, %d children (%d in slice):\n", indent, v, v.Count, len(v.ItemInfos)) 146 | for _, child := range v.ItemInfos { 147 | dumpBox(child, depth+1) 148 | } 149 | case *bmff.ItemInfoEntry: 150 | fmt.Printf("%s- %T, %+v\n", indent, v, v) 151 | if v.ItemType == "Exif" { 152 | exifItemID = v.ItemID 153 | } 154 | case *bmff.ItemPropertiesBox: 155 | fmt.Printf("%s- %T\n", indent, v) 156 | if v.PropertyContainer != nil { 157 | dumpBox(v.PropertyContainer, depth+1) 158 | } 159 | for _, child := range v.Associations { 160 | dumpBox(child, depth+1) 161 | } 162 | case *bmff.ItemPropertyAssociation: 163 | fmt.Printf("%s- %T: %d declared entries, %d parsed:\n", indent, v, v.EntryCount, len(v.Entries)) 164 | for _, ai := range v.Entries { 165 | fmt.Printf("%s for Item ID %d, %d associations declared, %d parsed:\n", indent, ai.ItemID, ai.AssociationsCount, len(ai.Associations)) 166 | for _, ass := range ai.Associations { 167 | fmt.Printf("%s index: %d, essential: %v\n", indent, ass.Index, ass.Essential) 168 | } 169 | } 170 | case *bmff.DataInformationBox: 171 | fmt.Printf("%s- %T\n", indent, v) 172 | for _, child := range v.Children { 173 | dumpBox(child, depth+1) 174 | } 175 | case *bmff.DataReferenceBox: 176 | fmt.Printf("%s- %T\n", indent, v) 177 | for _, child := range v.Children { 178 | dumpBox(child, depth+1) 179 | } 180 | case *bmff.ItemPropertyContainerBox: 181 | fmt.Printf("%s- %T\n", indent, v) 182 | for _, child := range v.Properties { 183 | dumpBox(child, depth+1) 184 | } 185 | case *bmff.ItemLocationBox: 186 | fmt.Printf("%s- %T: %d items declared, %d parsed:\n", indent, v, v.ItemCount, len(v.Items)) 187 | for _, lbe := range v.Items { 188 | fmt.Printf("%s %+v\n", indent, lbe) 189 | if exifItemID != 0 && lbe.ItemID == exifItemID { 190 | exifLoc = lbe 191 | } 192 | } 193 | 194 | case *bmff.ImageSpatialExtentsProperty: 195 | fmt.Printf("%s- %T dimensions: %d x %d\n", indent, v, v.ImageWidth, v.ImageHeight) 196 | default: 197 | fmt.Printf("%s- gotype: %T\n", indent, box2) 198 | } 199 | 200 | } 201 | -------------------------------------------------------------------------------- /media/heif/heif_test.go: -------------------------------------------------------------------------------- 1 | package heif 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "os" 7 | "testing" 8 | 9 | "github.com/rwcarlsen/goexif/exif" 10 | "github.com/rwcarlsen/goexif/tiff" 11 | ) 12 | 13 | func TestAll(t *testing.T) { 14 | f, err := os.Open("testdata/park.heic") 15 | if err != nil { 16 | t.Fatal(err) 17 | } 18 | defer f.Close() 19 | h := Open(f) 20 | 21 | // meta 22 | _, err = h.getMeta() 23 | if err != nil { 24 | t.Fatalf("getMeta: %v", err) 25 | } 26 | 27 | it, err := h.PrimaryItem() 28 | if err != nil { 29 | t.Fatalf("PrimaryItem: %v", err) 30 | } 31 | if want := uint32(49); it.ID != want { 32 | t.Errorf("PrimaryIem ID = %v; want %v", it.ID, want) 33 | } 34 | if it.Location == nil { 35 | t.Errorf("Item.Location is nil") 36 | } 37 | if it.Info == nil { 38 | t.Errorf("Item.Info is nil") 39 | } 40 | if len(it.Properties) == 0 { 41 | t.Errorf("Item.Properties is empty") 42 | } 43 | for _, prop := range it.Properties { 44 | t.Logf(" property: %q, %#v", prop.Type(), prop) 45 | } 46 | if w, h, ok := it.SpatialExtents(); !ok || w == 0 || h == 0 { 47 | t.Errorf("no spatial extents found") 48 | } else { 49 | t.Logf("dimensions: %v x %v", w, h) 50 | } 51 | 52 | // exif 53 | exbuf, err := h.EXIF() 54 | if err != nil { 55 | t.Errorf("EXIF: %v", err) 56 | } else { 57 | const magic = "Exif\x00\x00" 58 | if !bytes.HasPrefix(exbuf, []byte(magic)) { 59 | t.Errorf("Exif buffer doesn't start with %q: got %q", magic, exbuf) 60 | } 61 | x, err := exif.Decode(bytes.NewReader(exbuf)) 62 | if err != nil { 63 | t.Fatalf("EXIF decode: %v", err) 64 | } 65 | got := map[string]string{} 66 | if err := x.Walk(walkFunc(func(name exif.FieldName, tag *tiff.Tag) error { 67 | got[fmt.Sprint(name)] = fmt.Sprint(tag) 68 | return nil 69 | })); err != nil { 70 | t.Fatalf("EXIF walk: %v", err) 71 | } 72 | if g, w := len(got), 56; g < w { 73 | t.Errorf("saw %v EXIF tags; want at least %v", g, w) 74 | } 75 | if g, w := got["GPSLongitude"], `["122/1","21/1","3776/100"]`; g != w { 76 | t.Errorf("GPSLongitude = %#q; want %#q", g, w) 77 | } 78 | 79 | } 80 | } 81 | 82 | func TestRotations(t *testing.T) { 83 | f, err := os.Open("testdata/rotate.heic") 84 | if err != nil { 85 | t.Fatal(err) 86 | } 87 | defer f.Close() 88 | h := Open(f) 89 | it, err := h.PrimaryItem() 90 | if err != nil { 91 | t.Fatalf("PrimaryItem: %v", err) 92 | } 93 | if r := it.Rotations(); r != 3 { 94 | t.Errorf("Rotations = %v; want %v", r, 3) 95 | } 96 | sw, sh, ok := it.SpatialExtents() 97 | if !ok { 98 | t.Fatalf("expected spatial extents") 99 | } 100 | vw, vh, ok := it.VisualDimensions() 101 | if !ok { 102 | t.Fatalf("expected visual dimensions") 103 | } 104 | if vw != sh || vh != sw { 105 | t.Errorf("visual dimensions = %v, %v; want %v, %v", vw, vh, sh, sw) 106 | } 107 | } 108 | 109 | type walkFunc func(exif.FieldName, *tiff.Tag) error 110 | 111 | func (f walkFunc) Walk(name exif.FieldName, tag *tiff.Tag) error { 112 | return f(name, tag) 113 | } 114 | -------------------------------------------------------------------------------- /media/heif/testdata/park.heic: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go4org/go4/214862532bf518db360e7ecc2b8a94b66a10c176/media/heif/testdata/park.heic -------------------------------------------------------------------------------- /media/heif/testdata/rotate.heic: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go4org/go4/214862532bf518db360e7ecc2b8a94b66a10c176/media/heif/testdata/rotate.heic -------------------------------------------------------------------------------- /must/must.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 The Perkeep Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package must contains helpers that panic on failure. 18 | package must // import "go4.org/must" 19 | 20 | import "io" 21 | 22 | // Close calls c.Close and panics if it returns an error. 23 | // The panic value is the return value from Close. 24 | func Close(c io.Closer) { 25 | if err := c.Close(); err != nil { 26 | panic(err) 27 | } 28 | } 29 | 30 | // Do runs fn and panics if it returns an error. 31 | // The panic value is the return value from fn. 32 | func Do(fn func() error) { 33 | if err := fn(); err != nil { 34 | panic(err) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /net/throttle/throttle.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2012 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package throttle provides a net.Listener that returns 18 | // artificially-delayed connections for testing real-world 19 | // connectivity. 20 | package throttle // import "go4.org/net/throttle" 21 | 22 | import ( 23 | "fmt" 24 | "net" 25 | "sync" 26 | "time" 27 | ) 28 | 29 | const unitSize = 1400 // read/write chunk size. ~MTU size. 30 | 31 | type Rate struct { 32 | KBps int // or 0, to not rate-limit bandwidth 33 | Latency time.Duration 34 | } 35 | 36 | // byteTime returns the time required for n bytes. 37 | func (r Rate) byteTime(n int) time.Duration { 38 | if r.KBps == 0 { 39 | return 0 40 | } 41 | return time.Duration(float64(n)/1024/float64(r.KBps)) * time.Second 42 | } 43 | 44 | type Listener struct { 45 | net.Listener 46 | Down Rate // server Writes to Client 47 | Up Rate // server Reads from client 48 | } 49 | 50 | func (ln *Listener) Accept() (net.Conn, error) { 51 | c, err := ln.Listener.Accept() 52 | time.Sleep(ln.Up.Latency) 53 | if err != nil { 54 | return nil, err 55 | } 56 | tc := &conn{Conn: c, Down: ln.Down, Up: ln.Up} 57 | tc.start() 58 | return tc, nil 59 | } 60 | 61 | type nErr struct { 62 | n int 63 | err error 64 | } 65 | 66 | type writeReq struct { 67 | writeAt time.Time 68 | p []byte 69 | resc chan nErr 70 | } 71 | 72 | type conn struct { 73 | net.Conn 74 | Down Rate // for reads 75 | Up Rate // for writes 76 | 77 | wchan chan writeReq 78 | closeOnce sync.Once 79 | closeErr error 80 | } 81 | 82 | func (c *conn) start() { 83 | c.wchan = make(chan writeReq, 1024) 84 | go c.writeLoop() 85 | } 86 | 87 | func (c *conn) writeLoop() { 88 | for req := range c.wchan { 89 | time.Sleep(req.writeAt.Sub(time.Now())) 90 | var res nErr 91 | for len(req.p) > 0 && res.err == nil { 92 | writep := req.p 93 | if len(writep) > unitSize { 94 | writep = writep[:unitSize] 95 | } 96 | n, err := c.Conn.Write(writep) 97 | time.Sleep(c.Up.byteTime(len(writep))) 98 | res.n += n 99 | res.err = err 100 | req.p = req.p[n:] 101 | } 102 | req.resc <- res 103 | } 104 | } 105 | 106 | func (c *conn) Close() error { 107 | c.closeOnce.Do(func() { 108 | err := c.Conn.Close() 109 | close(c.wchan) 110 | c.closeErr = err 111 | }) 112 | return c.closeErr 113 | } 114 | 115 | func (c *conn) Write(p []byte) (n int, err error) { 116 | defer func() { 117 | if e := recover(); e != nil { 118 | n = 0 119 | err = fmt.Errorf("%v", err) 120 | return 121 | } 122 | }() 123 | resc := make(chan nErr, 1) 124 | c.wchan <- writeReq{time.Now().Add(c.Up.Latency), p, resc} 125 | res := <-resc 126 | return res.n, res.err 127 | } 128 | 129 | func (c *conn) Read(p []byte) (n int, err error) { 130 | const max = 1024 131 | if len(p) > max { 132 | p = p[:max] 133 | } 134 | n, err = c.Conn.Read(p) 135 | time.Sleep(c.Down.byteTime(n)) 136 | return 137 | } 138 | -------------------------------------------------------------------------------- /oauthutil/oauth.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 The Perkeep Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package oauthutil contains OAuth 2 related utilities. 18 | package oauthutil // import "go4.org/oauthutil" 19 | 20 | import ( 21 | "encoding/json" 22 | "errors" 23 | "fmt" 24 | "time" 25 | 26 | "go4.org/wkfs" 27 | "golang.org/x/oauth2" 28 | ) 29 | 30 | // TitleBarRedirectURL is the OAuth2 redirect URL to use when the authorization 31 | // code should be returned in the title bar of the browser, with the page text 32 | // prompting the user to copy the code and paste it in the application. 33 | const TitleBarRedirectURL = "urn:ietf:wg:oauth:2.0:oob" 34 | 35 | // ErrNoAuthCode is returned when Token() has not found any valid cached token 36 | // and TokenSource does not have an AuthCode for getting a new token. 37 | var ErrNoAuthCode = errors.New("oauthutil: unspecified TokenSource.AuthCode") 38 | 39 | // TokenSource is an implementation of oauth2.TokenSource. It uses CacheFile to store and 40 | // reuse the the acquired token, and AuthCode to provide the authorization code that will be 41 | // exchanged for a token otherwise. 42 | type TokenSource struct { 43 | Config *oauth2.Config 44 | 45 | // CacheFile is where the token will be stored JSON-encoded. Any call to Token 46 | // first tries to read a valid token from CacheFile. 47 | CacheFile string 48 | 49 | // AuthCode provides the authorization code that Token will exchange for a token. 50 | // It usually is a way to prompt the user for the code. If CacheFile does not provide 51 | // a token and AuthCode is nil, Token returns ErrNoAuthCode. 52 | AuthCode func() string 53 | } 54 | 55 | var errExpiredToken = errors.New("expired token") 56 | 57 | // cachedToken returns the token saved in cacheFile. It specifically returns 58 | // errTokenExpired if the token is expired. 59 | func cachedToken(cacheFile string) (*oauth2.Token, error) { 60 | tok := new(oauth2.Token) 61 | tokenData, err := wkfs.ReadFile(cacheFile) 62 | if err != nil { 63 | return nil, err 64 | } 65 | if err = json.Unmarshal(tokenData, tok); err != nil { 66 | return nil, err 67 | } 68 | if !tok.Valid() { 69 | if tok != nil && time.Now().After(tok.Expiry) { 70 | return nil, errExpiredToken 71 | } 72 | return nil, errors.New("invalid token") 73 | } 74 | return tok, nil 75 | } 76 | 77 | // Token first tries to find a valid token in CacheFile, and otherwise uses 78 | // Config and AuthCode to fetch a new token. This new token is saved in CacheFile 79 | // (if not blank). If CacheFile did not provide a token and AuthCode is nil, 80 | // ErrNoAuthCode is returned. 81 | func (src TokenSource) Token() (*oauth2.Token, error) { 82 | var tok *oauth2.Token 83 | var err error 84 | if src.CacheFile != "" { 85 | tok, err = cachedToken(src.CacheFile) 86 | if err == nil { 87 | return tok, nil 88 | } 89 | if err != errExpiredToken { 90 | fmt.Printf("Error getting token from %s: %v\n", src.CacheFile, err) 91 | } 92 | } 93 | if src.AuthCode == nil { 94 | return nil, ErrNoAuthCode 95 | } 96 | tok, err = src.Config.Exchange(oauth2.NoContext, src.AuthCode()) 97 | if err != nil { 98 | return nil, fmt.Errorf("could not exchange auth code for a token: %v", err) 99 | } 100 | if src.CacheFile == "" { 101 | return tok, nil 102 | } 103 | tokenData, err := json.Marshal(&tok) 104 | if err != nil { 105 | return nil, fmt.Errorf("could not encode token as json: %v", err) 106 | } 107 | if err := wkfs.WriteFile(src.CacheFile, tokenData, 0600); err != nil { 108 | return nil, fmt.Errorf("could not cache token in %v: %v", src.CacheFile, err) 109 | } 110 | return tok, nil 111 | } 112 | 113 | // NewRefreshTokenSource returns a token source that obtains its initial token 114 | // based on the provided config and the refresh token. 115 | func NewRefreshTokenSource(config *oauth2.Config, refreshToken string) oauth2.TokenSource { 116 | var noInitialToken *oauth2.Token = nil 117 | return oauth2.ReuseTokenSource(noInitialToken, config.TokenSource( 118 | oauth2.NoContext, // TODO: maybe accept a context later. 119 | &oauth2.Token{RefreshToken: refreshToken}, 120 | )) 121 | } 122 | -------------------------------------------------------------------------------- /osutil/exec_plan9.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The go4 Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // +build plan9 16 | 17 | package osutil 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | "path/filepath" 23 | "syscall" 24 | ) 25 | 26 | func executable() (string, error) { 27 | fn := fmt.Sprintf("/proc/%d/text", os.Getpid()) 28 | f, err := os.Open(fn) 29 | if err != nil { 30 | return "", err 31 | } 32 | defer f.Close() 33 | p, err := syscall.Fd2path(int(f.Fd())) 34 | return filepath.Clean(p), err 35 | } 36 | -------------------------------------------------------------------------------- /osutil/exec_procfs.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The go4 Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // +build linux netbsd openbsd dragonfly nacl 16 | 17 | package osutil 18 | 19 | import ( 20 | "errors" 21 | "os" 22 | "path/filepath" 23 | "runtime" 24 | ) 25 | 26 | func executable() (string, error) { 27 | var procfn string 28 | switch runtime.GOOS { 29 | default: 30 | return "", errors.New("Executable not implemented for " + runtime.GOOS) 31 | case "linux": 32 | procfn = "/proc/self/exe" 33 | case "netbsd": 34 | procfn = "/proc/curproc/exe" 35 | case "openbsd": 36 | procfn = "/proc/curproc/file" 37 | case "dragonfly": 38 | procfn = "/proc/curproc/file" 39 | } 40 | p, err := os.Readlink(procfn) 41 | return filepath.Clean(p), err 42 | } 43 | -------------------------------------------------------------------------------- /osutil/exec_solaris_amd64.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The go4 Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // +build amd64,solaris 16 | 17 | package osutil 18 | 19 | import ( 20 | "os" 21 | "syscall" 22 | "unsafe" 23 | ) 24 | 25 | //go:cgo_import_dynamic libc_getexecname getexecname "libc.so" 26 | //go:linkname libc_getexecname libc_getexecname 27 | 28 | var libc_getexecname uintptr 29 | 30 | func getexecname() (path unsafe.Pointer, err error) { 31 | r0, _, e1 := syscall.Syscall6(uintptr(unsafe.Pointer(&libc_getexecname)), 0, 0, 0, 0, 0, 0) 32 | path = unsafe.Pointer(r0) 33 | if e1 != 0 { 34 | err = syscall.Errno(e1) 35 | } 36 | return 37 | } 38 | 39 | func syscallGetexecname() (path string, err error) { 40 | ptr, err := getexecname() 41 | if err != nil { 42 | return "", err 43 | } 44 | bytes := (*[1 << 29]byte)(ptr)[:] 45 | for i, b := range bytes { 46 | if b == 0 { 47 | return string(bytes[:i]), nil 48 | } 49 | } 50 | panic("unreachable") 51 | } 52 | 53 | var initCwd, initCwdErr = os.Getwd() 54 | 55 | func executable() (string, error) { 56 | path, err := syscallGetexecname() 57 | if err != nil { 58 | return path, err 59 | } 60 | if len(path) > 0 && path[0] != '/' { 61 | if initCwdErr != nil { 62 | return path, initCwdErr 63 | } 64 | if len(path) > 2 && path[0:2] == "./" { 65 | // skip "./" 66 | path = path[2:] 67 | } 68 | return initCwd + "/" + path, nil 69 | } 70 | return path, nil 71 | } 72 | -------------------------------------------------------------------------------- /osutil/exec_sysctl.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The go4 Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // +build freebsd darwin 16 | 17 | package osutil 18 | 19 | import ( 20 | "bytes" 21 | "errors" 22 | "os" 23 | "path/filepath" 24 | "runtime" 25 | "syscall" 26 | "unsafe" 27 | ) 28 | 29 | var cacheWD, cacheWDErr = os.Getwd() 30 | 31 | func exec_darwin() (string, error) { 32 | mib := [4]int32{1 /* CTL_KERN */, 38 /* KERN_PROCARGS */, int32(os.Getpid()), -1} 33 | n := uintptr(0) 34 | // get length 35 | _, _, err := syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, 0, uintptr(unsafe.Pointer(&n)), 0, 0) 36 | if err != 0 { 37 | return "", err 38 | } 39 | if n == 0 { // shouldn't happen 40 | return "", nil 41 | } 42 | buf := make([]byte, n) 43 | _, _, err = syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, uintptr(unsafe.Pointer(&buf[0])), uintptr(unsafe.Pointer(&n)), 0, 0) 44 | if err != 0 { 45 | return "", err 46 | } 47 | if n == 0 { // shouldn't happen 48 | return "", nil 49 | } 50 | // Because KERN_PROC_ARGS returns a list of NULL separated items, 51 | // and we want the first one. 52 | parts := bytes.Split(buf[:n-1], []byte{0}) 53 | if len(parts) < 2 { 54 | return "", nil 55 | } 56 | p := string(parts[0]) 57 | if !filepath.IsAbs(p) { 58 | if cacheWDErr != nil { 59 | return p, cacheWDErr 60 | } 61 | p = filepath.Join(cacheWD, filepath.Clean(p)) 62 | } 63 | return filepath.EvalSymlinks(p) 64 | } 65 | 66 | func exec_freebsd() (string, error) { 67 | mib := [4]int32{1 /* CTL_KERN */, 14 /* KERN_PROC */, 12 /* KERN_PROC_PATHNAME */, -1} 68 | 69 | n := uintptr(0) 70 | // get length 71 | _, _, err := syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, 0, uintptr(unsafe.Pointer(&n)), 0, 0) 72 | if err != 0 { 73 | return "", err 74 | } 75 | if n == 0 { // shouldn't happen 76 | return "", nil 77 | } 78 | buf := make([]byte, n) 79 | _, _, err = syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, uintptr(unsafe.Pointer(&buf[0])), uintptr(unsafe.Pointer(&n)), 0, 0) 80 | if err != 0 { 81 | return "", err 82 | } 83 | if n == 0 { // shouldn't happen 84 | return "", nil 85 | } 86 | p := string(buf[:n-1]) 87 | if !filepath.IsAbs(p) { 88 | if cacheWDErr != nil { 89 | return p, cacheWDErr 90 | } 91 | p = filepath.Join(cacheWD, filepath.Clean(p)) 92 | } 93 | return filepath.EvalSymlinks(p) 94 | } 95 | 96 | func executable() (string, error) { 97 | switch runtime.GOOS { 98 | case "freebsd": 99 | return exec_freebsd() 100 | case "darwin": 101 | return exec_darwin() 102 | } 103 | return "", errors.New("unsupported OS") 104 | } 105 | -------------------------------------------------------------------------------- /osutil/exec_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The go4 Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package osutil 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | osexec "os/exec" 21 | "path/filepath" 22 | "runtime" 23 | "testing" 24 | ) 25 | 26 | const executable_EnvVar = "OSTEST_OUTPUT_EXECPATH" 27 | 28 | func TestExecutable(t *testing.T) { 29 | if runtime.GOOS == "nacl" { 30 | t.Skip() 31 | } 32 | ep, err := Executable() 33 | if err != nil { 34 | switch goos := runtime.GOOS; goos { 35 | case "openbsd": // procfs is not mounted by default 36 | t.Skipf("Executable failed on %s: %v, expected", goos, err) 37 | } 38 | t.Fatalf("Executable failed: %v", err) 39 | } 40 | // we want fn to be of the form "dir/prog" 41 | dir := filepath.Dir(filepath.Dir(ep)) 42 | fn, err := filepath.Rel(dir, ep) 43 | if err != nil { 44 | t.Fatalf("filepath.Rel: %v", err) 45 | } 46 | cmd := &osexec.Cmd{} 47 | // make child start with a relative program path 48 | cmd.Dir = dir 49 | cmd.Path = fn 50 | // forge argv[0] for child, so that we can verify we could correctly 51 | // get real path of the executable without influenced by argv[0]. 52 | cmd.Args = []string{"-", "-test.run=XXXX"} 53 | cmd.Env = []string{fmt.Sprintf("%s=1", executable_EnvVar)} 54 | out, err := cmd.CombinedOutput() 55 | if err != nil { 56 | t.Fatalf("exec(self) failed: %v", err) 57 | } 58 | outs := string(out) 59 | if !filepath.IsAbs(outs) { 60 | t.Fatalf("Child returned %q, want an absolute path", out) 61 | } 62 | if !sameFile(outs, ep) { 63 | t.Fatalf("Child returned %q, not the same file as %q", out, ep) 64 | } 65 | } 66 | 67 | func sameFile(fn1, fn2 string) bool { 68 | fi1, err := os.Stat(fn1) 69 | if err != nil { 70 | return false 71 | } 72 | fi2, err := os.Stat(fn2) 73 | if err != nil { 74 | return false 75 | } 76 | return os.SameFile(fi1, fi2) 77 | } 78 | 79 | func init() { 80 | if e := os.Getenv(executable_EnvVar); e != "" { 81 | // first chdir to another path 82 | dir := "/" 83 | if runtime.GOOS == "windows" { 84 | dir = filepath.VolumeName(".") 85 | } 86 | os.Chdir(dir) 87 | if ep, err := Executable(); err != nil { 88 | fmt.Fprint(os.Stderr, "ERROR: ", err) 89 | } else { 90 | fmt.Fprint(os.Stderr, ep) 91 | } 92 | os.Exit(0) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /osutil/exec_windows.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 The go4 Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package osutil 18 | 19 | import ( 20 | "path/filepath" 21 | "syscall" 22 | "unsafe" 23 | ) 24 | 25 | var ( 26 | modkernel32 = syscall.MustLoadDLL("kernel32.dll") 27 | procGetModuleFileNameW = modkernel32.MustFindProc("GetModuleFileNameW") 28 | ) 29 | 30 | func getModuleFileName(handle syscall.Handle) (string, error) { 31 | n := uint32(1024) 32 | var buf []uint16 33 | for { 34 | buf = make([]uint16, n) 35 | r, err := syscallGetModuleFileName(handle, &buf[0], n) 36 | if err != nil { 37 | return "", err 38 | } 39 | if r < n { 40 | break 41 | } 42 | // r == n means n not big enough 43 | n += 1024 44 | } 45 | return syscall.UTF16ToString(buf), nil 46 | } 47 | 48 | func executable() (string, error) { 49 | p, err := getModuleFileName(0) 50 | return filepath.Clean(p), err 51 | } 52 | 53 | func syscallGetModuleFileName(module syscall.Handle, fn *uint16, len uint32) (n uint32, err error) { 54 | r0, _, e1 := syscall.Syscall(procGetModuleFileNameW.Addr(), 3, uintptr(module), uintptr(unsafe.Pointer(fn)), uintptr(len)) 55 | n = uint32(r0) 56 | if n == 0 { 57 | if e1 != 0 { 58 | err = error(e1) 59 | } else { 60 | err = syscall.EINVAL 61 | } 62 | } 63 | return 64 | } 65 | -------------------------------------------------------------------------------- /osutil/osutil.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 The go4 Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package osutil contains os level functions. 18 | package osutil // import "go4.org/osutil" 19 | 20 | // capture executable on package init to work around various os issues if 21 | // captured after executable has been renamed. 22 | var execPath, execError = executable() 23 | 24 | // Executable returns the path name for the executable that starts the 25 | // current process. The result is the path that was used to start the 26 | // current process, but there is no guarantee that the path is still 27 | // pointing to the correct executable. 28 | // 29 | // OpenBSD is currently unsupported. 30 | func Executable() (string, error) { 31 | return execPath, execError 32 | } 33 | -------------------------------------------------------------------------------- /readerutil/bufreaderat.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The go4 Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package readerutil 18 | 19 | import "io" 20 | 21 | // NewBufferingReaderAt returns an io.ReaderAt that reads from r as 22 | // necessary and keeps a copy of all data read in memory. 23 | func NewBufferingReaderAt(r io.Reader) io.ReaderAt { 24 | return &bufReaderAt{r: r} 25 | } 26 | 27 | type bufReaderAt struct { 28 | r io.Reader 29 | buf []byte 30 | } 31 | 32 | func (br *bufReaderAt) ReadAt(p []byte, off int64) (n int, err error) { 33 | endOff := off + int64(len(p)) 34 | need := endOff - int64(len(br.buf)) 35 | if need > 0 { 36 | buf := make([]byte, need) 37 | var rn int 38 | rn, err = io.ReadFull(br.r, buf) 39 | br.buf = append(br.buf, buf[:rn]...) 40 | } 41 | if int64(len(br.buf)) >= off { 42 | n = copy(p, br.buf[off:]) 43 | } 44 | if n == len(p) { 45 | err = nil 46 | } 47 | return 48 | } 49 | -------------------------------------------------------------------------------- /readerutil/bufreaderat_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The go4 Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package readerutil 18 | 19 | import "testing" 20 | 21 | type trackingReader struct { 22 | off int 23 | reads int 24 | readBytes int 25 | } 26 | 27 | func (t *trackingReader) Read(p []byte) (n int, err error) { 28 | t.reads++ 29 | t.readBytes += len(p) 30 | for len(p) > 0 { 31 | p[0] = '0' + byte(t.off%10) 32 | t.off++ 33 | p = p[1:] 34 | n++ 35 | } 36 | return 37 | 38 | } 39 | 40 | func TestBufferingReaderAt(t *testing.T) { 41 | tr := new(trackingReader) 42 | ra := NewBufferingReaderAt(tr) 43 | for i, tt := range []struct { 44 | off int64 45 | want string 46 | wantReads int 47 | wantReadBytes int 48 | }{ 49 | {off: 0, want: "0123456789", wantReads: 1, wantReadBytes: 10}, 50 | {off: 5, want: "56789", wantReads: 1, wantReadBytes: 10}, // already buffered 51 | {off: 6, want: "67890", wantReads: 2, wantReadBytes: 11}, // need 1 more byte 52 | {off: 0, want: "0123456789", wantReads: 2, wantReadBytes: 11}, // already buffered 53 | } { 54 | got := make([]byte, len(tt.want)) 55 | n, err := ra.ReadAt(got, tt.off) 56 | if err != nil || n != len(tt.want) { 57 | t.Errorf("step %d: ReadAt = %v, %v; want %v, %v", i, n, err, len(tt.want), nil) 58 | continue 59 | } 60 | if string(got) != tt.want { 61 | t.Errorf("step %d: ReadAt read %q; want %q", i, got, tt.want) 62 | } 63 | if tr.reads != tt.wantReads { 64 | t.Errorf("step %d: num reads = %d; want %d", i, tr.reads, tt.wantReads) 65 | } 66 | if tr.readBytes != tt.wantReadBytes { 67 | t.Errorf("step %d: read bytes = %d; want %d", i, tr.reads, tt.wantReads) 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /readerutil/countingreader.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2011 The Go4 Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package readerutil 18 | 19 | import "io" 20 | 21 | // CountingReader wraps a Reader, incrementing N by the number of 22 | // bytes read. No locking is performed. 23 | type CountingReader struct { 24 | Reader io.Reader 25 | N *int64 26 | } 27 | 28 | func (cr CountingReader) Read(p []byte) (n int, err error) { 29 | n, err = cr.Reader.Read(p) 30 | *cr.N += int64(n) 31 | return 32 | } 33 | -------------------------------------------------------------------------------- /readerutil/fakeseeker.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014 The Perkeep Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package readerutil 18 | 19 | import ( 20 | "errors" 21 | "fmt" 22 | "io" 23 | "os" 24 | ) 25 | 26 | // fakeSeeker can seek to the ends but any read not at the current 27 | // position will fail. 28 | type fakeSeeker struct { 29 | r io.Reader 30 | size int64 31 | 32 | fakePos int64 33 | realPos int64 34 | } 35 | 36 | // NewFakeSeeker returns a ReadSeeker that can pretend to Seek (based 37 | // on the provided total size of the reader's content), but any reads 38 | // will fail if the fake seek position doesn't match reality. 39 | func NewFakeSeeker(r io.Reader, size int64) io.ReadSeeker { 40 | return &fakeSeeker{r: r, size: size} 41 | } 42 | 43 | func (fs *fakeSeeker) Seek(offset int64, whence int) (int64, error) { 44 | var newo int64 45 | switch whence { 46 | default: 47 | return 0, errors.New("invalid whence") 48 | case os.SEEK_SET: 49 | newo = offset 50 | case os.SEEK_CUR: 51 | newo = fs.fakePos + offset 52 | case os.SEEK_END: 53 | newo = fs.size + offset 54 | } 55 | if newo < 0 { 56 | return 0, errors.New("negative seek") 57 | } 58 | fs.fakePos = newo 59 | return newo, nil 60 | } 61 | 62 | func (fs *fakeSeeker) Read(p []byte) (n int, err error) { 63 | if fs.fakePos != fs.realPos { 64 | return 0, fmt.Errorf("attempt to read from fake seek offset %d; real offset is %d", fs.fakePos, fs.realPos) 65 | } 66 | n, err = fs.r.Read(p) 67 | fs.fakePos += int64(n) 68 | fs.realPos += int64(n) 69 | return 70 | } 71 | -------------------------------------------------------------------------------- /readerutil/fakeseeker_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014 The Perkeep Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package readerutil 18 | 19 | import ( 20 | "os" 21 | "strings" 22 | "testing" 23 | ) 24 | 25 | func TestFakeSeeker(t *testing.T) { 26 | rs := NewFakeSeeker(strings.NewReader("foobar"), 6) 27 | if pos, err := rs.Seek(0, os.SEEK_END); err != nil || pos != 6 { 28 | t.Fatalf("SEEK_END = %d, %v; want 6, nil", pos, err) 29 | } 30 | if pos, err := rs.Seek(0, os.SEEK_CUR); err != nil || pos != 6 { 31 | t.Fatalf("SEEK_CUR = %d, %v; want 6, nil", pos, err) 32 | } 33 | if pos, err := rs.Seek(0, os.SEEK_SET); err != nil || pos != 0 { 34 | t.Fatalf("SEEK_SET = %d, %v; want 0, nil", pos, err) 35 | } 36 | 37 | buf := make([]byte, 3) 38 | if n, err := rs.Read(buf); n != 3 || err != nil || string(buf) != "foo" { 39 | t.Fatalf("First read = %d, %v (buf = %q); want foo", n, err, buf) 40 | } 41 | if pos, err := rs.Seek(0, os.SEEK_CUR); err != nil || pos != 3 { 42 | t.Fatalf("Seek cur pos after first read = %d, %v; want 3, nil", pos, err) 43 | } 44 | if n, err := rs.Read(buf); n != 3 || err != nil || string(buf) != "bar" { 45 | t.Fatalf("Second read = %d, %v (buf = %q); want foo", n, err, buf) 46 | } 47 | 48 | if pos, err := rs.Seek(1, os.SEEK_SET); err != nil || pos != 1 { 49 | t.Fatalf("SEEK_SET = %d, %v; want 1, nil", pos, err) 50 | } 51 | const msg = "attempt to read from fake seek offset" 52 | if _, err := rs.Read(buf); err == nil || !strings.Contains(err.Error(), msg) { 53 | t.Fatalf("bogus Read after seek = %v; want something containing %q", err, msg) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /readerutil/multireaderat.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The go4 Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package readerutil 18 | 19 | import ( 20 | "io" 21 | "sort" 22 | ) 23 | 24 | // NewMultiReaderAt is like io.MultiReader but produces a ReaderAt 25 | // (and Size), instead of just a reader. 26 | func NewMultiReaderAt(parts ...SizeReaderAt) SizeReaderAt { 27 | m := &multiRA{ 28 | parts: make([]offsetAndSource, 0, len(parts)), 29 | } 30 | var off int64 31 | for _, p := range parts { 32 | m.parts = append(m.parts, offsetAndSource{off, p}) 33 | off += p.Size() 34 | } 35 | m.size = off 36 | return m 37 | } 38 | 39 | type offsetAndSource struct { 40 | off int64 41 | SizeReaderAt 42 | } 43 | 44 | type multiRA struct { 45 | parts []offsetAndSource 46 | size int64 47 | } 48 | 49 | func (m *multiRA) Size() int64 { return m.size } 50 | 51 | func (m *multiRA) ReadAt(p []byte, off int64) (n int, err error) { 52 | wantN := len(p) 53 | 54 | // Skip past the requested offset. 55 | skipParts := sort.Search(len(m.parts), func(i int) bool { 56 | // This function returns whether parts[i] will 57 | // contribute any bytes to our output. 58 | part := m.parts[i] 59 | return part.off+part.Size() > off 60 | }) 61 | parts := m.parts[skipParts:] 62 | 63 | // How far to skip in the first part. 64 | needSkip := off 65 | if len(parts) > 0 { 66 | needSkip -= parts[0].off 67 | } 68 | 69 | for len(parts) > 0 && len(p) > 0 { 70 | readP := p 71 | partSize := parts[0].Size() 72 | if int64(len(readP)) > partSize-needSkip { 73 | readP = readP[:partSize-needSkip] 74 | } 75 | pn, err0 := parts[0].ReadAt(readP, needSkip) 76 | if err0 != nil { 77 | return n, err0 78 | } 79 | n += pn 80 | p = p[pn:] 81 | if int64(pn)+needSkip == partSize { 82 | parts = parts[1:] 83 | } 84 | needSkip = 0 85 | } 86 | 87 | if n != wantN { 88 | err = io.ErrUnexpectedEOF 89 | } 90 | return 91 | } 92 | -------------------------------------------------------------------------------- /readerutil/multireaderat_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Go4 Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package readerutil 18 | 19 | import ( 20 | "io" 21 | "io/ioutil" 22 | "strings" 23 | "testing" 24 | ) 25 | 26 | func TestMultiReaderAt(t *testing.T) { 27 | sra := NewMultiReaderAt( 28 | io.NewSectionReader(strings.NewReader("xaaax"), 1, 3), 29 | io.NewSectionReader(strings.NewReader("xxbbbbxx"), 2, 3), 30 | io.NewSectionReader(strings.NewReader("cccx"), 0, 3), 31 | ) 32 | if sra.Size() != 9 { 33 | t.Fatalf("Size = %d; want 9", sra.Size()) 34 | } 35 | const full = "aaabbbccc" 36 | for start := 0; start < len(full); start++ { 37 | for end := start; end < len(full); end++ { 38 | want := full[start:end] 39 | got, err := ioutil.ReadAll(io.NewSectionReader(sra, int64(start), int64(end-start))) 40 | if err != nil { 41 | t.Fatal(err) 42 | } 43 | if string(got) != want { 44 | t.Errorf("for start=%d, end=%d: ReadAll = %q; want %q", start, end, got, want) 45 | } 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /readerutil/readersize.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2012 The Go4 Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package readerutil provides and operates on io.Readers. 18 | package readerutil // import "go4.org/readerutil" 19 | 20 | import ( 21 | "bytes" 22 | "io" 23 | "os" 24 | "strings" 25 | ) 26 | 27 | // Size tries to determine the length of r. If r is an io.Seeker, Size may seek 28 | // to guess the length. 29 | func Size(r io.Reader) (size int64, ok bool) { 30 | switch rt := r.(type) { 31 | case *bytes.Buffer: 32 | return int64(rt.Len()), true 33 | case *bytes.Reader: 34 | return int64(rt.Len()), true 35 | case *strings.Reader: 36 | return int64(rt.Len()), true 37 | case io.Seeker: 38 | pos, err := rt.Seek(0, os.SEEK_CUR) 39 | if err != nil { 40 | return 41 | } 42 | end, err := rt.Seek(0, os.SEEK_END) 43 | if err != nil { 44 | return 45 | } 46 | size = end - pos 47 | pos1, err := rt.Seek(pos, os.SEEK_SET) 48 | if err != nil || pos1 != pos { 49 | msg := "failed to restore seek position" 50 | if err != nil { 51 | msg += ": " + err.Error() 52 | } 53 | panic(msg) 54 | } 55 | return size, true 56 | } 57 | return 0, false 58 | } 59 | -------------------------------------------------------------------------------- /readerutil/readersize_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2012 The Go4 Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package readerutil 18 | 19 | import ( 20 | "bytes" 21 | "io" 22 | "io/ioutil" 23 | "os" 24 | "testing" 25 | ) 26 | 27 | const text = "HelloWorld" 28 | 29 | type testSrc struct { 30 | name string 31 | src io.Reader 32 | want int64 33 | } 34 | 35 | func (tsrc *testSrc) run(t *testing.T) { 36 | n, ok := Size(tsrc.src) 37 | if !ok { 38 | t.Fatalf("failed to read size for %q", tsrc.name) 39 | } 40 | if n != tsrc.want { 41 | t.Fatalf("wanted %v, got %v", tsrc.want, n) 42 | } 43 | } 44 | 45 | func TestBytesBuffer(t *testing.T) { 46 | buf := bytes.NewBuffer([]byte(text)) 47 | tsrc := &testSrc{"buffer", buf, int64(len(text))} 48 | tsrc.run(t) 49 | } 50 | 51 | func TestSeeker(t *testing.T) { 52 | f, err := ioutil.TempFile("", "camliTestReaderSize") 53 | if err != nil { 54 | t.Fatal(err) 55 | } 56 | defer os.Remove(f.Name()) 57 | defer f.Close() 58 | size, err := f.Write([]byte(text)) 59 | if err != nil { 60 | t.Fatal(err) 61 | } 62 | pos, err := f.Seek(5, 0) 63 | if err != nil { 64 | t.Fatal(err) 65 | } 66 | tsrc := &testSrc{"seeker", f, int64(size) - pos} 67 | tsrc.run(t) 68 | } 69 | -------------------------------------------------------------------------------- /readerutil/readerutil.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The go4 Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package readerutil contains io.Reader types. 18 | package readerutil // import "go4.org/readerutil" 19 | 20 | import ( 21 | "expvar" 22 | "io" 23 | ) 24 | 25 | // A SizeReaderAt is a ReaderAt with a Size method. 26 | // 27 | // An io.SectionReader implements SizeReaderAt. 28 | type SizeReaderAt interface { 29 | Size() int64 30 | io.ReaderAt 31 | } 32 | 33 | // A ReadSeekCloser can Read, Seek, and Close. 34 | type ReadSeekCloser interface { 35 | io.Reader 36 | io.Seeker 37 | io.Closer 38 | } 39 | 40 | type ReaderAtCloser interface { 41 | io.ReaderAt 42 | io.Closer 43 | } 44 | 45 | // TODO(wathiede): make sure all the stat readers work with code that 46 | // type asserts ReadFrom/WriteTo. 47 | 48 | type varStatReader struct { 49 | *expvar.Int 50 | r io.Reader 51 | } 52 | 53 | // NewStatsReader returns an io.Reader that will have the number of bytes 54 | // read from r added to v. 55 | func NewStatsReader(v *expvar.Int, r io.Reader) io.Reader { 56 | return &varStatReader{v, r} 57 | } 58 | 59 | func (v *varStatReader) Read(p []byte) (int, error) { 60 | n, err := v.r.Read(p) 61 | v.Int.Add(int64(n)) 62 | return n, err 63 | } 64 | 65 | type varStatReadSeeker struct { 66 | *expvar.Int 67 | rs io.ReadSeeker 68 | } 69 | 70 | // NewStatsReadSeeker returns an io.ReadSeeker that will have the number of bytes 71 | // read from rs added to v. 72 | func NewStatsReadSeeker(v *expvar.Int, rs io.ReadSeeker) io.ReadSeeker { 73 | return &varStatReadSeeker{v, rs} 74 | } 75 | 76 | func (v *varStatReadSeeker) Read(p []byte) (int, error) { 77 | n, err := v.rs.Read(p) 78 | v.Int.Add(int64(n)) 79 | return n, err 80 | } 81 | 82 | func (v *varStatReadSeeker) Seek(offset int64, whence int) (int64, error) { 83 | return v.rs.Seek(offset, whence) 84 | } 85 | -------------------------------------------------------------------------------- /readerutil/readerutil_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The Go4 Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package readerutil 18 | 19 | import ( 20 | "expvar" 21 | "fmt" 22 | "io" 23 | "io/ioutil" 24 | "strings" 25 | ) 26 | 27 | func ExampleNewStatsReader() { 28 | var ( 29 | // r is the io.Reader we'd like to count read from. 30 | r = strings.NewReader("Hello world") 31 | v = expvar.NewInt("read-bytes") 32 | sw = NewStatsReader(v, r) 33 | ) 34 | // Read from the wrapped io.Reader, StatReader will count the bytes. 35 | io.Copy(ioutil.Discard, sw) 36 | fmt.Printf("Read %s bytes\n", v.String()) 37 | // Output: Read 11 bytes 38 | } 39 | -------------------------------------------------------------------------------- /readerutil/singlereader/opener.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2013 The Go4 Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // package singlereader provides Open and Close operations, reusing existing 18 | // file descriptors when possible. 19 | package singlereader // import "go4.org/readerutil/singlereader" 20 | 21 | import ( 22 | "sync" 23 | 24 | "go4.org/readerutil" 25 | "go4.org/syncutil/singleflight" 26 | "go4.org/wkfs" 27 | ) 28 | 29 | var ( 30 | openerGroup singleflight.Group 31 | 32 | openFileMu sync.Mutex // guards openFiles 33 | openFiles = make(map[string]*openFile) 34 | ) 35 | 36 | type openFile struct { 37 | wkfs.File 38 | path string // map key of openFiles 39 | refCount int 40 | } 41 | 42 | type openFileHandle struct { 43 | closed bool 44 | *openFile 45 | } 46 | 47 | func (f *openFileHandle) Close() error { 48 | openFileMu.Lock() 49 | if f.closed { 50 | openFileMu.Unlock() 51 | return nil 52 | } 53 | f.closed = true 54 | f.refCount-- 55 | if f.refCount < 0 { 56 | panic("unexpected negative refcount") 57 | } 58 | zero := f.refCount == 0 59 | if zero { 60 | delete(openFiles, f.path) 61 | } 62 | openFileMu.Unlock() 63 | if !zero { 64 | return nil 65 | } 66 | return f.openFile.File.Close() 67 | } 68 | 69 | // Open opens the given file path for reading, reusing existing file descriptors 70 | // when possible. 71 | func Open(path string) (readerutil.ReaderAtCloser, error) { 72 | openFileMu.Lock() 73 | of := openFiles[path] 74 | if of != nil { 75 | of.refCount++ 76 | openFileMu.Unlock() 77 | return &openFileHandle{false, of}, nil 78 | } 79 | openFileMu.Unlock() // release the lock while we call os.Open 80 | 81 | winner := false // this goroutine made it into Do's func 82 | 83 | // Returns an *openFile 84 | resi, err := openerGroup.Do(path, func() (interface{}, error) { 85 | winner = true 86 | f, err := wkfs.Open(path) 87 | if err != nil { 88 | return nil, err 89 | } 90 | of := &openFile{ 91 | File: f, 92 | path: path, 93 | refCount: 1, 94 | } 95 | openFileMu.Lock() 96 | openFiles[path] = of 97 | openFileMu.Unlock() 98 | return of, nil 99 | }) 100 | if err != nil { 101 | return nil, err 102 | } 103 | of = resi.(*openFile) 104 | 105 | // If our os.Open was dup-suppressed, we have to increment our 106 | // reference count. 107 | if !winner { 108 | openFileMu.Lock() 109 | if of.refCount == 0 { 110 | // Winner already closed it. Try again (rare). 111 | openFileMu.Unlock() 112 | return Open(path) 113 | } 114 | of.refCount++ 115 | openFileMu.Unlock() 116 | } 117 | return &openFileHandle{false, of}, nil 118 | } 119 | -------------------------------------------------------------------------------- /readerutil/singlereader/opener_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2013 The Go4 Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package singlereader 18 | 19 | import ( 20 | "bytes" 21 | "fmt" 22 | "io/ioutil" 23 | "os" 24 | "runtime" 25 | "testing" 26 | ) 27 | 28 | func TestOpenSingle(t *testing.T) { 29 | if testing.Short() { 30 | t.Skip("skipping in short mode") 31 | } 32 | defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(4)) 33 | f, err := ioutil.TempFile("", "foo") 34 | if err != nil { 35 | t.Fatal(err) 36 | } 37 | defer os.Remove(f.Name()) 38 | contents := []byte("Some file contents") 39 | if _, err := f.Write(contents); err != nil { 40 | t.Fatal(err) 41 | } 42 | f.Close() 43 | 44 | const j = 4 45 | errc := make(chan error, j) 46 | for i := 1; i < j; i++ { 47 | go func() { 48 | buf := make([]byte, len(contents)) 49 | for i := 0; i < 400; i++ { 50 | rac, err := Open(f.Name()) 51 | if err != nil { 52 | errc <- err 53 | return 54 | } 55 | n, err := rac.ReadAt(buf, 0) 56 | if err != nil { 57 | errc <- err 58 | return 59 | } 60 | if n != len(contents) || !bytes.Equal(buf, contents) { 61 | errc <- fmt.Errorf("read %d, %q; want %d, %q", n, buf, len(contents), contents) 62 | return 63 | } 64 | if err := rac.Close(); err != nil { 65 | errc <- err 66 | return 67 | } 68 | } 69 | errc <- nil 70 | }() 71 | } 72 | for i := 1; i < j; i++ { 73 | if err := <-errc; err != nil { 74 | t.Error(err) 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /reflectutil/asm_b.s: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build !go1.8 6 | // +build go1.6 7 | // +build arm 8 | 9 | #include "textflag.h" 10 | #include "funcdata.h" 11 | 12 | // func typedmemmove(reflect_rtype, src unsafe.Pointer, size uintptr) 13 | TEXT ·typedmemmove(SB),(NOSPLIT|WRAPPER),$0-24 14 | B runtime·typedmemmove(SB) 15 | 16 | // func memmove(dst, src unsafe.Pointer, size uintptr) 17 | TEXT ·memmove(SB),(NOSPLIT|WRAPPER),$0-24 18 | B runtime·memmove(SB) 19 | -------------------------------------------------------------------------------- /reflectutil/asm_b_14.s: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build !go1.5 6 | // +build arm 7 | 8 | #include "textflag.h" 9 | #include "funcdata.h" 10 | 11 | // func memmove(dst, src unsafe.Pointer, size uintptr) 12 | TEXT ·memmove(SB),(NOSPLIT|WRAPPER),$0-24 13 | B runtime·memmove(SB) 14 | -------------------------------------------------------------------------------- /reflectutil/asm_jmp.s: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build !go1.8 6 | // +build go1.5,!js,!safe,!appengine 7 | // +build amd64 386 8 | 9 | #include "textflag.h" 10 | #include "funcdata.h" 11 | 12 | // func typedmemmove(reflect_rtype, src unsafe.Pointer, size uintptr) 13 | TEXT ·typedmemmove(SB),(NOSPLIT|WRAPPER),$0-24 14 | JMP runtime·typedmemmove(SB) 15 | 16 | // func memmove(dst, src unsafe.Pointer, size uintptr) 17 | TEXT ·memmove(SB),(NOSPLIT|WRAPPER),$0-24 18 | JMP runtime·memmove(SB) 19 | -------------------------------------------------------------------------------- /reflectutil/asm_jmp_14.s: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build !go1.5,!js,!safe,!appengine 6 | // +build amd64 386 7 | 8 | #include "textflag.h" 9 | #include "funcdata.h" 10 | 11 | // func memmove(dst, src unsafe.Pointer, size uintptr) 12 | TEXT ·memmove(SB),(NOSPLIT|WRAPPER),$0-24 13 | JMP runtime·memmove(SB) 14 | -------------------------------------------------------------------------------- /reflectutil/reflectutil.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package reflectutil contains reflect utilities. 6 | package reflectutil 7 | 8 | import "reflect" 9 | 10 | // hasPointers reports whether the given type contains any pointers, 11 | // including any internal pointers in slices, funcs, maps, channels, 12 | // etc. 13 | // 14 | // This function exists for Swapper's internal use, instead of reaching 15 | // into the runtime's *reflect._rtype kind&kindNoPointers flag. 16 | func hasPointers(t reflect.Type) bool { 17 | if t == nil { 18 | panic("nil Type") 19 | } 20 | k := t.Kind() 21 | if k <= reflect.Complex128 { 22 | return false 23 | } 24 | switch k { 25 | default: 26 | // chan, func, interface, map, ptr, slice, string, unsafepointer 27 | // And anything else. It's safer to err on the side of true. 28 | return true 29 | case reflect.Array: 30 | return hasPointers(t.Elem()) 31 | case reflect.Struct: 32 | num := t.NumField() 33 | for i := 0; i < num; i++ { 34 | if hasPointers(t.Field(i).Type) { 35 | return true 36 | } 37 | } 38 | return false 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /reflectutil/reflectutil_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package reflectutil 6 | 7 | import ( 8 | "reflect" 9 | "testing" 10 | ) 11 | 12 | func TestHasPointers(t *testing.T) { 13 | tests := []struct { 14 | val interface{} 15 | want bool 16 | }{ 17 | {false, false}, 18 | {int(1), false}, 19 | {int8(1), false}, 20 | {int16(1), false}, 21 | {int32(1), false}, 22 | {int64(1), false}, 23 | {uint(1), false}, 24 | {uint8(1), false}, 25 | {uint16(1), false}, 26 | {uint32(1), false}, 27 | {uint64(1), false}, 28 | {uintptr(1), false}, 29 | {float32(1.0), false}, 30 | {float64(1.0), false}, 31 | {complex64(1.0i), false}, 32 | {complex128(1.0i), false}, 33 | 34 | {[...]int{1, 2}, false}, 35 | {[...]*int{nil, nil}, true}, 36 | 37 | {make(chan bool), true}, 38 | 39 | {TestHasPointers, true}, 40 | 41 | {map[string]string{"foo": "bar"}, true}, 42 | 43 | {new(int), true}, 44 | 45 | {[]int{1, 2}, true}, 46 | 47 | {"foo", true}, 48 | 49 | {struct{}{}, false}, 50 | {struct{ int }{0}, false}, 51 | {struct { 52 | a int 53 | b bool 54 | }{0, false}, false}, 55 | {struct { 56 | a int 57 | b string 58 | }{0, ""}, true}, 59 | {struct{ *int }{nil}, true}, 60 | {struct { 61 | a *int 62 | b int 63 | }{nil, 0}, true}, 64 | } 65 | for i, tt := range tests { 66 | got := hasPointers(reflect.TypeOf(tt.val)) 67 | if got != tt.want { 68 | t.Errorf("%d. hasPointers(%T) = %v; want %v", i, tt.val, got, tt.want) 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /reflectutil/swapper.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package reflectutil 6 | 7 | import "reflect" 8 | 9 | // Swapper returns a function which swaps the elements in slice. 10 | // Swapper panics if the provided interface is not a slice. 11 | // 12 | // Its goal is to work safely and efficiently for all versions and 13 | // variants of Go: pre-Go1.5, Go1.5+, safe, unsafe, App Engine, 14 | // GopherJS, etc. 15 | // 16 | // Deprecated: this moved to the Go standard library. Use 17 | // reflect.Swapper in Go 1.8+ instead. 18 | func Swapper(slice interface{}) func(i, j int) { 19 | v := reflect.ValueOf(slice) 20 | if v.Kind() != reflect.Slice { 21 | panic(&reflect.ValueError{Method: "reflectutil.Swapper", Kind: v.Kind()}) 22 | } 23 | return swapper(v) 24 | } 25 | -------------------------------------------------------------------------------- /reflectutil/swapper_safe.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build !go1.8 6 | // +build js appengine safe ppc64 ppc64le arm64 mips mipsle mips64 mips64le 7 | 8 | package reflectutil 9 | 10 | import "reflect" 11 | 12 | func swapper(slice reflect.Value) func(i, j int) { 13 | tmp := reflect.New(slice.Type().Elem()).Elem() 14 | return func(i, j int) { 15 | v1 := slice.Index(i) 16 | v2 := slice.Index(j) 17 | tmp.Set(v1) 18 | v1.Set(v2) 19 | v2.Set(tmp) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /reflectutil/swapper_std.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build go1.8 6 | 7 | package reflectutil 8 | 9 | import "reflect" 10 | 11 | func swapper(slice reflect.Value) func(i, j int) { 12 | return reflect.Swapper(slice.Interface()) 13 | } 14 | -------------------------------------------------------------------------------- /reflectutil/swapper_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package reflectutil 6 | 7 | import ( 8 | "fmt" 9 | "reflect" 10 | "strconv" 11 | "testing" 12 | ) 13 | 14 | func TestSwapper(t *testing.T) { 15 | type I int 16 | var a, b, c I 17 | type pair struct { 18 | x, y int 19 | } 20 | type pairPtr struct { 21 | x, y int 22 | p *I 23 | } 24 | type S string 25 | 26 | tests := []struct { 27 | in interface{} 28 | i, j int 29 | want interface{} 30 | }{ 31 | { 32 | in: []int{1, 20, 300}, 33 | i: 0, 34 | j: 2, 35 | want: []int{300, 20, 1}, 36 | }, 37 | { 38 | in: []uintptr{1, 20, 300}, 39 | i: 0, 40 | j: 2, 41 | want: []uintptr{300, 20, 1}, 42 | }, 43 | { 44 | in: []int16{1, 20, 300}, 45 | i: 0, 46 | j: 2, 47 | want: []int16{300, 20, 1}, 48 | }, 49 | { 50 | in: []int8{1, 20, 100}, 51 | i: 0, 52 | j: 2, 53 | want: []int8{100, 20, 1}, 54 | }, 55 | { 56 | in: []*I{&a, &b, &c}, 57 | i: 0, 58 | j: 2, 59 | want: []*I{&c, &b, &a}, 60 | }, 61 | { 62 | in: []string{"eric", "sergey", "larry"}, 63 | i: 0, 64 | j: 2, 65 | want: []string{"larry", "sergey", "eric"}, 66 | }, 67 | { 68 | in: []S{"eric", "sergey", "larry"}, 69 | i: 0, 70 | j: 2, 71 | want: []S{"larry", "sergey", "eric"}, 72 | }, 73 | { 74 | in: []pair{{1, 2}, {3, 4}, {5, 6}}, 75 | i: 0, 76 | j: 2, 77 | want: []pair{{5, 6}, {3, 4}, {1, 2}}, 78 | }, 79 | { 80 | in: []pairPtr{{1, 2, &a}, {3, 4, &b}, {5, 6, &c}}, 81 | i: 0, 82 | j: 2, 83 | want: []pairPtr{{5, 6, &c}, {3, 4, &b}, {1, 2, &a}}, 84 | }, 85 | } 86 | for i, tt := range tests { 87 | inStr := fmt.Sprint(tt.in) 88 | Swapper(tt.in)(tt.i, tt.j) 89 | if !reflect.DeepEqual(tt.in, tt.want) { 90 | t.Errorf("%d. swapping %v and %v of %v = %v; want %v", i, tt.i, tt.j, inStr, tt.in, tt.want) 91 | } 92 | } 93 | } 94 | 95 | func BenchmarkSwap(b *testing.B) { 96 | const N = 1024 97 | strs := make([]string, N) 98 | for i := range strs { 99 | strs[i] = strconv.Itoa(i) 100 | } 101 | swap := Swapper(strs) 102 | 103 | b.ResetTimer() 104 | i, j := 0, 1 105 | for n := 0; n < b.N; n++ { 106 | i = (i + 1) % N 107 | j = (j + 2) % N 108 | swap(i, j) 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /reflectutil/swapper_unsafe.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build !go1.8 6 | // +build !ppc64,!ppc64le,!arm64,!mips,!mipsle,!mips64,!mips64le 7 | // +build !js,!appengine,!safe 8 | 9 | package reflectutil 10 | 11 | import ( 12 | "reflect" 13 | "unsafe" 14 | ) 15 | 16 | const ptrSize = 4 << (^uintptr(0) >> 63) // unsafe.Sizeof(uintptr(0)) but an ideal const 17 | 18 | // arrayAt returns the i-th element of p, a C-array whose elements are 19 | // eltSize wide (in bytes). 20 | func arrayAt(p unsafe.Pointer, i int, eltSize uintptr) unsafe.Pointer { 21 | return unsafe.Pointer(uintptr(p) + uintptr(i)*eltSize) 22 | } 23 | 24 | type sliceHeader struct { 25 | Data unsafe.Pointer 26 | Len int 27 | Cap int 28 | } 29 | 30 | func swapper(v reflect.Value) func(i, j int) { 31 | maxLen := uint(v.Len()) 32 | 33 | s := sliceHeader{unsafe.Pointer(v.Pointer()), int(maxLen), int(maxLen)} 34 | tt := v.Type() 35 | elemt := tt.Elem() 36 | typ := unsafe.Pointer(reflect.ValueOf(elemt).Pointer()) 37 | 38 | size := elemt.Size() 39 | hasPtr := hasPointers(elemt) 40 | 41 | // Some common & small cases, without using memmove: 42 | if hasPtr { 43 | if size == ptrSize { 44 | var ps []unsafe.Pointer 45 | *(*sliceHeader)(unsafe.Pointer(&ps)) = s 46 | return func(i, j int) { ps[i], ps[j] = ps[j], ps[i] } 47 | } 48 | if elemt.Kind() == reflect.String { 49 | var ss []string 50 | *(*sliceHeader)(unsafe.Pointer(&ss)) = s 51 | return func(i, j int) { ss[i], ss[j] = ss[j], ss[i] } 52 | } 53 | } else { 54 | switch size { 55 | case 8: 56 | var is []int64 57 | *(*sliceHeader)(unsafe.Pointer(&is)) = s 58 | return func(i, j int) { is[i], is[j] = is[j], is[i] } 59 | case 4: 60 | var is []int32 61 | *(*sliceHeader)(unsafe.Pointer(&is)) = s 62 | return func(i, j int) { is[i], is[j] = is[j], is[i] } 63 | case 2: 64 | var is []int16 65 | *(*sliceHeader)(unsafe.Pointer(&is)) = s 66 | return func(i, j int) { is[i], is[j] = is[j], is[i] } 67 | case 1: 68 | var is []int8 69 | *(*sliceHeader)(unsafe.Pointer(&is)) = s 70 | return func(i, j int) { is[i], is[j] = is[j], is[i] } 71 | } 72 | } 73 | 74 | // Allocate scratch space for swaps: 75 | tmpVal := reflect.New(elemt) 76 | tmp := unsafe.Pointer(tmpVal.Pointer()) 77 | 78 | // If no pointers (or Go 1.4 or below), we don't require typedmemmove: 79 | if !haveTypedMemmove || !hasPtr { 80 | return func(i, j int) { 81 | if uint(i) >= maxLen || uint(j) >= maxLen { 82 | panic("reflect: slice index out of range") 83 | } 84 | val1 := arrayAt(s.Data, i, size) 85 | val2 := arrayAt(s.Data, j, size) 86 | memmove(tmp, val1, size) 87 | memmove(val1, val2, size) 88 | memmove(val2, tmp, size) 89 | } 90 | } 91 | 92 | return func(i, j int) { 93 | if uint(i) >= maxLen || uint(j) >= maxLen { 94 | panic("reflect: slice index out of range") 95 | } 96 | val1 := arrayAt(s.Data, i, size) 97 | val2 := arrayAt(s.Data, j, size) 98 | typedmemmove(typ, tmp, val1) 99 | typedmemmove(typ, val1, val2) 100 | typedmemmove(typ, val2, tmp) 101 | } 102 | } 103 | 104 | // memmove copies size bytes from src to dst. 105 | // The memory must not contain any pointers. 106 | //go:noescape 107 | func memmove(dst, src unsafe.Pointer, size uintptr) 108 | -------------------------------------------------------------------------------- /reflectutil/swapper_unsafe_14.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build !ppc64,!ppc64le,!arm64,!mips,!mipsle,!mips64,!mips64le 6 | // +build !go1.5,!js,!appengine,!safe 7 | 8 | package reflectutil 9 | 10 | import "unsafe" 11 | 12 | const haveTypedMemmove = false 13 | 14 | func typedmemmove(reflect_rtype, dst, src unsafe.Pointer) { 15 | panic("never called") // only here so swapper_unsafe.go compiles 16 | } 17 | -------------------------------------------------------------------------------- /reflectutil/swapper_unsafe_15.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build !go1.8 6 | // +build !ppc64,!ppc64le,!arm64,!mips,!mipsle,!mips64,!mips64le 7 | // +build go1.5,!js,!appengine,!safe 8 | 9 | package reflectutil 10 | 11 | import "unsafe" 12 | 13 | const haveTypedMemmove = true 14 | 15 | // typedmemmove copies a value of type t to dst from src. 16 | //go:noescape 17 | func typedmemmove(reflect_rtype, dst, src unsafe.Pointer) 18 | -------------------------------------------------------------------------------- /rollsum/rollsum.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2011 The Perkeep Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package rollsum implements rolling checksums similar to apenwarr's bup, which 18 | // is similar to librsync. 19 | // 20 | // The bup project is at https://github.com/apenwarr/bup and its splitting in 21 | // particular is at https://github.com/apenwarr/bup/blob/master/lib/bup/bupsplit.c 22 | package rollsum // import "go4.org/rollsum" 23 | 24 | import ( 25 | "math/bits" 26 | ) 27 | 28 | const windowSize = 64 // Roll assumes windowSize is a power of 2 29 | const charOffset = 31 30 | 31 | const blobBits = 13 32 | const blobSize = 1 << blobBits // 8k 33 | 34 | type RollSum struct { 35 | s1, s2 uint32 36 | window [windowSize]uint8 37 | wofs int 38 | } 39 | 40 | func New() *RollSum { 41 | return &RollSum{ 42 | s1: windowSize * charOffset, 43 | s2: windowSize * (windowSize - 1) * charOffset, 44 | } 45 | } 46 | 47 | func (rs *RollSum) add(drop, add uint32) { 48 | s1 := rs.s1 + add - drop 49 | rs.s1 = s1 50 | rs.s2 += s1 - uint32(windowSize)*(drop+charOffset) 51 | } 52 | 53 | // Roll adds ch to the rolling sum. 54 | func (rs *RollSum) Roll(ch byte) { 55 | wp := &rs.window[rs.wofs] 56 | rs.add(uint32(*wp), uint32(ch)) 57 | *wp = ch 58 | rs.wofs = (rs.wofs + 1) & (windowSize - 1) 59 | } 60 | 61 | // OnSplit reports whether at least 13 consecutive trailing bits of 62 | // the current checksum are set the same way. 63 | func (rs *RollSum) OnSplit() bool { 64 | return (rs.s2 & (blobSize - 1)) == ((^0) & (blobSize - 1)) 65 | } 66 | 67 | // OnSplitWithBits reports whether at least n consecutive trailing bits 68 | // of the current checksum are set the same way. 69 | func (rs *RollSum) OnSplitWithBits(n uint32) bool { 70 | mask := (uint32(1) << n) - 1 71 | return rs.s2&mask == (^uint32(0))&mask 72 | } 73 | 74 | func (rs *RollSum) Bits() int { 75 | rsum := rs.Digest() >> (blobBits + 1) 76 | return blobBits + bits.TrailingZeros32(^rsum) 77 | } 78 | 79 | func (rs *RollSum) Digest() uint32 { 80 | return (rs.s1 << 16) | (rs.s2 & 0xffff) 81 | } 82 | -------------------------------------------------------------------------------- /rollsum/rollsum_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2011 The Perkeep Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package rollsum 18 | 19 | import ( 20 | "math/rand" 21 | "testing" 22 | ) 23 | 24 | func TestSum(t *testing.T) { 25 | var buf [100000]uint8 26 | rnd := rand.New(rand.NewSource(4)) 27 | for i := range buf { 28 | buf[i] = uint8(rnd.Intn(256)) 29 | } 30 | 31 | roll := func(offset, len int) *RollSum { 32 | rs := New() 33 | for count := offset; count < len; count++ { 34 | rs.Roll(buf[count]) 35 | } 36 | return rs 37 | } 38 | 39 | sum := func(offset, len int) uint32 { 40 | rs := roll(offset, len) 41 | return rs.Digest() 42 | } 43 | 44 | sum1a := sum(0, len(buf)) 45 | sum1b := sum(1, len(buf)) 46 | sum2a := sum(len(buf)-windowSize*5/2, len(buf)-windowSize) 47 | sum2b := sum(0, len(buf)-windowSize) 48 | sum3a := sum(0, windowSize+3) 49 | sum3b := sum(3, windowSize+3) 50 | 51 | if sum1a != sum1b { 52 | t.Errorf("sum1a=%d sum1b=%d", sum1a, sum1b) 53 | } 54 | if sum2a != sum2b { 55 | t.Errorf("sum2a=%d sum2b=%d", sum2a, sum2b) 56 | } 57 | if sum3a != sum3b { 58 | t.Errorf("sum3a=%d sum3b=%d", sum3a, sum3b) 59 | } 60 | 61 | end := 500 62 | rs := roll(0, windowSize) 63 | for i := 0; i < end; i++ { 64 | sumRoll := rs.Digest() 65 | newRoll := roll(i, i+windowSize).Digest() 66 | 67 | if sumRoll != newRoll { 68 | t.Errorf("Error: i=%d, buf[i]=%d, sumRoll=%d, newRoll=%d\n", i, buf[i], sumRoll, newRoll) 69 | } 70 | 71 | rs.Roll(buf[i+windowSize]) 72 | } 73 | } 74 | 75 | func BenchmarkRollsum(b *testing.B) { 76 | const bufSize = 5 << 20 77 | buf := make([]byte, bufSize) 78 | for i := range buf { 79 | buf[i] = byte(rand.Int63()) 80 | } 81 | 82 | b.ResetTimer() 83 | rs := New() 84 | splits := 0 85 | for i := 0; i < b.N; i++ { 86 | splits = 0 87 | for _, b := range buf { 88 | rs.Roll(b) 89 | if rs.OnSplit() { 90 | _ = rs.Bits() 91 | splits++ 92 | } 93 | } 94 | } 95 | b.SetBytes(bufSize) 96 | b.Logf("num splits = %d; every %d bytes", splits, int(float64(bufSize)/float64(splits))) 97 | } 98 | -------------------------------------------------------------------------------- /sort/example_interface_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2011 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package sort_test 6 | 7 | import ( 8 | "fmt" 9 | 10 | "go4.org/sort" 11 | ) 12 | 13 | type Person struct { 14 | Name string 15 | Age int 16 | } 17 | 18 | func (p Person) String() string { 19 | return fmt.Sprintf("%s: %d", p.Name, p.Age) 20 | } 21 | 22 | // ByAge implements sort.Interface for []Person based on 23 | // the Age field. 24 | type ByAge []Person 25 | 26 | func (a ByAge) Len() int { return len(a) } 27 | func (a ByAge) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 28 | func (a ByAge) Less(i, j int) bool { return a[i].Age < a[j].Age } 29 | 30 | func ExampleSort() { 31 | people := []Person{ 32 | {"Bob", 31}, 33 | {"John", 42}, 34 | {"Michael", 17}, 35 | {"Jenny", 26}, 36 | } 37 | 38 | fmt.Println(people) 39 | sort.Sort(ByAge(people)) 40 | fmt.Println(people) 41 | 42 | // Output: 43 | // [Bob: 31 John: 42 Michael: 17 Jenny: 26] 44 | // [Michael: 17 Jenny: 26 Bob: 31 John: 42] 45 | } 46 | -------------------------------------------------------------------------------- /sort/example_keys_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package sort_test 6 | 7 | import ( 8 | "fmt" 9 | "sort" 10 | ) 11 | 12 | // A couple of type definitions to make the units clear. 13 | type earthMass float64 14 | type au float64 15 | 16 | // A Planet defines the properties of a solar system object. 17 | type Planet struct { 18 | name string 19 | mass earthMass 20 | distance au 21 | } 22 | 23 | // By is the type of a "less" function that defines the ordering of its Planet arguments. 24 | type By func(p1, p2 *Planet) bool 25 | 26 | // Sort is a method on the function type, By, that sorts the argument slice according to the function. 27 | func (by By) Sort(planets []Planet) { 28 | ps := &planetSorter{ 29 | planets: planets, 30 | by: by, // The Sort method's receiver is the function (closure) that defines the sort order. 31 | } 32 | sort.Sort(ps) 33 | } 34 | 35 | // planetSorter joins a By function and a slice of Planets to be sorted. 36 | type planetSorter struct { 37 | planets []Planet 38 | by func(p1, p2 *Planet) bool // Closure used in the Less method. 39 | } 40 | 41 | // Len is part of sort.Interface. 42 | func (s *planetSorter) Len() int { 43 | return len(s.planets) 44 | } 45 | 46 | // Swap is part of sort.Interface. 47 | func (s *planetSorter) Swap(i, j int) { 48 | s.planets[i], s.planets[j] = s.planets[j], s.planets[i] 49 | } 50 | 51 | // Less is part of sort.Interface. It is implemented by calling the "by" closure in the sorter. 52 | func (s *planetSorter) Less(i, j int) bool { 53 | return s.by(&s.planets[i], &s.planets[j]) 54 | } 55 | 56 | var planets = []Planet{ 57 | {"Mercury", 0.055, 0.4}, 58 | {"Venus", 0.815, 0.7}, 59 | {"Earth", 1.0, 1.0}, 60 | {"Mars", 0.107, 1.5}, 61 | } 62 | 63 | // ExampleSortKeys demonstrates a technique for sorting a struct type using programmable sort criteria. 64 | func Example_sortKeys() { 65 | // Closures that order the Planet structure. 66 | name := func(p1, p2 *Planet) bool { 67 | return p1.name < p2.name 68 | } 69 | mass := func(p1, p2 *Planet) bool { 70 | return p1.mass < p2.mass 71 | } 72 | distance := func(p1, p2 *Planet) bool { 73 | return p1.distance < p2.distance 74 | } 75 | decreasingDistance := func(p1, p2 *Planet) bool { 76 | return !distance(p1, p2) 77 | } 78 | 79 | // Sort the planets by the various criteria. 80 | By(name).Sort(planets) 81 | fmt.Println("By name:", planets) 82 | 83 | By(mass).Sort(planets) 84 | fmt.Println("By mass:", planets) 85 | 86 | By(distance).Sort(planets) 87 | fmt.Println("By distance:", planets) 88 | 89 | By(decreasingDistance).Sort(planets) 90 | fmt.Println("By decreasing distance:", planets) 91 | 92 | // Output: By name: [{Earth 1 1} {Mars 0.107 1.5} {Mercury 0.055 0.4} {Venus 0.815 0.7}] 93 | // By mass: [{Mercury 0.055 0.4} {Mars 0.107 1.5} {Venus 0.815 0.7} {Earth 1 1}] 94 | // By distance: [{Mercury 0.055 0.4} {Venus 0.815 0.7} {Earth 1 1} {Mars 0.107 1.5}] 95 | // By decreasing distance: [{Mars 0.107 1.5} {Earth 1 1} {Venus 0.815 0.7} {Mercury 0.055 0.4}] 96 | } 97 | -------------------------------------------------------------------------------- /sort/example_multi_test.go: -------------------------------------------------------------------------------- 1 | // +build go1.6 2 | 3 | // Copyright 2013 The Go Authors. All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | package sort_test 8 | 9 | import ( 10 | "fmt" 11 | "sort" 12 | ) 13 | 14 | // A Change is a record of source code changes, recording user, language, and delta size. 15 | type Change struct { 16 | user string 17 | language string 18 | lines int 19 | } 20 | 21 | type lessFunc func(p1, p2 *Change) bool 22 | 23 | // multiSorter implements the Sort interface, sorting the changes within. 24 | type multiSorter struct { 25 | changes []Change 26 | less []lessFunc 27 | } 28 | 29 | // Sort sorts the argument slice according to the less functions passed to OrderedBy. 30 | func (ms *multiSorter) Sort(changes []Change) { 31 | ms.changes = changes 32 | sort.Sort(ms) 33 | } 34 | 35 | // OrderedBy returns a Sorter that sorts using the less functions, in order. 36 | // Call its Sort method to sort the data. 37 | func OrderedBy(less ...lessFunc) *multiSorter { 38 | return &multiSorter{ 39 | less: less, 40 | } 41 | } 42 | 43 | // Len is part of sort.Interface. 44 | func (ms *multiSorter) Len() int { 45 | return len(ms.changes) 46 | } 47 | 48 | // Swap is part of sort.Interface. 49 | func (ms *multiSorter) Swap(i, j int) { 50 | ms.changes[i], ms.changes[j] = ms.changes[j], ms.changes[i] 51 | } 52 | 53 | // Less is part of sort.Interface. It is implemented by looping along the 54 | // less functions until it finds a comparison that is either Less or 55 | // !Less. Note that it can call the less functions twice per call. We 56 | // could change the functions to return -1, 0, 1 and reduce the 57 | // number of calls for greater efficiency: an exercise for the reader. 58 | func (ms *multiSorter) Less(i, j int) bool { 59 | p, q := &ms.changes[i], &ms.changes[j] 60 | // Try all but the last comparison. 61 | var k int 62 | for k = 0; k < len(ms.less)-1; k++ { 63 | less := ms.less[k] 64 | switch { 65 | case less(p, q): 66 | // p < q, so we have a decision. 67 | return true 68 | case less(q, p): 69 | // p > q, so we have a decision. 70 | return false 71 | } 72 | // p == q; try the next comparison. 73 | } 74 | // All comparisons to here said "equal", so just return whatever 75 | // the final comparison reports. 76 | return ms.less[k](p, q) 77 | } 78 | 79 | var changes = []Change{ 80 | {"gri", "Go", 100}, 81 | {"ken", "C", 150}, 82 | {"glenda", "Go", 200}, 83 | {"rsc", "Go", 200}, 84 | {"r", "Go", 100}, 85 | {"ken", "Go", 200}, 86 | {"dmr", "C", 100}, 87 | {"r", "C", 150}, 88 | {"gri", "Smalltalk", 80}, 89 | } 90 | 91 | // ExampleMultiKeys demonstrates a technique for sorting a struct type using different 92 | // sets of multiple fields in the comparison. We chain together "Less" functions, each of 93 | // which compares a single field. 94 | func Example_sortMultiKeys() { 95 | // Closures that order the Change structure. 96 | user := func(c1, c2 *Change) bool { 97 | return c1.user < c2.user 98 | } 99 | language := func(c1, c2 *Change) bool { 100 | return c1.language < c2.language 101 | } 102 | increasingLines := func(c1, c2 *Change) bool { 103 | return c1.lines < c2.lines 104 | } 105 | decreasingLines := func(c1, c2 *Change) bool { 106 | return c1.lines > c2.lines // Note: > orders downwards. 107 | } 108 | 109 | // Simple use: Sort by user. 110 | OrderedBy(user).Sort(changes) 111 | fmt.Println("By user:", changes) 112 | 113 | // More examples. 114 | OrderedBy(user, increasingLines).Sort(changes) 115 | fmt.Println("By user,lines:", changes) 119 | 120 | OrderedBy(language, increasingLines).Sort(changes) 121 | fmt.Println("By language,lines: [{dmr C 100} {glenda Go 200} {gri Go 100} {gri Smalltalk 80} {ken Go 200} {ken C 150} {r C 150} {r Go 100} {rsc Go 200}] 130 | // By language,= 23 }) 27 | // returns the smallest index i such that data[i] >= 23. If the caller 28 | // wants to find whether 23 is in the slice, it must test data[i] == 23 29 | // separately. 30 | // 31 | // Searching data sorted in descending order would use the <= 32 | // operator instead of the >= operator. 33 | // 34 | // To complete the example above, the following code tries to find the value 35 | // x in an integer slice data sorted in ascending order: 36 | // 37 | // x := 23 38 | // i := sort.Search(len(data), func(i int) bool { return data[i] >= x }) 39 | // if i < len(data) && data[i] == x { 40 | // // x is present at data[i] 41 | // } else { 42 | // // x is not present in data, 43 | // // but i is the index where it would be inserted. 44 | // } 45 | // 46 | // As a more whimsical example, this program guesses your number: 47 | // 48 | // func GuessingGame() { 49 | // var s string 50 | // fmt.Printf("Pick an integer from 0 to 100.\n") 51 | // answer := sort.Search(100, func(i int) bool { 52 | // fmt.Printf("Is your number <= %d? ", i) 53 | // fmt.Scanf("%s", &s) 54 | // return s != "" && s[0] == 'y' 55 | // }) 56 | // fmt.Printf("Your number is %d.\n", answer) 57 | // } 58 | // 59 | func Search(n int, f func(int) bool) int { 60 | // Define f(-1) == false and f(n) == true. 61 | // Invariant: f(i-1) == false, f(j) == true. 62 | i, j := 0, n 63 | for i < j { 64 | h := i + (j-i)/2 // avoid overflow when computing h 65 | // i ≤ h < j 66 | if !f(h) { 67 | i = h + 1 // preserves f(i-1) == false 68 | } else { 69 | j = h // preserves f(j) == true 70 | } 71 | } 72 | // i == j, f(i-1) == false, and f(j) (= f(i)) == true => answer is i. 73 | return i 74 | } 75 | 76 | // Convenience wrappers for common cases. 77 | 78 | // SearchInts searches for x in a sorted slice of ints and returns the index 79 | // as specified by Search. The return value is the index to insert x if x is 80 | // not present (it could be len(a)). 81 | // The slice must be sorted in ascending order. 82 | // 83 | func SearchInts(a []int, x int) int { 84 | return Search(len(a), func(i int) bool { return a[i] >= x }) 85 | } 86 | 87 | // SearchFloat64s searches for x in a sorted slice of float64s and returns the index 88 | // as specified by Search. The return value is the index to insert x if x is not 89 | // present (it could be len(a)). 90 | // The slice must be sorted in ascending order. 91 | // 92 | func SearchFloat64s(a []float64, x float64) int { 93 | return Search(len(a), func(i int) bool { return a[i] >= x }) 94 | } 95 | 96 | // SearchStrings searches for x in a sorted slice of strings and returns the index 97 | // as specified by Search. The return value is the index to insert x if x is not 98 | // present (it could be len(a)). 99 | // The slice must be sorted in ascending order. 100 | // 101 | func SearchStrings(a []string, x string) int { 102 | return Search(len(a), func(i int) bool { return a[i] >= x }) 103 | } 104 | 105 | // Search returns the result of applying SearchInts to the receiver and x. 106 | func (p IntSlice) Search(x int) int { return SearchInts(p, x) } 107 | 108 | // Search returns the result of applying SearchFloat64s to the receiver and x. 109 | func (p Float64Slice) Search(x float64) int { return SearchFloat64s(p, x) } 110 | 111 | // Search returns the result of applying SearchStrings to the receiver and x. 112 | func (p StringSlice) Search(x string) int { return SearchStrings(p, x) } 113 | -------------------------------------------------------------------------------- /sort/search_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2010 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package sort_test 6 | 7 | import ( 8 | "runtime" 9 | . "sort" 10 | "testing" 11 | ) 12 | 13 | func f(a []int, x int) func(int) bool { 14 | return func(i int) bool { 15 | return a[i] >= x 16 | } 17 | } 18 | 19 | var data = []int{0: -10, 1: -5, 2: 0, 3: 1, 4: 2, 5: 3, 6: 5, 7: 7, 8: 11, 9: 100, 10: 100, 11: 100, 12: 1000, 13: 10000} 20 | 21 | var tests = []struct { 22 | name string 23 | n int 24 | f func(int) bool 25 | i int 26 | }{ 27 | {"empty", 0, nil, 0}, 28 | {"1 1", 1, func(i int) bool { return i >= 1 }, 1}, 29 | {"1 true", 1, func(i int) bool { return true }, 0}, 30 | {"1 false", 1, func(i int) bool { return false }, 1}, 31 | {"1e9 991", 1e9, func(i int) bool { return i >= 991 }, 991}, 32 | {"1e9 true", 1e9, func(i int) bool { return true }, 0}, 33 | {"1e9 false", 1e9, func(i int) bool { return false }, 1e9}, 34 | {"data -20", len(data), f(data, -20), 0}, 35 | {"data -10", len(data), f(data, -10), 0}, 36 | {"data -9", len(data), f(data, -9), 1}, 37 | {"data -6", len(data), f(data, -6), 1}, 38 | {"data -5", len(data), f(data, -5), 1}, 39 | {"data 3", len(data), f(data, 3), 5}, 40 | {"data 11", len(data), f(data, 11), 8}, 41 | {"data 99", len(data), f(data, 99), 9}, 42 | {"data 100", len(data), f(data, 100), 9}, 43 | {"data 101", len(data), f(data, 101), 12}, 44 | {"data 10000", len(data), f(data, 10000), 13}, 45 | {"data 10001", len(data), f(data, 10001), 14}, 46 | {"descending a", 7, func(i int) bool { return []int{99, 99, 59, 42, 7, 0, -1, -1}[i] <= 7 }, 4}, 47 | {"descending 7", 1e9, func(i int) bool { return 1e9-i <= 7 }, 1e9 - 7}, 48 | {"overflow", 2e9, func(i int) bool { return false }, 2e9}, 49 | } 50 | 51 | func TestSearch(t *testing.T) { 52 | for _, e := range tests { 53 | i := Search(e.n, e.f) 54 | if i != e.i { 55 | t.Errorf("%s: expected index %d; got %d", e.name, e.i, i) 56 | } 57 | } 58 | } 59 | 60 | // log2 computes the binary logarithm of x, rounded up to the next integer. 61 | // (log2(0) == 0, log2(1) == 0, log2(2) == 1, log2(3) == 2, etc.) 62 | // 63 | func log2(x int) int { 64 | n := 0 65 | for p := 1; p < x; p += p { 66 | // p == 2**n 67 | n++ 68 | } 69 | // p/2 < x <= p == 2**n 70 | return n 71 | } 72 | 73 | func TestSearchEfficiency(t *testing.T) { 74 | n := 100 75 | step := 1 76 | for exp := 2; exp < 10; exp++ { 77 | // n == 10**exp 78 | // step == 10**(exp-2) 79 | max := log2(n) 80 | for x := 0; x < n; x += step { 81 | count := 0 82 | i := Search(n, func(i int) bool { count++; return i >= x }) 83 | if i != x { 84 | t.Errorf("n = %d: expected index %d; got %d", n, x, i) 85 | } 86 | if count > max { 87 | t.Errorf("n = %d, x = %d: expected <= %d calls; got %d", n, x, max, count) 88 | } 89 | } 90 | n *= 10 91 | step *= 10 92 | } 93 | } 94 | 95 | // Smoke tests for convenience wrappers - not comprehensive. 96 | 97 | var fdata = []float64{0: -3.14, 1: 0, 2: 1, 3: 2, 4: 1000.7} 98 | var sdata = []string{0: "f", 1: "foo", 2: "foobar", 3: "x"} 99 | 100 | var wrappertests = []struct { 101 | name string 102 | result int 103 | i int 104 | }{ 105 | {"SearchInts", SearchInts(data, 11), 8}, 106 | {"SearchFloat64s", SearchFloat64s(fdata, 2.1), 4}, 107 | {"SearchStrings", SearchStrings(sdata, ""), 0}, 108 | {"IntSlice.Search", IntSlice(data).Search(0), 2}, 109 | {"Float64Slice.Search", Float64Slice(fdata).Search(2.0), 3}, 110 | {"StringSlice.Search", StringSlice(sdata).Search("x"), 3}, 111 | } 112 | 113 | func TestSearchWrappers(t *testing.T) { 114 | for _, e := range wrappertests { 115 | if e.result != e.i { 116 | t.Errorf("%s: expected index %d; got %d", e.name, e.i, e.result) 117 | } 118 | } 119 | } 120 | 121 | func runSearchWrappers() { 122 | SearchInts(data, 11) 123 | SearchFloat64s(fdata, 2.1) 124 | SearchStrings(sdata, "") 125 | IntSlice(data).Search(0) 126 | Float64Slice(fdata).Search(2.0) 127 | StringSlice(sdata).Search("x") 128 | } 129 | 130 | func TestSearchWrappersDontAlloc(t *testing.T) { 131 | if testing.Short() { 132 | t.Skip("skipping malloc count in short mode") 133 | } 134 | if runtime.GOMAXPROCS(0) > 1 { 135 | t.Skip("skipping; GOMAXPROCS>1") 136 | } 137 | allocs := testing.AllocsPerRun(100, runSearchWrappers) 138 | if allocs != 0 { 139 | t.Errorf("expected no allocs for runSearchWrappers, got %v", allocs) 140 | } 141 | } 142 | 143 | func BenchmarkSearchWrappers(b *testing.B) { 144 | for i := 0; i < b.N; i++ { 145 | runSearchWrappers() 146 | } 147 | } 148 | 149 | // Abstract exhaustive test: all sizes up to 100, 150 | // all possible return values. If there are any small 151 | // corner cases, this test exercises them. 152 | func TestSearchExhaustive(t *testing.T) { 153 | for size := 0; size <= 100; size++ { 154 | for targ := 0; targ <= size; targ++ { 155 | i := Search(size, func(i int) bool { return i >= targ }) 156 | if i != targ { 157 | t.Errorf("Search(%d, %d) = %d", size, targ, i) 158 | } 159 | } 160 | } 161 | } 162 | -------------------------------------------------------------------------------- /sort/zfuncversion.go: -------------------------------------------------------------------------------- 1 | // DO NOT EDIT; AUTO-GENERATED from sort.go using genzfunc.go 2 | 3 | // Copyright 2016 The Go Authors. All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | package sort 8 | 9 | // Auto-generated variant of sort.go:insertionSort 10 | func insertionSort_func(data lessSwap, a, b int) { 11 | for i := a + 1; i < b; i++ { 12 | for j := i; j > a && data.Less(j, j-1); j-- { 13 | data.Swap(j, j-1) 14 | } 15 | } 16 | } 17 | 18 | // Auto-generated variant of sort.go:siftDown 19 | func siftDown_func(data lessSwap, lo, hi, first int) { 20 | root := lo 21 | for { 22 | child := 2*root + 1 23 | if child >= hi { 24 | break 25 | } 26 | if child+1 < hi && data.Less(first+child, first+child+1) { 27 | child++ 28 | } 29 | if !data.Less(first+root, first+child) { 30 | return 31 | } 32 | data.Swap(first+root, first+child) 33 | root = child 34 | } 35 | } 36 | 37 | // Auto-generated variant of sort.go:heapSort 38 | func heapSort_func(data lessSwap, a, b int) { 39 | first := a 40 | lo := 0 41 | hi := b - a 42 | for i := (hi - 1) / 2; i >= 0; i-- { 43 | siftDown_func(data, i, hi, first) 44 | } 45 | for i := hi - 1; i >= 0; i-- { 46 | data.Swap(first, first+i) 47 | siftDown_func(data, lo, i, first) 48 | } 49 | } 50 | 51 | // Auto-generated variant of sort.go:medianOfThree 52 | func medianOfThree_func(data lessSwap, m1, m0, m2 int) { 53 | if data.Less(m1, m0) { 54 | data.Swap(m1, m0) 55 | } 56 | if data.Less(m2, m1) { 57 | data.Swap(m2, m1) 58 | if data.Less(m1, m0) { 59 | data.Swap(m1, m0) 60 | } 61 | } 62 | } 63 | 64 | // Auto-generated variant of sort.go:swapRange 65 | func swapRange_func(data lessSwap, a, b, n int) { 66 | for i := 0; i < n; i++ { 67 | data.Swap(a+i, b+i) 68 | } 69 | } 70 | 71 | // Auto-generated variant of sort.go:doPivot 72 | func doPivot_func(data lessSwap, lo, hi int) (midlo, midhi int) { 73 | m := lo + (hi-lo)/2 74 | if hi-lo > 40 { 75 | s := (hi - lo) / 8 76 | medianOfThree_func(data, lo, lo+s, lo+2*s) 77 | medianOfThree_func(data, m, m-s, m+s) 78 | medianOfThree_func(data, hi-1, hi-1-s, hi-1-2*s) 79 | } 80 | medianOfThree_func(data, lo, m, hi-1) 81 | pivot := lo 82 | a, c := lo+1, hi-1 83 | for ; a < c && data.Less(a, pivot); a++ { 84 | } 85 | b := a 86 | for { 87 | for ; b < c && !data.Less(pivot, b); b++ { 88 | } 89 | for ; b < c && data.Less(pivot, c-1); c-- { 90 | } 91 | if b >= c { 92 | break 93 | } 94 | data.Swap(b, c-1) 95 | b++ 96 | c-- 97 | } 98 | protect := hi-c < 5 99 | if !protect && hi-c < (hi-lo)/4 { 100 | dups := 0 101 | if !data.Less(pivot, hi-1) { 102 | data.Swap(c, hi-1) 103 | c++ 104 | dups++ 105 | } 106 | if !data.Less(b-1, pivot) { 107 | b-- 108 | dups++ 109 | } 110 | if !data.Less(m, pivot) { 111 | data.Swap(m, b-1) 112 | b-- 113 | dups++ 114 | } 115 | protect = dups > 1 116 | } 117 | if protect { 118 | for { 119 | for ; a < b && !data.Less(b-1, pivot); b-- { 120 | } 121 | for ; a < b && data.Less(a, pivot); a++ { 122 | } 123 | if a >= b { 124 | break 125 | } 126 | data.Swap(a, b-1) 127 | a++ 128 | b-- 129 | } 130 | } 131 | data.Swap(pivot, b-1) 132 | return b - 1, c 133 | } 134 | 135 | // Auto-generated variant of sort.go:quickSort 136 | func quickSort_func(data lessSwap, a, b, maxDepth int) { 137 | for b-a > 12 { 138 | if maxDepth == 0 { 139 | heapSort_func(data, a, b) 140 | return 141 | } 142 | maxDepth-- 143 | mlo, mhi := doPivot_func(data, a, b) 144 | if mlo-a < b-mhi { 145 | quickSort_func(data, a, mlo, maxDepth) 146 | a = mhi 147 | } else { 148 | quickSort_func(data, mhi, b, maxDepth) 149 | b = mlo 150 | } 151 | } 152 | if b-a > 1 { 153 | for i := a + 6; i < b; i++ { 154 | if data.Less(i, i-6) { 155 | data.Swap(i, i-6) 156 | } 157 | } 158 | insertionSort_func(data, a, b) 159 | } 160 | } 161 | 162 | // Auto-generated variant of sort.go:stable 163 | func stable_func(data lessSwap, n int) { 164 | blockSize := 20 165 | a, b := 0, blockSize 166 | for b <= n { 167 | insertionSort_func(data, a, b) 168 | a = b 169 | b += blockSize 170 | } 171 | insertionSort_func(data, a, n) 172 | for blockSize < n { 173 | a, b = 0, 2*blockSize 174 | for b <= n { 175 | symMerge_func(data, a, a+blockSize, b) 176 | a = b 177 | b += 2 * blockSize 178 | } 179 | if m := a + blockSize; m < n { 180 | symMerge_func(data, a, m, n) 181 | } 182 | blockSize *= 2 183 | } 184 | } 185 | 186 | // Auto-generated variant of sort.go:symMerge 187 | func symMerge_func(data lessSwap, a, m, b int) { 188 | if m-a == 1 { 189 | i := m 190 | j := b 191 | for i < j { 192 | h := i + (j-i)/2 193 | if data.Less(h, a) { 194 | i = h + 1 195 | } else { 196 | j = h 197 | } 198 | } 199 | for k := a; k < i-1; k++ { 200 | data.Swap(k, k+1) 201 | } 202 | return 203 | } 204 | if b-m == 1 { 205 | i := a 206 | j := m 207 | for i < j { 208 | h := i + (j-i)/2 209 | if !data.Less(m, h) { 210 | i = h + 1 211 | } else { 212 | j = h 213 | } 214 | } 215 | for k := m; k > i; k-- { 216 | data.Swap(k, k-1) 217 | } 218 | return 219 | } 220 | mid := a + (b-a)/2 221 | n := mid + m 222 | var start, r int 223 | if m > mid { 224 | start = n - b 225 | r = mid 226 | } else { 227 | start = a 228 | r = m 229 | } 230 | p := n - 1 231 | for start < r { 232 | c := start + (r-start)/2 233 | if !data.Less(p-c, c) { 234 | start = c + 1 235 | } else { 236 | r = c 237 | } 238 | } 239 | end := n - start 240 | if start < m && m < end { 241 | rotate_func(data, start, m, end) 242 | } 243 | if a < start && start < mid { 244 | symMerge_func(data, a, start, mid) 245 | } 246 | if mid < end && end < b { 247 | symMerge_func(data, mid, end, b) 248 | } 249 | } 250 | 251 | // Auto-generated variant of sort.go:rotate 252 | func rotate_func(data lessSwap, a, m, b int) { 253 | i := m - a 254 | j := b - m 255 | for i != j { 256 | if i > j { 257 | swapRange_func(data, m-i, m, j) 258 | i -= j 259 | } else { 260 | swapRange_func(data, m-i, m+j-i, i) 261 | j -= i 262 | } 263 | } 264 | swapRange_func(data, m-i, m, i) 265 | } 266 | -------------------------------------------------------------------------------- /strutil/intern.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2013 The Perkeep Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package strutil 18 | 19 | var internStr = map[string]string{} 20 | 21 | // RegisterCommonString adds common strings to the interned string 22 | // table. This should be called during init from the main 23 | // goroutine, not later at runtime. 24 | func RegisterCommonString(s ...string) { 25 | for _, v := range s { 26 | internStr[v] = v 27 | } 28 | } 29 | 30 | // StringFromBytes returns string(v), minimizing copies for common values of v 31 | // as previously registered with RegisterCommonString. 32 | func StringFromBytes(v []byte) string { 33 | // In Go 1.3, this string conversion in the map lookup does not allocate 34 | // to make a new string. We depend on Go 1.3, so this is always free: 35 | if s, ok := internStr[string(v)]; ok { 36 | return s 37 | } 38 | return string(v) 39 | } 40 | -------------------------------------------------------------------------------- /strutil/strconv.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2013 The Perkeep Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package strutil 18 | 19 | import ( 20 | "errors" 21 | "strconv" 22 | ) 23 | 24 | // ParseUintBytes is like strconv.ParseUint, but using a []byte. 25 | func ParseUintBytes(s []byte, base int, bitSize int) (n uint64, err error) { 26 | var cutoff, maxVal uint64 27 | 28 | if bitSize == 0 { 29 | bitSize = int(strconv.IntSize) 30 | } 31 | 32 | s0 := s 33 | switch { 34 | case len(s) < 1: 35 | err = strconv.ErrSyntax 36 | goto Error 37 | 38 | case 2 <= base && base <= 36: 39 | // valid base; nothing to do 40 | 41 | case base == 0: 42 | // Look for octal, hex prefix. 43 | switch { 44 | case s[0] == '0' && len(s) > 1 && (s[1] == 'x' || s[1] == 'X'): 45 | base = 16 46 | s = s[2:] 47 | if len(s) < 1 { 48 | err = strconv.ErrSyntax 49 | goto Error 50 | } 51 | case s[0] == '0': 52 | base = 8 53 | default: 54 | base = 10 55 | } 56 | 57 | default: 58 | err = errors.New("invalid base " + strconv.Itoa(base)) 59 | goto Error 60 | } 61 | 62 | n = 0 63 | cutoff = cutoff64(base) 64 | maxVal = 1<= base { 82 | n = 0 83 | err = strconv.ErrSyntax 84 | goto Error 85 | } 86 | 87 | if n >= cutoff { 88 | // n*base overflows 89 | n = 1<<64 - 1 90 | err = strconv.ErrRange 91 | goto Error 92 | } 93 | n *= uint64(base) 94 | 95 | n1 := n + uint64(v) 96 | if n1 < n || n1 > maxVal { 97 | // n+v overflows 98 | n = 1<<64 - 1 99 | err = strconv.ErrRange 100 | goto Error 101 | } 102 | n = n1 103 | } 104 | 105 | return n, nil 106 | 107 | Error: 108 | return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err} 109 | } 110 | 111 | // Return the first number n such that n*base >= 1<<64. 112 | func cutoff64(base int) uint64 { 113 | if base < 2 { 114 | return 0 115 | } 116 | return (1<<64-1)/uint64(base) + 1 117 | } 118 | -------------------------------------------------------------------------------- /strutil/strutil.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2013 The Perkeep Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package strutil contains string and byte processing functions. 18 | package strutil // import "go4.org/strutil" 19 | 20 | import ( 21 | "strings" 22 | "unicode" 23 | "unicode/utf8" 24 | ) 25 | 26 | // Fork of Go's implementation in pkg/strings/strings.go: 27 | // Generic split: splits after each instance of sep, 28 | // including sepSave bytes of sep in the subarrays. 29 | func genSplit(dst []string, s, sep string, sepSave, n int) []string { 30 | if n == 0 { 31 | return nil 32 | } 33 | if sep == "" { 34 | panic("sep is empty") 35 | } 36 | if n < 0 { 37 | n = strings.Count(s, sep) + 1 38 | } 39 | c := sep[0] 40 | start := 0 41 | na := 0 42 | for i := 0; i+len(sep) <= len(s) && na+1 < n; i++ { 43 | if s[i] == c && (len(sep) == 1 || s[i:i+len(sep)] == sep) { 44 | dst = append(dst, s[start:i+sepSave]) 45 | na++ 46 | start = i + len(sep) 47 | i += len(sep) - 1 48 | } 49 | } 50 | dst = append(dst, s[start:]) 51 | return dst 52 | } 53 | 54 | // AppendSplitN is like strings.SplitN but appends to and returns dst. 55 | // Unlike strings.SplitN, an empty separator is not supported. 56 | // The count n determines the number of substrings to return: 57 | // n > 0: at most n substrings; the last substring will be the unsplit remainder. 58 | // n == 0: the result is nil (zero substrings) 59 | // n < 0: all substrings 60 | func AppendSplitN(dst []string, s, sep string, n int) []string { 61 | return genSplit(dst, s, sep, 0, n) 62 | } 63 | 64 | // equalFoldRune compares a and b runes whether they fold equally. 65 | // 66 | // The code comes from strings.EqualFold, but shortened to only one rune. 67 | func equalFoldRune(sr, tr rune) bool { 68 | if sr == tr { 69 | return true 70 | } 71 | // Make sr < tr to simplify what follows. 72 | if tr < sr { 73 | sr, tr = tr, sr 74 | } 75 | // Fast check for ASCII. 76 | if tr < utf8.RuneSelf && 'A' <= sr && sr <= 'Z' { 77 | // ASCII, and sr is upper case. tr must be lower case. 78 | if tr == sr+'a'-'A' { 79 | return true 80 | } 81 | return false 82 | } 83 | 84 | // General case. SimpleFold(x) returns the next equivalent rune > x 85 | // or wraps around to smaller values. 86 | r := unicode.SimpleFold(sr) 87 | for r != sr && r < tr { 88 | r = unicode.SimpleFold(r) 89 | } 90 | if r == tr { 91 | return true 92 | } 93 | return false 94 | } 95 | 96 | // HasPrefixFold is like strings.HasPrefix but uses Unicode case-folding, 97 | // matching case insensitively. 98 | func HasPrefixFold(s, prefix string) bool { 99 | if prefix == "" { 100 | return true 101 | } 102 | for _, pr := range prefix { 103 | if s == "" { 104 | return false 105 | } 106 | // step with s, too 107 | sr, size := utf8.DecodeRuneInString(s) 108 | if sr == utf8.RuneError { 109 | return false 110 | } 111 | s = s[size:] 112 | if !equalFoldRune(sr, pr) { 113 | return false 114 | } 115 | } 116 | return true 117 | } 118 | 119 | // HasSuffixFold is like strings.HasSuffix but uses Unicode case-folding, 120 | // matching case insensitively. 121 | func HasSuffixFold(s, suffix string) bool { 122 | if suffix == "" { 123 | return true 124 | } 125 | // count the runes and bytes in s, but only till rune count of suffix 126 | bo, so := len(s), len(suffix) 127 | for bo > 0 && so > 0 { 128 | r, size := utf8.DecodeLastRuneInString(s[:bo]) 129 | if r == utf8.RuneError { 130 | return false 131 | } 132 | bo -= size 133 | 134 | sr, size := utf8.DecodeLastRuneInString(suffix[:so]) 135 | if sr == utf8.RuneError { 136 | return false 137 | } 138 | so -= size 139 | 140 | if !equalFoldRune(r, sr) { 141 | return false 142 | } 143 | } 144 | return so == 0 145 | } 146 | 147 | // ContainsFold is like strings.Contains but uses Unicode case-folding. 148 | func ContainsFold(s, substr string) bool { 149 | if substr == "" { 150 | return true 151 | } 152 | if s == "" { 153 | return false 154 | } 155 | firstRune := rune(substr[0]) 156 | if firstRune >= utf8.RuneSelf { 157 | firstRune, _ = utf8.DecodeRuneInString(substr) 158 | } 159 | for i, rune := range s { 160 | if equalFoldRune(rune, firstRune) && HasPrefixFold(s[i:], substr) { 161 | return true 162 | } 163 | } 164 | return false 165 | } 166 | 167 | // IsPlausibleJSON reports whether s likely contains a JSON object, without 168 | // actually parsing it. It's meant to be a light heuristic. 169 | func IsPlausibleJSON(s string) bool { 170 | return startsWithOpenBrace(s) && endsWithCloseBrace(s) 171 | } 172 | 173 | func isASCIIWhite(b byte) bool { return b == ' ' || b == '\n' || b == '\r' || b == '\t' } 174 | 175 | func startsWithOpenBrace(s string) bool { 176 | for len(s) > 0 { 177 | switch { 178 | case s[0] == '{': 179 | return true 180 | case isASCIIWhite(s[0]): 181 | s = s[1:] 182 | default: 183 | return false 184 | } 185 | } 186 | return false 187 | } 188 | 189 | func endsWithCloseBrace(s string) bool { 190 | for len(s) > 0 { 191 | last := len(s) - 1 192 | switch { 193 | case s[last] == '}': 194 | return true 195 | case isASCIIWhite(s[last]): 196 | s = s[:last] 197 | default: 198 | return false 199 | } 200 | } 201 | return false 202 | } 203 | -------------------------------------------------------------------------------- /syncutil/gate.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2013 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package syncutil 18 | 19 | // A Gate limits concurrency. 20 | type Gate struct { 21 | c chan struct{} 22 | } 23 | 24 | // NewGate returns a new gate that will only permit max operations at once. 25 | func NewGate(max int) *Gate { 26 | return &Gate{make(chan struct{}, max)} 27 | } 28 | 29 | // Start starts an operation, blocking until the gate has room. 30 | func (g *Gate) Start() { 31 | g.c <- struct{}{} 32 | } 33 | 34 | // Done finishes an operation. 35 | func (g *Gate) Done() { 36 | select { 37 | case <-g.c: 38 | default: 39 | panic("Done called more than Start") 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /syncutil/group.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2013 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package syncutil 18 | 19 | import "sync" 20 | 21 | // A Group is like a sync.WaitGroup and coordinates doing 22 | // multiple things at once. Its zero value is ready to use. 23 | type Group struct { 24 | wg sync.WaitGroup 25 | mu sync.Mutex // guards errs 26 | errs []error 27 | } 28 | 29 | // Go runs fn in its own goroutine, but does not wait for it to complete. 30 | // Call Err or Errs to wait for all the goroutines to complete. 31 | func (g *Group) Go(fn func() error) { 32 | g.wg.Add(1) 33 | go func() { 34 | defer g.wg.Done() 35 | err := fn() 36 | if err != nil { 37 | g.mu.Lock() 38 | defer g.mu.Unlock() 39 | g.errs = append(g.errs, err) 40 | } 41 | }() 42 | } 43 | 44 | // Wait waits for all the previous calls to Go to complete. 45 | func (g *Group) Wait() { 46 | g.wg.Wait() 47 | } 48 | 49 | // Err waits for all previous calls to Go to complete and returns the 50 | // first non-nil error, or nil. 51 | func (g *Group) Err() error { 52 | g.wg.Wait() 53 | if len(g.errs) > 0 { 54 | return g.errs[0] 55 | } 56 | return nil 57 | } 58 | 59 | // Errs waits for all previous calls to Go to complete and returns 60 | // all non-nil errors. 61 | func (g *Group) Errs() []error { 62 | g.wg.Wait() 63 | return g.errs 64 | } 65 | -------------------------------------------------------------------------------- /syncutil/once.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014 The Perkeep Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package syncutil 18 | 19 | import ( 20 | "sync" 21 | "sync/atomic" 22 | ) 23 | 24 | // A Once will perform a successful action exactly once. 25 | // 26 | // Unlike a sync.Once, this Once's func returns an error 27 | // and is re-armed on failure. 28 | type Once struct { 29 | m sync.Mutex 30 | done uint32 31 | } 32 | 33 | // Do calls the function f if and only if Do has not been invoked 34 | // without error for this instance of Once. In other words, given 35 | // var once Once 36 | // if once.Do(f) is called multiple times, only the first call will 37 | // invoke f, even if f has a different value in each invocation unless 38 | // f returns an error. A new instance of Once is required for each 39 | // function to execute. 40 | // 41 | // Do is intended for initialization that must be run exactly once. Since f 42 | // is niladic, it may be necessary to use a function literal to capture the 43 | // arguments to a function to be invoked by Do: 44 | // err := config.once.Do(func() error { return config.init(filename) }) 45 | func (o *Once) Do(f func() error) error { 46 | if atomic.LoadUint32(&o.done) == 1 { 47 | return nil 48 | } 49 | // Slow-path. 50 | o.m.Lock() 51 | defer o.m.Unlock() 52 | var err error 53 | if o.done == 0 { 54 | err = f() 55 | if err == nil { 56 | atomic.StoreUint32(&o.done, 1) 57 | } 58 | } 59 | return err 60 | } 61 | -------------------------------------------------------------------------------- /syncutil/once_test.go: -------------------------------------------------------------------------------- 1 | package syncutil 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | ) 7 | 8 | func TestOnce(t *testing.T) { 9 | timesRan := 0 10 | f := func() error { 11 | timesRan++ 12 | return nil 13 | } 14 | 15 | once := Once{} 16 | grp := Group{} 17 | 18 | for i := 0; i < 10; i++ { 19 | grp.Go(func() error { return once.Do(f) }) 20 | } 21 | 22 | if grp.Err() != nil { 23 | t.Errorf("Expected no errors, got %v", grp.Err()) 24 | } 25 | 26 | if timesRan != 1 { 27 | t.Errorf("Expected to run one time, ran %d", timesRan) 28 | } 29 | } 30 | 31 | // TestOnceErroring verifies we retry on every error, but stop after 32 | // the first success. 33 | func TestOnceErroring(t *testing.T) { 34 | timesRan := 0 35 | f := func() error { 36 | timesRan++ 37 | if timesRan < 3 { 38 | return errors.New("retry") 39 | } 40 | return nil 41 | } 42 | 43 | once := Once{} 44 | grp := Group{} 45 | 46 | for i := 0; i < 10; i++ { 47 | grp.Go(func() error { return once.Do(f) }) 48 | } 49 | 50 | if len(grp.Errs()) != 2 { 51 | t.Errorf("Expected two errors, got %d", len(grp.Errs())) 52 | } 53 | 54 | if timesRan != 3 { 55 | t.Errorf("Expected to run two times, ran %d", timesRan) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /syncutil/sem.go: -------------------------------------------------------------------------------- 1 | package syncutil 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "sync" 7 | ) 8 | 9 | type debugT bool 10 | 11 | var debug = debugT(false) 12 | 13 | func (d debugT) Printf(format string, args ...interface{}) { 14 | if bool(d) { 15 | log.Printf(format, args...) 16 | } 17 | } 18 | 19 | // Sem implements a semaphore that can have multiple units acquired/released 20 | // at a time. 21 | type Sem struct { 22 | c *sync.Cond // Protects size 23 | max, free int64 24 | } 25 | 26 | // NewSem creates a semaphore with max units available for acquisition. 27 | func NewSem(max int64) *Sem { 28 | return &Sem{ 29 | c: sync.NewCond(new(sync.Mutex)), 30 | free: max, 31 | max: max, 32 | } 33 | } 34 | 35 | // Acquire will deduct n units from the semaphore. If the deduction would 36 | // result in the available units falling below zero, the call will block until 37 | // another go routine returns units via a call to Release. If more units are 38 | // requested than the semaphore is configured to hold, error will be non-nil. 39 | func (s *Sem) Acquire(n int64) error { 40 | if n > s.max { 41 | return fmt.Errorf("sem: attempt to acquire more units than semaphore size %d > %d", n, s.max) 42 | } 43 | s.c.L.Lock() 44 | defer s.c.L.Unlock() 45 | for { 46 | debug.Printf("Acquire check max %d free %d, n %d", s.max, s.free, n) 47 | if s.free >= n { 48 | s.free -= n 49 | return nil 50 | } 51 | debug.Printf("Acquire Wait max %d free %d, n %d", s.max, s.free, n) 52 | s.c.Wait() 53 | } 54 | } 55 | 56 | // Release will return n units to the semaphore and notify any currently 57 | // blocking Acquire calls. 58 | func (s *Sem) Release(n int64) { 59 | s.c.L.Lock() 60 | defer s.c.L.Unlock() 61 | debug.Printf("Release max %d free %d, n %d", s.max, s.free, n) 62 | s.free += n 63 | s.c.Broadcast() 64 | } 65 | -------------------------------------------------------------------------------- /syncutil/sem_test.go: -------------------------------------------------------------------------------- 1 | package syncutil_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "go4.org/syncutil" 7 | ) 8 | 9 | func TestSem(t *testing.T) { 10 | s := syncutil.NewSem(5) 11 | 12 | if err := s.Acquire(2); err != nil { 13 | t.Fatal(err) 14 | } 15 | if err := s.Acquire(2); err != nil { 16 | t.Fatal(err) 17 | } 18 | 19 | go func() { 20 | s.Release(2) 21 | s.Release(2) 22 | }() 23 | if err := s.Acquire(5); err != nil { 24 | t.Fatal(err) 25 | } 26 | } 27 | 28 | func TestSemErr(t *testing.T) { 29 | s := syncutil.NewSem(5) 30 | if err := s.Acquire(6); err == nil { 31 | t.Fatal("Didn't get expected error for large acquire.") 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /syncutil/singleflight/singleflight.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2013 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package singleflight provides a duplicate function call suppression 18 | // mechanism. 19 | package singleflight // import "go4.org/syncutil/singleflight" 20 | 21 | import "sync" 22 | 23 | // call is an in-flight or completed Do call 24 | type call struct { 25 | wg sync.WaitGroup 26 | val interface{} 27 | err error 28 | } 29 | 30 | // Group represents a class of work and forms a namespace in which 31 | // units of work can be executed with duplicate suppression. 32 | type Group struct { 33 | mu sync.Mutex // protects m 34 | m map[string]*call // lazily initialized 35 | } 36 | 37 | // Do executes and returns the results of the given function, making 38 | // sure that only one execution is in-flight for a given key at a 39 | // time. If a duplicate comes in, the duplicate caller waits for the 40 | // original to complete and receives the same results. 41 | func (g *Group) Do(key string, fn func() (interface{}, error)) (interface{}, error) { 42 | g.mu.Lock() 43 | if g.m == nil { 44 | g.m = make(map[string]*call) 45 | } 46 | if c, ok := g.m[key]; ok { 47 | g.mu.Unlock() 48 | c.wg.Wait() 49 | return c.val, c.err 50 | } 51 | c := new(call) 52 | c.wg.Add(1) 53 | g.m[key] = c 54 | g.mu.Unlock() 55 | 56 | c.val, c.err = fn() 57 | c.wg.Done() 58 | 59 | g.mu.Lock() 60 | delete(g.m, key) 61 | g.mu.Unlock() 62 | 63 | return c.val, c.err 64 | } 65 | -------------------------------------------------------------------------------- /syncutil/singleflight/singleflight_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2013 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package singleflight 18 | 19 | import ( 20 | "errors" 21 | "fmt" 22 | "sync" 23 | "sync/atomic" 24 | "testing" 25 | "time" 26 | ) 27 | 28 | func TestDo(t *testing.T) { 29 | var g Group 30 | v, err := g.Do("key", func() (interface{}, error) { 31 | return "bar", nil 32 | }) 33 | if got, want := fmt.Sprintf("%v (%T)", v, v), "bar (string)"; got != want { 34 | t.Errorf("Do = %v; want %v", got, want) 35 | } 36 | if err != nil { 37 | t.Errorf("Do error = %v", err) 38 | } 39 | } 40 | 41 | func TestDoErr(t *testing.T) { 42 | var g Group 43 | someErr := errors.New("Some error") 44 | v, err := g.Do("key", func() (interface{}, error) { 45 | return nil, someErr 46 | }) 47 | if err != someErr { 48 | t.Errorf("Do error = %v; want someErr %v", err, someErr) 49 | } 50 | if v != nil { 51 | t.Errorf("unexpected non-nil value %#v", v) 52 | } 53 | } 54 | 55 | func TestDoDupSuppress(t *testing.T) { 56 | var g Group 57 | c := make(chan string) 58 | var calls int32 59 | fn := func() (interface{}, error) { 60 | atomic.AddInt32(&calls, 1) 61 | return <-c, nil 62 | } 63 | 64 | const n = 10 65 | var wg sync.WaitGroup 66 | for i := 0; i < n; i++ { 67 | wg.Add(1) 68 | go func() { 69 | v, err := g.Do("key", fn) 70 | if err != nil { 71 | t.Errorf("Do error: %v", err) 72 | } 73 | if v.(string) != "bar" { 74 | t.Errorf("got %q; want %q", v, "bar") 75 | } 76 | wg.Done() 77 | }() 78 | } 79 | time.Sleep(100 * time.Millisecond) // let goroutines above block 80 | c <- "bar" 81 | wg.Wait() 82 | if got := atomic.LoadInt32(&calls); got != 1 { 83 | t.Errorf("number of calls = %d; want 1", got) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /syncutil/syncdebug/syncdebug.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2013 The Perkeep Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package syncdebug contains facilities for debugging synchronization 18 | // problems. 19 | package syncdebug // import "go4.org/syncutil/syncdebug" 20 | 21 | import ( 22 | "bytes" 23 | "fmt" 24 | "log" 25 | "runtime" 26 | "sync" 27 | "sync/atomic" 28 | "time" 29 | 30 | "go4.org/strutil" 31 | ) 32 | 33 | // RWMutexTracker is a sync.RWMutex that tracks who owns the current 34 | // exclusive lock. It's used for debugging deadlocks. 35 | type RWMutexTracker struct { 36 | mu sync.RWMutex 37 | 38 | // Atomic counters for number waiting and having read and write locks. 39 | nwaitr int32 40 | nwaitw int32 41 | nhaver int32 42 | nhavew int32 // should always be 0 or 1 43 | 44 | logOnce sync.Once 45 | 46 | hmu sync.Mutex 47 | holder []byte 48 | holdr map[int64]bool // goroutines holding read lock 49 | } 50 | 51 | const stackBufSize = 16 << 20 52 | 53 | var stackBuf = make(chan []byte, 8) 54 | 55 | func getBuf() []byte { 56 | select { 57 | case b := <-stackBuf: 58 | return b[:stackBufSize] 59 | default: 60 | return make([]byte, stackBufSize) 61 | } 62 | } 63 | 64 | func putBuf(b []byte) { 65 | select { 66 | case stackBuf <- b: 67 | default: 68 | } 69 | } 70 | 71 | var goroutineSpace = []byte("goroutine ") 72 | 73 | // GoroutineID returns the current goroutine's ID. 74 | // Use of this function is almost always a terrible idea. 75 | // It is also very slow. 76 | // GoroutineID is intended only for debugging. 77 | // In particular, it is used by syncutil. 78 | func GoroutineID() int64 { 79 | b := getBuf() 80 | defer putBuf(b) 81 | b = b[:runtime.Stack(b, false)] 82 | // Parse the 4707 out of "goroutine 4707 [" 83 | b = bytes.TrimPrefix(b, goroutineSpace) 84 | i := bytes.IndexByte(b, ' ') 85 | if i < 0 { 86 | panic(fmt.Sprintf("No space found in %q", b)) 87 | } 88 | b = b[:i] 89 | n, err := strutil.ParseUintBytes(b, 10, 64) 90 | if err != nil { 91 | panic(fmt.Sprintf("Failed to parse goroutine ID out of %q: %v", b, err)) 92 | } 93 | return int64(n) 94 | } 95 | 96 | func (m *RWMutexTracker) startLogger() { 97 | go func() { 98 | var buf bytes.Buffer 99 | for { 100 | time.Sleep(1 * time.Second) 101 | buf.Reset() 102 | m.hmu.Lock() 103 | for gid := range m.holdr { 104 | fmt.Fprintf(&buf, " [%d]", gid) 105 | } 106 | m.hmu.Unlock() 107 | log.Printf("Mutex %p: waitW %d haveW %d waitR %d haveR %d %s", 108 | m, 109 | atomic.LoadInt32(&m.nwaitw), 110 | atomic.LoadInt32(&m.nhavew), 111 | atomic.LoadInt32(&m.nwaitr), 112 | atomic.LoadInt32(&m.nhaver), buf.Bytes()) 113 | } 114 | }() 115 | } 116 | 117 | func (m *RWMutexTracker) Lock() { 118 | m.logOnce.Do(m.startLogger) 119 | atomic.AddInt32(&m.nwaitw, 1) 120 | m.mu.Lock() 121 | atomic.AddInt32(&m.nwaitw, -1) 122 | atomic.AddInt32(&m.nhavew, 1) 123 | 124 | m.hmu.Lock() 125 | defer m.hmu.Unlock() 126 | if len(m.holder) == 0 { 127 | m.holder = make([]byte, stackBufSize) 128 | } 129 | m.holder = m.holder[:runtime.Stack(m.holder[:stackBufSize], false)] 130 | log.Printf("Lock at %s", string(m.holder)) 131 | } 132 | 133 | func (m *RWMutexTracker) Unlock() { 134 | m.hmu.Lock() 135 | m.holder = nil 136 | m.hmu.Unlock() 137 | 138 | atomic.AddInt32(&m.nhavew, -1) 139 | m.mu.Unlock() 140 | } 141 | 142 | func (m *RWMutexTracker) RLock() { 143 | m.logOnce.Do(m.startLogger) 144 | atomic.AddInt32(&m.nwaitr, 1) 145 | 146 | // Catch read-write-read lock. See if somebody (us? via 147 | // another goroutine?) already has a read lock, and then 148 | // somebody else is waiting to write, meaning our second read 149 | // will deadlock. 150 | if atomic.LoadInt32(&m.nhaver) > 0 && atomic.LoadInt32(&m.nwaitw) > 0 { 151 | buf := getBuf() 152 | buf = buf[:runtime.Stack(buf, false)] 153 | log.Printf("Potential R-W-R deadlock at: %s", buf) 154 | putBuf(buf) 155 | } 156 | 157 | m.mu.RLock() 158 | atomic.AddInt32(&m.nwaitr, -1) 159 | atomic.AddInt32(&m.nhaver, 1) 160 | 161 | gid := GoroutineID() 162 | m.hmu.Lock() 163 | defer m.hmu.Unlock() 164 | if m.holdr == nil { 165 | m.holdr = make(map[int64]bool) 166 | } 167 | if m.holdr[gid] { 168 | buf := getBuf() 169 | buf = buf[:runtime.Stack(buf, false)] 170 | log.Fatalf("Recursive call to RLock: %s", buf) 171 | } 172 | m.holdr[gid] = true 173 | } 174 | 175 | func stack() []byte { 176 | buf := make([]byte, 1024) 177 | return buf[:runtime.Stack(buf, false)] 178 | } 179 | 180 | func (m *RWMutexTracker) RUnlock() { 181 | atomic.AddInt32(&m.nhaver, -1) 182 | 183 | gid := GoroutineID() 184 | m.hmu.Lock() 185 | delete(m.holdr, gid) 186 | m.hmu.Unlock() 187 | 188 | m.mu.RUnlock() 189 | } 190 | 191 | // Holder returns the stack trace of the current exclusive lock holder's stack 192 | // when it acquired the lock (with Lock). It returns the empty string if the lock 193 | // is not currently held. 194 | func (m *RWMutexTracker) Holder() string { 195 | m.hmu.Lock() 196 | defer m.hmu.Unlock() 197 | return string(m.holder) 198 | } 199 | -------------------------------------------------------------------------------- /syncutil/syncdebug/syncdebug_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2013 The Perkeep Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package syncdebug 18 | 19 | import "testing" 20 | 21 | func TestGoroutineID(t *testing.T) { 22 | c := make(chan int64, 2) 23 | c <- GoroutineID() 24 | go func() { 25 | c <- GoroutineID() 26 | }() 27 | if a, b := <-c, <-c; a == b { 28 | t.Errorf("both goroutine IDs were %d; expected different", a) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /syncutil/syncutil.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014 The Perkeep Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package syncutil provides various synchronization utilities. 18 | package syncutil // import "go4.org/syncutil" 19 | -------------------------------------------------------------------------------- /testing/functest/functest_test.go: -------------------------------------------------------------------------------- 1 | package functest 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "testing" 7 | ) 8 | 9 | // trec is a testing.TB which logs Errorf calls to buf 10 | type trec struct { 11 | testing.TB // crash on unimplemented methods 12 | buf bytes.Buffer 13 | } 14 | 15 | func (t *trec) Errorf(format string, args ...interface{}) { 16 | t.buf.WriteString("ERR: ") 17 | fmt.Fprintf(&t.buf, format, args...) 18 | t.buf.WriteByte('\n') 19 | } 20 | 21 | func (t *trec) Logf(format string, args ...interface{}) { 22 | t.buf.WriteString("LOG: ") 23 | fmt.Fprintf(&t.buf, format, args...) 24 | t.buf.WriteByte('\n') 25 | } 26 | 27 | func (t *trec) String() string { return t.buf.String() } 28 | 29 | func add(a, b int) int { return a + b } 30 | 31 | func TestBasic(t *testing.T) { 32 | f := New(add) 33 | trec := new(trec) 34 | f.Test(trec, 35 | f.In(1, 2).Want(3), 36 | f.In(5, 6).Want(100), 37 | f.Case("also wrong").In(5, 6).Want(101), 38 | ) 39 | want := `ERR: add(5, 6) = 11; want 100 40 | ERR: also wrong: add(5, 6) = 11; want 101 41 | ` 42 | if got := trec.String(); got != want { 43 | t.Errorf("Output mismatch.\nGot:\n%v\nWant:\n%v\n", got, want) 44 | } 45 | } 46 | 47 | func TestBasic_Strings(t *testing.T) { 48 | concat := func(a, b string) string { return a + b } 49 | f := New(concat) 50 | f.Name = "concat" 51 | trec := new(trec) 52 | f.Test(trec, 53 | f.In("a", "b").Want("ab"), 54 | f.In("a", "b\x00").Want("ab"), 55 | ) 56 | want := `ERR: concat("a", "b\x00") = "ab\x00"; want "ab" 57 | ` 58 | if got := trec.String(); got != want { 59 | t.Errorf("Output mismatch.\nGot:\n%v\nWant:\n%v\n", got, want) 60 | } 61 | } 62 | 63 | func TestVariadic(t *testing.T) { 64 | sumVar := func(vals ...int) (sum int) { 65 | for _, v := range vals { 66 | sum += v 67 | } 68 | return 69 | } 70 | 71 | f := New(sumVar) 72 | f.Name = "sumVar" 73 | trec := new(trec) 74 | f.Test(trec, 75 | f.In().Want(0), 76 | f.In().Want(100), 77 | f.In(1).Want(1), 78 | f.In(1).Want(100), 79 | f.In(1, 2).Want(3), 80 | f.In(1, 2, 3).Want(6), 81 | f.In(1, 2, 3).Want(100), 82 | ) 83 | want := `ERR: sumVar() = 0; want 100 84 | ERR: sumVar(1) = 1; want 100 85 | ERR: sumVar(1, 2, 3) = 6; want 100 86 | ` 87 | if got := trec.String(); got != want { 88 | t.Errorf("Output mismatch.\nGot:\n%v\nWant:\n%v\n", got, want) 89 | } 90 | } 91 | 92 | func condPanic(doPanic bool, panicValue interface{}) { 93 | if doPanic { 94 | panic(panicValue) 95 | } 96 | } 97 | 98 | func TestPanic(t *testing.T) { 99 | f := New(condPanic) 100 | f.Name = "condPanic" 101 | trec := new(trec) 102 | f.Test(trec, 103 | f.In(false, nil), 104 | f.In(true, "boom").Check(func(res Result) error { 105 | trec.Logf("Got res: %+v", res) 106 | if res.Panic != "boom" { 107 | return fmt.Errorf("panic = %v; want boom", res.Panic) 108 | } 109 | return nil 110 | }), 111 | f.Case("panic with nil").In(true, nil), 112 | ) 113 | want := `LOG: Got res: {Result:[] Panic:boom Panicked:true} 114 | ERR: panic with nil: condPanic(true, ): panicked with 115 | ` 116 | if got := trec.String(); got != want { 117 | t.Errorf("Output mismatch.\nGot:\n%v\nWant:\n%v\n", got, want) 118 | } 119 | } 120 | 121 | func TestName_AutoFunc(t *testing.T) { 122 | testName(t, New(add), "add") 123 | } 124 | 125 | type SomeType struct{} 126 | 127 | func (t *SomeType) SomeMethod(int) int { return 123 } 128 | 129 | func TestName_AutoMethod(t *testing.T) { 130 | testName(t, New((*SomeType).SomeMethod), "SomeType.SomeMethod") 131 | } 132 | 133 | func testName(t *testing.T, f *Func, want string) { 134 | if f.Name != want { 135 | t.Errorf("name = %q; want %q", f.Name, want) 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /types/types.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2013 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package types provides various common types. 18 | package types // import "go4.org/types" 19 | 20 | import ( 21 | "bytes" 22 | "encoding/json" 23 | "fmt" 24 | "io" 25 | "io/ioutil" 26 | "strings" 27 | "sync" 28 | "time" 29 | ) 30 | 31 | var null_b = []byte("null") 32 | 33 | // NopCloser is an io.Closer that does nothing. 34 | var NopCloser io.Closer = CloseFunc(func() error { return nil }) 35 | 36 | // EmptyBody is a ReadCloser that returns EOF on Read and does nothing 37 | // on Close. 38 | var EmptyBody io.ReadCloser = ioutil.NopCloser(strings.NewReader("")) 39 | 40 | // Time3339 is a time.Time which encodes to and from JSON 41 | // as an RFC 3339 time in UTC. 42 | type Time3339 time.Time 43 | 44 | var ( 45 | _ json.Marshaler = Time3339{} 46 | _ json.Unmarshaler = (*Time3339)(nil) 47 | ) 48 | 49 | func (t Time3339) String() string { 50 | return time.Time(t).UTC().Format(time.RFC3339Nano) 51 | } 52 | 53 | func (t Time3339) MarshalJSON() ([]byte, error) { 54 | if t.Time().IsZero() { 55 | return null_b, nil 56 | } 57 | return json.Marshal(t.String()) 58 | } 59 | 60 | func (t *Time3339) UnmarshalJSON(b []byte) error { 61 | if bytes.Equal(b, null_b) { 62 | *t = Time3339{} 63 | return nil 64 | } 65 | if len(b) < 2 || b[0] != '"' || b[len(b)-1] != '"' { 66 | return fmt.Errorf("types: failed to unmarshal non-string value %q as an RFC 3339 time", b) 67 | } 68 | s := string(b[1 : len(b)-1]) 69 | if s == "" { 70 | *t = Time3339{} 71 | return nil 72 | } 73 | tm, err := time.Parse(time.RFC3339Nano, s) 74 | if err != nil { 75 | if strings.HasPrefix(s, "0000-00-00T00:00:00") { 76 | *t = Time3339{} 77 | return nil 78 | } 79 | return err 80 | } 81 | *t = Time3339(tm) 82 | return nil 83 | } 84 | 85 | // ParseTime3339OrZero parses a string in RFC3339 format. If it's invalid, 86 | // the zero time value is returned instead. 87 | func ParseTime3339OrZero(v string) Time3339 { 88 | t, err := time.Parse(time.RFC3339Nano, v) 89 | if err != nil { 90 | return Time3339{} 91 | } 92 | return Time3339(t) 93 | } 94 | 95 | func ParseTime3339OrNil(v string) *Time3339 { 96 | t, err := time.Parse(time.RFC3339Nano, v) 97 | if err != nil { 98 | return nil 99 | } 100 | tm := Time3339(t) 101 | return &tm 102 | } 103 | 104 | // Time returns the time as a time.Time with slightly less stutter 105 | // than a manual conversion. 106 | func (t Time3339) Time() time.Time { 107 | return time.Time(t) 108 | } 109 | 110 | // IsAnyZero returns whether the time is Go zero or Unix zero. 111 | func (t *Time3339) IsAnyZero() bool { 112 | return t == nil || time.Time(*t).IsZero() || time.Time(*t).Unix() == 0 113 | } 114 | 115 | // ByTime sorts times. 116 | type ByTime []time.Time 117 | 118 | func (s ByTime) Len() int { return len(s) } 119 | func (s ByTime) Less(i, j int) bool { return s[i].Before(s[j]) } 120 | func (s ByTime) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 121 | 122 | // NewOnceCloser returns a Closer wrapping c which only calls Close on c 123 | // once. Subsequent calls to Close return nil. 124 | func NewOnceCloser(c io.Closer) io.Closer { 125 | return &onceCloser{c: c} 126 | } 127 | 128 | type onceCloser struct { 129 | mu sync.Mutex 130 | c io.Closer 131 | } 132 | 133 | func (c *onceCloser) Close() error { 134 | c.mu.Lock() 135 | defer c.mu.Unlock() 136 | if c.c == nil { 137 | return nil 138 | } 139 | err := c.c.Close() 140 | c.c = nil 141 | return err 142 | } 143 | 144 | // CloseFunc implements io.Closer with a function. 145 | type CloseFunc func() error 146 | 147 | func (fn CloseFunc) Close() error { return fn() } 148 | -------------------------------------------------------------------------------- /types/types_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2013 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package types 18 | 19 | import ( 20 | "encoding/json" 21 | "strings" 22 | "testing" 23 | "time" 24 | ) 25 | 26 | func TestTime3339(t *testing.T) { 27 | tm := time.Unix(123, 456) 28 | t3 := Time3339(tm) 29 | type O struct { 30 | SomeTime Time3339 `json:"someTime"` 31 | } 32 | o := &O{SomeTime: t3} 33 | got, err := json.Marshal(o) 34 | if err != nil { 35 | t.Fatal(err) 36 | } 37 | goodEnc := "{\"someTime\":\"1970-01-01T00:02:03.000000456Z\"}" 38 | if string(got) != goodEnc { 39 | t.Errorf("Encoding wrong.\n Got: %q\nWant: %q", got, goodEnc) 40 | } 41 | ogot := &O{} 42 | err = json.Unmarshal([]byte(goodEnc), ogot) 43 | if err != nil { 44 | t.Fatal(err) 45 | } 46 | if !tm.Equal(ogot.SomeTime.Time()) { 47 | t.Errorf("Unmarshal got time %v; want %v", ogot.SomeTime.Time(), tm) 48 | } 49 | } 50 | 51 | func TestTime3339_Marshal(t *testing.T) { 52 | tests := []struct { 53 | in time.Time 54 | want string 55 | }{ 56 | {time.Time{}, "null"}, 57 | {time.Unix(1, 0), `"1970-01-01T00:00:01Z"`}, 58 | } 59 | for i, tt := range tests { 60 | got, err := Time3339(tt.in).MarshalJSON() 61 | if err != nil { 62 | t.Errorf("%d. marshal(%v) got error: %v", i, tt.in, err) 63 | continue 64 | } 65 | if string(got) != tt.want { 66 | t.Errorf("%d. marshal(%v) = %q; want %q", i, tt.in, got, tt.want) 67 | } 68 | } 69 | } 70 | 71 | func TestTime3339_empty(t *testing.T) { 72 | tests := []struct { 73 | enc string 74 | z bool 75 | }{ 76 | {enc: "null", z: true}, 77 | {enc: `""`, z: true}, 78 | {enc: "0000-00-00T00:00:00Z", z: true}, 79 | {enc: "0001-01-01T00:00:00Z", z: true}, 80 | {enc: "1970-01-01T00:00:00Z", z: true}, 81 | {enc: "2001-02-03T04:05:06Z", z: false}, 82 | {enc: "2001-02-03T04:05:06+06:00", z: false}, 83 | {enc: "2001-02-03T04:05:06-06:00", z: false}, 84 | {enc: "2001-02-03T04:05:06.123456789Z", z: false}, 85 | {enc: "2001-02-03T04:05:06.123456789+06:00", z: false}, 86 | {enc: "2001-02-03T04:05:06.123456789-06:00", z: false}, 87 | } 88 | for _, tt := range tests { 89 | var tm Time3339 90 | enc := tt.enc 91 | if strings.Contains(enc, "T") { 92 | enc = "\"" + enc + "\"" 93 | } 94 | err := json.Unmarshal([]byte(enc), &tm) 95 | if err != nil { 96 | t.Errorf("unmarshal %q = %v", enc, err) 97 | } 98 | if tm.IsAnyZero() != tt.z { 99 | t.Errorf("unmarshal %q = %v (%d), %v; zero=%v; want %v", tt.enc, tm.Time(), tm.Time().Unix(), err, 100 | !tt.z, tt.z) 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /wkfs/gcs/gcs.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014 The Perkeep Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package gcs registers a Google Cloud Storage filesystem at the 18 | // well-known /gcs/ filesystem path if the current machine is running 19 | // on Google Compute Engine. 20 | // 21 | // It was initially only meant for small files, and as such, it can only 22 | // read files smaller than 1MB for now. 23 | package gcs // import "go4.org/wkfs/gcs" 24 | 25 | import ( 26 | "bytes" 27 | "fmt" 28 | "io" 29 | "io/ioutil" 30 | "os" 31 | "path" 32 | "strings" 33 | "time" 34 | 35 | "cloud.google.com/go/compute/metadata" 36 | "cloud.google.com/go/storage" 37 | "go4.org/wkfs" 38 | "golang.org/x/net/context" 39 | "golang.org/x/oauth2" 40 | "golang.org/x/oauth2/google" 41 | "google.golang.org/api/option" 42 | ) 43 | 44 | // Max size for all files read, because we use a bytes.Reader as our file 45 | // reader, instead of storage.NewReader. This is because we get all wkfs.File 46 | // methods for free by embedding a bytes.Reader. This filesystem was only supposed 47 | // to be for configuration data only, so this is ok for now. 48 | const maxSize = 1 << 20 49 | 50 | func init() { 51 | if !metadata.OnGCE() { 52 | return 53 | } 54 | hc, err := google.DefaultClient(oauth2.NoContext) 55 | if err != nil { 56 | registerBrokenFS(fmt.Errorf("could not get http client for context: %v", err)) 57 | return 58 | } 59 | ctx := context.Background() 60 | sc, err := storage.NewClient(ctx, option.WithHTTPClient(hc)) 61 | if err != nil { 62 | registerBrokenFS(fmt.Errorf("could not get cloud storage client: %v", err)) 63 | return 64 | } 65 | wkfs.RegisterFS("/gcs/", &gcsFS{ 66 | ctx: ctx, 67 | sc: sc, 68 | }) 69 | } 70 | 71 | type gcsFS struct { 72 | ctx context.Context 73 | sc *storage.Client 74 | err error // sticky error 75 | } 76 | 77 | func registerBrokenFS(err error) { 78 | wkfs.RegisterFS("/gcs/", &gcsFS{ 79 | err: err, 80 | }) 81 | } 82 | 83 | func (fs *gcsFS) parseName(name string) (bucket, fileName string, err error) { 84 | if fs.err != nil { 85 | return "", "", fs.err 86 | } 87 | name = strings.TrimPrefix(name, "/gcs/") 88 | i := strings.Index(name, "/") 89 | if i < 0 { 90 | return name, "", nil 91 | } 92 | return name[:i], name[i+1:], nil 93 | } 94 | 95 | // Open opens the named file for reading. It returns an error if the file size 96 | // is larger than 1 << 20. 97 | func (fs *gcsFS) Open(name string) (wkfs.File, error) { 98 | bucket, fileName, err := fs.parseName(name) 99 | if err != nil { 100 | return nil, err 101 | } 102 | obj := fs.sc.Bucket(bucket).Object(fileName) 103 | attrs, err := obj.Attrs(fs.ctx) 104 | if err != nil { 105 | return nil, err 106 | } 107 | size := attrs.Size 108 | if size > maxSize { 109 | return nil, fmt.Errorf("file %s too large (%d bytes) for /gcs/ filesystem", name, size) 110 | } 111 | rc, err := obj.NewReader(fs.ctx) 112 | if err != nil { 113 | return nil, err 114 | } 115 | defer rc.Close() 116 | 117 | slurp, err := ioutil.ReadAll(io.LimitReader(rc, size)) 118 | if err != nil { 119 | return nil, err 120 | } 121 | return &file{ 122 | name: name, 123 | Reader: bytes.NewReader(slurp), 124 | }, nil 125 | } 126 | 127 | func (fs *gcsFS) Stat(name string) (os.FileInfo, error) { return fs.Lstat(name) } 128 | func (fs *gcsFS) Lstat(name string) (os.FileInfo, error) { 129 | bucket, fileName, err := fs.parseName(name) 130 | if err != nil { 131 | return nil, err 132 | } 133 | attrs, err := fs.sc.Bucket(bucket).Object(fileName).Attrs(fs.ctx) 134 | if err == storage.ErrObjectNotExist { 135 | return nil, os.ErrNotExist 136 | } 137 | if err != nil { 138 | return nil, err 139 | } 140 | return &statInfo{ 141 | name: attrs.Name, 142 | size: attrs.Size, 143 | }, nil 144 | } 145 | 146 | func (fs *gcsFS) MkdirAll(path string, perm os.FileMode) error { return nil } 147 | 148 | func (fs *gcsFS) OpenFile(name string, flag int, perm os.FileMode) (wkfs.FileWriter, error) { 149 | bucket, fileName, err := fs.parseName(name) 150 | if err != nil { 151 | return nil, err 152 | } 153 | switch flag { 154 | case os.O_WRONLY | os.O_CREATE | os.O_EXCL: 155 | case os.O_WRONLY | os.O_CREATE | os.O_TRUNC: 156 | default: 157 | return nil, fmt.Errorf("Unsupported OpenFlag flag mode %d on Google Cloud Storage", flag) 158 | } 159 | if flag&os.O_EXCL != 0 { 160 | if _, err := fs.Stat(name); err == nil { 161 | return nil, os.ErrExist 162 | } 163 | } 164 | // TODO(mpl): consider adding perm to the object's ObjectAttrs.Metadata 165 | return fs.sc.Bucket(bucket).Object(fileName).NewWriter(fs.ctx), nil 166 | } 167 | 168 | func (fs *gcsFS) Remove(name string) error { 169 | bucket, fileName, err := fs.parseName(name) 170 | if err != nil { 171 | return err 172 | } 173 | return fs.sc.Bucket(bucket).Object(fileName).Delete(fs.ctx) 174 | } 175 | 176 | type statInfo struct { 177 | name string 178 | size int64 179 | isDir bool 180 | modtime time.Time 181 | } 182 | 183 | func (si *statInfo) IsDir() bool { return si.isDir } 184 | func (si *statInfo) ModTime() time.Time { return si.modtime } 185 | func (si *statInfo) Mode() os.FileMode { return 0644 } 186 | func (si *statInfo) Name() string { return path.Base(si.name) } 187 | func (si *statInfo) Size() int64 { return si.size } 188 | func (si *statInfo) Sys() interface{} { return nil } 189 | 190 | type file struct { 191 | name string 192 | *bytes.Reader 193 | } 194 | 195 | func (*file) Close() error { return nil } 196 | func (f *file) Name() string { return path.Base(f.name) } 197 | func (f *file) Stat() (os.FileInfo, error) { 198 | panic("Stat not implemented on /gcs/ files yet") 199 | } 200 | -------------------------------------------------------------------------------- /wkfs/gcs/gcs_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 The Perkeep Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package gcs 18 | 19 | import ( 20 | "bytes" 21 | "flag" 22 | "io" 23 | "strings" 24 | "testing" 25 | 26 | "cloud.google.com/go/compute/metadata" 27 | "cloud.google.com/go/storage" 28 | "go4.org/wkfs" 29 | "golang.org/x/net/context" 30 | "google.golang.org/api/iterator" 31 | ) 32 | 33 | var flagBucket = flag.String("bucket", "", "Google Cloud Storage bucket where to run the tests. It should be empty.") 34 | 35 | func TestWriteRead(t *testing.T) { 36 | if !metadata.OnGCE() { 37 | t.Skipf("Not testing on GCE") 38 | } 39 | if *flagBucket == "" { 40 | t.Skipf("No bucket specified") 41 | } 42 | ctx := context.Background() 43 | cl, err := storage.NewClient(ctx) 44 | it := cl.Bucket(*flagBucket).Objects(ctx, nil) 45 | if _, err := it.Next(); err != iterator.Done { 46 | if err == nil { 47 | t.Fatalf("Bucket %v is not empty, aborting test.", *flagBucket) 48 | } 49 | t.Fatalf("unexpected bucket iteration error: %v", err) 50 | } 51 | 52 | // Write to camli-gcs_test.txt 53 | filename := "camli-gcs_test.txt" 54 | gcsPath := "/gcs/" + *flagBucket + "/" + filename 55 | f, err := wkfs.Create(gcsPath) 56 | if err != nil { 57 | t.Fatalf("error creating %v: %v", gcsPath, err) 58 | } 59 | defer func() { 60 | if err := wkfs.Remove(gcsPath); err != nil { 61 | t.Fatalf("error while cleaning up %v: %v", gcsPath, err) 62 | } 63 | }() 64 | 65 | data := "Hello World" 66 | if _, err := io.Copy(f, strings.NewReader(data)); err != nil { 67 | t.Fatalf("error writing to %v: %v", gcsPath, err) 68 | } 69 | if err := f.Close(); err != nil { 70 | t.Fatalf("error closing %v: %v", gcsPath, err) 71 | } 72 | 73 | // Read back from camli-gcs_test.txt 74 | g, err := wkfs.Open(gcsPath) 75 | if err != nil { 76 | t.Fatalf("error opening %v: %v", gcsPath, err) 77 | } 78 | defer g.Close() 79 | var buf bytes.Buffer 80 | if _, err := io.Copy(&buf, g); err != nil { 81 | t.Fatalf("error reading %v: %v", gcsPath, err) 82 | } 83 | if buf.String() != data { 84 | t.Fatalf("error with %v contents: got %v, wanted %v", gcsPath, buf.String(), data) 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /wkfs/wkfs.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2014 The Perkeep Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package wkfs implements the pluggable "well-known filesystem" abstraction layer. 18 | // 19 | // Instead of accessing files directly through the operating system 20 | // using os.Open or os.Stat, code should use wkfs.Open or wkfs.Stat, 21 | // which first try to intercept paths at well-known top-level 22 | // directories representing previously-registered mount types, 23 | // otherwise fall through to the operating system paths. 24 | // 25 | // Example of top-level well-known directories that might be 26 | // registered include /gcs/bucket/object for Google Cloud Storage or 27 | // /s3/bucket/object for AWS S3. 28 | package wkfs // import "go4.org/wkfs" 29 | 30 | import ( 31 | "io" 32 | "io/ioutil" 33 | "os" 34 | "strings" 35 | ) 36 | 37 | type File interface { 38 | io.Reader 39 | io.ReaderAt 40 | io.Closer 41 | io.Seeker 42 | Name() string 43 | Stat() (os.FileInfo, error) 44 | } 45 | 46 | type FileWriter interface { 47 | io.Writer 48 | io.Closer 49 | } 50 | 51 | func Open(name string) (File, error) { return fs(name).Open(name) } 52 | func Stat(name string) (os.FileInfo, error) { return fs(name).Stat(name) } 53 | func Lstat(name string) (os.FileInfo, error) { return fs(name).Lstat(name) } 54 | func MkdirAll(path string, perm os.FileMode) error { return fs(path).MkdirAll(path, perm) } 55 | func OpenFile(name string, flag int, perm os.FileMode) (FileWriter, error) { 56 | return fs(name).OpenFile(name, flag, perm) 57 | } 58 | func Remove(name string) error { return fs(name).Remove(name) } 59 | func Create(name string) (FileWriter, error) { 60 | // like os.Create but WRONLY instead of RDWR because we don't 61 | // expose a Reader here. 62 | return OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) 63 | } 64 | 65 | func fs(name string) FileSystem { 66 | for pfx, fs := range wkFS { 67 | if strings.HasPrefix(name, pfx) { 68 | return fs 69 | } 70 | } 71 | return osFS{} 72 | } 73 | 74 | type osFS struct{} 75 | 76 | func (osFS) Open(name string) (File, error) { return os.Open(name) } 77 | func (osFS) Stat(name string) (os.FileInfo, error) { return os.Stat(name) } 78 | func (osFS) Lstat(name string) (os.FileInfo, error) { return os.Lstat(name) } 79 | func (osFS) MkdirAll(path string, perm os.FileMode) error { return os.MkdirAll(path, perm) } 80 | func (osFS) OpenFile(name string, flag int, perm os.FileMode) (FileWriter, error) { 81 | return os.OpenFile(name, flag, perm) 82 | } 83 | func (osFS) Remove(name string) error { return os.Remove(name) } 84 | 85 | type FileSystem interface { 86 | Open(name string) (File, error) 87 | OpenFile(name string, flag int, perm os.FileMode) (FileWriter, error) 88 | Stat(name string) (os.FileInfo, error) 89 | Lstat(name string) (os.FileInfo, error) 90 | MkdirAll(path string, perm os.FileMode) error 91 | Remove(name string) error 92 | } 93 | 94 | // well-known filesystems 95 | var wkFS = map[string]FileSystem{} 96 | 97 | // RegisterFS registers a well-known filesystem. It intercepts 98 | // anything beginning with prefix (which must start and end with a 99 | // forward slash) and forwards it to fs. 100 | func RegisterFS(prefix string, fs FileSystem) { 101 | if !strings.HasPrefix(prefix, "/") || !strings.HasSuffix(prefix, "/") { 102 | panic("bogus prefix: " + prefix) 103 | } 104 | if _, dup := wkFS[prefix]; dup { 105 | panic("duplication registration of " + prefix) 106 | } 107 | wkFS[prefix] = fs 108 | } 109 | 110 | // WriteFile writes data to a file named by filename. 111 | // If the file does not exist, WriteFile creates it with permissions perm; 112 | // otherwise WriteFile truncates it before writing. 113 | func WriteFile(filename string, data []byte, perm os.FileMode) error { 114 | f, err := OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) 115 | if err != nil { 116 | return err 117 | } 118 | n, err := f.Write(data) 119 | if err == nil && n < len(data) { 120 | err = io.ErrShortWrite 121 | } 122 | if err1 := f.Close(); err == nil { 123 | err = err1 124 | } 125 | return err 126 | } 127 | 128 | func ReadFile(filename string) ([]byte, error) { 129 | f, err := Open(filename) 130 | if err != nil { 131 | return nil, err 132 | } 133 | defer f.Close() 134 | return ioutil.ReadAll(f) 135 | } 136 | -------------------------------------------------------------------------------- /writerutil/writerutil.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The go4 Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package writerutil contains io.Writer types. 18 | package writerutil // import "go4.org/writerutil" 19 | 20 | import ( 21 | "bytes" 22 | "strconv" 23 | ) 24 | 25 | // PrefixSuffixSaver is an io.Writer which retains the first N bytes 26 | // and the last N bytes written to it. The Bytes method reconstructs 27 | // it with a pretty error message. 28 | // It is copied from os/exec/exec.go of the Go stdlib. 29 | type PrefixSuffixSaver struct { 30 | N int // max size of prefix or suffix 31 | prefix []byte 32 | suffix []byte // ring buffer once len(suffix) == N 33 | suffixOff int // offset to write into suffix 34 | skipped int64 35 | 36 | // TODO(bradfitz): we could keep one large []byte and use part of it for 37 | // the prefix, reserve space for the '... Omitting N bytes ...' message, 38 | // then the ring buffer suffix, and just rearrange the ring buffer 39 | // suffix when Bytes() is called, but it doesn't seem worth it for 40 | // now just for error messages. It's only ~64KB anyway. 41 | } 42 | 43 | func (w *PrefixSuffixSaver) Write(p []byte) (n int, err error) { 44 | lenp := len(p) 45 | p = w.fill(&w.prefix, p) 46 | 47 | // Only keep the last w.N bytes of suffix data. 48 | if overage := len(p) - w.N; overage > 0 { 49 | p = p[overage:] 50 | w.skipped += int64(overage) 51 | } 52 | p = w.fill(&w.suffix, p) 53 | 54 | // w.suffix is full now if p is non-empty. Overwrite it in a circle. 55 | for len(p) > 0 { // 0, 1, or 2 iterations. 56 | n := copy(w.suffix[w.suffixOff:], p) 57 | p = p[n:] 58 | w.skipped += int64(n) 59 | w.suffixOff += n 60 | if w.suffixOff == w.N { 61 | w.suffixOff = 0 62 | } 63 | } 64 | return lenp, nil 65 | } 66 | 67 | // fill appends up to len(p) bytes of p to *dst, such that *dst does not 68 | // grow larger than w.N. It returns the un-appended suffix of p. 69 | func (w *PrefixSuffixSaver) fill(dst *[]byte, p []byte) (pRemain []byte) { 70 | if remain := w.N - len(*dst); remain > 0 { 71 | add := minInt(len(p), remain) 72 | *dst = append(*dst, p[:add]...) 73 | p = p[add:] 74 | } 75 | return p 76 | } 77 | 78 | // Bytes returns a slice of the bytes, or a copy of the bytes, retained by w. 79 | // If more bytes than could be retained were written to w, it returns a 80 | // concatenation of the N first bytes, a message for how many bytes were dropped, 81 | // and the N last bytes. 82 | func (w *PrefixSuffixSaver) Bytes() []byte { 83 | if w.suffix == nil { 84 | return w.prefix 85 | } 86 | if w.skipped == 0 { 87 | return append(w.prefix, w.suffix...) 88 | } 89 | var buf bytes.Buffer 90 | buf.Grow(len(w.prefix) + len(w.suffix) + 50) 91 | buf.Write(w.prefix) 92 | buf.WriteString("\n... omitting ") 93 | buf.WriteString(strconv.FormatInt(w.skipped, 10)) 94 | buf.WriteString(" bytes ...\n") 95 | buf.Write(w.suffix[w.suffixOff:]) 96 | buf.Write(w.suffix[:w.suffixOff]) 97 | return buf.Bytes() 98 | } 99 | 100 | func minInt(a, b int) int { 101 | if a < b { 102 | return a 103 | } 104 | return b 105 | } 106 | -------------------------------------------------------------------------------- /writerutil/writerutil_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2016 The go4 Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package writerutil 18 | 19 | import ( 20 | "io" 21 | "testing" 22 | ) 23 | 24 | func TestPrefixSuffixSaver(t *testing.T) { 25 | tests := []struct { 26 | N int 27 | writes []string 28 | want string 29 | }{ 30 | { 31 | N: 2, 32 | writes: nil, 33 | want: "", 34 | }, 35 | { 36 | N: 2, 37 | writes: []string{"a"}, 38 | want: "a", 39 | }, 40 | { 41 | N: 2, 42 | writes: []string{"abc", "d"}, 43 | want: "abcd", 44 | }, 45 | { 46 | N: 2, 47 | writes: []string{"abc", "d", "e"}, 48 | want: "ab\n... omitting 1 bytes ...\nde", 49 | }, 50 | { 51 | N: 2, 52 | writes: []string{"ab______________________yz"}, 53 | want: "ab\n... omitting 22 bytes ...\nyz", 54 | }, 55 | { 56 | N: 2, 57 | writes: []string{"ab_______________________y", "z"}, 58 | want: "ab\n... omitting 23 bytes ...\nyz", 59 | }, 60 | } 61 | for i, tt := range tests { 62 | w := &PrefixSuffixSaver{N: tt.N} 63 | for _, s := range tt.writes { 64 | n, err := io.WriteString(w, s) 65 | if err != nil || n != len(s) { 66 | t.Errorf("%d. WriteString(%q) = %v, %v; want %v, %v", i, s, n, err, len(s), nil) 67 | } 68 | } 69 | if got := string(w.Bytes()); got != tt.want { 70 | t.Errorf("%d. Bytes = %q; want %q", i, got, tt.want) 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /xdgdir/example_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The go4 Authors 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package xdgdir_test 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | 23 | "go4.org/xdgdir" 24 | ) 25 | 26 | func Example() { 27 | // Print the absolute path of the current user's XDG_CONFIG_DIR. 28 | fmt.Println(xdgdir.Config.Path()) 29 | 30 | // Read a file from $XDG_CONFIG_DIR/myconfig.json. 31 | // This will search for a file named "myconfig.json" inside 32 | // $XDG_CONFIG_DIR and then each entry inside $XDG_CONFIG_DIRS. 33 | // It opens and returns the first file it finds, or returns an error. 34 | if f, err := xdgdir.Data.Create("myconfig.json"); err == nil { 35 | fmt.Fprintln(f, "Hello, World!") 36 | if err := f.Close(); err != nil { 37 | fmt.Fprintln(os.Stderr, err) 38 | } 39 | } else { 40 | fmt.Fprintln(os.Stderr, err) 41 | } 42 | 43 | // Write a file to $XDG_DATA_DIR/myapp/foo.txt 44 | if f, err := xdgdir.Data.Create("myapp/foo.txt"); err == nil { 45 | fmt.Fprintln(f, "Hello, World!") 46 | f.Close() 47 | } else { 48 | fmt.Fprintln(os.Stderr, err) 49 | } 50 | } 51 | --------------------------------------------------------------------------------