├── LICENSE ├── PATENTS ├── README.md ├── codereview.cfg ├── go.mod ├── go.sum ├── gosumcheck ├── main.go ├── test.bash └── test.sum ├── internal └── lazyregexp │ └── lazyre.go ├── modfile ├── print.go ├── read.go ├── read_test.go ├── rule.go ├── rule_test.go ├── testdata │ ├── block.golden │ ├── block.in │ ├── comment.golden │ ├── comment.in │ ├── empty.golden │ ├── empty.in │ ├── goline.golden │ ├── goline.in │ ├── gopkg.in.golden │ ├── issue70632.golden │ ├── issue70632.in │ ├── module.golden │ ├── module.in │ ├── replace.golden │ ├── replace.in │ ├── replace2.golden │ ├── replace2.in │ ├── retract.golden │ ├── retract.in │ ├── rule1.golden │ └── work │ │ ├── comment.golden │ │ ├── comment.in │ │ ├── empty.golden │ │ ├── empty.in │ │ ├── goline.golden │ │ ├── goline.in │ │ ├── replace.golden │ │ ├── replace.in │ │ ├── replace2.golden │ │ ├── replace2.in │ │ ├── use.golden │ │ └── use.in ├── work.go └── work_test.go ├── module ├── module.go ├── module_test.go ├── pseudo.go └── pseudo_test.go ├── semver ├── semver.go └── semver_test.go ├── sumdb ├── cache.go ├── client.go ├── client_test.go ├── dirhash │ ├── hash.go │ └── hash_test.go ├── note │ ├── example_test.go │ ├── note.go │ └── note_test.go ├── server.go ├── storage │ ├── mem.go │ ├── mem_test.go │ ├── storage.go │ └── test.go ├── test.go └── tlog │ ├── ct_test.go │ ├── note.go │ ├── note_test.go │ ├── tile.go │ ├── tile_test.go │ ├── tlog.go │ └── tlog_test.go └── zip ├── testdata ├── check_dir │ ├── empty.txt │ ├── various.txt │ ├── various_go123.txt │ └── various_go124.txt ├── check_files │ ├── empty.txt │ ├── various.txt │ ├── various_go123.txt │ └── various_go124.txt ├── check_zip │ ├── empty.txt │ └── various.txt ├── create │ ├── bad_file_path.txt │ ├── bad_gomod_case.txt │ ├── bad_mod_path.txt │ ├── bad_mod_path_version_suffix.txt │ ├── bad_version.txt │ ├── dup_file.txt │ ├── dup_file_and_dir.txt │ ├── empty.txt │ ├── exclude_cap_go_mod_submodule.txt │ ├── exclude_submodule.txt │ ├── exclude_vendor.txt │ ├── exclude_vendor_go124.txt │ ├── file_case_conflict.txt │ ├── go_mod_dir.txt │ ├── invalid_utf8_mod_path.txt │ └── simple.txt ├── create_from_dir │ ├── bad_file_path.txt │ ├── bad_gomod_case.txt │ ├── bad_mod_path.txt │ ├── bad_mod_path_version_suffix.txt │ ├── bad_version.txt │ ├── empty.txt │ ├── exclude_submodule.txt │ ├── exclude_vcs.txt │ ├── exclude_vendor.txt │ ├── exclude_vendor_go124.txt │ ├── go_mod_dir.txt │ ├── invalid_utf8_mod_path.txt │ └── simple.txt └── unzip │ ├── bad_file_path.txt │ ├── bad_gomod_case.txt │ ├── bad_mod_path.txt │ ├── bad_mod_path_version_suffix.txt │ ├── bad_submodule.txt │ ├── bad_version.txt │ ├── cap_go_mod_not_submodule.txt │ ├── dup_file.txt │ ├── dup_file_and_dir.txt │ ├── empty.txt │ ├── file_case_conflict.txt │ ├── go_mod_dir.txt │ ├── invalid_utf8_mod_path.txt │ ├── prefix_only.txt │ └── simple.txt ├── vendor_test.go ├── zip.go └── zip_test.go /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2009 The Go Authors. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google LLC nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /PATENTS: -------------------------------------------------------------------------------- 1 | Additional IP Rights Grant (Patents) 2 | 3 | "This implementation" means the copyrightable works distributed by 4 | Google as part of the Go project. 5 | 6 | Google hereby grants to You a perpetual, worldwide, non-exclusive, 7 | no-charge, royalty-free, irrevocable (except as stated in this section) 8 | patent license to make, have made, use, offer to sell, sell, import, 9 | transfer and otherwise run, modify and propagate the contents of this 10 | implementation of Go, where such license applies only to those patent 11 | claims, both currently owned or controlled by Google and acquired in 12 | the future, licensable by Google that are necessarily infringed by this 13 | implementation of Go. This grant does not include claims that would be 14 | infringed only as a consequence of further modification of this 15 | implementation. If you or your agent or exclusive licensee institute or 16 | order or agree to the institution of patent litigation against any 17 | entity (including a cross-claim or counterclaim in a lawsuit) alleging 18 | that this implementation of Go or any code incorporated within this 19 | implementation of Go constitutes direct or contributory patent 20 | infringement, or inducement of patent infringement, then any patent 21 | rights granted to you under this License for this implementation of Go 22 | shall terminate as of the date such litigation is filed. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mod 2 | 3 | [![PkgGoDev](https://pkg.go.dev/badge/golang.org/x/mod)](https://pkg.go.dev/golang.org/x/mod) 4 | 5 | This repository holds packages for writing tools 6 | that work directly with Go module mechanics. 7 | That is, it is for direct manipulation of Go modules themselves. 8 | 9 | It is NOT about supporting general development tools that 10 | need to do things like load packages in module mode. 11 | That use case, where modules are incidental rather than the focus, 12 | should remain in [x/tools](https://pkg.go.dev/golang.org/x/tools), 13 | specifically [x/tools/go/packages](https://pkg.go.dev/golang.org/x/tools/go/packages). 14 | 15 | The specific case of loading packages should still be done by 16 | invoking the go command, which remains the single point of 17 | truth for package loading algorithms. 18 | -------------------------------------------------------------------------------- /codereview.cfg: -------------------------------------------------------------------------------- 1 | issuerepo: golang/go 2 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module golang.org/x/mod 2 | 3 | go 1.23.0 4 | 5 | require golang.org/x/tools v0.13.0 // tagx:ignore 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= 2 | golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= 3 | -------------------------------------------------------------------------------- /gosumcheck/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Gosumcheck checks a go.sum file against a go.sum database server. 6 | // 7 | // Usage: 8 | // 9 | // gosumcheck [-h H] [-k key] [-u url] [-v] go.sum 10 | // 11 | // The -h flag changes the tile height (default 8). 12 | // 13 | // The -k flag changes the go.sum database server key. 14 | // 15 | // The -u flag overrides the URL of the server (usually set from the key name). 16 | // 17 | // The -v flag enables verbose output. 18 | // In particular, it causes gosumcheck to report 19 | // the URL and elapsed time for each server request. 20 | // 21 | // WARNING! WARNING! WARNING! 22 | // 23 | // Gosumcheck is meant as a proof of concept demo and should not be 24 | // used in production scripts or continuous integration testing. 25 | // It does not cache any downloaded information from run to run, 26 | // making it expensive and also keeping it from detecting server 27 | // misbehavior or successful HTTPS man-in-the-middle timeline forks. 28 | // 29 | // To discourage misuse in automated settings, gosumcheck does not 30 | // set any exit status to report whether any problems were found. 31 | package main 32 | 33 | import ( 34 | "flag" 35 | "fmt" 36 | "io" 37 | "log" 38 | "net/http" 39 | "os" 40 | "os/exec" 41 | "strings" 42 | "sync" 43 | "time" 44 | 45 | "golang.org/x/mod/sumdb" 46 | ) 47 | 48 | func usage() { 49 | fmt.Fprintf(os.Stderr, "usage: gosumcheck [-h H] [-k key] [-u url] [-v] go.sum...\n") 50 | os.Exit(2) 51 | } 52 | 53 | var ( 54 | height = flag.Int("h", 8, "tile height") 55 | vkey = flag.String("k", "sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8", "key") 56 | url = flag.String("u", "", "url to server (overriding name)") 57 | vflag = flag.Bool("v", false, "enable verbose output") 58 | ) 59 | 60 | func main() { 61 | log.SetPrefix("notecheck: ") 62 | log.SetFlags(0) 63 | 64 | flag.Usage = usage 65 | flag.Parse() 66 | if flag.NArg() < 1 { 67 | usage() 68 | } 69 | 70 | client := sumdb.NewClient(new(clientOps)) 71 | 72 | // Look in environment explicitly, so that if 'go env' is old and 73 | // doesn't know about GONOSUMDB, we at least get anything 74 | // set in the environment. 75 | env := os.Getenv("GONOSUMDB") 76 | if env == "" { 77 | out, err := exec.Command("go", "env", "GONOSUMDB").CombinedOutput() 78 | if err != nil { 79 | log.Fatalf("go env GONOSUMDB: %v\n%s", err, out) 80 | } 81 | env = strings.TrimSpace(string(out)) 82 | } 83 | client.SetGONOSUMDB(env) 84 | 85 | for _, arg := range flag.Args() { 86 | data, err := os.ReadFile(arg) 87 | if err != nil { 88 | log.Fatal(err) 89 | } 90 | checkGoSum(client, arg, data) 91 | } 92 | } 93 | 94 | func checkGoSum(client *sumdb.Client, name string, data []byte) { 95 | lines := strings.Split(string(data), "\n") 96 | if lines[len(lines)-1] != "" { 97 | log.Printf("error: final line missing newline") 98 | return 99 | } 100 | lines = lines[:len(lines)-1] 101 | 102 | errs := make([]string, len(lines)) 103 | var wg sync.WaitGroup 104 | for i, line := range lines { 105 | wg.Add(1) 106 | go func(i int, line string) { 107 | defer wg.Done() 108 | f := strings.Fields(line) 109 | if len(f) != 3 { 110 | errs[i] = "invalid number of fields" 111 | return 112 | } 113 | 114 | dbLines, err := client.Lookup(f[0], f[1]) 115 | if err != nil { 116 | if err == sumdb.ErrGONOSUMDB { 117 | errs[i] = fmt.Sprintf("%s@%s: %v", f[0], f[1], err) 118 | } else { 119 | // Otherwise Lookup properly adds the prefix itself. 120 | errs[i] = err.Error() 121 | } 122 | return 123 | } 124 | hashAlgPrefix := f[0] + " " + f[1] + " " + f[2][:strings.Index(f[2], ":")+1] 125 | for _, dbLine := range dbLines { 126 | if dbLine == line { 127 | return 128 | } 129 | if strings.HasPrefix(dbLine, hashAlgPrefix) { 130 | errs[i] = fmt.Sprintf("%s@%s hash mismatch: have %s, want %s", f[0], f[1], line, dbLine) 131 | return 132 | } 133 | } 134 | errs[i] = fmt.Sprintf("%s@%s hash algorithm mismatch: have %s, want one of:\n\t%s", f[0], f[1], line, strings.Join(dbLines, "\n\t")) 135 | }(i, line) 136 | } 137 | wg.Wait() 138 | 139 | for i, err := range errs { 140 | if err != "" { 141 | fmt.Printf("%s:%d: %s\n", name, i+1, err) 142 | } 143 | } 144 | } 145 | 146 | type clientOps struct{} 147 | 148 | func (*clientOps) ReadConfig(file string) ([]byte, error) { 149 | if file == "key" { 150 | return []byte(*vkey), nil 151 | } 152 | if strings.HasSuffix(file, "/latest") { 153 | // Looking for cached latest tree head. 154 | // Empty result means empty tree. 155 | return []byte{}, nil 156 | } 157 | return nil, fmt.Errorf("unknown config %s", file) 158 | } 159 | 160 | func (*clientOps) WriteConfig(file string, old, new []byte) error { 161 | // Ignore writes. 162 | return nil 163 | } 164 | 165 | func (*clientOps) ReadCache(file string) ([]byte, error) { 166 | return nil, fmt.Errorf("no cache") 167 | } 168 | 169 | func (*clientOps) WriteCache(file string, data []byte) { 170 | // Ignore writes. 171 | } 172 | 173 | func (*clientOps) Log(msg string) { 174 | log.Print(msg) 175 | } 176 | 177 | func (*clientOps) SecurityError(msg string) { 178 | log.Fatal(msg) 179 | } 180 | 181 | func init() { 182 | http.DefaultClient.Timeout = 1 * time.Minute 183 | } 184 | 185 | func (*clientOps) ReadRemote(path string) ([]byte, error) { 186 | name := *vkey 187 | if i := strings.Index(name, "+"); i >= 0 { 188 | name = name[:i] 189 | } 190 | start := time.Now() 191 | target := "https://" + name + path 192 | if *url != "" { 193 | target = *url + path 194 | } 195 | resp, err := http.Get(target) 196 | if err != nil { 197 | return nil, err 198 | } 199 | defer resp.Body.Close() 200 | if resp.StatusCode != 200 { 201 | return nil, fmt.Errorf("GET %v: %v", target, resp.Status) 202 | } 203 | data, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) 204 | if err != nil { 205 | return nil, err 206 | } 207 | if *vflag { 208 | fmt.Fprintf(os.Stderr, "%.3fs %s\n", time.Since(start).Seconds(), target) 209 | } 210 | return data, nil 211 | } 212 | -------------------------------------------------------------------------------- /gosumcheck/test.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | go build -o gosumcheck.exe 5 | export GONOSUMDB=*/text # rsc.io/text but not golang.org/x/text 6 | ./gosumcheck.exe "$@" -v test.sum 7 | rm -f ./gosumcheck.exe 8 | echo PASS 9 | -------------------------------------------------------------------------------- /gosumcheck/test.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c h1:qgOY6WgZOaTkIIMiVjBQcw93ERBE4m30iBm00nkL0i8= 2 | golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 3 | rsc.io/quote v1.5.2 h1:w5fcysjrx7yqtD/aO+QwRjYZOKnaM9Uh2b40tElTs3Y= 4 | rsc.io/quote v1.5.2/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0= 5 | rsc.io/text v1.5.2 h1:w5fcysjrx7yqtD/aO+QwRjYZOKnaM9Uh2b40tElTs3Y= 6 | rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= 7 | -------------------------------------------------------------------------------- /internal/lazyregexp/lazyre.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package lazyregexp is a thin wrapper over regexp, allowing the use of global 6 | // regexp variables without forcing them to be compiled at init. 7 | package lazyregexp 8 | 9 | import ( 10 | "os" 11 | "regexp" 12 | "strings" 13 | "sync" 14 | ) 15 | 16 | // Regexp is a wrapper around [regexp.Regexp], where the underlying regexp will be 17 | // compiled the first time it is needed. 18 | type Regexp struct { 19 | str string 20 | once sync.Once 21 | rx *regexp.Regexp 22 | } 23 | 24 | func (r *Regexp) re() *regexp.Regexp { 25 | r.once.Do(r.build) 26 | return r.rx 27 | } 28 | 29 | func (r *Regexp) build() { 30 | r.rx = regexp.MustCompile(r.str) 31 | r.str = "" 32 | } 33 | 34 | func (r *Regexp) FindSubmatch(s []byte) [][]byte { 35 | return r.re().FindSubmatch(s) 36 | } 37 | 38 | func (r *Regexp) FindStringSubmatch(s string) []string { 39 | return r.re().FindStringSubmatch(s) 40 | } 41 | 42 | func (r *Regexp) FindStringSubmatchIndex(s string) []int { 43 | return r.re().FindStringSubmatchIndex(s) 44 | } 45 | 46 | func (r *Regexp) ReplaceAllString(src, repl string) string { 47 | return r.re().ReplaceAllString(src, repl) 48 | } 49 | 50 | func (r *Regexp) FindString(s string) string { 51 | return r.re().FindString(s) 52 | } 53 | 54 | func (r *Regexp) FindAllString(s string, n int) []string { 55 | return r.re().FindAllString(s, n) 56 | } 57 | 58 | func (r *Regexp) MatchString(s string) bool { 59 | return r.re().MatchString(s) 60 | } 61 | 62 | func (r *Regexp) SubexpNames() []string { 63 | return r.re().SubexpNames() 64 | } 65 | 66 | var inTest = len(os.Args) > 0 && strings.HasSuffix(strings.TrimSuffix(os.Args[0], ".exe"), ".test") 67 | 68 | // New creates a new lazy regexp, delaying the compiling work until it is first 69 | // needed. If the code is being run as part of tests, the regexp compiling will 70 | // happen immediately. 71 | func New(str string) *Regexp { 72 | lr := &Regexp{str: str} 73 | if inTest { 74 | // In tests, always compile the regexps early. 75 | lr.re() 76 | } 77 | return lr 78 | } 79 | -------------------------------------------------------------------------------- /modfile/print.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Module file printer. 6 | 7 | package modfile 8 | 9 | import ( 10 | "bytes" 11 | "fmt" 12 | "strings" 13 | ) 14 | 15 | // Format returns a go.mod file as a byte slice, formatted in standard style. 16 | func Format(f *FileSyntax) []byte { 17 | pr := &printer{} 18 | pr.file(f) 19 | 20 | // remove trailing blank lines 21 | b := pr.Bytes() 22 | for len(b) > 0 && b[len(b)-1] == '\n' && (len(b) == 1 || b[len(b)-2] == '\n') { 23 | b = b[:len(b)-1] 24 | } 25 | return b 26 | } 27 | 28 | // A printer collects the state during printing of a file or expression. 29 | type printer struct { 30 | bytes.Buffer // output buffer 31 | comment []Comment // pending end-of-line comments 32 | margin int // left margin (indent), a number of tabs 33 | } 34 | 35 | // printf prints to the buffer. 36 | func (p *printer) printf(format string, args ...interface{}) { 37 | fmt.Fprintf(p, format, args...) 38 | } 39 | 40 | // indent returns the position on the current line, in bytes, 0-indexed. 41 | func (p *printer) indent() int { 42 | b := p.Bytes() 43 | n := 0 44 | for n < len(b) && b[len(b)-1-n] != '\n' { 45 | n++ 46 | } 47 | return n 48 | } 49 | 50 | // newline ends the current line, flushing end-of-line comments. 51 | func (p *printer) newline() { 52 | if len(p.comment) > 0 { 53 | p.printf(" ") 54 | for i, com := range p.comment { 55 | if i > 0 { 56 | p.trim() 57 | p.printf("\n") 58 | for i := 0; i < p.margin; i++ { 59 | p.printf("\t") 60 | } 61 | } 62 | p.printf("%s", strings.TrimSpace(com.Token)) 63 | } 64 | p.comment = p.comment[:0] 65 | } 66 | 67 | p.trim() 68 | if b := p.Bytes(); len(b) == 0 || (len(b) >= 2 && b[len(b)-1] == '\n' && b[len(b)-2] == '\n') { 69 | // skip the blank line at top of file or after a blank line 70 | } else { 71 | p.printf("\n") 72 | } 73 | for i := 0; i < p.margin; i++ { 74 | p.printf("\t") 75 | } 76 | } 77 | 78 | // trim removes trailing spaces and tabs from the current line. 79 | func (p *printer) trim() { 80 | // Remove trailing spaces and tabs from line we're about to end. 81 | b := p.Bytes() 82 | n := len(b) 83 | for n > 0 && (b[n-1] == '\t' || b[n-1] == ' ') { 84 | n-- 85 | } 86 | p.Truncate(n) 87 | } 88 | 89 | // file formats the given file into the print buffer. 90 | func (p *printer) file(f *FileSyntax) { 91 | for _, com := range f.Before { 92 | p.printf("%s", strings.TrimSpace(com.Token)) 93 | p.newline() 94 | } 95 | 96 | for i, stmt := range f.Stmt { 97 | switch x := stmt.(type) { 98 | case *CommentBlock: 99 | // comments already handled 100 | p.expr(x) 101 | 102 | default: 103 | p.expr(x) 104 | p.newline() 105 | } 106 | 107 | for _, com := range stmt.Comment().After { 108 | p.printf("%s", strings.TrimSpace(com.Token)) 109 | p.newline() 110 | } 111 | 112 | if i+1 < len(f.Stmt) { 113 | p.newline() 114 | } 115 | } 116 | } 117 | 118 | func (p *printer) expr(x Expr) { 119 | // Emit line-comments preceding this expression. 120 | if before := x.Comment().Before; len(before) > 0 { 121 | // Want to print a line comment. 122 | // Line comments must be at the current margin. 123 | p.trim() 124 | if p.indent() > 0 { 125 | // There's other text on the line. Start a new line. 126 | p.printf("\n") 127 | } 128 | // Re-indent to margin. 129 | for i := 0; i < p.margin; i++ { 130 | p.printf("\t") 131 | } 132 | for _, com := range before { 133 | p.printf("%s", strings.TrimSpace(com.Token)) 134 | p.newline() 135 | } 136 | } 137 | 138 | switch x := x.(type) { 139 | default: 140 | panic(fmt.Errorf("printer: unexpected type %T", x)) 141 | 142 | case *CommentBlock: 143 | // done 144 | 145 | case *LParen: 146 | p.printf("(") 147 | case *RParen: 148 | p.printf(")") 149 | 150 | case *Line: 151 | p.tokens(x.Token) 152 | 153 | case *LineBlock: 154 | p.tokens(x.Token) 155 | p.printf(" ") 156 | p.expr(&x.LParen) 157 | p.margin++ 158 | for _, l := range x.Line { 159 | p.newline() 160 | p.expr(l) 161 | } 162 | p.margin-- 163 | p.newline() 164 | p.expr(&x.RParen) 165 | } 166 | 167 | // Queue end-of-line comments for printing when we 168 | // reach the end of the line. 169 | p.comment = append(p.comment, x.Comment().Suffix...) 170 | } 171 | 172 | func (p *printer) tokens(tokens []string) { 173 | sep := "" 174 | for _, t := range tokens { 175 | if t == "," || t == ")" || t == "]" || t == "}" { 176 | sep = "" 177 | } 178 | p.printf("%s%s", sep, t) 179 | sep = " " 180 | if t == "(" || t == "[" || t == "{" { 181 | sep = "" 182 | } 183 | } 184 | } 185 | -------------------------------------------------------------------------------- /modfile/testdata/block.golden: -------------------------------------------------------------------------------- 1 | // comment 2 | x "y" z 3 | 4 | // block 5 | block ( // block-eol 6 | // x-before-line 7 | 8 | "x" (y // x-eol 9 | "x") y // y-eol 10 | "x1" 11 | "x2" 12 | // line 13 | "x3" 14 | "x4" 15 | 16 | "x5" 17 | 18 | // y-line 19 | "y" // y-eol 20 | 21 | "z" // z-eol 22 | ) // block-eol2 23 | 24 | block1 ( 25 | ) 26 | 27 | block2 (x y z) 28 | 29 | block3 "w" ( 30 | ) // empty block 31 | 32 | block4 "x" () "y" // not a block 33 | 34 | block5 ("z" // also not a block 35 | 36 | // eof 37 | -------------------------------------------------------------------------------- /modfile/testdata/block.in: -------------------------------------------------------------------------------- 1 | // comment 2 | x "y" z 3 | 4 | // block 5 | block ( // block-eol 6 | // x-before-line 7 | 8 | "x" ( y // x-eol 9 | "x" ) y // y-eol 10 | "x1" 11 | "x2" 12 | // line 13 | "x3" 14 | "x4" 15 | 16 | "x5" 17 | 18 | // y-line 19 | "y" // y-eol 20 | 21 | "z" // z-eol 22 | ) // block-eol2 23 | 24 | 25 | block1() 26 | 27 | block2 (x y z) 28 | 29 | block3 "w" ( ) // empty block 30 | block4 "x" ( ) "y" // not a block 31 | block5 ( "z" // also not a block 32 | 33 | // eof 34 | -------------------------------------------------------------------------------- /modfile/testdata/comment.golden: -------------------------------------------------------------------------------- 1 | // comment 2 | module "x" // eol 3 | 4 | // mid comment 5 | 6 | // comment 2 7 | // comment 2 line 2 8 | module "y" // eoy 9 | 10 | // comment 3 11 | -------------------------------------------------------------------------------- /modfile/testdata/comment.in: -------------------------------------------------------------------------------- 1 | // comment 2 | module "x" // eol 3 | // mid comment 4 | 5 | // comment 2 6 | // comment 2 line 2 7 | module "y" // eoy 8 | // comment 3 9 | -------------------------------------------------------------------------------- /modfile/testdata/empty.golden: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golang/mod/9d3333156f465c85f68264344b5c08fbcf5fcacb/modfile/testdata/empty.golden -------------------------------------------------------------------------------- /modfile/testdata/empty.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golang/mod/9d3333156f465c85f68264344b5c08fbcf5fcacb/modfile/testdata/empty.in -------------------------------------------------------------------------------- /modfile/testdata/goline.golden: -------------------------------------------------------------------------------- 1 | go 1.2.3 2 | 3 | toolchain default 4 | -------------------------------------------------------------------------------- /modfile/testdata/goline.in: -------------------------------------------------------------------------------- 1 | go 1.2.3 2 | toolchain default 3 | -------------------------------------------------------------------------------- /modfile/testdata/gopkg.in.golden: -------------------------------------------------------------------------------- 1 | module x 2 | 3 | require ( 4 | gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528 5 | gopkg.in/yaml.v2 v2.2.1 6 | ) 7 | -------------------------------------------------------------------------------- /modfile/testdata/issue70632.golden: -------------------------------------------------------------------------------- 1 | module tidy 2 | 3 | go 1.23.0 4 | 5 | require ( 6 | golang.org/x/time v0.8.0 7 | ) 8 | -------------------------------------------------------------------------------- /modfile/testdata/issue70632.in: -------------------------------------------------------------------------------- 1 | module tidy 2 | 3 | go 1.23.0 4 | 5 | require ( 6 | 7 | "golang.org/x/time" v0.8.0 8 | 9 | 10 | 11 | 12 | ) 13 | -------------------------------------------------------------------------------- /modfile/testdata/module.golden: -------------------------------------------------------------------------------- 1 | module abc 2 | -------------------------------------------------------------------------------- /modfile/testdata/module.in: -------------------------------------------------------------------------------- 1 | module "abc" 2 | -------------------------------------------------------------------------------- /modfile/testdata/replace.golden: -------------------------------------------------------------------------------- 1 | module abc 2 | 3 | replace xyz v1.2.3 => /tmp/z 4 | 5 | replace xyz v1.3.4 => my/xyz v1.3.4-me 6 | 7 | replace ( 8 | w v1.0.0 => "./a," 9 | w v1.0.1 => "./a()" 10 | w v1.0.2 => "./a[]" 11 | w v1.0.3 => "./a{}" 12 | ) 13 | -------------------------------------------------------------------------------- /modfile/testdata/replace.in: -------------------------------------------------------------------------------- 1 | module "abc" 2 | 3 | replace "xyz" v1.2.3 => "/tmp/z" 4 | 5 | replace "xyz" v1.3.4 => "my/xyz" v1.3.4-me 6 | 7 | replace ( 8 | "w" v1.0.0 => "./a," 9 | "w" v1.0.1 => "./a()" 10 | "w" v1.0.2 => "./a[]" 11 | "w" v1.0.3 => "./a{}" 12 | ) 13 | -------------------------------------------------------------------------------- /modfile/testdata/replace2.golden: -------------------------------------------------------------------------------- 1 | module abc 2 | 3 | replace ( 4 | xyz v1.2.3 => /tmp/z 5 | xyz v1.3.4 => my/xyz v1.3.4-me 6 | xyz v1.4.5 => "/tmp/my dir" 7 | xyz v1.5.6 => my/xyz v1.5.6 8 | 9 | xyz => my/other/xyz v1.5.4 10 | ) 11 | -------------------------------------------------------------------------------- /modfile/testdata/replace2.in: -------------------------------------------------------------------------------- 1 | module "abc" 2 | 3 | replace ( 4 | "xyz" v1.2.3 => "/tmp/z" 5 | "xyz" v1.3.4 => "my/xyz" "v1.3.4-me" 6 | xyz "v1.4.5" => "/tmp/my dir" 7 | xyz v1.5.6 => my/xyz v1.5.6 8 | 9 | xyz => my/other/xyz v1.5.4 10 | ) 11 | -------------------------------------------------------------------------------- /modfile/testdata/retract.golden: -------------------------------------------------------------------------------- 1 | module abc 2 | 3 | retract v1.2.3 4 | 5 | retract [v1.2.3, v1.2.4] 6 | 7 | retract ( 8 | v1.2.3 9 | 10 | [v1.2.3, v1.2.4] 11 | ) 12 | -------------------------------------------------------------------------------- /modfile/testdata/retract.in: -------------------------------------------------------------------------------- 1 | module abc 2 | 3 | retract "v1.2.3" 4 | 5 | retract [ "v1.2.3" , "v1.2.4" ] 6 | 7 | retract ( 8 | "v1.2.3" 9 | 10 | [ "v1.2.3" , "v1.2.4" ] 11 | ) 12 | -------------------------------------------------------------------------------- /modfile/testdata/rule1.golden: -------------------------------------------------------------------------------- 1 | module "x" 2 | 3 | module "y" 4 | 5 | require "x" 6 | 7 | require x 8 | -------------------------------------------------------------------------------- /modfile/testdata/work/comment.golden: -------------------------------------------------------------------------------- 1 | // comment 2 | use x // eol 3 | 4 | // mid comment 5 | 6 | // comment 2 7 | // comment 2 line 2 8 | use y // eoy 9 | 10 | // comment 3 11 | -------------------------------------------------------------------------------- /modfile/testdata/work/comment.in: -------------------------------------------------------------------------------- 1 | // comment 2 | use "x" // eol 3 | // mid comment 4 | 5 | // comment 2 6 | // comment 2 line 2 7 | use "y" // eoy 8 | // comment 3 9 | -------------------------------------------------------------------------------- /modfile/testdata/work/empty.golden: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golang/mod/9d3333156f465c85f68264344b5c08fbcf5fcacb/modfile/testdata/work/empty.golden -------------------------------------------------------------------------------- /modfile/testdata/work/empty.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golang/mod/9d3333156f465c85f68264344b5c08fbcf5fcacb/modfile/testdata/work/empty.in -------------------------------------------------------------------------------- /modfile/testdata/work/goline.golden: -------------------------------------------------------------------------------- 1 | go 1.2.3 2 | 3 | toolchain default 4 | -------------------------------------------------------------------------------- /modfile/testdata/work/goline.in: -------------------------------------------------------------------------------- 1 | go 1.2.3 2 | toolchain default 3 | -------------------------------------------------------------------------------- /modfile/testdata/work/replace.golden: -------------------------------------------------------------------------------- 1 | use abc 2 | 3 | replace xyz v1.2.3 => /tmp/z 4 | 5 | replace xyz v1.3.4 => my/xyz v1.3.4-me 6 | 7 | replace ( 8 | w v1.0.0 => "./a," 9 | w v1.0.1 => "./a()" 10 | w v1.0.2 => "./a[]" 11 | w v1.0.3 => "./a{}" 12 | ) 13 | -------------------------------------------------------------------------------- /modfile/testdata/work/replace.in: -------------------------------------------------------------------------------- 1 | use "abc" 2 | 3 | replace "xyz" v1.2.3 => "/tmp/z" 4 | 5 | replace "xyz" v1.3.4 => "my/xyz" v1.3.4-me 6 | 7 | replace ( 8 | "w" v1.0.0 => "./a," 9 | "w" v1.0.1 => "./a()" 10 | "w" v1.0.2 => "./a[]" 11 | "w" v1.0.3 => "./a{}" 12 | ) 13 | -------------------------------------------------------------------------------- /modfile/testdata/work/replace2.golden: -------------------------------------------------------------------------------- 1 | use abc 2 | 3 | replace ( 4 | xyz v1.2.3 => /tmp/z 5 | xyz v1.3.4 => my/xyz v1.3.4-me 6 | xyz v1.4.5 => "/tmp/my dir" 7 | xyz v1.5.6 => my/xyz v1.5.6 8 | 9 | xyz => my/other/xyz v1.5.4 10 | ) 11 | -------------------------------------------------------------------------------- /modfile/testdata/work/replace2.in: -------------------------------------------------------------------------------- 1 | use "abc" 2 | 3 | replace ( 4 | "xyz" v1.2.3 => "/tmp/z" 5 | "xyz" v1.3.4 => "my/xyz" "v1.3.4-me" 6 | xyz "v1.4.5" => "/tmp/my dir" 7 | xyz v1.5.6 => my/xyz v1.5.6 8 | 9 | xyz => my/other/xyz v1.5.4 10 | ) 11 | -------------------------------------------------------------------------------- /modfile/testdata/work/use.golden: -------------------------------------------------------------------------------- 1 | use ../foo 2 | 3 | use ( 4 | /bar 5 | 6 | baz 7 | ) 8 | -------------------------------------------------------------------------------- /modfile/testdata/work/use.in: -------------------------------------------------------------------------------- 1 | use "../foo" 2 | 3 | use ( 4 | "/bar" 5 | 6 | "baz" 7 | ) 8 | -------------------------------------------------------------------------------- /modfile/work.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package modfile 6 | 7 | import ( 8 | "fmt" 9 | "slices" 10 | "strings" 11 | ) 12 | 13 | // A WorkFile is the parsed, interpreted form of a go.work file. 14 | type WorkFile struct { 15 | Go *Go 16 | Toolchain *Toolchain 17 | Godebug []*Godebug 18 | Use []*Use 19 | Replace []*Replace 20 | 21 | Syntax *FileSyntax 22 | } 23 | 24 | // A Use is a single directory statement. 25 | type Use struct { 26 | Path string // Use path of module. 27 | ModulePath string // Module path in the comment. 28 | Syntax *Line 29 | } 30 | 31 | // ParseWork parses and returns a go.work file. 32 | // 33 | // file is the name of the file, used in positions and errors. 34 | // 35 | // data is the content of the file. 36 | // 37 | // fix is an optional function that canonicalizes module versions. 38 | // If fix is nil, all module versions must be canonical ([module.CanonicalVersion] 39 | // must return the same string). 40 | func ParseWork(file string, data []byte, fix VersionFixer) (*WorkFile, error) { 41 | fs, err := parse(file, data) 42 | if err != nil { 43 | return nil, err 44 | } 45 | f := &WorkFile{ 46 | Syntax: fs, 47 | } 48 | var errs ErrorList 49 | 50 | for _, x := range fs.Stmt { 51 | switch x := x.(type) { 52 | case *Line: 53 | f.add(&errs, x, x.Token[0], x.Token[1:], fix) 54 | 55 | case *LineBlock: 56 | if len(x.Token) > 1 { 57 | errs = append(errs, Error{ 58 | Filename: file, 59 | Pos: x.Start, 60 | Err: fmt.Errorf("unknown block type: %s", strings.Join(x.Token, " ")), 61 | }) 62 | continue 63 | } 64 | switch x.Token[0] { 65 | default: 66 | errs = append(errs, Error{ 67 | Filename: file, 68 | Pos: x.Start, 69 | Err: fmt.Errorf("unknown block type: %s", strings.Join(x.Token, " ")), 70 | }) 71 | continue 72 | case "godebug", "use", "replace": 73 | for _, l := range x.Line { 74 | f.add(&errs, l, x.Token[0], l.Token, fix) 75 | } 76 | } 77 | } 78 | } 79 | 80 | if len(errs) > 0 { 81 | return nil, errs 82 | } 83 | return f, nil 84 | } 85 | 86 | // Cleanup cleans up the file f after any edit operations. 87 | // To avoid quadratic behavior, modifications like [WorkFile.DropRequire] 88 | // clear the entry but do not remove it from the slice. 89 | // Cleanup cleans out all the cleared entries. 90 | func (f *WorkFile) Cleanup() { 91 | w := 0 92 | for _, r := range f.Use { 93 | if r.Path != "" { 94 | f.Use[w] = r 95 | w++ 96 | } 97 | } 98 | f.Use = f.Use[:w] 99 | 100 | w = 0 101 | for _, r := range f.Replace { 102 | if r.Old.Path != "" { 103 | f.Replace[w] = r 104 | w++ 105 | } 106 | } 107 | f.Replace = f.Replace[:w] 108 | 109 | f.Syntax.Cleanup() 110 | } 111 | 112 | func (f *WorkFile) AddGoStmt(version string) error { 113 | if !GoVersionRE.MatchString(version) { 114 | return fmt.Errorf("invalid language version %q", version) 115 | } 116 | if f.Go == nil { 117 | stmt := &Line{Token: []string{"go", version}} 118 | f.Go = &Go{ 119 | Version: version, 120 | Syntax: stmt, 121 | } 122 | // Find the first non-comment-only block and add 123 | // the go statement before it. That will keep file comments at the top. 124 | i := 0 125 | for i = 0; i < len(f.Syntax.Stmt); i++ { 126 | if _, ok := f.Syntax.Stmt[i].(*CommentBlock); !ok { 127 | break 128 | } 129 | } 130 | f.Syntax.Stmt = append(append(f.Syntax.Stmt[:i:i], stmt), f.Syntax.Stmt[i:]...) 131 | } else { 132 | f.Go.Version = version 133 | f.Syntax.updateLine(f.Go.Syntax, "go", version) 134 | } 135 | return nil 136 | } 137 | 138 | func (f *WorkFile) AddToolchainStmt(name string) error { 139 | if !ToolchainRE.MatchString(name) { 140 | return fmt.Errorf("invalid toolchain name %q", name) 141 | } 142 | if f.Toolchain == nil { 143 | stmt := &Line{Token: []string{"toolchain", name}} 144 | f.Toolchain = &Toolchain{ 145 | Name: name, 146 | Syntax: stmt, 147 | } 148 | // Find the go line and add the toolchain line after it. 149 | // Or else find the first non-comment-only block and add 150 | // the toolchain line before it. That will keep file comments at the top. 151 | i := 0 152 | for i = 0; i < len(f.Syntax.Stmt); i++ { 153 | if line, ok := f.Syntax.Stmt[i].(*Line); ok && len(line.Token) > 0 && line.Token[0] == "go" { 154 | i++ 155 | goto Found 156 | } 157 | } 158 | for i = 0; i < len(f.Syntax.Stmt); i++ { 159 | if _, ok := f.Syntax.Stmt[i].(*CommentBlock); !ok { 160 | break 161 | } 162 | } 163 | Found: 164 | f.Syntax.Stmt = append(append(f.Syntax.Stmt[:i:i], stmt), f.Syntax.Stmt[i:]...) 165 | } else { 166 | f.Toolchain.Name = name 167 | f.Syntax.updateLine(f.Toolchain.Syntax, "toolchain", name) 168 | } 169 | return nil 170 | } 171 | 172 | // DropGoStmt deletes the go statement from the file. 173 | func (f *WorkFile) DropGoStmt() { 174 | if f.Go != nil { 175 | f.Go.Syntax.markRemoved() 176 | f.Go = nil 177 | } 178 | } 179 | 180 | // DropToolchainStmt deletes the toolchain statement from the file. 181 | func (f *WorkFile) DropToolchainStmt() { 182 | if f.Toolchain != nil { 183 | f.Toolchain.Syntax.markRemoved() 184 | f.Toolchain = nil 185 | } 186 | } 187 | 188 | // AddGodebug sets the first godebug line for key to value, 189 | // preserving any existing comments for that line and removing all 190 | // other godebug lines for key. 191 | // 192 | // If no line currently exists for key, AddGodebug adds a new line 193 | // at the end of the last godebug block. 194 | func (f *WorkFile) AddGodebug(key, value string) error { 195 | need := true 196 | for _, g := range f.Godebug { 197 | if g.Key == key { 198 | if need { 199 | g.Value = value 200 | f.Syntax.updateLine(g.Syntax, "godebug", key+"="+value) 201 | need = false 202 | } else { 203 | g.Syntax.markRemoved() 204 | *g = Godebug{} 205 | } 206 | } 207 | } 208 | 209 | if need { 210 | f.addNewGodebug(key, value) 211 | } 212 | return nil 213 | } 214 | 215 | // addNewGodebug adds a new godebug key=value line at the end 216 | // of the last godebug block, regardless of any existing godebug lines for key. 217 | func (f *WorkFile) addNewGodebug(key, value string) { 218 | line := f.Syntax.addLine(nil, "godebug", key+"="+value) 219 | g := &Godebug{ 220 | Key: key, 221 | Value: value, 222 | Syntax: line, 223 | } 224 | f.Godebug = append(f.Godebug, g) 225 | } 226 | 227 | func (f *WorkFile) DropGodebug(key string) error { 228 | for _, g := range f.Godebug { 229 | if g.Key == key { 230 | g.Syntax.markRemoved() 231 | *g = Godebug{} 232 | } 233 | } 234 | return nil 235 | } 236 | 237 | func (f *WorkFile) AddUse(diskPath, modulePath string) error { 238 | need := true 239 | for _, d := range f.Use { 240 | if d.Path == diskPath { 241 | if need { 242 | d.ModulePath = modulePath 243 | f.Syntax.updateLine(d.Syntax, "use", AutoQuote(diskPath)) 244 | need = false 245 | } else { 246 | d.Syntax.markRemoved() 247 | *d = Use{} 248 | } 249 | } 250 | } 251 | 252 | if need { 253 | f.AddNewUse(diskPath, modulePath) 254 | } 255 | return nil 256 | } 257 | 258 | func (f *WorkFile) AddNewUse(diskPath, modulePath string) { 259 | line := f.Syntax.addLine(nil, "use", AutoQuote(diskPath)) 260 | f.Use = append(f.Use, &Use{Path: diskPath, ModulePath: modulePath, Syntax: line}) 261 | } 262 | 263 | func (f *WorkFile) SetUse(dirs []*Use) { 264 | need := make(map[string]string) 265 | for _, d := range dirs { 266 | need[d.Path] = d.ModulePath 267 | } 268 | 269 | for _, d := range f.Use { 270 | if modulePath, ok := need[d.Path]; ok { 271 | d.ModulePath = modulePath 272 | } else { 273 | d.Syntax.markRemoved() 274 | *d = Use{} 275 | } 276 | } 277 | 278 | // TODO(#45713): Add module path to comment. 279 | 280 | for diskPath, modulePath := range need { 281 | f.AddNewUse(diskPath, modulePath) 282 | } 283 | f.SortBlocks() 284 | } 285 | 286 | func (f *WorkFile) DropUse(path string) error { 287 | for _, d := range f.Use { 288 | if d.Path == path { 289 | d.Syntax.markRemoved() 290 | *d = Use{} 291 | } 292 | } 293 | return nil 294 | } 295 | 296 | func (f *WorkFile) AddReplace(oldPath, oldVers, newPath, newVers string) error { 297 | return addReplace(f.Syntax, &f.Replace, oldPath, oldVers, newPath, newVers) 298 | } 299 | 300 | func (f *WorkFile) DropReplace(oldPath, oldVers string) error { 301 | for _, r := range f.Replace { 302 | if r.Old.Path == oldPath && r.Old.Version == oldVers { 303 | r.Syntax.markRemoved() 304 | *r = Replace{} 305 | } 306 | } 307 | return nil 308 | } 309 | 310 | func (f *WorkFile) SortBlocks() { 311 | f.removeDups() // otherwise sorting is unsafe 312 | 313 | for _, stmt := range f.Syntax.Stmt { 314 | block, ok := stmt.(*LineBlock) 315 | if !ok { 316 | continue 317 | } 318 | slices.SortStableFunc(block.Line, compareLine) 319 | } 320 | } 321 | 322 | // removeDups removes duplicate replace directives. 323 | // 324 | // Later replace directives take priority. 325 | // 326 | // require directives are not de-duplicated. That's left up to higher-level 327 | // logic (MVS). 328 | // 329 | // retract directives are not de-duplicated since comments are 330 | // meaningful, and versions may be retracted multiple times. 331 | func (f *WorkFile) removeDups() { 332 | removeDups(f.Syntax, nil, &f.Replace, nil, nil) 333 | } 334 | -------------------------------------------------------------------------------- /modfile/work_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package modfile 6 | 7 | import ( 8 | "bytes" 9 | "os" 10 | "path/filepath" 11 | "strings" 12 | "testing" 13 | ) 14 | 15 | // TODO(#45713): Update these tests once AddUse sets the module path. 16 | var workAddUseTests = []struct { 17 | desc string 18 | in string 19 | path string 20 | modulePath string 21 | out string 22 | }{ 23 | { 24 | `empty`, 25 | ``, 26 | `foo`, `bar`, 27 | `use foo`, 28 | }, 29 | { 30 | `go_stmt_only`, 31 | `go 1.17 32 | `, 33 | `foo`, `bar`, 34 | `go 1.17 35 | use foo 36 | `, 37 | }, 38 | { 39 | `use_line_present`, 40 | `go 1.17 41 | use baz`, 42 | `foo`, `bar`, 43 | `go 1.17 44 | use ( 45 | baz 46 | foo 47 | ) 48 | `, 49 | }, 50 | { 51 | `use_block_present`, 52 | `go 1.17 53 | use ( 54 | baz 55 | quux 56 | ) 57 | `, 58 | `foo`, `bar`, 59 | `go 1.17 60 | use ( 61 | baz 62 | quux 63 | foo 64 | ) 65 | `, 66 | }, 67 | { 68 | `use_and_replace_present`, 69 | `go 1.17 70 | use baz 71 | replace a => ./b 72 | `, 73 | `foo`, `bar`, 74 | `go 1.17 75 | use ( 76 | baz 77 | foo 78 | ) 79 | replace a => ./b 80 | `, 81 | }, 82 | } 83 | 84 | var workDropUseTests = []struct { 85 | desc string 86 | in string 87 | path string 88 | out string 89 | }{ 90 | { 91 | `empty`, 92 | ``, 93 | `foo`, 94 | ``, 95 | }, 96 | { 97 | `go_stmt_only`, 98 | `go 1.17 99 | `, 100 | `foo`, 101 | `go 1.17 102 | `, 103 | }, 104 | { 105 | `single_use`, 106 | `go 1.17 107 | use foo`, 108 | `foo`, 109 | `go 1.17 110 | `, 111 | }, 112 | { 113 | `use_block`, 114 | `go 1.17 115 | use ( 116 | foo 117 | bar 118 | baz 119 | )`, 120 | `bar`, 121 | `go 1.17 122 | use ( 123 | foo 124 | baz 125 | )`, 126 | }, 127 | { 128 | `use_multi`, 129 | `go 1.17 130 | use ( 131 | foo 132 | bar 133 | baz 134 | ) 135 | use foo 136 | use quux 137 | use foo`, 138 | `foo`, 139 | `go 1.17 140 | use ( 141 | bar 142 | baz 143 | ) 144 | use quux`, 145 | }, 146 | } 147 | 148 | var workAddGoTests = []struct { 149 | desc string 150 | in string 151 | version string 152 | out string 153 | }{ 154 | { 155 | `empty`, 156 | ``, 157 | `1.17`, 158 | `go 1.17 159 | `, 160 | }, 161 | { 162 | `comment`, 163 | `// this is a comment`, 164 | `1.17`, 165 | `// this is a comment 166 | 167 | go 1.17`, 168 | }, 169 | { 170 | `use_after_replace`, 171 | ` 172 | replace example.com/foo => ../bar 173 | use foo 174 | `, 175 | `1.17`, 176 | ` 177 | go 1.17 178 | replace example.com/foo => ../bar 179 | use foo 180 | `, 181 | }, 182 | { 183 | `use_before_replace`, 184 | `use foo 185 | replace example.com/foo => ../bar 186 | `, 187 | `1.17`, 188 | ` 189 | go 1.17 190 | use foo 191 | replace example.com/foo => ../bar 192 | `, 193 | }, 194 | { 195 | `use_only`, 196 | `use foo 197 | `, 198 | `1.17`, 199 | ` 200 | go 1.17 201 | use foo 202 | `, 203 | }, 204 | { 205 | `already_have_go`, 206 | `go 1.17 207 | `, 208 | `1.18`, 209 | ` 210 | go 1.18 211 | `, 212 | }, 213 | } 214 | 215 | var workAddToolchainTests = []struct { 216 | desc string 217 | in string 218 | version string 219 | out string 220 | }{ 221 | { 222 | `empty`, 223 | ``, 224 | `go1.17`, 225 | `toolchain go1.17 226 | `, 227 | }, 228 | { 229 | `aftergo`, 230 | `// this is a comment 231 | use foo 232 | 233 | go 1.17 234 | 235 | use bar 236 | `, 237 | `go1.17`, 238 | `// this is a comment 239 | use foo 240 | 241 | go 1.17 242 | 243 | toolchain go1.17 244 | 245 | use bar 246 | `, 247 | }, 248 | { 249 | `already_have_toolchain`, 250 | `go 1.17 251 | 252 | toolchain go1.18 253 | `, 254 | `go1.19`, 255 | `go 1.17 256 | 257 | toolchain go1.19 258 | `, 259 | }, 260 | } 261 | 262 | var workSortBlocksTests = []struct { 263 | desc, in, out string 264 | }{ 265 | { 266 | `use_duplicates_not_removed`, 267 | `go 1.17 268 | use foo 269 | use bar 270 | use ( 271 | foo 272 | )`, 273 | `go 1.17 274 | use foo 275 | use bar 276 | use ( 277 | foo 278 | )`, 279 | }, 280 | { 281 | `replace_duplicates_removed`, 282 | `go 1.17 283 | use foo 284 | replace x.y/z v1.0.0 => ./a 285 | replace x.y/z v1.1.0 => ./b 286 | replace ( 287 | x.y/z v1.0.0 => ./c 288 | ) 289 | `, 290 | `go 1.17 291 | use foo 292 | replace x.y/z v1.1.0 => ./b 293 | replace ( 294 | x.y/z v1.0.0 => ./c 295 | ) 296 | `, 297 | }, 298 | } 299 | 300 | func TestAddUse(t *testing.T) { 301 | for _, tt := range workAddUseTests { 302 | t.Run(tt.desc, func(t *testing.T) { 303 | testWorkEdit(t, tt.in, tt.out, func(f *WorkFile) error { 304 | return f.AddUse(tt.path, tt.modulePath) 305 | }) 306 | }) 307 | } 308 | } 309 | 310 | func TestDropUse(t *testing.T) { 311 | for _, tt := range workDropUseTests { 312 | t.Run(tt.desc, func(t *testing.T) { 313 | testWorkEdit(t, tt.in, tt.out, func(f *WorkFile) error { 314 | if err := f.DropUse(tt.path); err != nil { 315 | return err 316 | } 317 | f.Cleanup() 318 | return nil 319 | }) 320 | }) 321 | } 322 | } 323 | 324 | func TestWorkAddGo(t *testing.T) { 325 | for _, tt := range workAddGoTests { 326 | t.Run(tt.desc, func(t *testing.T) { 327 | testWorkEdit(t, tt.in, tt.out, func(f *WorkFile) error { 328 | return f.AddGoStmt(tt.version) 329 | }) 330 | }) 331 | } 332 | } 333 | 334 | func TestWorkAddToolchain(t *testing.T) { 335 | for _, tt := range workAddToolchainTests { 336 | t.Run(tt.desc, func(t *testing.T) { 337 | testWorkEdit(t, tt.in, tt.out, func(f *WorkFile) error { 338 | return f.AddToolchainStmt(tt.version) 339 | }) 340 | }) 341 | } 342 | } 343 | 344 | func TestWorkSortBlocks(t *testing.T) { 345 | for _, tt := range workSortBlocksTests { 346 | t.Run(tt.desc, func(t *testing.T) { 347 | testWorkEdit(t, tt.in, tt.out, func(f *WorkFile) error { 348 | f.SortBlocks() 349 | return nil 350 | }) 351 | }) 352 | } 353 | } 354 | 355 | func TestWorkAddGodebug(t *testing.T) { 356 | for _, tt := range addGodebugTests { 357 | t.Run(tt.desc, func(t *testing.T) { 358 | in := strings.ReplaceAll(tt.in, "module m", "use foo") 359 | out := strings.ReplaceAll(tt.out, "module m", "use foo") 360 | testWorkEdit(t, in, out, func(f *WorkFile) error { 361 | err := f.AddGodebug(tt.key, tt.value) 362 | f.Cleanup() 363 | return err 364 | }) 365 | }) 366 | } 367 | } 368 | 369 | func TestWorkDropGodebug(t *testing.T) { 370 | for _, tt := range dropGodebugTests { 371 | t.Run(tt.desc, func(t *testing.T) { 372 | in := strings.ReplaceAll(tt.in, "module m", "use foo") 373 | out := strings.ReplaceAll(tt.out, "module m", "use foo") 374 | testWorkEdit(t, in, out, func(f *WorkFile) error { 375 | f.DropGodebug(tt.key) 376 | f.Cleanup() 377 | return nil 378 | }) 379 | }) 380 | } 381 | } 382 | 383 | // Test that when files in the testdata directory are parsed 384 | // and printed and parsed again, we get the same parse tree 385 | // both times. 386 | func TestWorkPrintParse(t *testing.T) { 387 | outs, err := filepath.Glob("testdata/work/*") 388 | if err != nil { 389 | t.Fatal(err) 390 | } 391 | for _, out := range outs { 392 | out := out 393 | name := filepath.Base(out) 394 | t.Run(name, func(t *testing.T) { 395 | t.Parallel() 396 | data, err := os.ReadFile(out) 397 | if err != nil { 398 | t.Fatal(err) 399 | } 400 | 401 | base := "testdata/work/" + filepath.Base(out) 402 | f, err := parse(base, data) 403 | if err != nil { 404 | t.Fatalf("parsing original: %v", err) 405 | } 406 | 407 | ndata := Format(f) 408 | f2, err := parse(base, ndata) 409 | if err != nil { 410 | t.Fatalf("parsing reformatted: %v", err) 411 | } 412 | 413 | eq := eqchecker{file: base} 414 | if err := eq.check(f, f2); err != nil { 415 | t.Errorf("not equal (parse/Format/parse): %v", err) 416 | } 417 | 418 | pf1, err := ParseWork(base, data, nil) 419 | if err != nil { 420 | t.Errorf("should parse %v: %v", base, err) 421 | } 422 | if err == nil { 423 | pf2, err := ParseWork(base, ndata, nil) 424 | if err != nil { 425 | t.Fatalf("Parsing reformatted: %v", err) 426 | } 427 | eq := eqchecker{file: base} 428 | if err := eq.check(pf1, pf2); err != nil { 429 | t.Errorf("not equal (parse/Format/Parse): %v", err) 430 | } 431 | 432 | ndata2 := Format(pf1.Syntax) 433 | pf3, err := ParseWork(base, ndata2, nil) 434 | if err != nil { 435 | t.Fatalf("Parsing reformatted2: %v", err) 436 | } 437 | eq = eqchecker{file: base} 438 | if err := eq.check(pf1, pf3); err != nil { 439 | t.Errorf("not equal (Parse/Format/Parse): %v", err) 440 | } 441 | ndata = ndata2 442 | } 443 | 444 | if strings.HasSuffix(out, ".in") { 445 | golden, err := os.ReadFile(strings.TrimSuffix(out, ".in") + ".golden") 446 | if err != nil { 447 | t.Fatal(err) 448 | } 449 | if !bytes.Equal(ndata, golden) { 450 | t.Errorf("formatted %s incorrectly: diff shows -golden, +ours", base) 451 | tdiff(t, string(golden), string(ndata)) 452 | return 453 | } 454 | } 455 | }) 456 | } 457 | } 458 | 459 | func testWorkEdit(t *testing.T, in, want string, transform func(f *WorkFile) error) *WorkFile { 460 | t.Helper() 461 | parse := ParseWork 462 | f, err := parse("in", []byte(in), nil) 463 | if err != nil { 464 | t.Fatal(err) 465 | } 466 | g, err := parse("out", []byte(want), nil) 467 | if err != nil { 468 | t.Fatal(err) 469 | } 470 | golden := Format(g.Syntax) 471 | 472 | if err := transform(f); err != nil { 473 | t.Fatal(err) 474 | } 475 | out := Format(f.Syntax) 476 | if err != nil { 477 | t.Fatal(err) 478 | } 479 | if !bytes.Equal(out, golden) { 480 | t.Errorf("have:\n%s\nwant:\n%s", out, golden) 481 | } 482 | 483 | return f 484 | } 485 | -------------------------------------------------------------------------------- /module/module_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package module 6 | 7 | import "testing" 8 | 9 | var checkTests = []struct { 10 | path string 11 | version string 12 | ok bool 13 | }{ 14 | {"rsc.io/quote", "0.1.0", false}, 15 | {"rsc io/quote", "v1.0.0", false}, 16 | 17 | {"github.com/go-yaml/yaml", "v0.8.0", true}, 18 | {"github.com/go-yaml/yaml", "v1.0.0", true}, 19 | {"github.com/go-yaml/yaml", "v2.0.0", false}, 20 | {"github.com/go-yaml/yaml", "v2.1.5", false}, 21 | {"github.com/go-yaml/yaml", "v3.0.0", false}, 22 | 23 | {"github.com/go-yaml/yaml/v2", "v1.0.0", false}, 24 | {"github.com/go-yaml/yaml/v2", "v2.0.0", true}, 25 | {"github.com/go-yaml/yaml/v2", "v2.1.5", true}, 26 | {"github.com/go-yaml/yaml/v2", "v3.0.0", false}, 27 | 28 | {"gopkg.in/yaml.v0", "v0.8.0", true}, 29 | {"gopkg.in/yaml.v0", "v1.0.0", false}, 30 | {"gopkg.in/yaml.v0", "v2.0.0", false}, 31 | {"gopkg.in/yaml.v0", "v2.1.5", false}, 32 | {"gopkg.in/yaml.v0", "v3.0.0", false}, 33 | 34 | {"gopkg.in/yaml.v1", "v0.8.0", false}, 35 | {"gopkg.in/yaml.v1", "v1.0.0", true}, 36 | {"gopkg.in/yaml.v1", "v2.0.0", false}, 37 | {"gopkg.in/yaml.v1", "v2.1.5", false}, 38 | {"gopkg.in/yaml.v1", "v3.0.0", false}, 39 | 40 | // For gopkg.in, .v1 means v1 only (not v0). 41 | // But early versions of vgo still generated v0 pseudo-versions for it. 42 | // Even though now we'd generate those as v1 pseudo-versions, 43 | // we accept the old pseudo-versions to avoid breaking existing go.mod files. 44 | // For example gopkg.in/yaml.v2@v2.2.1's go.mod requires check.v1 at a v0 pseudo-version. 45 | {"gopkg.in/check.v1", "v0.0.0", false}, 46 | {"gopkg.in/check.v1", "v0.0.0-20160102150405-abcdef123456", true}, 47 | 48 | {"gopkg.in/yaml.v2", "v1.0.0", false}, 49 | {"gopkg.in/yaml.v2", "v2.0.0", true}, 50 | {"gopkg.in/yaml.v2", "v2.1.5", true}, 51 | {"gopkg.in/yaml.v2", "v3.0.0", false}, 52 | 53 | {"rsc.io/quote", "v17.0.0", false}, 54 | {"rsc.io/quote", "v17.0.0+incompatible", true}, 55 | } 56 | 57 | func TestCheck(t *testing.T) { 58 | for _, tt := range checkTests { 59 | err := Check(tt.path, tt.version) 60 | if tt.ok && err != nil { 61 | t.Errorf("Check(%q, %q) = %v, wanted nil error", tt.path, tt.version, err) 62 | } else if !tt.ok && err == nil { 63 | t.Errorf("Check(%q, %q) succeeded, wanted error", tt.path, tt.version) 64 | } 65 | } 66 | } 67 | 68 | var checkPathTests = []struct { 69 | path string 70 | ok bool 71 | importOK bool 72 | fileOK bool 73 | }{ 74 | {"x.y/z", true, true, true}, 75 | {"x.y", true, true, true}, 76 | 77 | {"", false, false, false}, 78 | {"x.y/\xFFz", false, false, false}, 79 | {"/x.y/z", false, false, false}, 80 | {"x./z", false, false, false}, 81 | {".x/z", false, true, true}, 82 | {"-x/z", false, false, true}, 83 | {"x..y/z", true, true, true}, 84 | {"x.y/z/../../w", false, false, false}, 85 | {"x.y//z", false, false, false}, 86 | {"x.y/z//w", false, false, false}, 87 | {"x.y/z/", false, false, false}, 88 | 89 | {"x.y/z/v0", false, true, true}, 90 | {"x.y/z/v1", false, true, true}, 91 | {"x.y/z/v2", true, true, true}, 92 | {"x.y/z/v2.0", false, true, true}, 93 | {"X.y/z", false, true, true}, 94 | 95 | {"!x.y/z", false, false, true}, 96 | {"_x.y/z", false, true, true}, 97 | {"x.y!/z", false, false, true}, 98 | {"x.y\"/z", false, false, false}, 99 | {"x.y#/z", false, false, true}, 100 | {"x.y$/z", false, false, true}, 101 | {"x.y%/z", false, false, true}, 102 | {"x.y&/z", false, false, true}, 103 | {"x.y'/z", false, false, false}, 104 | {"x.y(/z", false, false, true}, 105 | {"x.y)/z", false, false, true}, 106 | {"x.y*/z", false, false, false}, 107 | {"x.y+/z", false, true, true}, 108 | {"x.y,/z", false, false, true}, 109 | {"x.y-/z", true, true, true}, 110 | {"x.y./zt", false, false, false}, 111 | {"x.y:/z", false, false, false}, 112 | {"x.y;/z", false, false, false}, 113 | {"x.y/z", false, false, false}, 116 | {"x.y?/z", false, false, false}, 117 | {"x.y@/z", false, false, true}, 118 | {"x.y[/z", false, false, true}, 119 | {"x.y\\/z", false, false, false}, 120 | {"x.y]/z", false, false, true}, 121 | {"x.y^/z", false, false, true}, 122 | {"x.y_/z", false, true, true}, 123 | {"x.y`/z", false, false, false}, 124 | {"x.y{/z", false, false, true}, 125 | {"x.y}/z", false, false, true}, 126 | {"x.y~/z", false, true, true}, 127 | {"x.y/z!", false, false, true}, 128 | {"x.y/z\"", false, false, false}, 129 | {"x.y/z#", false, false, true}, 130 | {"x.y/z$", false, false, true}, 131 | {"x.y/z%", false, false, true}, 132 | {"x.y/z&", false, false, true}, 133 | {"x.y/z'", false, false, false}, 134 | {"x.y/z(", false, false, true}, 135 | {"x.y/z)", false, false, true}, 136 | {"x.y/z*", false, false, false}, 137 | {"x.y/z++", false, true, true}, 138 | {"x.y/z,", false, false, true}, 139 | {"x.y/z-", true, true, true}, 140 | {"x.y/z.t", true, true, true}, 141 | {"x.y/z/t", true, true, true}, 142 | {"x.y/z:", false, false, false}, 143 | {"x.y/z;", false, false, false}, 144 | {"x.y/z<", false, false, false}, 145 | {"x.y/z=", false, false, true}, 146 | {"x.y/z>", false, false, false}, 147 | {"x.y/z?", false, false, false}, 148 | {"x.y/z@", false, false, true}, 149 | {"x.y/z[", false, false, true}, 150 | {"x.y/z\\", false, false, false}, 151 | {"x.y/z]", false, false, true}, 152 | {"x.y/z^", false, false, true}, 153 | {"x.y/z_", true, true, true}, 154 | {"x.y/z`", false, false, false}, 155 | {"x.y/z{", false, false, true}, 156 | {"x.y/z}", false, false, true}, 157 | {"x.y/z~", true, true, true}, 158 | {"x.y/x.foo", true, true, true}, 159 | {"x.y/aux.foo", false, false, false}, 160 | {"x.y/prn", false, false, false}, 161 | {"x.y/prn2", true, true, true}, 162 | {"x.y/com", true, true, true}, 163 | {"x.y/com1", false, false, false}, 164 | {"x.y/com1.txt", false, false, false}, 165 | {"x.y/calm1", true, true, true}, 166 | {"x.y/z~", true, true, true}, 167 | {"x.y/z~0", false, false, true}, 168 | {"x.y/z~09", false, false, true}, 169 | {"x.y/z09", true, true, true}, 170 | {"x.y/z09~", true, true, true}, 171 | {"x.y/z09~09z", true, true, true}, 172 | {"x.y/z09~09z~09", false, false, true}, 173 | {"github.com/!123/logrus", false, false, true}, 174 | 175 | // TODO: CL 41822 allowed Unicode letters in old "go get" 176 | // without due consideration of the implications, and only on github.com (!). 177 | // For now, we disallow non-ASCII characters in module mode, 178 | // in both module paths and general import paths, 179 | // until we can get the implications right. 180 | // When we do, we'll enable them everywhere, not just for GitHub. 181 | {"github.com/user/unicode/испытание", false, false, true}, 182 | 183 | {"../x", false, false, false}, 184 | {"./y", false, false, false}, 185 | {"x:y", false, false, false}, 186 | {`\temp\foo`, false, false, false}, 187 | {".gitignore", false, true, true}, 188 | {".github/ISSUE_TEMPLATE", false, true, true}, 189 | {"x☺y", false, false, false}, 190 | } 191 | 192 | func TestCheckPath(t *testing.T) { 193 | for _, tt := range checkPathTests { 194 | err := CheckPath(tt.path) 195 | if tt.ok && err != nil { 196 | t.Errorf("CheckPath(%q) = %v, wanted nil error", tt.path, err) 197 | } else if !tt.ok && err == nil { 198 | t.Errorf("CheckPath(%q) succeeded, wanted error", tt.path) 199 | } 200 | 201 | err = CheckImportPath(tt.path) 202 | if tt.importOK && err != nil { 203 | t.Errorf("CheckImportPath(%q) = %v, wanted nil error", tt.path, err) 204 | } else if !tt.importOK && err == nil { 205 | t.Errorf("CheckImportPath(%q) succeeded, wanted error", tt.path) 206 | } 207 | 208 | err = CheckFilePath(tt.path) 209 | if tt.fileOK && err != nil { 210 | t.Errorf("CheckFilePath(%q) = %v, wanted nil error", tt.path, err) 211 | } else if !tt.fileOK && err == nil { 212 | t.Errorf("CheckFilePath(%q) succeeded, wanted error", tt.path) 213 | } 214 | } 215 | } 216 | 217 | var splitPathVersionTests = []struct { 218 | pathPrefix string 219 | version string 220 | }{ 221 | {"x.y/z", ""}, 222 | {"x.y/z", "/v2"}, 223 | {"x.y/z", "/v3"}, 224 | {"x.y/v", ""}, 225 | {"gopkg.in/yaml", ".v0"}, 226 | {"gopkg.in/yaml", ".v1"}, 227 | {"gopkg.in/yaml", ".v2"}, 228 | {"gopkg.in/yaml", ".v3"}, 229 | } 230 | 231 | func TestSplitPathVersion(t *testing.T) { 232 | for _, tt := range splitPathVersionTests { 233 | pathPrefix, version, ok := SplitPathVersion(tt.pathPrefix + tt.version) 234 | if pathPrefix != tt.pathPrefix || version != tt.version || !ok { 235 | t.Errorf("SplitPathVersion(%q) = %q, %q, %v, want %q, %q, true", tt.pathPrefix+tt.version, pathPrefix, version, ok, tt.pathPrefix, tt.version) 236 | } 237 | } 238 | 239 | for _, tt := range checkPathTests { 240 | pathPrefix, version, ok := SplitPathVersion(tt.path) 241 | if pathPrefix+version != tt.path { 242 | t.Errorf("SplitPathVersion(%q) = %q, %q, %v, doesn't add to input", tt.path, pathPrefix, version, ok) 243 | } 244 | } 245 | } 246 | 247 | var escapeTests = []struct { 248 | path string 249 | esc string // empty means same as path 250 | }{ 251 | {path: "ascii.com/abcdefghijklmnopqrstuvwxyz.-/~_0123456789"}, 252 | {path: "github.com/GoogleCloudPlatform/omega", esc: "github.com/!google!cloud!platform/omega"}, 253 | } 254 | 255 | func TestEscapePath(t *testing.T) { 256 | // Check invalid paths. 257 | for _, tt := range checkPathTests { 258 | if !tt.ok { 259 | _, err := EscapePath(tt.path) 260 | if err == nil { 261 | t.Errorf("EscapePath(%q): succeeded, want error (invalid path)", tt.path) 262 | } 263 | } 264 | } 265 | 266 | // Check encodings. 267 | for _, tt := range escapeTests { 268 | esc, err := EscapePath(tt.path) 269 | if err != nil { 270 | t.Errorf("EscapePath(%q): unexpected error: %v", tt.path, err) 271 | continue 272 | } 273 | want := tt.esc 274 | if want == "" { 275 | want = tt.path 276 | } 277 | if esc != want { 278 | t.Errorf("EscapePath(%q) = %q, want %q", tt.path, esc, want) 279 | } 280 | } 281 | } 282 | 283 | var badUnescape = []string{ 284 | "github.com/GoogleCloudPlatform/omega", 285 | "github.com/!google!cloud!platform!/omega", 286 | "github.com/!0google!cloud!platform/omega", 287 | "github.com/!_google!cloud!platform/omega", 288 | "github.com/!!google!cloud!platform/omega", 289 | "", 290 | } 291 | 292 | func TestUnescapePath(t *testing.T) { 293 | // Check invalid decodings. 294 | for _, bad := range badUnescape { 295 | _, err := UnescapePath(bad) 296 | if err == nil { 297 | t.Errorf("UnescapePath(%q): succeeded, want error (invalid decoding)", bad) 298 | } 299 | } 300 | 301 | // Check invalid paths (or maybe decodings). 302 | for _, tt := range checkPathTests { 303 | if !tt.ok { 304 | path, err := UnescapePath(tt.path) 305 | if err == nil { 306 | t.Errorf("UnescapePath(%q) = %q, want error (invalid path)", tt.path, path) 307 | } 308 | } 309 | } 310 | 311 | // Check encodings. 312 | for _, tt := range escapeTests { 313 | esc := tt.esc 314 | if esc == "" { 315 | esc = tt.path 316 | } 317 | path, err := UnescapePath(esc) 318 | if err != nil { 319 | t.Errorf("UnescapePath(%q): unexpected error: %v", esc, err) 320 | continue 321 | } 322 | if path != tt.path { 323 | t.Errorf("UnescapePath(%q) = %q, want %q", esc, path, tt.path) 324 | } 325 | } 326 | } 327 | 328 | func TestMatchPathMajor(t *testing.T) { 329 | for _, test := range []struct { 330 | v, pathMajor string 331 | want bool 332 | }{ 333 | {"v0.0.0", "", true}, 334 | {"v0.0.0", "/v2", false}, 335 | {"v0.0.0", ".v0", true}, 336 | {"v0.0.0-20190510104115-cbcb75029529", ".v1", true}, 337 | {"v1.0.0", "/v2", false}, 338 | {"v1.0.0", ".v1", true}, 339 | {"v1.0.0", ".v1-unstable", true}, 340 | {"v2.0.0+incompatible", "", true}, 341 | {"v2.0.0", "", false}, 342 | {"v2.0.0", "/v2", true}, 343 | {"v2.0.0", ".v2", true}, 344 | } { 345 | if got := MatchPathMajor(test.v, test.pathMajor); got != test.want { 346 | t.Errorf("MatchPathMajor(%q, %q) = %v, want %v", test.v, test.pathMajor, got, test.want) 347 | } 348 | } 349 | } 350 | 351 | func TestMatchPrefixPatterns(t *testing.T) { 352 | for _, test := range []struct { 353 | globs, target string 354 | want bool 355 | }{ 356 | {"", "rsc.io/quote", false}, 357 | {"/", "rsc.io/quote", false}, 358 | {"*/quote", "rsc.io/quote", true}, 359 | {"*/quo", "rsc.io/quote", false}, 360 | {"*/quo??", "rsc.io/quote", true}, 361 | {"*/quo*", "rsc.io/quote", true}, 362 | {"*quo*", "rsc.io/quote", false}, 363 | {"rsc.io", "rsc.io/quote", true}, 364 | {"*.io", "rsc.io/quote", true}, 365 | {"rsc.io/", "rsc.io/quote", true}, 366 | {"rsc", "rsc.io/quote", false}, 367 | {"rsc*", "rsc.io/quote", true}, 368 | 369 | {"rsc.io", "rsc.io/quote/v3", true}, 370 | {"*/quote", "rsc.io/quote/v3", true}, 371 | {"*/quote/", "rsc.io/quote/v3", true}, 372 | {"*/quote/*", "rsc.io/quote/v3", true}, 373 | {"*/quote/*/", "rsc.io/quote/v3", true}, 374 | {"*/v3", "rsc.io/quote/v3", false}, 375 | {"*/*/v3", "rsc.io/quote/v3", true}, 376 | {"*/*/*", "rsc.io/quote/v3", true}, 377 | {"*/*/*/", "rsc.io/quote/v3", true}, 378 | {"*/*/*", "rsc.io/quote", false}, 379 | {"*/*/*/", "rsc.io/quote", false}, 380 | 381 | {"*/*/*,,", "rsc.io/quote", false}, 382 | {"*/*/*,,*/quote", "rsc.io/quote", true}, 383 | {",,*/quote", "rsc.io/quote", true}, 384 | } { 385 | if got := MatchPrefixPatterns(test.globs, test.target); got != test.want { 386 | t.Errorf("MatchPrefixPatterns(%q, %q) = %t, want %t", test.globs, test.target, got, test.want) 387 | } 388 | } 389 | } 390 | -------------------------------------------------------------------------------- /module/pseudo.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Pseudo-versions 6 | // 7 | // Code authors are expected to tag the revisions they want users to use, 8 | // including prereleases. However, not all authors tag versions at all, 9 | // and not all commits a user might want to try will have tags. 10 | // A pseudo-version is a version with a special form that allows us to 11 | // address an untagged commit and order that version with respect to 12 | // other versions we might encounter. 13 | // 14 | // A pseudo-version takes one of the general forms: 15 | // 16 | // (1) vX.0.0-yyyymmddhhmmss-abcdef123456 17 | // (2) vX.Y.(Z+1)-0.yyyymmddhhmmss-abcdef123456 18 | // (3) vX.Y.(Z+1)-0.yyyymmddhhmmss-abcdef123456+incompatible 19 | // (4) vX.Y.Z-pre.0.yyyymmddhhmmss-abcdef123456 20 | // (5) vX.Y.Z-pre.0.yyyymmddhhmmss-abcdef123456+incompatible 21 | // 22 | // If there is no recently tagged version with the right major version vX, 23 | // then form (1) is used, creating a space of pseudo-versions at the bottom 24 | // of the vX version range, less than any tagged version, including the unlikely v0.0.0. 25 | // 26 | // If the most recent tagged version before the target commit is vX.Y.Z or vX.Y.Z+incompatible, 27 | // then the pseudo-version uses form (2) or (3), making it a prerelease for the next 28 | // possible semantic version after vX.Y.Z. The leading 0 segment in the prerelease string 29 | // ensures that the pseudo-version compares less than possible future explicit prereleases 30 | // like vX.Y.(Z+1)-rc1 or vX.Y.(Z+1)-1. 31 | // 32 | // If the most recent tagged version before the target commit is vX.Y.Z-pre or vX.Y.Z-pre+incompatible, 33 | // then the pseudo-version uses form (4) or (5), making it a slightly later prerelease. 34 | 35 | package module 36 | 37 | import ( 38 | "errors" 39 | "fmt" 40 | "strings" 41 | "time" 42 | 43 | "golang.org/x/mod/internal/lazyregexp" 44 | "golang.org/x/mod/semver" 45 | ) 46 | 47 | var pseudoVersionRE = lazyregexp.New(`^v[0-9]+\.(0\.0-|\d+\.\d+-([^+]*\.)?0\.)\d{14}-[A-Za-z0-9]+(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$`) 48 | 49 | const PseudoVersionTimestampFormat = "20060102150405" 50 | 51 | // PseudoVersion returns a pseudo-version for the given major version ("v1") 52 | // preexisting older tagged version ("" or "v1.2.3" or "v1.2.3-pre"), revision time, 53 | // and revision identifier (usually a 12-byte commit hash prefix). 54 | func PseudoVersion(major, older string, t time.Time, rev string) string { 55 | if major == "" { 56 | major = "v0" 57 | } 58 | segment := fmt.Sprintf("%s-%s", t.UTC().Format(PseudoVersionTimestampFormat), rev) 59 | build := semver.Build(older) 60 | older = semver.Canonical(older) 61 | if older == "" { 62 | return major + ".0.0-" + segment // form (1) 63 | } 64 | if semver.Prerelease(older) != "" { 65 | return older + ".0." + segment + build // form (4), (5) 66 | } 67 | 68 | // Form (2), (3). 69 | // Extract patch from vMAJOR.MINOR.PATCH 70 | i := strings.LastIndex(older, ".") + 1 71 | v, patch := older[:i], older[i:] 72 | 73 | // Reassemble. 74 | return v + incDecimal(patch) + "-0." + segment + build 75 | } 76 | 77 | // ZeroPseudoVersion returns a pseudo-version with a zero timestamp and 78 | // revision, which may be used as a placeholder. 79 | func ZeroPseudoVersion(major string) string { 80 | return PseudoVersion(major, "", time.Time{}, "000000000000") 81 | } 82 | 83 | // incDecimal returns the decimal string incremented by 1. 84 | func incDecimal(decimal string) string { 85 | // Scan right to left turning 9s to 0s until you find a digit to increment. 86 | digits := []byte(decimal) 87 | i := len(digits) - 1 88 | for ; i >= 0 && digits[i] == '9'; i-- { 89 | digits[i] = '0' 90 | } 91 | if i >= 0 { 92 | digits[i]++ 93 | } else { 94 | // digits is all zeros 95 | digits[0] = '1' 96 | digits = append(digits, '0') 97 | } 98 | return string(digits) 99 | } 100 | 101 | // decDecimal returns the decimal string decremented by 1, or the empty string 102 | // if the decimal is all zeroes. 103 | func decDecimal(decimal string) string { 104 | // Scan right to left turning 0s to 9s until you find a digit to decrement. 105 | digits := []byte(decimal) 106 | i := len(digits) - 1 107 | for ; i >= 0 && digits[i] == '0'; i-- { 108 | digits[i] = '9' 109 | } 110 | if i < 0 { 111 | // decimal is all zeros 112 | return "" 113 | } 114 | if i == 0 && digits[i] == '1' && len(digits) > 1 { 115 | digits = digits[1:] 116 | } else { 117 | digits[i]-- 118 | } 119 | return string(digits) 120 | } 121 | 122 | // IsPseudoVersion reports whether v is a pseudo-version. 123 | func IsPseudoVersion(v string) bool { 124 | return strings.Count(v, "-") >= 2 && semver.IsValid(v) && pseudoVersionRE.MatchString(v) 125 | } 126 | 127 | // IsZeroPseudoVersion returns whether v is a pseudo-version with a zero base, 128 | // timestamp, and revision, as returned by [ZeroPseudoVersion]. 129 | func IsZeroPseudoVersion(v string) bool { 130 | return v == ZeroPseudoVersion(semver.Major(v)) 131 | } 132 | 133 | // PseudoVersionTime returns the time stamp of the pseudo-version v. 134 | // It returns an error if v is not a pseudo-version or if the time stamp 135 | // embedded in the pseudo-version is not a valid time. 136 | func PseudoVersionTime(v string) (time.Time, error) { 137 | _, timestamp, _, _, err := parsePseudoVersion(v) 138 | if err != nil { 139 | return time.Time{}, err 140 | } 141 | t, err := time.Parse("20060102150405", timestamp) 142 | if err != nil { 143 | return time.Time{}, &InvalidVersionError{ 144 | Version: v, 145 | Pseudo: true, 146 | Err: fmt.Errorf("malformed time %q", timestamp), 147 | } 148 | } 149 | return t, nil 150 | } 151 | 152 | // PseudoVersionRev returns the revision identifier of the pseudo-version v. 153 | // It returns an error if v is not a pseudo-version. 154 | func PseudoVersionRev(v string) (rev string, err error) { 155 | _, _, rev, _, err = parsePseudoVersion(v) 156 | return 157 | } 158 | 159 | // PseudoVersionBase returns the canonical parent version, if any, upon which 160 | // the pseudo-version v is based. 161 | // 162 | // If v has no parent version (that is, if it is "vX.0.0-[…]"), 163 | // PseudoVersionBase returns the empty string and a nil error. 164 | func PseudoVersionBase(v string) (string, error) { 165 | base, _, _, build, err := parsePseudoVersion(v) 166 | if err != nil { 167 | return "", err 168 | } 169 | 170 | switch pre := semver.Prerelease(base); pre { 171 | case "": 172 | // vX.0.0-yyyymmddhhmmss-abcdef123456 → "" 173 | if build != "" { 174 | // Pseudo-versions of the form vX.0.0-yyyymmddhhmmss-abcdef123456+incompatible 175 | // are nonsensical: the "vX.0.0-" prefix implies that there is no parent tag, 176 | // but the "+incompatible" suffix implies that the major version of 177 | // the parent tag is not compatible with the module's import path. 178 | // 179 | // There are a few such entries in the index generated by proxy.golang.org, 180 | // but we believe those entries were generated by the proxy itself. 181 | return "", &InvalidVersionError{ 182 | Version: v, 183 | Pseudo: true, 184 | Err: fmt.Errorf("lacks base version, but has build metadata %q", build), 185 | } 186 | } 187 | return "", nil 188 | 189 | case "-0": 190 | // vX.Y.(Z+1)-0.yyyymmddhhmmss-abcdef123456 → vX.Y.Z 191 | // vX.Y.(Z+1)-0.yyyymmddhhmmss-abcdef123456+incompatible → vX.Y.Z+incompatible 192 | base = strings.TrimSuffix(base, pre) 193 | i := strings.LastIndexByte(base, '.') 194 | if i < 0 { 195 | panic("base from parsePseudoVersion missing patch number: " + base) 196 | } 197 | patch := decDecimal(base[i+1:]) 198 | if patch == "" { 199 | // vX.0.0-0 is invalid, but has been observed in the wild in the index 200 | // generated by requests to proxy.golang.org. 201 | // 202 | // NOTE(bcmills): I cannot find a historical bug that accounts for 203 | // pseudo-versions of this form, nor have I seen such versions in any 204 | // actual go.mod files. If we find actual examples of this form and a 205 | // reasonable theory of how they came into existence, it seems fine to 206 | // treat them as equivalent to vX.0.0 (especially since the invalid 207 | // pseudo-versions have lower precedence than the real ones). For now, we 208 | // reject them. 209 | return "", &InvalidVersionError{ 210 | Version: v, 211 | Pseudo: true, 212 | Err: fmt.Errorf("version before %s would have negative patch number", base), 213 | } 214 | } 215 | return base[:i+1] + patch + build, nil 216 | 217 | default: 218 | // vX.Y.Z-pre.0.yyyymmddhhmmss-abcdef123456 → vX.Y.Z-pre 219 | // vX.Y.Z-pre.0.yyyymmddhhmmss-abcdef123456+incompatible → vX.Y.Z-pre+incompatible 220 | if !strings.HasSuffix(base, ".0") { 221 | panic(`base from parsePseudoVersion missing ".0" before date: ` + base) 222 | } 223 | return strings.TrimSuffix(base, ".0") + build, nil 224 | } 225 | } 226 | 227 | var errPseudoSyntax = errors.New("syntax error") 228 | 229 | func parsePseudoVersion(v string) (base, timestamp, rev, build string, err error) { 230 | if !IsPseudoVersion(v) { 231 | return "", "", "", "", &InvalidVersionError{ 232 | Version: v, 233 | Pseudo: true, 234 | Err: errPseudoSyntax, 235 | } 236 | } 237 | build = semver.Build(v) 238 | v = strings.TrimSuffix(v, build) 239 | j := strings.LastIndex(v, "-") 240 | v, rev = v[:j], v[j+1:] 241 | i := strings.LastIndex(v, "-") 242 | if j := strings.LastIndex(v, "."); j > i { 243 | base = v[:j] // "vX.Y.Z-pre.0" or "vX.Y.(Z+1)-0" 244 | timestamp = v[j+1:] 245 | } else { 246 | base = v[:i] // "vX.0.0" 247 | timestamp = v[i+1:] 248 | } 249 | return base, timestamp, rev, build, nil 250 | } 251 | -------------------------------------------------------------------------------- /module/pseudo_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package module 6 | 7 | import ( 8 | "testing" 9 | "time" 10 | ) 11 | 12 | var pseudoTests = []struct { 13 | major string 14 | older string 15 | version string 16 | }{ 17 | {"", "", "v0.0.0-20060102150405-hash"}, 18 | {"v0", "", "v0.0.0-20060102150405-hash"}, 19 | {"v1", "", "v1.0.0-20060102150405-hash"}, 20 | {"v2", "", "v2.0.0-20060102150405-hash"}, 21 | {"unused", "v0.0.0", "v0.0.1-0.20060102150405-hash"}, 22 | {"unused", "v1.2.3", "v1.2.4-0.20060102150405-hash"}, 23 | {"unused", "v1.2.99999999999999999", "v1.2.100000000000000000-0.20060102150405-hash"}, 24 | {"unused", "v1.2.3-pre", "v1.2.3-pre.0.20060102150405-hash"}, 25 | {"unused", "v1.3.0-pre", "v1.3.0-pre.0.20060102150405-hash"}, 26 | {"unused", "v0.0.0--", "v0.0.0--.0.20060102150405-hash"}, 27 | {"unused", "v1.0.0+metadata", "v1.0.1-0.20060102150405-hash+metadata"}, 28 | {"unused", "v2.0.0+incompatible", "v2.0.1-0.20060102150405-hash+incompatible"}, 29 | {"unused", "v2.3.0-pre+incompatible", "v2.3.0-pre.0.20060102150405-hash+incompatible"}, 30 | } 31 | 32 | var pseudoTime = time.Date(2006, 1, 2, 15, 4, 5, 0, time.UTC) 33 | 34 | func TestPseudoVersion(t *testing.T) { 35 | for _, tt := range pseudoTests { 36 | v := PseudoVersion(tt.major, tt.older, pseudoTime, "hash") 37 | if v != tt.version { 38 | t.Errorf("PseudoVersion(%q, %q, ...) = %v, want %v", tt.major, tt.older, v, tt.version) 39 | } 40 | } 41 | } 42 | 43 | func TestIsPseudoVersion(t *testing.T) { 44 | for _, tt := range pseudoTests { 45 | if !IsPseudoVersion(tt.version) { 46 | t.Errorf("IsPseudoVersion(%q) = false, want true", tt.version) 47 | } 48 | if IsPseudoVersion(tt.older) { 49 | t.Errorf("IsPseudoVersion(%q) = true, want false", tt.older) 50 | } 51 | } 52 | } 53 | 54 | func TestPseudoVersionTime(t *testing.T) { 55 | for _, tt := range pseudoTests { 56 | tm, err := PseudoVersionTime(tt.version) 57 | if tm != pseudoTime || err != nil { 58 | t.Errorf("PseudoVersionTime(%q) = %v, %v, want %v, nil", tt.version, tm.Format(time.RFC3339), err, pseudoTime.Format(time.RFC3339)) 59 | } 60 | tm, err = PseudoVersionTime(tt.older) 61 | if tm != (time.Time{}) || err == nil { 62 | t.Errorf("PseudoVersionTime(%q) = %v, %v, want %v, error", tt.older, tm.Format(time.RFC3339), err, time.Time{}.Format(time.RFC3339)) 63 | } 64 | } 65 | } 66 | 67 | func TestInvalidPseudoVersionTime(t *testing.T) { 68 | const v = "---" 69 | if _, err := PseudoVersionTime(v); err == nil { 70 | t.Error("expected error, got nil instead") 71 | } 72 | } 73 | 74 | func TestPseudoVersionRev(t *testing.T) { 75 | for _, tt := range pseudoTests { 76 | rev, err := PseudoVersionRev(tt.version) 77 | if rev != "hash" || err != nil { 78 | t.Errorf("PseudoVersionRev(%q) = %q, %v, want %q, nil", tt.older, rev, err, "hash") 79 | } 80 | rev, err = PseudoVersionRev(tt.older) 81 | if rev != "" || err == nil { 82 | t.Errorf("PseudoVersionRev(%q) = %q, %v, want %q, error", tt.older, rev, err, "") 83 | } 84 | } 85 | } 86 | 87 | func TestPseudoVersionBase(t *testing.T) { 88 | for _, tt := range pseudoTests { 89 | base, err := PseudoVersionBase(tt.version) 90 | if err != nil { 91 | t.Errorf("PseudoVersionBase(%q): %v", tt.version, err) 92 | } else if base != tt.older { 93 | t.Errorf("PseudoVersionBase(%q) = %q; want %q", tt.version, base, tt.older) 94 | } 95 | } 96 | } 97 | 98 | func TestInvalidPseudoVersionBase(t *testing.T) { 99 | for _, in := range []string{ 100 | "v0.0.0", 101 | "v0.0.0-", // malformed: empty prerelease 102 | "v0.0.0-0.20060102150405-hash", // Z+1 == 0 103 | "v0.1.0-0.20060102150405-hash", // Z+1 == 0 104 | "v1.0.0-0.20060102150405-hash", // Z+1 == 0 105 | "v0.0.0-20060102150405-hash+incompatible", // "+incompatible without base version 106 | "v0.0.0-20060102150405-hash+metadata", // other metadata without base version 107 | } { 108 | base, err := PseudoVersionBase(in) 109 | if err == nil || base != "" { 110 | t.Errorf(`PseudoVersionBase(%q) = %q, %v; want "", error`, in, base, err) 111 | } 112 | } 113 | } 114 | 115 | func TestIncDecimal(t *testing.T) { 116 | cases := []struct { 117 | in, want string 118 | }{ 119 | {"0", "1"}, 120 | {"1", "2"}, 121 | {"99", "100"}, 122 | {"100", "101"}, 123 | {"101", "102"}, 124 | } 125 | 126 | for _, tc := range cases { 127 | got := incDecimal(tc.in) 128 | if got != tc.want { 129 | t.Fatalf("incDecimal(%q) = %q; want %q", tc.in, tc.want, got) 130 | } 131 | } 132 | } 133 | 134 | func TestDecDecimal(t *testing.T) { 135 | cases := []struct { 136 | in, want string 137 | }{ 138 | {"", ""}, 139 | {"0", ""}, 140 | {"00", ""}, 141 | {"1", "0"}, 142 | {"2", "1"}, 143 | {"99", "98"}, 144 | {"100", "99"}, 145 | {"101", "100"}, 146 | } 147 | 148 | for _, tc := range cases { 149 | got := decDecimal(tc.in) 150 | if got != tc.want { 151 | t.Fatalf("decDecimal(%q) = %q; want %q", tc.in, tc.want, got) 152 | } 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /semver/semver.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package semver implements comparison of semantic version strings. 6 | // In this package, semantic version strings must begin with a leading "v", 7 | // as in "v1.0.0". 8 | // 9 | // The general form of a semantic version string accepted by this package is 10 | // 11 | // vMAJOR[.MINOR[.PATCH[-PRERELEASE][+BUILD]]] 12 | // 13 | // where square brackets indicate optional parts of the syntax; 14 | // MAJOR, MINOR, and PATCH are decimal integers without extra leading zeros; 15 | // PRERELEASE and BUILD are each a series of non-empty dot-separated identifiers 16 | // using only alphanumeric characters and hyphens; and 17 | // all-numeric PRERELEASE identifiers must not have leading zeros. 18 | // 19 | // This package follows Semantic Versioning 2.0.0 (see semver.org) 20 | // with two exceptions. First, it requires the "v" prefix. Second, it recognizes 21 | // vMAJOR and vMAJOR.MINOR (with no prerelease or build suffixes) 22 | // as shorthands for vMAJOR.0.0 and vMAJOR.MINOR.0. 23 | package semver 24 | 25 | import ( 26 | "slices" 27 | "strings" 28 | ) 29 | 30 | // parsed returns the parsed form of a semantic version string. 31 | type parsed struct { 32 | major string 33 | minor string 34 | patch string 35 | short string 36 | prerelease string 37 | build string 38 | } 39 | 40 | // IsValid reports whether v is a valid semantic version string. 41 | func IsValid(v string) bool { 42 | _, ok := parse(v) 43 | return ok 44 | } 45 | 46 | // Canonical returns the canonical formatting of the semantic version v. 47 | // It fills in any missing .MINOR or .PATCH and discards build metadata. 48 | // Two semantic versions compare equal only if their canonical formattings 49 | // are identical strings. 50 | // The canonical invalid semantic version is the empty string. 51 | func Canonical(v string) string { 52 | p, ok := parse(v) 53 | if !ok { 54 | return "" 55 | } 56 | if p.build != "" { 57 | return v[:len(v)-len(p.build)] 58 | } 59 | if p.short != "" { 60 | return v + p.short 61 | } 62 | return v 63 | } 64 | 65 | // Major returns the major version prefix of the semantic version v. 66 | // For example, Major("v2.1.0") == "v2". 67 | // If v is an invalid semantic version string, Major returns the empty string. 68 | func Major(v string) string { 69 | pv, ok := parse(v) 70 | if !ok { 71 | return "" 72 | } 73 | return v[:1+len(pv.major)] 74 | } 75 | 76 | // MajorMinor returns the major.minor version prefix of the semantic version v. 77 | // For example, MajorMinor("v2.1.0") == "v2.1". 78 | // If v is an invalid semantic version string, MajorMinor returns the empty string. 79 | func MajorMinor(v string) string { 80 | pv, ok := parse(v) 81 | if !ok { 82 | return "" 83 | } 84 | i := 1 + len(pv.major) 85 | if j := i + 1 + len(pv.minor); j <= len(v) && v[i] == '.' && v[i+1:j] == pv.minor { 86 | return v[:j] 87 | } 88 | return v[:i] + "." + pv.minor 89 | } 90 | 91 | // Prerelease returns the prerelease suffix of the semantic version v. 92 | // For example, Prerelease("v2.1.0-pre+meta") == "-pre". 93 | // If v is an invalid semantic version string, Prerelease returns the empty string. 94 | func Prerelease(v string) string { 95 | pv, ok := parse(v) 96 | if !ok { 97 | return "" 98 | } 99 | return pv.prerelease 100 | } 101 | 102 | // Build returns the build suffix of the semantic version v. 103 | // For example, Build("v2.1.0+meta") == "+meta". 104 | // If v is an invalid semantic version string, Build returns the empty string. 105 | func Build(v string) string { 106 | pv, ok := parse(v) 107 | if !ok { 108 | return "" 109 | } 110 | return pv.build 111 | } 112 | 113 | // Compare returns an integer comparing two versions according to 114 | // semantic version precedence. 115 | // The result will be 0 if v == w, -1 if v < w, or +1 if v > w. 116 | // 117 | // An invalid semantic version string is considered less than a valid one. 118 | // All invalid semantic version strings compare equal to each other. 119 | func Compare(v, w string) int { 120 | pv, ok1 := parse(v) 121 | pw, ok2 := parse(w) 122 | if !ok1 && !ok2 { 123 | return 0 124 | } 125 | if !ok1 { 126 | return -1 127 | } 128 | if !ok2 { 129 | return +1 130 | } 131 | if c := compareInt(pv.major, pw.major); c != 0 { 132 | return c 133 | } 134 | if c := compareInt(pv.minor, pw.minor); c != 0 { 135 | return c 136 | } 137 | if c := compareInt(pv.patch, pw.patch); c != 0 { 138 | return c 139 | } 140 | return comparePrerelease(pv.prerelease, pw.prerelease) 141 | } 142 | 143 | // Max canonicalizes its arguments and then returns the version string 144 | // that compares greater. 145 | // 146 | // Deprecated: use [Compare] instead. In most cases, returning a canonicalized 147 | // version is not expected or desired. 148 | func Max(v, w string) string { 149 | v = Canonical(v) 150 | w = Canonical(w) 151 | if Compare(v, w) > 0 { 152 | return v 153 | } 154 | return w 155 | } 156 | 157 | // ByVersion implements [sort.Interface] for sorting semantic version strings. 158 | type ByVersion []string 159 | 160 | func (vs ByVersion) Len() int { return len(vs) } 161 | func (vs ByVersion) Swap(i, j int) { vs[i], vs[j] = vs[j], vs[i] } 162 | func (vs ByVersion) Less(i, j int) bool { return compareVersion(vs[i], vs[j]) < 0 } 163 | 164 | // Sort sorts a list of semantic version strings using [Compare] and falls back 165 | // to use [strings.Compare] if both versions are considered equal. 166 | func Sort(list []string) { 167 | slices.SortFunc(list, compareVersion) 168 | } 169 | 170 | func compareVersion(a, b string) int { 171 | cmp := Compare(a, b) 172 | if cmp != 0 { 173 | return cmp 174 | } 175 | return strings.Compare(a, b) 176 | } 177 | 178 | func parse(v string) (p parsed, ok bool) { 179 | if v == "" || v[0] != 'v' { 180 | return 181 | } 182 | p.major, v, ok = parseInt(v[1:]) 183 | if !ok { 184 | return 185 | } 186 | if v == "" { 187 | p.minor = "0" 188 | p.patch = "0" 189 | p.short = ".0.0" 190 | return 191 | } 192 | if v[0] != '.' { 193 | ok = false 194 | return 195 | } 196 | p.minor, v, ok = parseInt(v[1:]) 197 | if !ok { 198 | return 199 | } 200 | if v == "" { 201 | p.patch = "0" 202 | p.short = ".0" 203 | return 204 | } 205 | if v[0] != '.' { 206 | ok = false 207 | return 208 | } 209 | p.patch, v, ok = parseInt(v[1:]) 210 | if !ok { 211 | return 212 | } 213 | if len(v) > 0 && v[0] == '-' { 214 | p.prerelease, v, ok = parsePrerelease(v) 215 | if !ok { 216 | return 217 | } 218 | } 219 | if len(v) > 0 && v[0] == '+' { 220 | p.build, v, ok = parseBuild(v) 221 | if !ok { 222 | return 223 | } 224 | } 225 | if v != "" { 226 | ok = false 227 | return 228 | } 229 | ok = true 230 | return 231 | } 232 | 233 | func parseInt(v string) (t, rest string, ok bool) { 234 | if v == "" { 235 | return 236 | } 237 | if v[0] < '0' || '9' < v[0] { 238 | return 239 | } 240 | i := 1 241 | for i < len(v) && '0' <= v[i] && v[i] <= '9' { 242 | i++ 243 | } 244 | if v[0] == '0' && i != 1 { 245 | return 246 | } 247 | return v[:i], v[i:], true 248 | } 249 | 250 | func parsePrerelease(v string) (t, rest string, ok bool) { 251 | // "A pre-release version MAY be denoted by appending a hyphen and 252 | // a series of dot separated identifiers immediately following the patch version. 253 | // Identifiers MUST comprise only ASCII alphanumerics and hyphen [0-9A-Za-z-]. 254 | // Identifiers MUST NOT be empty. Numeric identifiers MUST NOT include leading zeroes." 255 | if v == "" || v[0] != '-' { 256 | return 257 | } 258 | i := 1 259 | start := 1 260 | for i < len(v) && v[i] != '+' { 261 | if !isIdentChar(v[i]) && v[i] != '.' { 262 | return 263 | } 264 | if v[i] == '.' { 265 | if start == i || isBadNum(v[start:i]) { 266 | return 267 | } 268 | start = i + 1 269 | } 270 | i++ 271 | } 272 | if start == i || isBadNum(v[start:i]) { 273 | return 274 | } 275 | return v[:i], v[i:], true 276 | } 277 | 278 | func parseBuild(v string) (t, rest string, ok bool) { 279 | if v == "" || v[0] != '+' { 280 | return 281 | } 282 | i := 1 283 | start := 1 284 | for i < len(v) { 285 | if !isIdentChar(v[i]) && v[i] != '.' { 286 | return 287 | } 288 | if v[i] == '.' { 289 | if start == i { 290 | return 291 | } 292 | start = i + 1 293 | } 294 | i++ 295 | } 296 | if start == i { 297 | return 298 | } 299 | return v[:i], v[i:], true 300 | } 301 | 302 | func isIdentChar(c byte) bool { 303 | return 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' || c == '-' 304 | } 305 | 306 | func isBadNum(v string) bool { 307 | i := 0 308 | for i < len(v) && '0' <= v[i] && v[i] <= '9' { 309 | i++ 310 | } 311 | return i == len(v) && i > 1 && v[0] == '0' 312 | } 313 | 314 | func isNum(v string) bool { 315 | i := 0 316 | for i < len(v) && '0' <= v[i] && v[i] <= '9' { 317 | i++ 318 | } 319 | return i == len(v) 320 | } 321 | 322 | func compareInt(x, y string) int { 323 | if x == y { 324 | return 0 325 | } 326 | if len(x) < len(y) { 327 | return -1 328 | } 329 | if len(x) > len(y) { 330 | return +1 331 | } 332 | if x < y { 333 | return -1 334 | } else { 335 | return +1 336 | } 337 | } 338 | 339 | func comparePrerelease(x, y string) int { 340 | // "When major, minor, and patch are equal, a pre-release version has 341 | // lower precedence than a normal version. 342 | // Example: 1.0.0-alpha < 1.0.0. 343 | // Precedence for two pre-release versions with the same major, minor, 344 | // and patch version MUST be determined by comparing each dot separated 345 | // identifier from left to right until a difference is found as follows: 346 | // identifiers consisting of only digits are compared numerically and 347 | // identifiers with letters or hyphens are compared lexically in ASCII 348 | // sort order. Numeric identifiers always have lower precedence than 349 | // non-numeric identifiers. A larger set of pre-release fields has a 350 | // higher precedence than a smaller set, if all of the preceding 351 | // identifiers are equal. 352 | // Example: 1.0.0-alpha < 1.0.0-alpha.1 < 1.0.0-alpha.beta < 353 | // 1.0.0-beta < 1.0.0-beta.2 < 1.0.0-beta.11 < 1.0.0-rc.1 < 1.0.0." 354 | if x == y { 355 | return 0 356 | } 357 | if x == "" { 358 | return +1 359 | } 360 | if y == "" { 361 | return -1 362 | } 363 | for x != "" && y != "" { 364 | x = x[1:] // skip - or . 365 | y = y[1:] // skip - or . 366 | var dx, dy string 367 | dx, x = nextIdent(x) 368 | dy, y = nextIdent(y) 369 | if dx != dy { 370 | ix := isNum(dx) 371 | iy := isNum(dy) 372 | if ix != iy { 373 | if ix { 374 | return -1 375 | } else { 376 | return +1 377 | } 378 | } 379 | if ix { 380 | if len(dx) < len(dy) { 381 | return -1 382 | } 383 | if len(dx) > len(dy) { 384 | return +1 385 | } 386 | } 387 | if dx < dy { 388 | return -1 389 | } else { 390 | return +1 391 | } 392 | } 393 | } 394 | if x == "" { 395 | return -1 396 | } else { 397 | return +1 398 | } 399 | } 400 | 401 | func nextIdent(x string) (dx, rest string) { 402 | i := 0 403 | for i < len(x) && x[i] != '.' { 404 | i++ 405 | } 406 | return x[:i], x[i:] 407 | } 408 | -------------------------------------------------------------------------------- /semver/semver_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package semver 6 | 7 | import ( 8 | "math/rand" 9 | "slices" 10 | "sort" 11 | "strings" 12 | "testing" 13 | ) 14 | 15 | var tests = []struct { 16 | in string 17 | out string 18 | }{ 19 | {"bad", ""}, 20 | {"v1-alpha.beta.gamma", ""}, 21 | {"v1-pre", ""}, 22 | {"v1+meta", ""}, 23 | {"v1-pre+meta", ""}, 24 | {"v1.2-pre", ""}, 25 | {"v1.2+meta", ""}, 26 | {"v1.2-pre+meta", ""}, 27 | {"v1.0.0-alpha", "v1.0.0-alpha"}, 28 | {"v1.0.0-alpha.1", "v1.0.0-alpha.1"}, 29 | {"v1.0.0-alpha.beta", "v1.0.0-alpha.beta"}, 30 | {"v1.0.0-beta", "v1.0.0-beta"}, 31 | {"v1.0.0-beta.2", "v1.0.0-beta.2"}, 32 | {"v1.0.0-beta.11", "v1.0.0-beta.11"}, 33 | {"v1.0.0-rc.1", "v1.0.0-rc.1"}, 34 | {"v1", "v1.0.0"}, 35 | {"v1.0", "v1.0.0"}, 36 | {"v1.0.0", "v1.0.0"}, 37 | {"v1.2", "v1.2.0"}, 38 | {"v1.2.0", "v1.2.0"}, 39 | {"v1.2.3-456", "v1.2.3-456"}, 40 | {"v1.2.3-456.789", "v1.2.3-456.789"}, 41 | {"v1.2.3-456-789", "v1.2.3-456-789"}, 42 | {"v1.2.3-456a", "v1.2.3-456a"}, 43 | {"v1.2.3-pre", "v1.2.3-pre"}, 44 | {"v1.2.3-pre+meta", "v1.2.3-pre"}, 45 | {"v1.2.3-pre.1", "v1.2.3-pre.1"}, 46 | {"v1.2.3-zzz", "v1.2.3-zzz"}, 47 | {"v1.2.3", "v1.2.3"}, 48 | {"v1.2.3+meta", "v1.2.3"}, 49 | {"v1.2.3+meta-pre", "v1.2.3"}, 50 | {"v1.2.3+meta-pre.sha.256a", "v1.2.3"}, 51 | } 52 | 53 | func TestIsValid(t *testing.T) { 54 | for _, tt := range tests { 55 | ok := IsValid(tt.in) 56 | if ok != (tt.out != "") { 57 | t.Errorf("IsValid(%q) = %v, want %v", tt.in, ok, !ok) 58 | } 59 | } 60 | } 61 | 62 | func TestCanonical(t *testing.T) { 63 | for _, tt := range tests { 64 | out := Canonical(tt.in) 65 | if out != tt.out { 66 | t.Errorf("Canonical(%q) = %q, want %q", tt.in, out, tt.out) 67 | } 68 | } 69 | } 70 | 71 | func TestMajor(t *testing.T) { 72 | for _, tt := range tests { 73 | out := Major(tt.in) 74 | want := "" 75 | if i := strings.Index(tt.out, "."); i >= 0 { 76 | want = tt.out[:i] 77 | } 78 | if out != want { 79 | t.Errorf("Major(%q) = %q, want %q", tt.in, out, want) 80 | } 81 | } 82 | } 83 | 84 | func TestMajorMinor(t *testing.T) { 85 | for _, tt := range tests { 86 | out := MajorMinor(tt.in) 87 | var want string 88 | if tt.out != "" { 89 | want = tt.in 90 | if i := strings.Index(want, "+"); i >= 0 { 91 | want = want[:i] 92 | } 93 | if i := strings.Index(want, "-"); i >= 0 { 94 | want = want[:i] 95 | } 96 | switch strings.Count(want, ".") { 97 | case 0: 98 | want += ".0" 99 | case 1: 100 | // ok 101 | case 2: 102 | want = want[:strings.LastIndex(want, ".")] 103 | } 104 | } 105 | if out != want { 106 | t.Errorf("MajorMinor(%q) = %q, want %q", tt.in, out, want) 107 | } 108 | } 109 | } 110 | 111 | func TestPrerelease(t *testing.T) { 112 | for _, tt := range tests { 113 | pre := Prerelease(tt.in) 114 | var want string 115 | if tt.out != "" { 116 | if i := strings.Index(tt.out, "-"); i >= 0 { 117 | want = tt.out[i:] 118 | } 119 | } 120 | if pre != want { 121 | t.Errorf("Prerelease(%q) = %q, want %q", tt.in, pre, want) 122 | } 123 | } 124 | } 125 | 126 | func TestBuild(t *testing.T) { 127 | for _, tt := range tests { 128 | build := Build(tt.in) 129 | var want string 130 | if tt.out != "" { 131 | if i := strings.Index(tt.in, "+"); i >= 0 { 132 | want = tt.in[i:] 133 | } 134 | } 135 | if build != want { 136 | t.Errorf("Build(%q) = %q, want %q", tt.in, build, want) 137 | } 138 | } 139 | } 140 | 141 | func TestCompare(t *testing.T) { 142 | for i, ti := range tests { 143 | for j, tj := range tests { 144 | cmp := Compare(ti.in, tj.in) 145 | var want int 146 | if ti.out == tj.out { 147 | want = 0 148 | } else if i < j { 149 | want = -1 150 | } else { 151 | want = +1 152 | } 153 | if cmp != want { 154 | t.Errorf("Compare(%q, %q) = %d, want %d", ti.in, tj.in, cmp, want) 155 | } 156 | } 157 | } 158 | } 159 | 160 | func TestSort(t *testing.T) { 161 | versions := make([]string, len(tests)) 162 | for i, test := range tests { 163 | versions[i] = test.in 164 | } 165 | rand.Shuffle(len(versions), func(i, j int) { versions[i], versions[j] = versions[j], versions[i] }) 166 | Sort(versions) 167 | if !sort.IsSorted(ByVersion(versions)) { 168 | t.Errorf("list is not sorted:\n%s", strings.Join(versions, "\n")) 169 | } 170 | 171 | golden := []string{ 172 | "bad", 173 | "v1+meta", 174 | "v1-alpha.beta.gamma", 175 | "v1-pre", 176 | "v1-pre+meta", 177 | "v1.2+meta", 178 | "v1.2-pre", 179 | "v1.2-pre+meta", 180 | "v1.0.0-alpha", 181 | "v1.0.0-alpha.1", 182 | "v1.0.0-alpha.beta", 183 | "v1.0.0-beta", 184 | "v1.0.0-beta.2", 185 | "v1.0.0-beta.11", 186 | "v1.0.0-rc.1", 187 | "v1", 188 | "v1.0", 189 | "v1.0.0", 190 | "v1.2", 191 | "v1.2.0", 192 | "v1.2.3-456", 193 | "v1.2.3-456.789", 194 | "v1.2.3-456-789", 195 | "v1.2.3-456a", 196 | "v1.2.3-pre", 197 | "v1.2.3-pre+meta", 198 | "v1.2.3-pre.1", 199 | "v1.2.3-zzz", 200 | "v1.2.3", 201 | "v1.2.3+meta", 202 | "v1.2.3+meta-pre", 203 | "v1.2.3+meta-pre.sha.256a", 204 | } 205 | if !slices.Equal(versions, golden) { 206 | t.Errorf("list is not sorted correctly\ngot:\n%v\nwant:\n%v", versions, golden) 207 | } 208 | } 209 | 210 | func TestMax(t *testing.T) { 211 | for i, ti := range tests { 212 | for j, tj := range tests { 213 | max := Max(ti.in, tj.in) 214 | want := Canonical(ti.in) 215 | if i < j { 216 | want = Canonical(tj.in) 217 | } 218 | if max != want { 219 | t.Errorf("Max(%q, %q) = %q, want %q", ti.in, tj.in, max, want) 220 | } 221 | } 222 | } 223 | } 224 | 225 | var ( 226 | v1 = "v1.0.0+metadata-dash" 227 | v2 = "v1.0.0+metadata-dash1" 228 | ) 229 | 230 | func BenchmarkCompare(b *testing.B) { 231 | for i := 0; i < b.N; i++ { 232 | if Compare(v1, v2) != 0 { 233 | b.Fatalf("bad compare") 234 | } 235 | } 236 | } 237 | -------------------------------------------------------------------------------- /sumdb/cache.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Parallel cache. 6 | // This file is copied from cmd/go/internal/par. 7 | 8 | package sumdb 9 | 10 | import ( 11 | "sync" 12 | "sync/atomic" 13 | ) 14 | 15 | // parCache runs an action once per key and caches the result. 16 | type parCache struct { 17 | m sync.Map 18 | } 19 | 20 | type cacheEntry struct { 21 | done uint32 22 | mu sync.Mutex 23 | result interface{} 24 | } 25 | 26 | // Do calls the function f if and only if Do is being called for the first time with this key. 27 | // No call to Do with a given key returns until the one call to f returns. 28 | // Do returns the value returned by the one call to f. 29 | func (c *parCache) Do(key interface{}, f func() interface{}) interface{} { 30 | entryIface, ok := c.m.Load(key) 31 | if !ok { 32 | entryIface, _ = c.m.LoadOrStore(key, new(cacheEntry)) 33 | } 34 | e := entryIface.(*cacheEntry) 35 | if atomic.LoadUint32(&e.done) == 0 { 36 | e.mu.Lock() 37 | if atomic.LoadUint32(&e.done) == 0 { 38 | e.result = f() 39 | atomic.StoreUint32(&e.done, 1) 40 | } 41 | e.mu.Unlock() 42 | } 43 | return e.result 44 | } 45 | 46 | // Get returns the cached result associated with key. 47 | // It returns nil if there is no such result. 48 | // If the result for key is being computed, Get does not wait for the computation to finish. 49 | func (c *parCache) Get(key interface{}) interface{} { 50 | entryIface, ok := c.m.Load(key) 51 | if !ok { 52 | return nil 53 | } 54 | e := entryIface.(*cacheEntry) 55 | if atomic.LoadUint32(&e.done) == 0 { 56 | return nil 57 | } 58 | return e.result 59 | } 60 | -------------------------------------------------------------------------------- /sumdb/client_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package sumdb 6 | 7 | import ( 8 | "bytes" 9 | "fmt" 10 | "strings" 11 | "sync" 12 | "testing" 13 | 14 | "golang.org/x/mod/sumdb/note" 15 | "golang.org/x/mod/sumdb/tlog" 16 | ) 17 | 18 | const ( 19 | testName = "localhost.localdev/sumdb" 20 | testVerifierKey = "localhost.localdev/sumdb+00000c67+AcTrnkbUA+TU4heY3hkjiSES/DSQniBqIeQ/YppAUtK6" 21 | testSignerKey = "PRIVATE+KEY+localhost.localdev/sumdb+00000c67+AXu6+oaVaOYuQOFrf1V59JK1owcFlJcHwwXHDfDGxSPk" 22 | ) 23 | 24 | func TestClientLookup(t *testing.T) { 25 | tc := newTestClient(t) 26 | tc.mustHaveLatest(1) 27 | 28 | // Basic lookup. 29 | tc.mustLookup("rsc.io/sampler", "v1.3.0", "rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=") 30 | tc.mustHaveLatest(3) 31 | 32 | // Everything should now be cached, both for the original package and its /go.mod. 33 | tc.getOK = false 34 | tc.mustLookup("rsc.io/sampler", "v1.3.0", "rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=") 35 | tc.mustLookup("rsc.io/sampler", "v1.3.0/go.mod", "rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=") 36 | tc.mustHaveLatest(3) 37 | tc.getOK = true 38 | tc.getTileOK = false // the cache has what we need 39 | 40 | // Lookup with multiple returned lines. 41 | tc.mustLookup("rsc.io/quote", "v1.5.2", "rsc.io/quote v1.5.2 h1:w5fcysjrx7yqtD/aO+QwRjYZOKnaM9Uh2b40tElTs3Y=\nrsc.io/quote v1.5.2 h2:xyzzy") 42 | tc.mustHaveLatest(3) 43 | 44 | // Lookup with need for !-encoding. 45 | // rsc.io/Quote is the only record written after rsc.io/samper, 46 | // so it is the only one that should need more tiles. 47 | tc.getTileOK = true 48 | tc.mustLookup("rsc.io/Quote", "v1.5.2", "rsc.io/Quote v1.5.2 h1:uppercase!=") 49 | tc.mustHaveLatest(4) 50 | } 51 | 52 | func TestClientBadTiles(t *testing.T) { 53 | tc := newTestClient(t) 54 | 55 | flipBits := func() { 56 | for url, data := range tc.remote { 57 | if strings.Contains(url, "/tile/") { 58 | for i := range data { 59 | data[i] ^= 0x80 60 | } 61 | } 62 | } 63 | } 64 | 65 | // Bad tiles in initial download. 66 | tc.mustHaveLatest(1) 67 | flipBits() 68 | _, err := tc.client.Lookup("rsc.io/sampler", "v1.3.0") 69 | tc.mustError(err, "rsc.io/sampler@v1.3.0: initializing sumdb.Client: checking tree#1: downloaded inconsistent tile") 70 | flipBits() 71 | tc.newClient() 72 | tc.mustLookup("rsc.io/sampler", "v1.3.0", "rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=") 73 | 74 | // Bad tiles after initial download. 75 | flipBits() 76 | _, err = tc.client.Lookup("rsc.io/Quote", "v1.5.2") 77 | tc.mustError(err, "rsc.io/Quote@v1.5.2: checking tree#3 against tree#4: downloaded inconsistent tile") 78 | flipBits() 79 | tc.newClient() 80 | tc.mustLookup("rsc.io/Quote", "v1.5.2", "rsc.io/Quote v1.5.2 h1:uppercase!=") 81 | 82 | // Bad starting tree hash looks like bad tiles. 83 | tc.newClient() 84 | text := tlog.FormatTree(tlog.Tree{N: 1, Hash: tlog.Hash{}}) 85 | data, err := note.Sign(¬e.Note{Text: string(text)}, tc.signer) 86 | if err != nil { 87 | tc.t.Fatal(err) 88 | } 89 | tc.config[testName+"/latest"] = data 90 | _, err = tc.client.Lookup("rsc.io/sampler", "v1.3.0") 91 | tc.mustError(err, "rsc.io/sampler@v1.3.0: initializing sumdb.Client: checking tree#1: downloaded inconsistent tile") 92 | } 93 | 94 | func TestClientFork(t *testing.T) { 95 | tc := newTestClient(t) 96 | tc2 := tc.fork() 97 | 98 | tc.addRecord("rsc.io/pkg1@v1.5.2", `rsc.io/pkg1 v1.5.2 h1:hash!= 99 | `) 100 | tc.addRecord("rsc.io/pkg1@v1.5.4", `rsc.io/pkg1 v1.5.4 h1:hash!= 101 | `) 102 | tc.mustLookup("rsc.io/pkg1", "v1.5.2", "rsc.io/pkg1 v1.5.2 h1:hash!=") 103 | 104 | tc2.addRecord("rsc.io/pkg1@v1.5.3", `rsc.io/pkg1 v1.5.3 h1:hash!= 105 | `) 106 | tc2.addRecord("rsc.io/pkg1@v1.5.4", `rsc.io/pkg1 v1.5.4 h1:hash!= 107 | `) 108 | tc2.mustLookup("rsc.io/pkg1", "v1.5.4", "rsc.io/pkg1 v1.5.4 h1:hash!=") 109 | 110 | key := "/lookup/rsc.io/pkg1@v1.5.2" 111 | tc2.remote[key] = tc.remote[key] 112 | _, err := tc2.client.Lookup("rsc.io/pkg1", "v1.5.2") 113 | tc2.mustError(err, ErrSecurity.Error()) 114 | 115 | /* 116 | SECURITY ERROR 117 | go.sum database server misbehavior detected! 118 | 119 | old database: 120 | go.sum database tree! 121 | 5 122 | nWzN20+pwMt62p7jbv1/NlN95ePTlHijabv5zO/s36w= 123 | 124 | — localhost.localdev/sumdb AAAMZ5/2FVAdMH58kmnz/0h299pwyskEbzDzoa2/YaPdhvLya4YWDFQQxu2TQb5GpwAH4NdWnTwuhILafisyf3CNbgg= 125 | 126 | new database: 127 | go.sum database tree 128 | 6 129 | wc4SkQt52o5W2nQ8To2ARs+mWuUJjss+sdleoiqxMmM= 130 | 131 | — localhost.localdev/sumdb AAAMZ6oRNswlEZ6ZZhxrCvgl1MBy+nusq4JU+TG6Fe2NihWLqOzb+y2c2kzRLoCr4tvw9o36ucQEnhc20e4nA4Qc/wc= 132 | 133 | proof of misbehavior: 134 | T7i+H/8ER4nXOiw4Bj0koZOkGjkxoNvlI34GpvhHhQg= 135 | Nsuejv72de9hYNM5bqFv8rv3gm3zJQwv/DT/WNbLDLA= 136 | mOmqqZ1aI/lzS94oq/JSbj7pD8Rv9S+xDyi12BtVSHo= 137 | /7Aw5jVSMM9sFjQhaMg+iiDYPMk6decH7QLOGrL9Lx0= 138 | */ 139 | 140 | wants := []string{ 141 | "SECURITY ERROR", 142 | "go.sum database server misbehavior detected!", 143 | "old database:\n\tgo.sum database tree\n\t5\n", 144 | "— localhost.localdev/sumdb AAAMZ5/2FVAd", 145 | "new database:\n\tgo.sum database tree\n\t6\n", 146 | "— localhost.localdev/sumdb AAAMZ6oRNswl", 147 | "proof of misbehavior:\n\tT7i+H/8ER4nXOiw4Bj0k", 148 | } 149 | text := tc2.security.String() 150 | for _, want := range wants { 151 | if !strings.Contains(text, want) { 152 | t.Fatalf("cannot find %q in security text:\n%s", want, text) 153 | } 154 | } 155 | } 156 | 157 | func TestClientGONOSUMDB(t *testing.T) { 158 | tc := newTestClient(t) 159 | tc.client.SetGONOSUMDB("p,*/q") 160 | tc.client.Lookup("rsc.io/sampler", "v1.3.0") // initialize before we turn off network 161 | tc.getOK = false 162 | 163 | ok := []string{ 164 | "abc", 165 | "a/p", 166 | "pq", 167 | "q", 168 | "n/o/p/q", 169 | } 170 | skip := []string{ 171 | "p", 172 | "p/x", 173 | "x/q", 174 | "x/q/z", 175 | } 176 | 177 | for _, path := range ok { 178 | _, err := tc.client.Lookup(path, "v1.0.0") 179 | if err == ErrGONOSUMDB { 180 | t.Errorf("Lookup(%q): ErrGONOSUMDB, wanted failed actual lookup", path) 181 | } 182 | } 183 | for _, path := range skip { 184 | _, err := tc.client.Lookup(path, "v1.0.0") 185 | if err != ErrGONOSUMDB { 186 | t.Errorf("Lookup(%q): %v, wanted ErrGONOSUMDB", path, err) 187 | } 188 | } 189 | } 190 | 191 | // A testClient is a self-contained client-side testing environment. 192 | type testClient struct { 193 | t *testing.T // active test 194 | client *Client // client being tested 195 | tileHeight int // tile height to use (default 2) 196 | getOK bool // should tc.GetURL succeed? 197 | getTileOK bool // should tc.GetURL of tiles succeed? 198 | treeSize int64 199 | hashes []tlog.Hash 200 | remote map[string][]byte 201 | signer note.Signer 202 | 203 | // mu protects config, cache, log, security 204 | // during concurrent use of the exported methods 205 | // by the client itself (testClient is the Client's ClientOps, 206 | // and the Client methods can both read and write these fields). 207 | // Unexported methods invoked directly by the test 208 | // (for example, addRecord) need not hold the mutex: 209 | // for proper test execution those methods should only 210 | // be called when the Client is idle and not using its ClientOps. 211 | // Not holding the mutex in those methods ensures 212 | // that if a mistake is made, go test -race will report it. 213 | // (Holding the mutex would eliminate the race report but 214 | // not the underlying problem.) 215 | // Similarly, the get map is not protected by the mutex, 216 | // because the Client methods only read it. 217 | mu sync.Mutex // prot 218 | config map[string][]byte 219 | cache map[string][]byte 220 | security bytes.Buffer 221 | } 222 | 223 | // newTestClient returns a new testClient that will call t.Fatal on error 224 | // and has a few records already available on the remote server. 225 | func newTestClient(t *testing.T) *testClient { 226 | tc := &testClient{ 227 | t: t, 228 | tileHeight: 2, 229 | getOK: true, 230 | getTileOK: true, 231 | config: make(map[string][]byte), 232 | cache: make(map[string][]byte), 233 | remote: make(map[string][]byte), 234 | } 235 | 236 | tc.config["key"] = []byte(testVerifierKey + "\n") 237 | var err error 238 | tc.signer, err = note.NewSigner(testSignerKey) 239 | if err != nil { 240 | t.Fatal(err) 241 | } 242 | 243 | tc.newClient() 244 | 245 | tc.addRecord("rsc.io/quote@v1.5.2", `rsc.io/quote v1.5.2 h1:w5fcysjrx7yqtD/aO+QwRjYZOKnaM9Uh2b40tElTs3Y= 246 | rsc.io/quote v1.5.2/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0= 247 | rsc.io/quote v1.5.2 h2:xyzzy 248 | `) 249 | 250 | tc.addRecord("golang.org/x/text@v0.0.0-20170915032832-14c0d48ead0c", `golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c h1:qgOY6WgZOaTkIIMiVjBQcw93ERBE4m30iBm00nkL0i8= 251 | golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 252 | `) 253 | tc.addRecord("rsc.io/sampler@v1.3.0", `rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= 254 | rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= 255 | `) 256 | tc.config[testName+"/latest"] = tc.signTree(1) 257 | 258 | tc.addRecord("rsc.io/!quote@v1.5.2", `rsc.io/Quote v1.5.2 h1:uppercase!= 259 | `) 260 | return tc 261 | } 262 | 263 | // newClient resets the Client associated with tc. 264 | // This clears any in-memory cache from the Client 265 | // but not tc's on-disk cache. 266 | func (tc *testClient) newClient() { 267 | tc.client = NewClient(tc) 268 | tc.client.SetTileHeight(tc.tileHeight) 269 | } 270 | 271 | // mustLookup does a lookup for path@vers and checks that the lines that come back match want. 272 | func (tc *testClient) mustLookup(path, vers, want string) { 273 | tc.t.Helper() 274 | lines, err := tc.client.Lookup(path, vers) 275 | if err != nil { 276 | tc.t.Fatal(err) 277 | } 278 | if strings.Join(lines, "\n") != want { 279 | tc.t.Fatalf("Lookup(%q, %q):\n\t%s\nwant:\n\t%s", path, vers, strings.Join(lines, "\n\t"), strings.Replace(want, "\n", "\n\t", -1)) 280 | } 281 | } 282 | 283 | // mustHaveLatest checks that the on-disk configuration 284 | // for latest is a tree of size n. 285 | func (tc *testClient) mustHaveLatest(n int64) { 286 | tc.t.Helper() 287 | 288 | latest := tc.config[testName+"/latest"] 289 | lines := strings.Split(string(latest), "\n") 290 | if len(lines) < 2 || lines[1] != fmt.Sprint(n) { 291 | tc.t.Fatalf("/latest should have tree %d, but has:\n%s", n, latest) 292 | } 293 | } 294 | 295 | // mustError checks that err's error string contains the text. 296 | func (tc *testClient) mustError(err error, text string) { 297 | tc.t.Helper() 298 | if err == nil || !strings.Contains(err.Error(), text) { 299 | tc.t.Fatalf("err = %v, want %q", err, text) 300 | } 301 | } 302 | 303 | // fork returns a copy of tc. 304 | // Changes made to the new copy or to tc are not reflected in the other. 305 | func (tc *testClient) fork() *testClient { 306 | tc2 := &testClient{ 307 | t: tc.t, 308 | getOK: tc.getOK, 309 | getTileOK: tc.getTileOK, 310 | tileHeight: tc.tileHeight, 311 | treeSize: tc.treeSize, 312 | hashes: append([]tlog.Hash{}, tc.hashes...), 313 | signer: tc.signer, 314 | config: copyMap(tc.config), 315 | cache: copyMap(tc.cache), 316 | remote: copyMap(tc.remote), 317 | } 318 | tc2.newClient() 319 | return tc2 320 | } 321 | 322 | func copyMap(m map[string][]byte) map[string][]byte { 323 | m2 := make(map[string][]byte) 324 | for k, v := range m { 325 | m2[k] = v 326 | } 327 | return m2 328 | } 329 | 330 | // ReadHashes is tc's implementation of tlog.HashReader, for use with 331 | // tlog.TreeHash and so on. 332 | func (tc *testClient) ReadHashes(indexes []int64) ([]tlog.Hash, error) { 333 | var list []tlog.Hash 334 | for _, id := range indexes { 335 | list = append(list, tc.hashes[id]) 336 | } 337 | return list, nil 338 | } 339 | 340 | // addRecord adds a log record using the given (!-encoded) key and data. 341 | func (tc *testClient) addRecord(key, data string) { 342 | tc.t.Helper() 343 | 344 | // Create record, add hashes to log tree. 345 | id := tc.treeSize 346 | tc.treeSize++ 347 | rec, err := tlog.FormatRecord(id, []byte(data)) 348 | if err != nil { 349 | tc.t.Fatal(err) 350 | } 351 | hashes, err := tlog.StoredHashesForRecordHash(id, tlog.RecordHash([]byte(data)), tc) 352 | if err != nil { 353 | tc.t.Fatal(err) 354 | } 355 | tc.hashes = append(tc.hashes, hashes...) 356 | 357 | // Create lookup result. 358 | tc.remote["/lookup/"+key] = append(rec, tc.signTree(tc.treeSize)...) 359 | 360 | // Create new tiles. 361 | tiles := tlog.NewTiles(tc.tileHeight, id, tc.treeSize) 362 | for _, tile := range tiles { 363 | data, err := tlog.ReadTileData(tile, tc) 364 | if err != nil { 365 | tc.t.Fatal(err) 366 | } 367 | tc.remote["/"+tile.Path()] = data 368 | // TODO delete old partial tiles 369 | } 370 | } 371 | 372 | // signTree returns the signed head for the tree of the given size. 373 | func (tc *testClient) signTree(size int64) []byte { 374 | h, err := tlog.TreeHash(size, tc) 375 | if err != nil { 376 | tc.t.Fatal(err) 377 | } 378 | text := tlog.FormatTree(tlog.Tree{N: size, Hash: h}) 379 | data, err := note.Sign(¬e.Note{Text: string(text)}, tc.signer) 380 | if err != nil { 381 | tc.t.Fatal(err) 382 | } 383 | return data 384 | } 385 | 386 | // ReadRemote is for tc's implementation of Client. 387 | func (tc *testClient) ReadRemote(path string) ([]byte, error) { 388 | // No mutex here because only the Client should be running 389 | // and the Client cannot change tc.get. 390 | if !tc.getOK { 391 | return nil, fmt.Errorf("disallowed remote read %s", path) 392 | } 393 | if strings.Contains(path, "/tile/") && !tc.getTileOK { 394 | return nil, fmt.Errorf("disallowed remote tile read %s", path) 395 | } 396 | 397 | data, ok := tc.remote[path] 398 | if !ok { 399 | return nil, fmt.Errorf("no remote path %s", path) 400 | } 401 | return data, nil 402 | } 403 | 404 | // ReadConfig is for tc's implementation of Client. 405 | func (tc *testClient) ReadConfig(file string) ([]byte, error) { 406 | tc.mu.Lock() 407 | defer tc.mu.Unlock() 408 | 409 | data, ok := tc.config[file] 410 | if !ok { 411 | return nil, fmt.Errorf("no config %s", file) 412 | } 413 | return data, nil 414 | } 415 | 416 | // WriteConfig is for tc's implementation of Client. 417 | func (tc *testClient) WriteConfig(file string, old, new []byte) error { 418 | tc.mu.Lock() 419 | defer tc.mu.Unlock() 420 | 421 | data := tc.config[file] 422 | if !bytes.Equal(old, data) { 423 | return ErrWriteConflict 424 | } 425 | tc.config[file] = new 426 | return nil 427 | } 428 | 429 | // ReadCache is for tc's implementation of Client. 430 | func (tc *testClient) ReadCache(file string) ([]byte, error) { 431 | tc.mu.Lock() 432 | defer tc.mu.Unlock() 433 | 434 | data, ok := tc.cache[file] 435 | if !ok { 436 | return nil, fmt.Errorf("no cache %s", file) 437 | } 438 | return data, nil 439 | } 440 | 441 | // WriteCache is for tc's implementation of Client. 442 | func (tc *testClient) WriteCache(file string, data []byte) { 443 | tc.mu.Lock() 444 | defer tc.mu.Unlock() 445 | 446 | tc.cache[file] = data 447 | } 448 | 449 | // Log is for tc's implementation of Client. 450 | func (tc *testClient) Log(msg string) { 451 | tc.t.Log(msg) 452 | } 453 | 454 | // SecurityError is for tc's implementation of Client. 455 | func (tc *testClient) SecurityError(msg string) { 456 | tc.mu.Lock() 457 | defer tc.mu.Unlock() 458 | 459 | fmt.Fprintf(&tc.security, "%s\n", strings.TrimRight(msg, "\n")) 460 | } 461 | -------------------------------------------------------------------------------- /sumdb/dirhash/hash.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package dirhash defines hashes over directory trees. 6 | // These hashes are recorded in go.sum files and in the Go checksum database, 7 | // to allow verifying that a newly-downloaded module has the expected content. 8 | package dirhash 9 | 10 | import ( 11 | "archive/zip" 12 | "crypto/sha256" 13 | "encoding/base64" 14 | "errors" 15 | "fmt" 16 | "io" 17 | "os" 18 | "path/filepath" 19 | "slices" 20 | "strings" 21 | ) 22 | 23 | // DefaultHash is the default hash function used in new go.sum entries. 24 | var DefaultHash Hash = Hash1 25 | 26 | // A Hash is a directory hash function. 27 | // It accepts a list of files along with a function that opens the content of each file. 28 | // It opens, reads, hashes, and closes each file and returns the overall directory hash. 29 | type Hash func(files []string, open func(string) (io.ReadCloser, error)) (string, error) 30 | 31 | // Hash1 is the "h1:" directory hash function, using SHA-256. 32 | // 33 | // Hash1 is "h1:" followed by the base64-encoded SHA-256 hash of a summary 34 | // prepared as if by the Unix command: 35 | // 36 | // sha256sum $(find . -type f | sort) | sha256sum 37 | // 38 | // More precisely, the hashed summary contains a single line for each file in the list, 39 | // ordered by [slices.Sort] applied to the file names, where each line consists of 40 | // the hexadecimal SHA-256 hash of the file content, 41 | // two spaces (U+0020), the file name, and a newline (U+000A). 42 | // 43 | // File names with newlines (U+000A) are disallowed. 44 | func Hash1(files []string, open func(string) (io.ReadCloser, error)) (string, error) { 45 | h := sha256.New() 46 | files = append([]string(nil), files...) 47 | slices.Sort(files) 48 | for _, file := range files { 49 | if strings.Contains(file, "\n") { 50 | return "", errors.New("dirhash: filenames with newlines are not supported") 51 | } 52 | r, err := open(file) 53 | if err != nil { 54 | return "", err 55 | } 56 | hf := sha256.New() 57 | _, err = io.Copy(hf, r) 58 | r.Close() 59 | if err != nil { 60 | return "", err 61 | } 62 | fmt.Fprintf(h, "%x %s\n", hf.Sum(nil), file) 63 | } 64 | return "h1:" + base64.StdEncoding.EncodeToString(h.Sum(nil)), nil 65 | } 66 | 67 | // HashDir returns the hash of the local file system directory dir, 68 | // replacing the directory name itself with prefix in the file names 69 | // used in the hash function. 70 | func HashDir(dir, prefix string, hash Hash) (string, error) { 71 | files, err := DirFiles(dir, prefix) 72 | if err != nil { 73 | return "", err 74 | } 75 | osOpen := func(name string) (io.ReadCloser, error) { 76 | return os.Open(filepath.Join(dir, strings.TrimPrefix(name, prefix))) 77 | } 78 | return hash(files, osOpen) 79 | } 80 | 81 | // DirFiles returns the list of files in the tree rooted at dir, 82 | // replacing the directory name dir with prefix in each name. 83 | // The resulting names always use forward slashes. 84 | func DirFiles(dir, prefix string) ([]string, error) { 85 | var files []string 86 | dir = filepath.Clean(dir) 87 | err := filepath.Walk(dir, func(file string, info os.FileInfo, err error) error { 88 | if err != nil { 89 | return err 90 | } 91 | if info.IsDir() { 92 | return nil 93 | } else if file == dir { 94 | return fmt.Errorf("%s is not a directory", dir) 95 | } 96 | 97 | rel := file 98 | if dir != "." { 99 | rel = file[len(dir)+1:] 100 | } 101 | f := filepath.Join(prefix, rel) 102 | files = append(files, filepath.ToSlash(f)) 103 | return nil 104 | }) 105 | if err != nil { 106 | return nil, err 107 | } 108 | return files, nil 109 | } 110 | 111 | // HashZip returns the hash of the file content in the named zip file. 112 | // Only the file names and their contents are included in the hash: 113 | // the exact zip file format encoding, compression method, 114 | // per-file modification times, and other metadata are ignored. 115 | func HashZip(zipfile string, hash Hash) (string, error) { 116 | z, err := zip.OpenReader(zipfile) 117 | if err != nil { 118 | return "", err 119 | } 120 | defer z.Close() 121 | var files []string 122 | zfiles := make(map[string]*zip.File) 123 | for _, file := range z.File { 124 | files = append(files, file.Name) 125 | zfiles[file.Name] = file 126 | } 127 | zipOpen := func(name string) (io.ReadCloser, error) { 128 | f := zfiles[name] 129 | if f == nil { 130 | return nil, fmt.Errorf("file %q not found in zip", name) // should never happen 131 | } 132 | return f.Open() 133 | } 134 | return hash(files, zipOpen) 135 | } 136 | -------------------------------------------------------------------------------- /sumdb/dirhash/hash_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package dirhash 6 | 7 | import ( 8 | "archive/zip" 9 | "crypto/sha256" 10 | "encoding/base64" 11 | "fmt" 12 | "io" 13 | "os" 14 | "path/filepath" 15 | "strings" 16 | "testing" 17 | ) 18 | 19 | func h(s string) string { 20 | return fmt.Sprintf("%x", sha256.Sum256([]byte(s))) 21 | } 22 | 23 | func htop(k string, s string) string { 24 | sum := sha256.Sum256([]byte(s)) 25 | return k + ":" + base64.StdEncoding.EncodeToString(sum[:]) 26 | } 27 | 28 | func TestHash1(t *testing.T) { 29 | files := []string{"xyz", "abc"} 30 | open := func(name string) (io.ReadCloser, error) { 31 | return io.NopCloser(strings.NewReader("data for " + name)), nil 32 | } 33 | want := htop("h1", fmt.Sprintf("%s %s\n%s %s\n", h("data for abc"), "abc", h("data for xyz"), "xyz")) 34 | out, err := Hash1(files, open) 35 | if err != nil { 36 | t.Fatal(err) 37 | } 38 | if out != want { 39 | t.Errorf("Hash1(...) = %s, want %s", out, want) 40 | } 41 | 42 | _, err = Hash1([]string{"xyz", "a\nbc"}, open) 43 | if err == nil { 44 | t.Error("Hash1: expected error on newline in filenames") 45 | } 46 | } 47 | 48 | func TestHashDir(t *testing.T) { 49 | dir := t.TempDir() 50 | if err := os.WriteFile(filepath.Join(dir, "xyz"), []byte("data for xyz"), 0666); err != nil { 51 | t.Fatal(err) 52 | } 53 | if err := os.WriteFile(filepath.Join(dir, "abc"), []byte("data for abc"), 0666); err != nil { 54 | t.Fatal(err) 55 | } 56 | want := htop("h1", fmt.Sprintf("%s %s\n%s %s\n", h("data for abc"), "prefix/abc", h("data for xyz"), "prefix/xyz")) 57 | out, err := HashDir(dir, "prefix", Hash1) 58 | if err != nil { 59 | t.Fatalf("HashDir: %v", err) 60 | } 61 | if out != want { 62 | t.Errorf("HashDir(...) = %s, want %s", out, want) 63 | } 64 | } 65 | 66 | func TestHashZip(t *testing.T) { 67 | f, err := os.CreateTemp(t.TempDir(), "dirhash-test-") 68 | if err != nil { 69 | t.Fatal(err) 70 | } 71 | defer f.Close() 72 | 73 | z := zip.NewWriter(f) 74 | w, err := z.Create("prefix/xyz") 75 | if err != nil { 76 | t.Fatal(err) 77 | } 78 | w.Write([]byte("data for xyz")) 79 | w, err = z.Create("prefix/abc") 80 | if err != nil { 81 | t.Fatal(err) 82 | } 83 | w.Write([]byte("data for abc")) 84 | if err := z.Close(); err != nil { 85 | t.Fatal(err) 86 | } 87 | if err := f.Close(); err != nil { 88 | t.Fatal(err) 89 | } 90 | 91 | want := htop("h1", fmt.Sprintf("%s %s\n%s %s\n", h("data for abc"), "prefix/abc", h("data for xyz"), "prefix/xyz")) 92 | out, err := HashZip(f.Name(), Hash1) 93 | if err != nil { 94 | t.Fatalf("HashDir: %v", err) 95 | } 96 | if out != want { 97 | t.Errorf("HashDir(...) = %s, want %s", out, want) 98 | } 99 | } 100 | 101 | func TestDirFiles(t *testing.T) { 102 | t.Run("valid directory with files", func(t *testing.T) { 103 | dir := t.TempDir() 104 | if err := os.WriteFile(filepath.Join(dir, "xyz"), []byte("data for xyz"), 0666); err != nil { 105 | t.Fatal(err) 106 | } 107 | if err := os.WriteFile(filepath.Join(dir, "abc"), []byte("data for abc"), 0666); err != nil { 108 | t.Fatal(err) 109 | } 110 | if err := os.Mkdir(filepath.Join(dir, "subdir"), 0777); err != nil { 111 | t.Fatal(err) 112 | } 113 | if err := os.WriteFile(filepath.Join(dir, "subdir", "xyz"), []byte("data for subdir xyz"), 0666); err != nil { 114 | t.Fatal(err) 115 | } 116 | prefix := "foo/bar@v2.3.4" 117 | out, err := DirFiles(dir, prefix) 118 | if err != nil { 119 | t.Fatalf("DirFiles: %v", err) 120 | } 121 | for _, file := range out { 122 | if !strings.HasPrefix(file, prefix) { 123 | t.Errorf("Dir file = %s, want prefix %s", file, prefix) 124 | } 125 | } 126 | }) 127 | 128 | t.Run("invalid directory", func(t *testing.T) { 129 | path := filepath.Join(t.TempDir(), "not-a-directory.txt") 130 | if err := os.WriteFile(path, []byte("This is a file."), 0644); err != nil { 131 | t.Fatal(err) 132 | } 133 | defer os.RemoveAll(path) 134 | 135 | out, err := DirFiles(path, "") 136 | if err == nil { 137 | t.Errorf("DirFiles(...) = %v, expected an error", err) 138 | } 139 | if len(out) > 0 { 140 | t.Errorf("DirFiles(...) = unexpected files %s", out) 141 | } 142 | }) 143 | } 144 | -------------------------------------------------------------------------------- /sumdb/note/example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package note_test 6 | 7 | import ( 8 | "fmt" 9 | "io" 10 | "os" 11 | 12 | "golang.org/x/mod/sumdb/note" 13 | ) 14 | 15 | func ExampleSign() { 16 | skey := "PRIVATE+KEY+PeterNeumann+c74f20a3+AYEKFALVFGyNhPJEMzD1QIDr+Y7hfZx09iUvxdXHKDFz" 17 | text := "If you think cryptography is the answer to your problem,\n" + 18 | "then you don't know what your problem is.\n" 19 | 20 | signer, err := note.NewSigner(skey) 21 | if err != nil { 22 | fmt.Println(err) 23 | return 24 | } 25 | 26 | msg, err := note.Sign(¬e.Note{Text: text}, signer) 27 | if err != nil { 28 | fmt.Println(err) 29 | return 30 | } 31 | os.Stdout.Write(msg) 32 | 33 | // Output: 34 | // If you think cryptography is the answer to your problem, 35 | // then you don't know what your problem is. 36 | // 37 | // — PeterNeumann x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM= 38 | } 39 | 40 | func ExampleOpen() { 41 | vkey := "PeterNeumann+c74f20a3+ARpc2QcUPDhMQegwxbzhKqiBfsVkmqq/LDE4izWy10TW" 42 | msg := []byte("If you think cryptography is the answer to your problem,\n" + 43 | "then you don't know what your problem is.\n" + 44 | "\n" + 45 | "— PeterNeumann x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM=\n") 46 | 47 | verifier, err := note.NewVerifier(vkey) 48 | if err != nil { 49 | fmt.Println(err) 50 | return 51 | } 52 | verifiers := note.VerifierList(verifier) 53 | 54 | n, err := note.Open(msg, verifiers) 55 | if err != nil { 56 | fmt.Println(err) 57 | return 58 | } 59 | fmt.Printf("%s (%08x):\n%s", n.Sigs[0].Name, n.Sigs[0].Hash, n.Text) 60 | 61 | // Output: 62 | // PeterNeumann (c74f20a3): 63 | // If you think cryptography is the answer to your problem, 64 | // then you don't know what your problem is. 65 | } 66 | 67 | var rand = struct { 68 | Reader io.Reader 69 | }{ 70 | zeroReader{}, 71 | } 72 | 73 | type zeroReader struct{} 74 | 75 | func (zeroReader) Read(buf []byte) (int, error) { 76 | for i := range buf { 77 | buf[i] = 0 78 | } 79 | return len(buf), nil 80 | } 81 | 82 | func ExampleSign_add_signatures() { 83 | vkey := "PeterNeumann+c74f20a3+ARpc2QcUPDhMQegwxbzhKqiBfsVkmqq/LDE4izWy10TW" 84 | msg := []byte("If you think cryptography is the answer to your problem,\n" + 85 | "then you don't know what your problem is.\n" + 86 | "\n" + 87 | "— PeterNeumann x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM=\n") 88 | 89 | verifier, err := note.NewVerifier(vkey) 90 | if err != nil { 91 | fmt.Println(err) 92 | return 93 | } 94 | verifiers := note.VerifierList(verifier) 95 | 96 | n, err := note.Open(msg, verifiers) 97 | if err != nil { 98 | fmt.Println(err) 99 | return 100 | } 101 | 102 | skey, vkey, err := note.GenerateKey(rand.Reader, "EnochRoot") 103 | if err != nil { 104 | fmt.Println(err) 105 | return 106 | } 107 | _ = vkey // give to verifiers 108 | 109 | me, err := note.NewSigner(skey) 110 | if err != nil { 111 | fmt.Println(err) 112 | return 113 | } 114 | 115 | msg, err = note.Sign(n, me) 116 | if err != nil { 117 | fmt.Println(err) 118 | return 119 | } 120 | os.Stdout.Write(msg) 121 | 122 | // Output: 123 | // If you think cryptography is the answer to your problem, 124 | // then you don't know what your problem is. 125 | // 126 | // — PeterNeumann x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM= 127 | // — EnochRoot rwz+eBzmZa0SO3NbfRGzPCpDckykFXSdeX+MNtCOXm2/5n2tiOHp+vAF1aGrQ5ovTG01oOTGwnWLox33WWd1RvMc+QQ= 128 | } 129 | -------------------------------------------------------------------------------- /sumdb/note/note_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package note 6 | 7 | import ( 8 | "crypto/ed25519" 9 | "crypto/rand" 10 | "errors" 11 | "strings" 12 | "testing" 13 | "testing/iotest" 14 | ) 15 | 16 | func TestNewVerifier(t *testing.T) { 17 | vkey := "PeterNeumann+c74f20a3+ARpc2QcUPDhMQegwxbzhKqiBfsVkmqq/LDE4izWy10TW" 18 | _, err := NewVerifier(vkey) 19 | if err != nil { 20 | t.Fatal(err) 21 | } 22 | 23 | // Check various manglings are not accepted. 24 | badKey := func(k string) { 25 | _, err := NewVerifier(k) 26 | if err == nil { 27 | t.Errorf("NewVerifier(%q) succeeded, should have failed", k) 28 | } 29 | } 30 | 31 | b := []byte(vkey) 32 | for i := 0; i <= len(b); i++ { 33 | for j := i + 1; j <= len(b); j++ { 34 | if i != 0 || j != len(b) { 35 | badKey(string(b[i:j])) 36 | } 37 | } 38 | } 39 | for i := 0; i < len(b); i++ { 40 | b[i]++ 41 | badKey(string(b)) 42 | b[i]-- 43 | } 44 | 45 | badKey("PeterNeumann+cc469956+ARpc2QcUPDhMQegwxbzhKqiBfsVkmqq/LDE4izWy10TWBADKEY==") // wrong length key, with adjusted key hash 46 | badKey("PeterNeumann+173116ae+ZRpc2QcUPDhMQegwxbzhKqiBfsVkmqq/LDE4izWy10TW") // unknown algorithm, with adjusted key hash 47 | } 48 | 49 | func TestNewSigner(t *testing.T) { 50 | skey := "PRIVATE+KEY+PeterNeumann+c74f20a3+AYEKFALVFGyNhPJEMzD1QIDr+Y7hfZx09iUvxdXHKDFz" 51 | _, err := NewSigner(skey) 52 | if err != nil { 53 | t.Fatal(err) 54 | } 55 | 56 | // Check various manglings are not accepted. 57 | b := []byte(skey) 58 | for i := 0; i <= len(b); i++ { 59 | for j := i + 1; j <= len(b); j++ { 60 | if i == 0 && j == len(b) { 61 | continue 62 | } 63 | _, err := NewSigner(string(b[i:j])) 64 | if err == nil { 65 | t.Errorf("NewSigner(%q) succeeded, should have failed", b[i:j]) 66 | } 67 | } 68 | } 69 | for i := 0; i < len(b); i++ { 70 | b[i]++ 71 | _, err := NewSigner(string(b)) 72 | if err == nil { 73 | t.Errorf("NewSigner(%q) succeeded, should have failed", b) 74 | } 75 | b[i]-- 76 | } 77 | } 78 | 79 | func testSignerAndVerifier(t *testing.T, Name string, signer Signer, verifier Verifier) { 80 | if name := signer.Name(); name != Name { 81 | t.Errorf("signer.Name() = %q, want %q", name, Name) 82 | } 83 | if name := verifier.Name(); name != Name { 84 | t.Errorf("verifier.Name() = %q, want %q", name, Name) 85 | } 86 | shash := signer.KeyHash() 87 | vhash := verifier.KeyHash() 88 | if shash != vhash { 89 | t.Errorf("signer.KeyHash() = %#08x != verifier.KeyHash() = %#08x", shash, vhash) 90 | } 91 | 92 | msg := []byte("hi") 93 | sig, err := signer.Sign(msg) 94 | if err != nil { 95 | t.Fatalf("signer.Sign: %v", err) 96 | } 97 | if !verifier.Verify(msg, sig) { 98 | t.Fatalf("verifier.Verify failed on signature returned by signer.Sign") 99 | } 100 | sig[0]++ 101 | if verifier.Verify(msg, sig) { 102 | t.Fatalf("verifier.Verify succeeded on corrupt signature") 103 | } 104 | sig[0]-- 105 | msg[0]++ 106 | if verifier.Verify(msg, sig) { 107 | t.Fatalf("verifier.Verify succeeded on corrupt message") 108 | } 109 | } 110 | 111 | func TestGenerateKey(t *testing.T) { 112 | // Generate key pair, make sure it is all self-consistent. 113 | const Name = "EnochRoot" 114 | 115 | skey, vkey, err := GenerateKey(rand.Reader, Name) 116 | if err != nil { 117 | t.Fatalf("GenerateKey: %v", err) 118 | } 119 | signer, err := NewSigner(skey) 120 | if err != nil { 121 | t.Fatalf("NewSigner: %v", err) 122 | } 123 | verifier, err := NewVerifier(vkey) 124 | if err != nil { 125 | t.Fatalf("NewVerifier: %v", err) 126 | } 127 | 128 | testSignerAndVerifier(t, Name, signer, verifier) 129 | 130 | // Check that GenerateKey returns error from rand reader. 131 | _, _, err = GenerateKey(iotest.TimeoutReader(iotest.OneByteReader(rand.Reader)), Name) 132 | if err == nil { 133 | t.Fatalf("GenerateKey succeeded with error-returning rand reader") 134 | } 135 | } 136 | 137 | func TestFromEd25519(t *testing.T) { 138 | const Name = "EnochRoot" 139 | 140 | pub, priv, err := ed25519.GenerateKey(rand.Reader) 141 | if err != nil { 142 | t.Fatalf("GenerateKey: %v", err) 143 | } 144 | signer, err := newSignerFromEd25519Seed(Name, priv.Seed()) 145 | if err != nil { 146 | t.Fatalf("newSignerFromEd25519Seed: %v", err) 147 | } 148 | vkey, err := NewEd25519VerifierKey(Name, pub) 149 | if err != nil { 150 | t.Fatalf("NewEd25519VerifierKey: %v", err) 151 | } 152 | verifier, err := NewVerifier(vkey) 153 | if err != nil { 154 | t.Fatalf("NewVerifier: %v", err) 155 | } 156 | 157 | testSignerAndVerifier(t, Name, signer, verifier) 158 | 159 | // Check that wrong key sizes return errors. 160 | _, err = NewEd25519VerifierKey(Name, pub[:len(pub)-1]) 161 | if err == nil { 162 | t.Errorf("NewEd25519VerifierKey succeeded with a seed of the wrong size") 163 | } 164 | } 165 | 166 | // newSignerFromEd25519Seed constructs a new signer from a verifier name and a 167 | // crypto/ed25519 private key seed. 168 | func newSignerFromEd25519Seed(name string, seed []byte) (Signer, error) { 169 | if len(seed) != ed25519.SeedSize { 170 | return nil, errors.New("invalid seed size") 171 | } 172 | priv := ed25519.NewKeyFromSeed(seed) 173 | pub := priv[32:] 174 | 175 | pubkey := append([]byte{algEd25519}, pub...) 176 | hash := keyHash(name, pubkey) 177 | 178 | s := &signer{ 179 | name: name, 180 | hash: hash, 181 | sign: func(msg []byte) ([]byte, error) { 182 | return ed25519.Sign(priv, msg), nil 183 | }, 184 | } 185 | return s, nil 186 | } 187 | 188 | func TestSign(t *testing.T) { 189 | skey := "PRIVATE+KEY+PeterNeumann+c74f20a3+AYEKFALVFGyNhPJEMzD1QIDr+Y7hfZx09iUvxdXHKDFz" 190 | text := "If you think cryptography is the answer to your problem,\n" + 191 | "then you don't know what your problem is.\n" 192 | 193 | signer, err := NewSigner(skey) 194 | if err != nil { 195 | t.Fatal(err) 196 | } 197 | 198 | msg, err := Sign(&Note{Text: text}, signer) 199 | if err != nil { 200 | t.Fatal(err) 201 | } 202 | 203 | want := `If you think cryptography is the answer to your problem, 204 | then you don't know what your problem is. 205 | 206 | — PeterNeumann x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM= 207 | ` 208 | if string(msg) != want { 209 | t.Errorf("Sign: wrong output\nhave:\n%s\nwant:\n%s", msg, want) 210 | } 211 | 212 | // Check that existing signature is replaced by new one. 213 | msg, err = Sign(&Note{Text: text, Sigs: []Signature{{Name: "PeterNeumann", Hash: 0xc74f20a3, Base64: "BADSIGN="}}}, signer) 214 | if err != nil { 215 | t.Fatal(err) 216 | } 217 | if string(msg) != want { 218 | t.Errorf("Sign replacing signature: wrong output\nhave:\n%s\nwant:\n%s", msg, want) 219 | } 220 | 221 | // Check various bad inputs. 222 | _, err = Sign(&Note{Text: "abc"}, signer) 223 | if err == nil || err.Error() != "malformed note" { 224 | t.Fatalf("Sign with short text: %v, want malformed note error", err) 225 | } 226 | 227 | _, err = Sign(&Note{Text: text, Sigs: []Signature{{Name: "a+b", Base64: "ABCD"}}}) 228 | if err == nil || err.Error() != "malformed note" { 229 | t.Fatalf("Sign with bad name: %v, want malformed note error", err) 230 | } 231 | 232 | _, err = Sign(&Note{Text: text, Sigs: []Signature{{Name: "PeterNeumann", Hash: 0xc74f20a3, Base64: "BADHASH="}}}) 233 | if err == nil || err.Error() != "malformed note" { 234 | t.Fatalf("Sign with bad pre-filled signature: %v, want malformed note error", err) 235 | } 236 | 237 | _, err = Sign(&Note{Text: text}, &badSigner{signer}) 238 | if err == nil || err.Error() != "invalid signer" { 239 | t.Fatalf("Sign with bad signer: %v, want invalid signer error", err) 240 | } 241 | 242 | _, err = Sign(&Note{Text: text}, &errSigner{signer}) 243 | if err != errSurprise { 244 | t.Fatalf("Sign with failing signer: %v, want errSurprise", err) 245 | } 246 | } 247 | 248 | func TestVerifierList(t *testing.T) { 249 | peterKey := "PeterNeumann+c74f20a3+ARpc2QcUPDhMQegwxbzhKqiBfsVkmqq/LDE4izWy10TW" 250 | peterVerifier, err := NewVerifier(peterKey) 251 | if err != nil { 252 | t.Fatal(err) 253 | } 254 | 255 | enochKey := "EnochRoot+af0cfe78+ATtqJ7zOtqQtYqOo0CpvDXNlMhV3HeJDpjrASKGLWdop" 256 | enochVerifier, err := NewVerifier(enochKey) 257 | if err != nil { 258 | t.Fatal(err) 259 | } 260 | 261 | list := VerifierList(peterVerifier, enochVerifier, enochVerifier) 262 | v, err := list.Verifier("PeterNeumann", 0xc74f20a3) 263 | if v != peterVerifier || err != nil { 264 | t.Fatalf("list.Verifier(peter) = %v, %v, want %v, nil", v, err, peterVerifier) 265 | } 266 | v, err = list.Verifier("PeterNeumann", 0xc74f20a4) 267 | if v != nil || err == nil || err.Error() != "unknown key PeterNeumann+c74f20a4" { 268 | t.Fatalf("list.Verifier(peter bad hash) = %v, %v, want nil, unknown key error", v, err) 269 | } 270 | 271 | v, err = list.Verifier("PeterNeuman", 0xc74f20a3) 272 | if v != nil || err == nil || err.Error() != "unknown key PeterNeuman+c74f20a3" { 273 | t.Fatalf("list.Verifier(peter bad name) = %v, %v, want nil, unknown key error", v, err) 274 | } 275 | v, err = list.Verifier("EnochRoot", 0xaf0cfe78) 276 | if v != nil || err == nil || err.Error() != "ambiguous key EnochRoot+af0cfe78" { 277 | t.Fatalf("list.Verifier(enoch) = %v, %v, want nil, ambiguous key error", v, err) 278 | } 279 | } 280 | 281 | type badSigner struct { 282 | Signer 283 | } 284 | 285 | func (b *badSigner) Name() string { 286 | return "bad name" 287 | } 288 | 289 | var errSurprise = errors.New("surprise!") 290 | 291 | type errSigner struct { 292 | Signer 293 | } 294 | 295 | func (e *errSigner) Sign([]byte) ([]byte, error) { 296 | return nil, errSurprise 297 | } 298 | 299 | type fixedVerifier struct{ v Verifier } 300 | 301 | func (v fixedVerifier) Verifier(name string, hash uint32) (Verifier, error) { 302 | return v.v, nil 303 | } 304 | 305 | func TestOpen(t *testing.T) { 306 | peterKey := "PeterNeumann+c74f20a3+ARpc2QcUPDhMQegwxbzhKqiBfsVkmqq/LDE4izWy10TW" 307 | peterVerifier, err := NewVerifier(peterKey) 308 | if err != nil { 309 | t.Fatal(err) 310 | } 311 | 312 | enochKey := "EnochRoot+af0cfe78+ATtqJ7zOtqQtYqOo0CpvDXNlMhV3HeJDpjrASKGLWdop" 313 | enochVerifier, err := NewVerifier(enochKey) 314 | if err != nil { 315 | t.Fatal(err) 316 | } 317 | 318 | text := `If you think cryptography is the answer to your problem, 319 | then you don't know what your problem is. 320 | ` 321 | peterSig := "— PeterNeumann x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM=\n" 322 | enochSig := "— EnochRoot rwz+eBzmZa0SO3NbfRGzPCpDckykFXSdeX+MNtCOXm2/5n2tiOHp+vAF1aGrQ5ovTG01oOTGwnWLox33WWd1RvMc+QQ=\n" 323 | 324 | peter := Signature{"PeterNeumann", 0xc74f20a3, "x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM="} 325 | enoch := Signature{"EnochRoot", 0xaf0cfe78, "rwz+eBzmZa0SO3NbfRGzPCpDckykFXSdeX+MNtCOXm2/5n2tiOHp+vAF1aGrQ5ovTG01oOTGwnWLox33WWd1RvMc+QQ="} 326 | 327 | // Check one signature verified, one not. 328 | n, err := Open([]byte(text+"\n"+peterSig+enochSig), VerifierList(peterVerifier)) 329 | if err != nil { 330 | t.Fatal(err) 331 | } 332 | if n.Text != text { 333 | t.Errorf("n.Text = %q, want %q", n.Text, text) 334 | } 335 | if len(n.Sigs) != 1 || n.Sigs[0] != peter { 336 | t.Errorf("n.Sigs:\nhave %v\nwant %v", n.Sigs, []Signature{peter}) 337 | } 338 | if len(n.UnverifiedSigs) != 1 || n.UnverifiedSigs[0] != enoch { 339 | t.Errorf("n.UnverifiedSigs:\nhave %v\nwant %v", n.Sigs, []Signature{peter}) 340 | } 341 | 342 | // Check both verified. 343 | n, err = Open([]byte(text+"\n"+peterSig+enochSig), VerifierList(peterVerifier, enochVerifier)) 344 | if err != nil { 345 | t.Fatal(err) 346 | } 347 | if len(n.Sigs) != 2 || n.Sigs[0] != peter || n.Sigs[1] != enoch { 348 | t.Errorf("n.Sigs:\nhave %v\nwant %v", n.Sigs, []Signature{peter, enoch}) 349 | } 350 | if len(n.UnverifiedSigs) != 0 { 351 | t.Errorf("n.UnverifiedSigs:\nhave %v\nwant %v", n.Sigs, []Signature{}) 352 | } 353 | 354 | // Check both unverified. 355 | n, err = Open([]byte(text+"\n"+peterSig+enochSig), VerifierList()) 356 | if n != nil || err == nil { 357 | t.Fatalf("Open unverified = %v, %v, want nil, error", n, err) 358 | } 359 | e, ok := err.(*UnverifiedNoteError) 360 | if !ok { 361 | t.Fatalf("Open unverified: err is %T, want *UnverifiedNoteError", err) 362 | } 363 | if err.Error() != "note has no verifiable signatures" { 364 | t.Fatalf("Open unverified: err.Error() = %q, want %q", err.Error(), "note has no verifiable signatures") 365 | } 366 | 367 | n = e.Note 368 | if n == nil { 369 | t.Fatalf("Open unverified: missing note in UnverifiedNoteError") 370 | } 371 | if len(n.Sigs) != 0 { 372 | t.Errorf("n.Sigs:\nhave %v\nwant %v", n.Sigs, []Signature{}) 373 | } 374 | if len(n.UnverifiedSigs) != 2 || n.UnverifiedSigs[0] != peter || n.UnverifiedSigs[1] != enoch { 375 | t.Errorf("n.UnverifiedSigs:\nhave %v\nwant %v", n.Sigs, []Signature{peter, enoch}) 376 | } 377 | 378 | // Check duplicated verifier. 379 | _, err = Open([]byte(text+"\n"+enochSig), VerifierList(enochVerifier, peterVerifier, enochVerifier)) 380 | if err == nil || err.Error() != "ambiguous key EnochRoot+af0cfe78" { 381 | t.Fatalf("Open with duplicated verifier: err=%v, want ambiguous key", err) 382 | } 383 | 384 | // Check unused duplicated verifier. 385 | _, err = Open([]byte(text+"\n"+peterSig), VerifierList(enochVerifier, peterVerifier, enochVerifier)) 386 | if err != nil { 387 | t.Fatal(err) 388 | } 389 | 390 | // Check too many signatures. 391 | n, err = Open([]byte(text+"\n"+strings.Repeat(peterSig, 101)), VerifierList(peterVerifier)) 392 | if n != nil || err == nil || err.Error() != "malformed note" { 393 | t.Fatalf("Open too many verified signatures = %v, %v, want nil, malformed note error", n, err) 394 | } 395 | n, err = Open([]byte(text+"\n"+strings.Repeat(peterSig, 101)), VerifierList()) 396 | if n != nil || err == nil || err.Error() != "malformed note" { 397 | t.Fatalf("Open too many verified signatures = %v, %v, want nil, malformed note error", n, err) 398 | } 399 | 400 | // Invalid signature. 401 | n, err = Open([]byte(text+"\n"+peterSig[:60]+"ABCD"+peterSig[60:]), VerifierList(peterVerifier)) 402 | if n != nil || err == nil || err.Error() != "invalid signature for key PeterNeumann+c74f20a3" { 403 | t.Fatalf("Open too many verified signatures = %v, %v, want nil, invalid signature error", n, err) 404 | } 405 | 406 | // Duplicated verified and unverified signatures. 407 | enochABCD := Signature{"EnochRoot", 0xaf0cfe78, "rwz+eBzmZa0SO3NbfRGzPCpDckykFXSdeX+MNtCOXm2/5n" + "ABCD" + "2tiOHp+vAF1aGrQ5ovTG01oOTGwnWLox33WWd1RvMc+QQ="} 408 | n, err = Open([]byte(text+"\n"+peterSig+peterSig+enochSig+enochSig+enochSig[:60]+"ABCD"+enochSig[60:]), VerifierList(peterVerifier)) 409 | if err != nil { 410 | t.Fatal(err) 411 | } 412 | if len(n.Sigs) != 1 || n.Sigs[0] != peter { 413 | t.Errorf("n.Sigs:\nhave %v\nwant %v", n.Sigs, []Signature{peter}) 414 | } 415 | if len(n.UnverifiedSigs) != 2 || n.UnverifiedSigs[0] != enoch || n.UnverifiedSigs[1] != enochABCD { 416 | t.Errorf("n.UnverifiedSigs:\nhave %v\nwant %v", n.UnverifiedSigs, []Signature{enoch, enochABCD}) 417 | } 418 | 419 | // Invalid encoded message syntax. 420 | badMsgs := []string{ 421 | text, 422 | text + "\n", 423 | text + "\n" + peterSig[:len(peterSig)-1], 424 | "\x01" + text + "\n" + peterSig, 425 | "\xff" + text + "\n" + peterSig, 426 | text + "\n" + "— Bad Name x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM=", 427 | text + "\n" + peterSig + "Unexpected line.\n", 428 | } 429 | for _, msg := range badMsgs { 430 | n, err := Open([]byte(msg), VerifierList(peterVerifier)) 431 | if n != nil || err == nil || err.Error() != "malformed note" { 432 | t.Fatalf("Open bad msg = %v, %v, want nil, malformed note error\nmsg:\n%s", n, err, msg) 433 | } 434 | } 435 | 436 | // Verifiers returns a Verifier for the wrong name or hash. 437 | misnamedSig := strings.Replace(peterSig, "PeterNeumann", "CarmenSandiego", -1) 438 | _, err = Open([]byte(text+"\n"+misnamedSig), fixedVerifier{peterVerifier}) 439 | if err != errMismatchedVerifier { 440 | t.Fatalf("Open with wrong Verifier, err=%v, want errMismatchedVerifier", err) 441 | } 442 | wrongHash := strings.Replace(peterSig, "x08g", "xxxx", -1) 443 | _, err = Open([]byte(text+"\n"+wrongHash), fixedVerifier{peterVerifier}) 444 | if err != errMismatchedVerifier { 445 | t.Fatalf("Open with wrong Verifier, err=%v, want errMismatchedVerifier", err) 446 | } 447 | } 448 | 449 | func BenchmarkOpen(b *testing.B) { 450 | vkey := "PeterNeumann+c74f20a3+ARpc2QcUPDhMQegwxbzhKqiBfsVkmqq/LDE4izWy10TW" 451 | msg := []byte("If you think cryptography is the answer to your problem,\n" + 452 | "then you don't know what your problem is.\n" + 453 | "\n" + 454 | "— PeterNeumann x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM=\n") 455 | 456 | verifier, err := NewVerifier(vkey) 457 | if err != nil { 458 | b.Fatal(err) 459 | } 460 | verifiers := VerifierList(verifier) 461 | verifiers0 := VerifierList() 462 | 463 | // Try with 0 signatures and 1 signature so we can tell how much each signature adds. 464 | 465 | b.Run("Sig0", func(b *testing.B) { 466 | for i := 0; i < b.N; i++ { 467 | _, err := Open(msg, verifiers0) 468 | e, ok := err.(*UnverifiedNoteError) 469 | if !ok { 470 | b.Fatal("expected UnverifiedNoteError") 471 | } 472 | n := e.Note 473 | if len(n.Sigs) != 0 || len(n.UnverifiedSigs) != 1 { 474 | b.Fatal("wrong signature count") 475 | } 476 | } 477 | }) 478 | 479 | b.Run("Sig1", func(b *testing.B) { 480 | for i := 0; i < b.N; i++ { 481 | n, err := Open(msg, verifiers) 482 | if err != nil { 483 | b.Fatal(err) 484 | } 485 | if len(n.Sigs) != 1 || len(n.UnverifiedSigs) != 0 { 486 | b.Fatal("wrong signature count") 487 | } 488 | } 489 | }) 490 | } 491 | -------------------------------------------------------------------------------- /sumdb/server.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package sumdb implements the HTTP protocols for serving or accessing a module checksum database. 6 | package sumdb 7 | 8 | import ( 9 | "bytes" 10 | "context" 11 | "net/http" 12 | "os" 13 | "strings" 14 | 15 | "golang.org/x/mod/internal/lazyregexp" 16 | "golang.org/x/mod/module" 17 | "golang.org/x/mod/sumdb/tlog" 18 | ) 19 | 20 | // A ServerOps provides the external operations 21 | // (underlying database access and so on) needed by the [Server]. 22 | type ServerOps interface { 23 | // Signed returns the signed hash of the latest tree. 24 | Signed(ctx context.Context) ([]byte, error) 25 | 26 | // ReadRecords returns the content for the n records id through id+n-1. 27 | ReadRecords(ctx context.Context, id, n int64) ([][]byte, error) 28 | 29 | // Lookup looks up a record for the given module, 30 | // returning the record ID. 31 | Lookup(ctx context.Context, m module.Version) (int64, error) 32 | 33 | // ReadTileData reads the content of tile t. 34 | // It is only invoked for hash tiles (t.L ≥ 0). 35 | ReadTileData(ctx context.Context, t tlog.Tile) ([]byte, error) 36 | } 37 | 38 | // A Server is the checksum database HTTP server, 39 | // which implements http.Handler and should be invoked 40 | // to serve the paths listed in [ServerPaths]. 41 | type Server struct { 42 | ops ServerOps 43 | } 44 | 45 | // NewServer returns a new Server using the given operations. 46 | func NewServer(ops ServerOps) *Server { 47 | return &Server{ops: ops} 48 | } 49 | 50 | // ServerPaths are the URL paths the Server can (and should) serve. 51 | // 52 | // Typically a server will do: 53 | // 54 | // srv := sumdb.NewServer(ops) 55 | // for _, path := range sumdb.ServerPaths { 56 | // http.Handle(path, srv) 57 | // } 58 | var ServerPaths = []string{ 59 | "/lookup/", 60 | "/latest", 61 | "/tile/", 62 | } 63 | 64 | var modVerRE = lazyregexp.New(`^[^@]+@v[0-9]+\.[0-9]+\.[0-9]+(-[^@]*)?(\+incompatible)?$`) 65 | 66 | func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { 67 | ctx := r.Context() 68 | 69 | switch { 70 | default: 71 | http.NotFound(w, r) 72 | 73 | case strings.HasPrefix(r.URL.Path, "/lookup/"): 74 | mod := strings.TrimPrefix(r.URL.Path, "/lookup/") 75 | if !modVerRE.MatchString(mod) { 76 | http.Error(w, "invalid module@version syntax", http.StatusBadRequest) 77 | return 78 | } 79 | i := strings.Index(mod, "@") 80 | escPath, escVers := mod[:i], mod[i+1:] 81 | path, err := module.UnescapePath(escPath) 82 | if err != nil { 83 | reportError(w, err) 84 | return 85 | } 86 | vers, err := module.UnescapeVersion(escVers) 87 | if err != nil { 88 | reportError(w, err) 89 | return 90 | } 91 | id, err := s.ops.Lookup(ctx, module.Version{Path: path, Version: vers}) 92 | if err != nil { 93 | reportError(w, err) 94 | return 95 | } 96 | records, err := s.ops.ReadRecords(ctx, id, 1) 97 | if err != nil { 98 | // This should never happen - the lookup says the record exists. 99 | http.Error(w, err.Error(), http.StatusInternalServerError) 100 | return 101 | } 102 | if len(records) != 1 { 103 | http.Error(w, "invalid record count returned by ReadRecords", http.StatusInternalServerError) 104 | return 105 | } 106 | msg, err := tlog.FormatRecord(id, records[0]) 107 | if err != nil { 108 | http.Error(w, err.Error(), http.StatusInternalServerError) 109 | return 110 | } 111 | signed, err := s.ops.Signed(ctx) 112 | if err != nil { 113 | http.Error(w, err.Error(), http.StatusInternalServerError) 114 | return 115 | } 116 | w.Header().Set("Content-Type", "text/plain; charset=UTF-8") 117 | w.Write(msg) 118 | w.Write(signed) 119 | 120 | case r.URL.Path == "/latest": 121 | data, err := s.ops.Signed(ctx) 122 | if err != nil { 123 | http.Error(w, err.Error(), http.StatusInternalServerError) 124 | return 125 | } 126 | w.Header().Set("Content-Type", "text/plain; charset=UTF-8") 127 | w.Write(data) 128 | 129 | case strings.HasPrefix(r.URL.Path, "/tile/"): 130 | t, err := tlog.ParseTilePath(r.URL.Path[1:]) 131 | if err != nil { 132 | http.Error(w, "invalid tile syntax", http.StatusBadRequest) 133 | return 134 | } 135 | if t.L == -1 { 136 | // Record data. 137 | start := t.N << uint(t.H) 138 | records, err := s.ops.ReadRecords(ctx, start, int64(t.W)) 139 | if err != nil { 140 | reportError(w, err) 141 | return 142 | } 143 | if len(records) != t.W { 144 | http.Error(w, "invalid record count returned by ReadRecords", http.StatusInternalServerError) 145 | return 146 | } 147 | var data []byte 148 | for i, text := range records { 149 | msg, err := tlog.FormatRecord(start+int64(i), text) 150 | if err != nil { 151 | http.Error(w, err.Error(), http.StatusInternalServerError) 152 | return 153 | } 154 | // Data tiles contain formatted records without the first line with record ID. 155 | _, msg, _ = bytes.Cut(msg, []byte{'\n'}) 156 | data = append(data, msg...) 157 | } 158 | w.Header().Set("Content-Type", "text/plain; charset=UTF-8") 159 | w.Write(data) 160 | return 161 | } 162 | 163 | data, err := s.ops.ReadTileData(ctx, t) 164 | if err != nil { 165 | reportError(w, err) 166 | return 167 | } 168 | w.Header().Set("Content-Type", "application/octet-stream") 169 | w.Write(data) 170 | } 171 | } 172 | 173 | // reportError reports err to w. 174 | // If it's a not-found, the reported error is 404. 175 | // Otherwise it is an internal server error. 176 | // The caller must only call reportError in contexts where 177 | // a not-found err should be reported as 404. 178 | func reportError(w http.ResponseWriter, err error) { 179 | if os.IsNotExist(err) { 180 | http.Error(w, err.Error(), http.StatusNotFound) 181 | return 182 | } 183 | http.Error(w, err.Error(), http.StatusInternalServerError) 184 | } 185 | -------------------------------------------------------------------------------- /sumdb/storage/mem.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package storage 6 | 7 | import ( 8 | "context" 9 | "errors" 10 | "math/rand" 11 | "sync" 12 | ) 13 | 14 | // Mem is an in-memory implementation of [Storage]. 15 | // It is meant for tests and does not store any data to persistent storage. 16 | // 17 | // The zero value is an empty Mem ready for use. 18 | type Mem struct { 19 | mu sync.RWMutex 20 | table map[string]string 21 | } 22 | 23 | // A memTx is a transaction in a Mem. 24 | type memTx struct { 25 | m *Mem 26 | writes []Write 27 | } 28 | 29 | // errRetry is an internal sentinel indicating that the transaction should be retried. 30 | // It is never returned to the caller. 31 | var errRetry = errors.New("retry") 32 | 33 | // ReadOnly runs f in a read-only transaction. 34 | func (m *Mem) ReadOnly(ctx context.Context, f func(context.Context, Transaction) error) error { 35 | tx := &memTx{m: m} 36 | for { 37 | err := func() error { 38 | m.mu.Lock() 39 | defer m.mu.Unlock() 40 | 41 | if err := f(ctx, tx); err != nil { 42 | return err 43 | } 44 | // Spurious retry with 10% probability. 45 | if rand.Intn(10) == 0 { 46 | return errRetry 47 | } 48 | return nil 49 | }() 50 | if err != errRetry { 51 | return err 52 | } 53 | } 54 | } 55 | 56 | // ReadWrite runs f in a read-write transaction. 57 | func (m *Mem) ReadWrite(ctx context.Context, f func(context.Context, Transaction) error) error { 58 | tx := &memTx{m: m} 59 | for { 60 | err := func() error { 61 | m.mu.Lock() 62 | defer m.mu.Unlock() 63 | 64 | tx.writes = []Write{} 65 | if err := f(ctx, tx); err != nil { 66 | return err 67 | } 68 | // Spurious retry with 10% probability. 69 | if rand.Intn(10) == 0 { 70 | return errRetry 71 | } 72 | if m.table == nil { 73 | m.table = make(map[string]string) 74 | } 75 | for _, w := range tx.writes { 76 | if w.Value == "" { 77 | delete(m.table, w.Key) 78 | } else { 79 | m.table[w.Key] = w.Value 80 | } 81 | } 82 | return nil 83 | }() 84 | if err != errRetry { 85 | return err 86 | } 87 | } 88 | } 89 | 90 | // ReadValues returns the values associated with the given keys. 91 | func (tx *memTx) ReadValues(ctx context.Context, keys []string) ([]string, error) { 92 | vals := make([]string, len(keys)) 93 | for i, key := range keys { 94 | vals[i] = tx.m.table[key] 95 | } 96 | return vals, nil 97 | } 98 | 99 | // ReadValue returns the value associated with the single key. 100 | func (tx *memTx) ReadValue(ctx context.Context, key string) (string, error) { 101 | return tx.m.table[key], nil 102 | } 103 | 104 | // BufferWrites buffers a list of writes to be applied 105 | // to the table when the transaction commits. 106 | // The changes are not visible to reads within the transaction. 107 | // The map argument is not used after the call returns. 108 | func (tx *memTx) BufferWrites(list []Write) error { 109 | if tx.writes == nil { 110 | panic("BufferWrite on read-only transaction") 111 | } 112 | tx.writes = append(tx.writes, list...) 113 | return nil 114 | } 115 | -------------------------------------------------------------------------------- /sumdb/storage/mem_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package storage 6 | 7 | import ( 8 | "context" 9 | "testing" 10 | ) 11 | 12 | func TestMem(t *testing.T) { 13 | TestStorage(t, context.Background(), new(Mem)) 14 | } 15 | -------------------------------------------------------------------------------- /sumdb/storage/storage.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package storage defines storage interfaces for and a basic implementation of a checksum database. 6 | package storage 7 | 8 | import "context" 9 | 10 | // A Storage is a transaction key-value storage system. 11 | type Storage interface { 12 | // ReadOnly runs f in a read-only transaction. 13 | // It is equivalent to ReadWrite except that the 14 | // transaction's BufferWrite method will fail unconditionally. 15 | // (The implementation may be able to optimize the 16 | // transaction if it knows at the start that no writes will happen.) 17 | ReadOnly(ctx context.Context, f func(context.Context, Transaction) error) error 18 | 19 | // ReadWrite runs f in a read-write transaction. 20 | // If f returns an error, the transaction aborts and returns that error. 21 | // If f returns nil, the transaction attempts to commit and then return nil. 22 | // Otherwise it tries again. Note that f may be called multiple times and that 23 | // the result only describes the effect of the final call to f. 24 | // The caller must take care not to use any state computed during 25 | // earlier calls to f, or even the last call to f when an error is returned. 26 | ReadWrite(ctx context.Context, f func(context.Context, Transaction) error) error 27 | } 28 | 29 | // A Transaction provides read and write operations within a transaction, 30 | // as executed by [Storage]'s ReadOnly or ReadWrite methods. 31 | type Transaction interface { 32 | // ReadValue reads the value associated with a single key. 33 | // If there is no value associated with that key, ReadKey returns an empty value. 34 | // An error is only returned for problems accessing the storage. 35 | ReadValue(ctx context.Context, key string) (value string, err error) 36 | 37 | // ReadValues reads the values associated with the given keys. 38 | // If there is no value stored for a given key, ReadValues returns an empty value for that key. 39 | // An error is only returned for problems accessing the storage. 40 | ReadValues(ctx context.Context, keys []string) (values []string, err error) 41 | 42 | // BufferWrites buffers the given writes, 43 | // to be applied at the end of the transaction. 44 | // BufferWrites panics if this is a ReadOnly transaction. 45 | // It returns an error if it detects any other problems. 46 | // The behavior of multiple writes buffered using the same key 47 | // is undefined: it may return an error or not. 48 | BufferWrites(writes []Write) error 49 | } 50 | 51 | // A Write is a single change to be applied at the end of a read-write transaction. 52 | // A Write with an empty value deletes the value associated with the given key. 53 | type Write struct { 54 | Key string 55 | Value string 56 | } 57 | -------------------------------------------------------------------------------- /sumdb/storage/test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package storage 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | "io" 11 | "testing" 12 | ) 13 | 14 | // TestStorage tests a Storage implementation. 15 | func TestStorage(t *testing.T, ctx context.Context, storage Storage) { 16 | s := storage 17 | 18 | // Insert records. 19 | err := s.ReadWrite(ctx, func(ctx context.Context, tx Transaction) error { 20 | for i := 0; i < 10; i++ { 21 | err := tx.BufferWrites([]Write{ 22 | {Key: fmt.Sprint(i), Value: fmt.Sprint(-i)}, 23 | {Key: fmt.Sprint(1000 + i), Value: fmt.Sprint(-1000 - i)}, 24 | }) 25 | if err != nil { 26 | t.Fatal(err) 27 | } 28 | } 29 | return nil 30 | }) 31 | if err != nil { 32 | t.Fatal(err) 33 | } 34 | 35 | // Read the records back. 36 | testRead := func() { 37 | err := s.ReadOnly(ctx, func(ctx context.Context, tx Transaction) error { 38 | for i := int64(0); i < 1010; i++ { 39 | if i == 10 { 40 | i = 1000 41 | } 42 | val, err := tx.ReadValue(ctx, fmt.Sprint(i)) 43 | if err != nil { 44 | t.Fatalf("reading %v: %v", i, err) 45 | } 46 | if want := fmt.Sprint(-i); val != want { 47 | t.Fatalf("ReadValue %v = %q, want %v", i, val, want) 48 | } 49 | } 50 | return nil 51 | }) 52 | if err != nil { 53 | t.Fatal(err) 54 | } 55 | } 56 | testRead() 57 | 58 | // Buffered writes in failed transaction should not be applied. 59 | err = s.ReadWrite(ctx, func(ctx context.Context, tx Transaction) error { 60 | tx.BufferWrites([]Write{ 61 | {Key: fmt.Sprint(0), Value: ""}, // delete 62 | {Key: fmt.Sprint(1), Value: "overwrite"}, // overwrite 63 | }) 64 | if err != nil { 65 | t.Fatal(err) 66 | } 67 | return io.ErrUnexpectedEOF 68 | }) 69 | if err != io.ErrUnexpectedEOF { 70 | t.Fatalf("ReadWrite returned %v, want ErrUnexpectedEOF", err) 71 | } 72 | 73 | // All same values should still be there. 74 | testRead() 75 | } 76 | -------------------------------------------------------------------------------- /sumdb/test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package sumdb 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | "sync" 11 | 12 | "golang.org/x/mod/module" 13 | "golang.org/x/mod/sumdb/note" 14 | "golang.org/x/mod/sumdb/tlog" 15 | ) 16 | 17 | // NewTestServer constructs a new [TestServer] 18 | // that will sign its tree with the given signer key 19 | // (see [golang.org/x/mod/sumdb/note]) 20 | // and fetch new records as needed by calling gosum. 21 | func NewTestServer(signer string, gosum func(path, vers string) ([]byte, error)) *TestServer { 22 | return &TestServer{signer: signer, gosum: gosum} 23 | } 24 | 25 | // A TestServer is an in-memory implementation of [ServerOps] for testing. 26 | type TestServer struct { 27 | signer string 28 | gosum func(path, vers string) ([]byte, error) 29 | 30 | mu sync.Mutex 31 | hashes testHashes 32 | records [][]byte 33 | lookup map[string]int64 34 | } 35 | 36 | // testHashes implements tlog.HashReader, reading from a slice. 37 | type testHashes []tlog.Hash 38 | 39 | func (h testHashes) ReadHashes(indexes []int64) ([]tlog.Hash, error) { 40 | var list []tlog.Hash 41 | for _, id := range indexes { 42 | list = append(list, h[id]) 43 | } 44 | return list, nil 45 | } 46 | 47 | func (s *TestServer) Signed(ctx context.Context) ([]byte, error) { 48 | s.mu.Lock() 49 | defer s.mu.Unlock() 50 | 51 | size := int64(len(s.records)) 52 | h, err := tlog.TreeHash(size, s.hashes) 53 | if err != nil { 54 | return nil, err 55 | } 56 | text := tlog.FormatTree(tlog.Tree{N: size, Hash: h}) 57 | signer, err := note.NewSigner(s.signer) 58 | if err != nil { 59 | return nil, err 60 | } 61 | return note.Sign(¬e.Note{Text: string(text)}, signer) 62 | } 63 | 64 | func (s *TestServer) ReadRecords(ctx context.Context, id, n int64) ([][]byte, error) { 65 | s.mu.Lock() 66 | defer s.mu.Unlock() 67 | 68 | var list [][]byte 69 | for i := int64(0); i < n; i++ { 70 | if id+i >= int64(len(s.records)) { 71 | return nil, fmt.Errorf("missing records") 72 | } 73 | list = append(list, s.records[id+i]) 74 | } 75 | return list, nil 76 | } 77 | 78 | func (s *TestServer) Lookup(ctx context.Context, m module.Version) (int64, error) { 79 | key := m.String() 80 | s.mu.Lock() 81 | id, ok := s.lookup[key] 82 | s.mu.Unlock() 83 | if ok { 84 | return id, nil 85 | } 86 | 87 | // Look up module and compute go.sum lines. 88 | data, err := s.gosum(m.Path, m.Version) 89 | if err != nil { 90 | return 0, err 91 | } 92 | 93 | s.mu.Lock() 94 | defer s.mu.Unlock() 95 | 96 | // We ran the fetch without the lock. 97 | // If another fetch happened and committed, use it instead. 98 | id, ok = s.lookup[key] 99 | if ok { 100 | return id, nil 101 | } 102 | 103 | // Add record. 104 | id = int64(len(s.records)) 105 | s.records = append(s.records, data) 106 | if s.lookup == nil { 107 | s.lookup = make(map[string]int64) 108 | } 109 | s.lookup[key] = id 110 | hashes, err := tlog.StoredHashesForRecordHash(id, tlog.RecordHash(data), s.hashes) 111 | if err != nil { 112 | panic(err) 113 | } 114 | s.hashes = append(s.hashes, hashes...) 115 | 116 | return id, nil 117 | } 118 | 119 | func (s *TestServer) ReadTileData(ctx context.Context, t tlog.Tile) ([]byte, error) { 120 | s.mu.Lock() 121 | defer s.mu.Unlock() 122 | 123 | return tlog.ReadTileData(t, s.hashes) 124 | } 125 | -------------------------------------------------------------------------------- /sumdb/tlog/ct_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package tlog 6 | 7 | import ( 8 | "encoding/json" 9 | "fmt" 10 | "io" 11 | "net/http" 12 | "net/url" 13 | "os" 14 | "testing" 15 | ) 16 | 17 | func TestCertificateTransparency(t *testing.T) { 18 | // Test that we can verify actual Certificate Transparency proofs. 19 | // (The other tests check that we can verify our own proofs; 20 | // this is a test that the two are compatible.) 21 | 22 | if testing.Short() { 23 | t.Skip("skipping in -short mode") 24 | } 25 | 26 | var root ctTree 27 | httpGET(t, "http://ct.googleapis.com/logs/argon2020/ct/v1/get-sth", &root) 28 | 29 | var leaf ctEntries 30 | httpGET(t, "http://ct.googleapis.com/logs/argon2020/ct/v1/get-entries?start=10000&end=10000", &leaf) 31 | hash := RecordHash(leaf.Entries[0].Data) 32 | 33 | var rp ctRecordProof 34 | httpGET(t, "http://ct.googleapis.com/logs/argon2020/ct/v1/get-proof-by-hash?tree_size="+fmt.Sprint(root.Size)+"&hash="+url.QueryEscape(hash.String()), &rp) 35 | 36 | err := CheckRecord(rp.Proof, root.Size, root.Hash, 10000, hash) 37 | if err != nil { 38 | t.Fatal(err) 39 | } 40 | 41 | var tp ctTreeProof 42 | httpGET(t, "http://ct.googleapis.com/logs/argon2020/ct/v1/get-sth-consistency?first=3654490&second="+fmt.Sprint(root.Size), &tp) 43 | 44 | oh, _ := ParseHash("AuIZ5V6sDUj1vn3Y1K85oOaQ7y+FJJKtyRTl1edIKBQ=") 45 | err = CheckTree(tp.Proof, root.Size, root.Hash, 3654490, oh) 46 | if err != nil { 47 | t.Fatal(err) 48 | } 49 | } 50 | 51 | type ctTree struct { 52 | Size int64 `json:"tree_size"` 53 | Hash Hash `json:"sha256_root_hash"` 54 | } 55 | 56 | type ctEntries struct { 57 | Entries []*ctEntry 58 | } 59 | 60 | type ctEntry struct { 61 | Data []byte `json:"leaf_input"` 62 | } 63 | 64 | type ctRecordProof struct { 65 | Index int64 `json:"leaf_index"` 66 | Proof RecordProof `json:"audit_path"` 67 | } 68 | 69 | type ctTreeProof struct { 70 | Proof TreeProof `json:"consistency"` 71 | } 72 | 73 | func httpGET(t *testing.T, url string, targ interface{}) { 74 | if testing.Verbose() { 75 | println() 76 | println(url) 77 | } 78 | resp, err := http.Get(url) 79 | if err != nil { 80 | t.Fatal(err) 81 | } 82 | defer resp.Body.Close() 83 | data, err := io.ReadAll(resp.Body) 84 | if err != nil { 85 | t.Fatal(err) 86 | } 87 | if testing.Verbose() { 88 | os.Stdout.Write(data) 89 | } 90 | err = json.Unmarshal(data, targ) 91 | if err != nil { 92 | println(url) 93 | os.Stdout.Write(data) 94 | t.Fatal(err) 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /sumdb/tlog/note.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package tlog 6 | 7 | import ( 8 | "bytes" 9 | "encoding/base64" 10 | "errors" 11 | "fmt" 12 | "strconv" 13 | "strings" 14 | "unicode/utf8" 15 | ) 16 | 17 | // A Tree is a tree description, to be signed by a go.sum database server. 18 | type Tree struct { 19 | N int64 20 | Hash Hash 21 | } 22 | 23 | // FormatTree formats a tree description for inclusion in a note. 24 | // 25 | // The encoded form is three lines, each ending in a newline (U+000A): 26 | // 27 | // go.sum database tree 28 | // N 29 | // Hash 30 | // 31 | // where N is in decimal and Hash is in base64. 32 | // 33 | // A future backwards-compatible encoding may add additional lines, 34 | // which the parser can ignore. 35 | // A future backwards-incompatible encoding would use a different 36 | // first line (for example, "go.sum database tree v2"). 37 | func FormatTree(tree Tree) []byte { 38 | return []byte(fmt.Sprintf("go.sum database tree\n%d\n%s\n", tree.N, tree.Hash)) 39 | } 40 | 41 | var errMalformedTree = errors.New("malformed tree note") 42 | var treePrefix = []byte("go.sum database tree\n") 43 | 44 | // ParseTree parses a formatted tree root description. 45 | func ParseTree(text []byte) (tree Tree, err error) { 46 | // The message looks like: 47 | // 48 | // go.sum database tree 49 | // 2 50 | // nND/nri/U0xuHUrYSy0HtMeal2vzD9V4k/BO79C+QeI= 51 | // 52 | // For forwards compatibility, extra text lines after the encoding are ignored. 53 | if !bytes.HasPrefix(text, treePrefix) || bytes.Count(text, []byte("\n")) < 3 || len(text) > 1e6 { 54 | return Tree{}, errMalformedTree 55 | } 56 | 57 | lines := strings.SplitN(string(text), "\n", 4) 58 | n, err := strconv.ParseInt(lines[1], 10, 64) 59 | if err != nil || n < 0 || lines[1] != strconv.FormatInt(n, 10) { 60 | return Tree{}, errMalformedTree 61 | } 62 | 63 | h, err := base64.StdEncoding.DecodeString(lines[2]) 64 | if err != nil || len(h) != HashSize { 65 | return Tree{}, errMalformedTree 66 | } 67 | 68 | var hash Hash 69 | copy(hash[:], h) 70 | return Tree{n, hash}, nil 71 | } 72 | 73 | var errMalformedRecord = errors.New("malformed record data") 74 | 75 | // FormatRecord formats a record for serving to a client 76 | // in a lookup response. 77 | // 78 | // The encoded form is the record ID as a single number, 79 | // then the text of the record, and then a terminating blank line. 80 | // Record text must be valid UTF-8 and must not contain any ASCII control 81 | // characters (those below U+0020) other than newline (U+000A). 82 | // It must end in a terminating newline and not contain any blank lines. 83 | // 84 | // Responses to data tiles consist of concatenated formatted records from each of 85 | // which the first line, with the record ID, is removed. 86 | func FormatRecord(id int64, text []byte) (msg []byte, err error) { 87 | if !isValidRecordText(text) { 88 | return nil, errMalformedRecord 89 | } 90 | msg = []byte(fmt.Sprintf("%d\n", id)) 91 | msg = append(msg, text...) 92 | msg = append(msg, '\n') 93 | return msg, nil 94 | } 95 | 96 | // isValidRecordText reports whether text is syntactically valid record text. 97 | func isValidRecordText(text []byte) bool { 98 | var last rune 99 | for i := 0; i < len(text); { 100 | r, size := utf8.DecodeRune(text[i:]) 101 | if r < 0x20 && r != '\n' || r == utf8.RuneError && size == 1 || last == '\n' && r == '\n' { 102 | return false 103 | } 104 | i += size 105 | last = r 106 | } 107 | if last != '\n' { 108 | return false 109 | } 110 | return true 111 | } 112 | 113 | // ParseRecord parses a record description at the start of text, 114 | // stopping immediately after the terminating blank line. 115 | // It returns the record id, the record text, and the remainder of text. 116 | func ParseRecord(msg []byte) (id int64, text, rest []byte, err error) { 117 | // Leading record id. 118 | i := bytes.IndexByte(msg, '\n') 119 | if i < 0 { 120 | return 0, nil, nil, errMalformedRecord 121 | } 122 | id, err = strconv.ParseInt(string(msg[:i]), 10, 64) 123 | if err != nil { 124 | return 0, nil, nil, errMalformedRecord 125 | } 126 | msg = msg[i+1:] 127 | 128 | // Record text. 129 | i = bytes.Index(msg, []byte("\n\n")) 130 | if i < 0 { 131 | return 0, nil, nil, errMalformedRecord 132 | } 133 | text, rest = msg[:i+1], msg[i+2:] 134 | if !isValidRecordText(text) { 135 | return 0, nil, nil, errMalformedRecord 136 | } 137 | return id, text, rest, nil 138 | } 139 | -------------------------------------------------------------------------------- /sumdb/tlog/note_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package tlog 6 | 7 | import ( 8 | "strings" 9 | "testing" 10 | ) 11 | 12 | func TestFormatTree(t *testing.T) { 13 | n := int64(123456789012) 14 | h := RecordHash([]byte("hello world")) 15 | golden := "go.sum database tree\n123456789012\nTszzRgjTG6xce+z2AG31kAXYKBgQVtCSCE40HmuwBb0=\n" 16 | b := FormatTree(Tree{n, h}) 17 | if string(b) != golden { 18 | t.Errorf("FormatTree(...) = %q, want %q", b, golden) 19 | } 20 | } 21 | 22 | func TestParseTree(t *testing.T) { 23 | in := "go.sum database tree\n123456789012\nTszzRgjTG6xce+z2AG31kAXYKBgQVtCSCE40HmuwBb0=\n" 24 | goldH := RecordHash([]byte("hello world")) 25 | goldN := int64(123456789012) 26 | tree, err := ParseTree([]byte(in)) 27 | if tree.N != goldN || tree.Hash != goldH || err != nil { 28 | t.Fatalf("ParseTree(...) = Tree{%d, %v}, %v, want Tree{%d, %v}, nil", tree.N, tree.Hash, err, goldN, goldH) 29 | } 30 | 31 | // Check invalid trees. 32 | var badTrees = []string{ 33 | "not-" + in, 34 | "go.sum database tree\n0xabcdef\nTszzRgjTG6xce+z2AG31kAXYKBgQVtCSCE40HmuwBb0=\n", 35 | "go.sum database tree\n123456789012\nTszzRgjTG6xce+z2AG31kAXYKBgQVtCSCE40HmuwBTOOBIG=\n", 36 | } 37 | for _, bad := range badTrees { 38 | _, err := ParseTree([]byte(bad)) 39 | if err == nil { 40 | t.Fatalf("ParseTree(%q) succeeded, want failure", in) 41 | } 42 | } 43 | 44 | // Check junk on end is ignored. 45 | var goodTrees = []string{ 46 | in + "JOE", 47 | in + "JOE\n", 48 | in + strings.Repeat("JOE\n", 1000), 49 | } 50 | for _, good := range goodTrees { 51 | _, err := ParseTree([]byte(good)) 52 | if tree.N != goldN || tree.Hash != goldH || err != nil { 53 | t.Fatalf("ParseTree(...+%q) = Tree{%d, %v}, %v, want Tree{%d, %v}, nil", good[len(in):], tree.N, tree.Hash, err, goldN, goldH) 54 | } 55 | } 56 | } 57 | 58 | func TestFormatRecord(t *testing.T) { 59 | id := int64(123456789012) 60 | text := "hello, world\n" 61 | golden := "123456789012\nhello, world\n\n" 62 | msg, err := FormatRecord(id, []byte(text)) 63 | if err != nil { 64 | t.Fatalf("FormatRecord: %v", err) 65 | } 66 | if string(msg) != golden { 67 | t.Fatalf("FormatRecord(...) = %q, want %q", msg, golden) 68 | } 69 | 70 | var badTexts = []string{ 71 | "", 72 | "hello\nworld", 73 | "hello\n\nworld\n", 74 | "hello\x01world\n", 75 | } 76 | for _, bad := range badTexts { 77 | msg, err := FormatRecord(id, []byte(bad)) 78 | if err == nil { 79 | t.Errorf("FormatRecord(id, %q) = %q, want error", bad, msg) 80 | } 81 | } 82 | } 83 | 84 | func TestParseRecord(t *testing.T) { 85 | in := "123456789012\nhello, world\n\njunk on end\x01\xff" 86 | goldID := int64(123456789012) 87 | goldText := "hello, world\n" 88 | goldRest := "junk on end\x01\xff" 89 | id, text, rest, err := ParseRecord([]byte(in)) 90 | if id != goldID || string(text) != goldText || string(rest) != goldRest || err != nil { 91 | t.Fatalf("ParseRecord(%q) = %d, %q, %q, %v, want %d, %q, %q, nil", in, id, text, rest, err, goldID, goldText, goldRest) 92 | } 93 | 94 | in = "123456789012\nhello, world\n\n" 95 | id, text, rest, err = ParseRecord([]byte(in)) 96 | if id != goldID || string(text) != goldText || len(rest) != 0 || err != nil { 97 | t.Fatalf("ParseRecord(%q) = %d, %q, %q, %v, want %d, %q, %q, nil", in, id, text, rest, err, goldID, goldText, "") 98 | } 99 | if rest == nil { 100 | t.Fatalf("ParseRecord(%q): rest = []byte(nil), want []byte{}", in) 101 | } 102 | 103 | // Check invalid records. 104 | var badRecords = []string{ 105 | "not-" + in, 106 | "123\nhello\x01world\n\n", 107 | "123\nhello\xffworld\n\n", 108 | "123\nhello world\n", 109 | "0x123\nhello world\n\n", 110 | } 111 | for _, bad := range badRecords { 112 | id, text, rest, err := ParseRecord([]byte(bad)) 113 | if err == nil { 114 | t.Fatalf("ParseRecord(%q) = %d, %q, %q, nil, want error", in, id, text, rest) 115 | } 116 | } 117 | } 118 | 119 | // FuzzParseTree tests that ParseTree never crashes 120 | func FuzzParseTree(f *testing.F) { 121 | f.Add([]byte("go.sum database tree\n123456789012\nTszzRgjTG6xce+z2AG31kAXYKBgQVtCSCE40HmuwBb0=\n")) 122 | f.Fuzz(func(t *testing.T, text []byte) { 123 | ParseTree(text) 124 | }) 125 | } 126 | 127 | // FuzzParseRecord tests that ParseRecord never crashes 128 | func FuzzParseRecord(f *testing.F) { 129 | f.Add([]byte("12345\nhello\n\n")) 130 | f.Fuzz(func(t *testing.T, msg []byte) { 131 | ParseRecord(msg) 132 | }) 133 | } 134 | -------------------------------------------------------------------------------- /sumdb/tlog/tile.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package tlog 6 | 7 | import ( 8 | "fmt" 9 | "strconv" 10 | "strings" 11 | ) 12 | 13 | // A Tile is a description of a transparency log tile. 14 | // A tile of height H at level L offset N lists W consecutive hashes 15 | // at level H*L of the tree starting at offset N*(2**H). 16 | // A complete tile lists 2**H hashes; a partial tile lists fewer. 17 | // Note that a tile represents the entire subtree of height H 18 | // with those hashes as the leaves. The levels above H*L 19 | // can be reconstructed by hashing the leaves. 20 | // 21 | // Each Tile can be encoded as a “tile coordinate path” 22 | // of the form tile/H/L/NNN[.p/W]. 23 | // The .p/W suffix is present only for partial tiles, meaning W < 2**H. 24 | // The NNN element is an encoding of N into 3-digit path elements. 25 | // All but the last path element begins with an "x". 26 | // For example, 27 | // Tile{H: 3, L: 4, N: 1234067, W: 1}'s path 28 | // is tile/3/4/x001/x234/067.p/1, and 29 | // Tile{H: 3, L: 4, N: 1234067, W: 8}'s path 30 | // is tile/3/4/x001/x234/067. 31 | // See the [Tile.Path] method and the [ParseTilePath] function. 32 | // 33 | // The special level L=-1 holds raw record data instead of hashes. 34 | // In this case, the level encodes into a tile path as the path element 35 | // "data" instead of "-1". 36 | // 37 | // See also https://golang.org/design/25530-sumdb#checksum-database 38 | // and https://research.swtch.com/tlog#tiling_a_log. 39 | type Tile struct { 40 | H int // height of tile (1 ≤ H ≤ 30) 41 | L int // level in tiling (-1 ≤ L ≤ 63) 42 | N int64 // number within level (0 ≤ N, unbounded) 43 | W int // width of tile (1 ≤ W ≤ 2**H; 2**H is complete tile) 44 | } 45 | 46 | // TileForIndex returns the tile of fixed height h ≥ 1 47 | // and least width storing the given hash storage index. 48 | // 49 | // If h ≤ 0, [TileForIndex] panics. 50 | func TileForIndex(h int, index int64) Tile { 51 | if h <= 0 { 52 | panic(fmt.Sprintf("TileForIndex: invalid height %d", h)) 53 | } 54 | t, _, _ := tileForIndex(h, index) 55 | return t 56 | } 57 | 58 | // tileForIndex returns the tile of height h ≥ 1 59 | // storing the given hash index, which can be 60 | // reconstructed using tileHash(data[start:end]). 61 | func tileForIndex(h int, index int64) (t Tile, start, end int) { 62 | level, n := SplitStoredHashIndex(index) 63 | t.H = h 64 | t.L = level / h 65 | level -= t.L * h // now level within tile 66 | t.N = n << uint(level) >> uint(t.H) 67 | n -= t.N << uint(t.H) >> uint(level) // now n within tile at level 68 | t.W = int((n + 1) << uint(level)) 69 | return t, int(n< 30 || t.L < 0 || t.L >= 64 || t.W < 1 || t.W > 1<>(H*level) > 0; level++ { 116 | oldN := oldTreeSize >> (H * level) 117 | newN := newTreeSize >> (H * level) 118 | if oldN == newN { 119 | continue 120 | } 121 | for n := oldN >> H; n < newN>>H; n++ { 122 | tiles = append(tiles, Tile{H: h, L: int(level), N: n, W: 1 << H}) 123 | } 124 | n := newN >> H 125 | if w := int(newN - n< 0 { 126 | tiles = append(tiles, Tile{H: h, L: int(level), N: n, W: w}) 127 | } 128 | } 129 | return tiles 130 | } 131 | 132 | // ReadTileData reads the hashes for tile t from r 133 | // and returns the corresponding tile data. 134 | func ReadTileData(t Tile, r HashReader) ([]byte, error) { 135 | size := t.W 136 | if size == 0 { 137 | size = 1 << uint(t.H) 138 | } 139 | start := t.N << uint(t.H) 140 | indexes := make([]int64, size) 141 | for i := 0; i < size; i++ { 142 | indexes[i] = StoredHashIndex(t.H*t.L, start+int64(i)) 143 | } 144 | 145 | hashes, err := r.ReadHashes(indexes) 146 | if err != nil { 147 | return nil, err 148 | } 149 | if len(hashes) != len(indexes) { 150 | return nil, fmt.Errorf("tlog: ReadHashes(%d indexes) = %d hashes", len(indexes), len(hashes)) 151 | } 152 | 153 | tile := make([]byte, size*HashSize) 154 | for i := 0; i < size; i++ { 155 | copy(tile[i*HashSize:], hashes[i][:]) 156 | } 157 | return tile, nil 158 | } 159 | 160 | // To limit the size of any particular directory listing, 161 | // we encode the (possibly very large) number N 162 | // by encoding three digits at a time. 163 | // For example, 123456789 encodes as x123/x456/789. 164 | // Each directory has at most 1000 each xNNN, NNN, and NNN.p children, 165 | // so there are at most 3000 entries in any one directory. 166 | const pathBase = 1000 167 | 168 | // Path returns a tile coordinate path describing t. 169 | func (t Tile) Path() string { 170 | n := t.N 171 | nStr := fmt.Sprintf("%03d", n%pathBase) 172 | for n >= pathBase { 173 | n /= pathBase 174 | nStr = fmt.Sprintf("x%03d/%s", n%pathBase, nStr) 175 | } 176 | pStr := "" 177 | if t.W != 1< 30 { 203 | return Tile{}, &badPathError{path} 204 | } 205 | w := 1 << uint(h) 206 | if dotP := f[len(f)-2]; strings.HasSuffix(dotP, ".p") { 207 | ww, err := strconv.Atoi(f[len(f)-1]) 208 | if err != nil || ww <= 0 || ww >= w { 209 | return Tile{}, &badPathError{path} 210 | } 211 | w = ww 212 | f[len(f)-2] = dotP[:len(dotP)-len(".p")] 213 | f = f[:len(f)-1] 214 | } 215 | f = f[3:] 216 | n := int64(0) 217 | for _, s := range f { 218 | nn, err := strconv.Atoi(strings.TrimPrefix(s, "x")) 219 | if err != nil || nn < 0 || nn >= pathBase { 220 | return Tile{}, &badPathError{path} 221 | } 222 | n = n*pathBase + int64(nn) 223 | } 224 | if isData { 225 | l = -1 226 | } 227 | t := Tile{H: h, L: l, N: n, W: w} 228 | if path != t.Path() { 229 | return Tile{}, &badPathError{path} 230 | } 231 | return t, nil 232 | } 233 | 234 | type badPathError struct { 235 | path string 236 | } 237 | 238 | func (e *badPathError) Error() string { 239 | return fmt.Sprintf("malformed tile path %q", e.path) 240 | } 241 | 242 | // A TileReader reads tiles from a go.sum database log. 243 | type TileReader interface { 244 | // Height returns the height of the available tiles. 245 | Height() int 246 | 247 | // ReadTiles returns the data for each requested tile. 248 | // If ReadTiles returns err == nil, it must also return 249 | // a data record for each tile (len(data) == len(tiles)) 250 | // and each data record must be the correct length 251 | // (len(data[i]) == tiles[i].W*HashSize). 252 | // 253 | // An implementation of ReadTiles typically reads 254 | // them from an on-disk cache or else from a remote 255 | // tile server. Tile data downloaded from a server should 256 | // be considered suspect and not saved into a persistent 257 | // on-disk cache before returning from ReadTiles. 258 | // When the client confirms the validity of the tile data, 259 | // it will call SaveTiles to signal that they can be safely 260 | // written to persistent storage. 261 | // See also https://research.swtch.com/tlog#authenticating_tiles. 262 | ReadTiles(tiles []Tile) (data [][]byte, err error) 263 | 264 | // SaveTiles informs the TileReader that the tile data 265 | // returned by ReadTiles has been confirmed as valid 266 | // and can be saved in persistent storage (on disk). 267 | SaveTiles(tiles []Tile, data [][]byte) 268 | } 269 | 270 | // TileHashReader returns a HashReader that satisfies requests 271 | // by loading tiles of the given tree. 272 | // 273 | // The returned [HashReader] checks that loaded tiles are 274 | // valid for the given tree. Therefore, any hashes returned 275 | // by the HashReader are already proven to be in the tree. 276 | func TileHashReader(tree Tree, tr TileReader) HashReader { 277 | return &tileHashReader{tree: tree, tr: tr} 278 | } 279 | 280 | type tileHashReader struct { 281 | tree Tree 282 | tr TileReader 283 | } 284 | 285 | // tileParent returns t's k'th tile parent in the tiles for a tree of size n. 286 | // If there is no such parent, tileParent returns Tile{}. 287 | func tileParent(t Tile, k int, n int64) Tile { 288 | t.L += k 289 | t.N >>= uint(k * t.H) 290 | t.W = 1 << uint(t.H) 291 | if max := n >> uint(t.L*t.H); t.N<= max { 292 | if t.N<= max { 293 | return Tile{} 294 | } 295 | t.W = int(max - t.N<= StoredHashIndex(0, r.tree.N) { 329 | return nil, fmt.Errorf("indexes not in tree") 330 | } 331 | 332 | tile, _, _ := tileForIndex(h, x) 333 | 334 | // Walk up parent tiles until we find one we've requested. 335 | // That one will be authenticated. 336 | k := 0 337 | for ; ; k++ { 338 | p := tileParent(tile, k, r.tree.N) 339 | if j, ok := tileOrder[p]; ok { 340 | if k == 0 { 341 | indexTileOrder[i] = j 342 | } 343 | break 344 | } 345 | } 346 | 347 | // Walk back down recording child tiles after parents. 348 | // This loop ends by revisiting the tile for this index 349 | // (tileParent(tile, 0, r.tree.N)) unless k == 0, in which 350 | // case the previous loop did it. 351 | for k--; k >= 0; k-- { 352 | p := tileParent(tile, k, r.tree.N) 353 | if p.W != 1<= 0; i-- { 388 | h, err := HashFromTile(tiles[stxTileOrder[i]], data[stxTileOrder[i]], stx[i]) 389 | if err != nil { 390 | return nil, err 391 | } 392 | th = NodeHash(h, th) 393 | } 394 | if th != r.tree.Hash { 395 | // The tiles do not support the tree hash. 396 | // We know at least one is wrong, but not which one. 397 | return nil, fmt.Errorf("downloaded inconsistent tile") 398 | } 399 | 400 | // Authenticate full tiles against their parents. 401 | for i := len(stx); i < len(tiles); i++ { 402 | tile := tiles[i] 403 | p := tileParent(tile, 1, r.tree.N) 404 | j, ok := tileOrder[p] 405 | if !ok { 406 | return nil, fmt.Errorf("bad math in tileHashReader %d %v: lost parent of %v", r.tree.N, indexes, tile) 407 | } 408 | h, err := HashFromTile(p, data[j], StoredHashIndex(p.L*p.H, tile.N)) 409 | if err != nil { 410 | return nil, fmt.Errorf("bad math in tileHashReader %d %v: lost hash of %v: %v", r.tree.N, indexes, tile, err) 411 | } 412 | if h != tileHash(data[i]) { 413 | return nil, fmt.Errorf("downloaded inconsistent tile") 414 | } 415 | } 416 | 417 | // Now we have all the tiles needed for the requested hashes, 418 | // and we've authenticated the full tile set against the trusted tree hash. 419 | r.tr.SaveTiles(tiles, data) 420 | 421 | // Pull out the requested hashes. 422 | hashes := make([]Hash, len(indexes)) 423 | for i, x := range indexes { 424 | j := indexTileOrder[i] 425 | h, err := HashFromTile(tiles[j], data[j], x) 426 | if err != nil { 427 | return nil, fmt.Errorf("bad math in tileHashReader %d %v: lost hash %v: %v", r.tree.N, indexes, x, err) 428 | } 429 | hashes[i] = h 430 | } 431 | 432 | return hashes, nil 433 | } 434 | -------------------------------------------------------------------------------- /sumdb/tlog/tile_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package tlog 6 | 7 | import ( 8 | "fmt" 9 | "testing" 10 | ) 11 | 12 | // FuzzParseTilePath tests that ParseTilePath never crashes 13 | func FuzzParseTilePath(f *testing.F) { 14 | f.Add("tile/4/0/001") 15 | f.Add("tile/4/0/001.p/5") 16 | f.Add("tile/3/5/x123/x456/078") 17 | f.Add("tile/3/5/x123/x456/078.p/2") 18 | f.Add("tile/1/0/x003/x057/500") 19 | f.Add("tile/3/5/123/456/078") 20 | f.Add("tile/3/-1/123/456/078") 21 | f.Add("tile/1/data/x003/x057/500") 22 | f.Fuzz(func(t *testing.T, path string) { 23 | ParseTilePath(path) 24 | }) 25 | } 26 | 27 | func TestNewTilesForSize(t *testing.T) { 28 | for _, tt := range []struct { 29 | old, new int64 30 | want int 31 | }{ 32 | {1, 1, 0}, 33 | {100, 101, 1}, 34 | {1023, 1025, 3}, 35 | {1024, 1030, 1}, 36 | {1030, 2000, 1}, 37 | {1030, 10000, 10}, 38 | {49516517, 49516586, 3}, 39 | } { 40 | t.Run(fmt.Sprintf("%d-%d", tt.old, tt.new), func(t *testing.T) { 41 | tiles := NewTiles(10, tt.old, tt.new) 42 | if got := len(tiles); got != tt.want { 43 | t.Errorf("got %d, want %d", got, tt.want) 44 | for _, tile := range tiles { 45 | t.Logf("%+v", tile) 46 | } 47 | } 48 | }) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /sumdb/tlog/tlog_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package tlog 6 | 7 | import ( 8 | "bytes" 9 | "crypto/sha256" 10 | "fmt" 11 | "testing" 12 | ) 13 | 14 | type testHashStorage []Hash 15 | 16 | func (t testHashStorage) ReadHash(level int, n int64) (Hash, error) { 17 | return t[StoredHashIndex(level, n)], nil 18 | } 19 | 20 | func (t testHashStorage) ReadHashes(index []int64) ([]Hash, error) { 21 | // It's not required by HashReader that indexes be in increasing order, 22 | // but check that the functions we are testing only ever ask for 23 | // indexes in increasing order. 24 | for i := 1; i < len(index); i++ { 25 | if index[i-1] >= index[i] { 26 | panic("indexes out of order") 27 | } 28 | } 29 | 30 | out := make([]Hash, len(index)) 31 | for i, x := range index { 32 | out[i] = t[x] 33 | } 34 | return out, nil 35 | } 36 | 37 | type testTilesStorage struct { 38 | unsaved int 39 | m map[Tile][]byte 40 | } 41 | 42 | func (t testTilesStorage) Height() int { 43 | return 2 44 | } 45 | 46 | func (t *testTilesStorage) SaveTiles(tiles []Tile, data [][]byte) { 47 | t.unsaved -= len(tiles) 48 | } 49 | 50 | func (t *testTilesStorage) ReadTiles(tiles []Tile) ([][]byte, error) { 51 | out := make([][]byte, len(tiles)) 52 | for i, tile := range tiles { 53 | out[i] = t.m[tile] 54 | } 55 | t.unsaved += len(tiles) 56 | return out, nil 57 | } 58 | 59 | func TestTree(t *testing.T) { 60 | var trees []Hash 61 | var leafhashes []Hash 62 | var storage testHashStorage 63 | tiles := make(map[Tile][]byte) 64 | const testH = 2 65 | for i := int64(0); i < 100; i++ { 66 | data := []byte(fmt.Sprintf("leaf %d", i)) 67 | hashes, err := StoredHashes(i, data, storage) 68 | if err != nil { 69 | t.Fatal(err) 70 | } 71 | leafhashes = append(leafhashes, RecordHash(data)) 72 | oldStorage := len(storage) 73 | storage = append(storage, hashes...) 74 | if count := StoredHashCount(i + 1); count != int64(len(storage)) { 75 | t.Errorf("StoredHashCount(%d) = %d, have %d StoredHashes", i+1, count, len(storage)) 76 | } 77 | th, err := TreeHash(i+1, storage) 78 | if err != nil { 79 | t.Fatal(err) 80 | } 81 | 82 | for _, tile := range NewTiles(testH, i, i+1) { 83 | data, err := ReadTileData(tile, storage) 84 | if err != nil { 85 | t.Fatal(err) 86 | } 87 | old := Tile{H: tile.H, L: tile.L, N: tile.N, W: tile.W - 1} 88 | oldData := tiles[old] 89 | if len(oldData) != len(data)-HashSize || !bytes.Equal(oldData, data[:len(oldData)]) { 90 | t.Fatalf("tile %v not extending earlier tile %v", tile.Path(), old.Path()) 91 | } 92 | tiles[tile] = data 93 | } 94 | for _, tile := range NewTiles(testH, 0, i+1) { 95 | data, err := ReadTileData(tile, storage) 96 | if err != nil { 97 | t.Fatal(err) 98 | } 99 | if !bytes.Equal(tiles[tile], data) { 100 | t.Fatalf("mismatch at %+v", tile) 101 | } 102 | } 103 | for _, tile := range NewTiles(testH, i/2, i+1) { 104 | data, err := ReadTileData(tile, storage) 105 | if err != nil { 106 | t.Fatal(err) 107 | } 108 | if !bytes.Equal(tiles[tile], data) { 109 | t.Fatalf("mismatch at %+v", tile) 110 | } 111 | } 112 | 113 | // Check that all the new hashes are readable from their tiles. 114 | for j := oldStorage; j < len(storage); j++ { 115 | tile := TileForIndex(testH, int64(j)) 116 | data, ok := tiles[tile] 117 | if !ok { 118 | t.Log(NewTiles(testH, 0, i+1)) 119 | t.Fatalf("TileForIndex(%d, %d) = %v, not yet stored (i=%d, stored %d)", testH, j, tile.Path(), i, len(storage)) 120 | continue 121 | } 122 | h, err := HashFromTile(tile, data, int64(j)) 123 | if err != nil { 124 | t.Fatal(err) 125 | } 126 | if h != storage[j] { 127 | t.Errorf("HashFromTile(%v, %d) = %v, want %v", tile.Path(), int64(j), h, storage[j]) 128 | } 129 | } 130 | 131 | trees = append(trees, th) 132 | 133 | // Check that leaf proofs work, for all trees and leaves so far. 134 | for j := int64(0); j <= i; j++ { 135 | p, err := ProveRecord(i+1, j, storage) 136 | if err != nil { 137 | t.Fatalf("ProveRecord(%d, %d): %v", i+1, j, err) 138 | } 139 | if err := CheckRecord(p, i+1, th, j, leafhashes[j]); err != nil { 140 | t.Fatalf("CheckRecord(%d, %d): %v", i+1, j, err) 141 | } 142 | for k := range p { 143 | p[k][0] ^= 1 144 | if err := CheckRecord(p, i+1, th, j, leafhashes[j]); err == nil { 145 | t.Fatalf("CheckRecord(%d, %d) succeeded with corrupt proof hash #%d!", i+1, j, k) 146 | } 147 | p[k][0] ^= 1 148 | } 149 | } 150 | 151 | // Check that leaf proofs work using TileReader. 152 | // To prove a leaf that way, all you have to do is read and verify its hash. 153 | storage := &testTilesStorage{m: tiles} 154 | thr := TileHashReader(Tree{i + 1, th}, storage) 155 | for j := int64(0); j <= i; j++ { 156 | h, err := thr.ReadHashes([]int64{StoredHashIndex(0, j)}) 157 | if err != nil { 158 | t.Fatalf("TileHashReader(%d).ReadHashes(%d): %v", i+1, j, err) 159 | } 160 | if h[0] != leafhashes[j] { 161 | t.Fatalf("TileHashReader(%d).ReadHashes(%d) returned wrong hash", i+1, j) 162 | } 163 | 164 | // Even though reading the hash suffices, 165 | // check we can generate the proof too. 166 | p, err := ProveRecord(i+1, j, thr) 167 | if err != nil { 168 | t.Fatalf("ProveRecord(%d, %d, TileHashReader(%d)): %v", i+1, j, i+1, err) 169 | } 170 | if err := CheckRecord(p, i+1, th, j, leafhashes[j]); err != nil { 171 | t.Fatalf("CheckRecord(%d, %d, TileHashReader(%d)): %v", i+1, j, i+1, err) 172 | } 173 | } 174 | if storage.unsaved != 0 { 175 | t.Fatalf("TileHashReader(%d) did not save %d tiles", i+1, storage.unsaved) 176 | } 177 | 178 | // Check that ReadHashes will give an error if the index is not in the tree. 179 | if _, err := thr.ReadHashes([]int64{(i + 1) * 2}); err == nil { 180 | t.Fatalf("TileHashReader(%d).ReadHashes(%d) for index not in tree , want err", i, i+1) 181 | } 182 | if storage.unsaved != 0 { 183 | t.Fatalf("TileHashReader(%d) did not save %d tiles", i+1, storage.unsaved) 184 | } 185 | 186 | // Check that tree proofs work, for all trees so far, using TileReader. 187 | // To prove a tree that way, all you have to do is compute and verify its hash. 188 | for j := int64(0); j <= i; j++ { 189 | h, err := TreeHash(j+1, thr) 190 | if err != nil { 191 | t.Fatalf("TreeHash(%d, TileHashReader(%d)): %v", j, i+1, err) 192 | } 193 | if h != trees[j] { 194 | t.Fatalf("TreeHash(%d, TileHashReader(%d)) = %x, want %x (%v)", j, i+1, h[:], trees[j][:], trees[j]) 195 | } 196 | 197 | // Even though computing the subtree hash suffices, 198 | // check that we can generate the proof too. 199 | p, err := ProveTree(i+1, j+1, thr) 200 | if err != nil { 201 | t.Fatalf("ProveTree(%d, %d): %v", i+1, j+1, err) 202 | } 203 | if err := CheckTree(p, i+1, th, j+1, trees[j]); err != nil { 204 | t.Fatalf("CheckTree(%d, %d): %v [%v]", i+1, j+1, err, p) 205 | } 206 | for k := range p { 207 | p[k][0] ^= 1 208 | if err := CheckTree(p, i+1, th, j+1, trees[j]); err == nil { 209 | t.Fatalf("CheckTree(%d, %d) succeeded with corrupt proof hash #%d!", i+1, j+1, k) 210 | } 211 | p[k][0] ^= 1 212 | } 213 | } 214 | if storage.unsaved != 0 { 215 | t.Fatalf("TileHashReader(%d) did not save %d tiles", i+1, storage.unsaved) 216 | } 217 | } 218 | } 219 | 220 | func TestSplitStoredHashIndex(t *testing.T) { 221 | for l := 0; l < 10; l++ { 222 | for n := int64(0); n < 100; n++ { 223 | x := StoredHashIndex(l, n) 224 | l1, n1 := SplitStoredHashIndex(x) 225 | if l1 != l || n1 != n { 226 | t.Fatalf("StoredHashIndex(%d, %d) = %d, but SplitStoredHashIndex(%d) = %d, %d", l, n, x, x, l1, n1) 227 | } 228 | } 229 | } 230 | } 231 | 232 | // TODO(rsc): Test invalid paths too, like "tile/3/5/123/456/078". 233 | var tilePaths = []struct { 234 | path string 235 | tile Tile 236 | }{ 237 | {"tile/4/0/001", Tile{4, 0, 1, 16}}, 238 | {"tile/4/0/001.p/5", Tile{4, 0, 1, 5}}, 239 | {"tile/3/5/x123/x456/078", Tile{3, 5, 123456078, 8}}, 240 | {"tile/3/5/x123/x456/078.p/2", Tile{3, 5, 123456078, 2}}, 241 | {"tile/1/0/x003/x057/500", Tile{1, 0, 3057500, 2}}, 242 | {"tile/3/5/123/456/078", Tile{}}, 243 | {"tile/3/-1/123/456/078", Tile{}}, 244 | {"tile/1/data/x003/x057/500", Tile{1, -1, 3057500, 2}}, 245 | } 246 | 247 | func TestTilePath(t *testing.T) { 248 | for _, tt := range tilePaths { 249 | if tt.tile.H > 0 { 250 | p := tt.tile.Path() 251 | if p != tt.path { 252 | t.Errorf("%+v.Path() = %q, want %q", tt.tile, p, tt.path) 253 | } 254 | } 255 | tile, err := ParseTilePath(tt.path) 256 | if err != nil { 257 | if tt.tile.H == 0 { 258 | // Expected error. 259 | continue 260 | } 261 | t.Errorf("ParseTilePath(%q): %v", tt.path, err) 262 | } else if tile != tt.tile { 263 | if tt.tile.H == 0 { 264 | t.Errorf("ParseTilePath(%q): expected error, got %+v", tt.path, tt.tile) 265 | continue 266 | } 267 | t.Errorf("ParseTilePath(%q) = %+v, want %+v", tt.path, tile, tt.tile) 268 | } 269 | } 270 | } 271 | 272 | func TestEmptyTree(t *testing.T) { 273 | h, err := TreeHash(0, nil) 274 | if err != nil { 275 | t.Fatal(err) 276 | } 277 | if h != sha256.Sum256(nil) { 278 | t.Fatalf("TreeHash(0) = %x, want SHA-256('')", h) 279 | } 280 | } 281 | -------------------------------------------------------------------------------- /zip/testdata/check_dir/empty.txt: -------------------------------------------------------------------------------- 1 | -- want -- 2 | valid: 3 | 4 | omitted: 5 | 6 | invalid: 7 | -------------------------------------------------------------------------------- /zip/testdata/check_dir/various.txt: -------------------------------------------------------------------------------- 1 | -- want -- 2 | valid: 3 | $work/valid.go 4 | $work/vendor/modules.txt 5 | 6 | omitted: 7 | $work/.hg_archival.txt: file is inserted by 'hg archive' and is always omitted 8 | $work/.git: directory is a version control repository 9 | $work/pkg/vendor/vendor.go: file is in vendor directory 10 | $work/sub: directory is in another module 11 | $work/vendor/x/y: file is in vendor directory 12 | 13 | invalid: 14 | $work/GO.MOD: go.mod files must have lowercase names 15 | $work/invalid.go': malformed file path "invalid.go'": invalid char '\'' 16 | -- valid.go -- 17 | -- GO.MOD -- 18 | -- invalid.go' -- 19 | -- vendor/modules.txt -- 20 | -- vendor/x/y -- 21 | -- sub/go.mod -- 22 | -- .hg_archival.txt -- 23 | -- .git/x -- 24 | -- pkg/vendor/vendor.go -- 25 | -------------------------------------------------------------------------------- /zip/testdata/check_dir/various_go123.txt: -------------------------------------------------------------------------------- 1 | -- want -- 2 | valid: 3 | $work/go.mod 4 | $work/valid.go 5 | $work/vendor/modules.txt 6 | 7 | omitted: 8 | $work/.hg_archival.txt: file is inserted by 'hg archive' and is always omitted 9 | $work/.git: directory is a version control repository 10 | $work/pkg/vendor/vendor.go: file is in vendor directory 11 | $work/sub: directory is in another module 12 | $work/vendor/x/y: file is in vendor directory 13 | 14 | invalid: 15 | $work/invalid.go': malformed file path "invalid.go'": invalid char '\'' 16 | -- valid.go -- 17 | -- go.mod -- 18 | go 1.23 19 | -- invalid.go' -- 20 | -- vendor/modules.txt -- 21 | -- vendor/x/y -- 22 | -- sub/go.mod -- 23 | -- .hg_archival.txt -- 24 | -- .git/x -- 25 | -- pkg/vendor/vendor.go -- 26 | -------------------------------------------------------------------------------- /zip/testdata/check_dir/various_go124.txt: -------------------------------------------------------------------------------- 1 | -- want -- 2 | valid: 3 | $work/go.mod 4 | $work/pkg/vendor/vendor.go 5 | $work/valid.go 6 | 7 | omitted: 8 | $work/.hg_archival.txt: file is inserted by 'hg archive' and is always omitted 9 | $work/.git: directory is a version control repository 10 | $work/sub: directory is in another module 11 | $work/vendor/modules.txt: file is in vendor directory 12 | $work/vendor/x/y: file is in vendor directory 13 | 14 | invalid: 15 | $work/invalid.go': malformed file path "invalid.go'": invalid char '\'' 16 | -- go.mod -- 17 | go 1.24 18 | -- valid.go -- 19 | -- invalid.go' -- 20 | -- vendor/modules.txt -- 21 | -- vendor/x/y -- 22 | -- sub/go.mod -- 23 | go 1.23 24 | -- .hg_archival.txt -- 25 | -- .git/x -- 26 | -- pkg/vendor/vendor.go -- 27 | -------------------------------------------------------------------------------- /zip/testdata/check_files/empty.txt: -------------------------------------------------------------------------------- 1 | -- want -- 2 | valid: 3 | 4 | omitted: 5 | 6 | invalid: 7 | -------------------------------------------------------------------------------- /zip/testdata/check_files/various.txt: -------------------------------------------------------------------------------- 1 | -- want -- 2 | valid: 3 | valid.go 4 | vendor/modules.txt 5 | 6 | omitted: 7 | vendor/x/y: file is in vendor directory 8 | sub/go.mod: file is in another module 9 | .hg_archival.txt: file is inserted by 'hg archive' and is always omitted 10 | pkg/vendor/vendor.go: file is in vendor directory 11 | 12 | invalid: 13 | not/../clean: file path is not clean 14 | GO.MOD: go.mod files must have lowercase names 15 | invalid.go': malformed file path "invalid.go'": invalid char '\'' 16 | valid.go: multiple entries for file "valid.go" 17 | -- valid.go -- 18 | -- not/../clean -- 19 | -- GO.MOD -- 20 | -- invalid.go' -- 21 | -- vendor/x/y -- 22 | -- vendor/modules.txt -- 23 | -- sub/go.mod -- 24 | -- .hg_archival.txt -- 25 | -- valid.go -- 26 | duplicate 27 | -- valid.go -- 28 | another duplicate 29 | -- pkg/vendor/vendor.go -- 30 | -------------------------------------------------------------------------------- /zip/testdata/check_files/various_go123.txt: -------------------------------------------------------------------------------- 1 | -- want -- 2 | valid: 3 | valid.go 4 | go.mod 5 | vendor/modules.txt 6 | 7 | omitted: 8 | vendor/x/y: file is in vendor directory 9 | sub/go.mod: file is in another module 10 | .hg_archival.txt: file is inserted by 'hg archive' and is always omitted 11 | pkg/vendor/vendor.go: file is in vendor directory 12 | 13 | invalid: 14 | not/../clean: file path is not clean 15 | invalid.go': malformed file path "invalid.go'": invalid char '\'' 16 | valid.go: multiple entries for file "valid.go" 17 | -- valid.go -- 18 | -- not/../clean -- 19 | -- go.mod -- 20 | go 1.23 21 | -- invalid.go' -- 22 | -- vendor/x/y -- 23 | -- vendor/modules.txt -- 24 | -- sub/go.mod -- 25 | -- .hg_archival.txt -- 26 | -- valid.go -- 27 | duplicate 28 | -- valid.go -- 29 | another duplicate 30 | -- pkg/vendor/vendor.go -- 31 | -------------------------------------------------------------------------------- /zip/testdata/check_files/various_go124.txt: -------------------------------------------------------------------------------- 1 | -- want -- 2 | valid: 3 | valid.go 4 | go.mod 5 | pkg/vendor/vendor.go 6 | 7 | omitted: 8 | vendor/x/y: file is in vendor directory 9 | vendor/modules.txt: file is in vendor directory 10 | sub/go.mod: file is in another module 11 | .hg_archival.txt: file is inserted by 'hg archive' and is always omitted 12 | 13 | invalid: 14 | not/../clean: file path is not clean 15 | invalid.go': malformed file path "invalid.go'": invalid char '\'' 16 | valid.go: multiple entries for file "valid.go" 17 | -- valid.go -- 18 | -- not/../clean -- 19 | -- go.mod -- 20 | go 1.24 21 | -- invalid.go' -- 22 | -- vendor/x/y -- 23 | -- vendor/modules.txt -- 24 | -- sub/go.mod -- 25 | go 1.23 26 | -- .hg_archival.txt -- 27 | -- valid.go -- 28 | duplicate 29 | -- valid.go -- 30 | another duplicate 31 | -- pkg/vendor/vendor.go -- 32 | -------------------------------------------------------------------------------- /zip/testdata/check_zip/empty.txt: -------------------------------------------------------------------------------- 1 | path=example.com/empty 2 | version=v1.0.0 3 | -- want -- 4 | valid: 5 | 6 | omitted: 7 | 8 | invalid: 9 | -------------------------------------------------------------------------------- /zip/testdata/check_zip/various.txt: -------------------------------------------------------------------------------- 1 | path=example.com/various 2 | version=v1.0.0 3 | -- want -- 4 | valid: 5 | example.com/various@v1.0.0/valid.go 6 | 7 | omitted: 8 | 9 | invalid: 10 | noprefix: path does not have prefix "example.com/various@v1.0.0/" 11 | example.com/various@v1.0.0/not/../clean: file path is not clean 12 | example.com/various@v1.0.0/invalid.go': malformed file path "invalid.go'": invalid char '\'' 13 | example.com/various@v1.0.0/GO.MOD: go.mod files must have lowercase names 14 | example.com/various@v1.0.0/valid.go: multiple entries for file "valid.go" 15 | -- noprefix -- 16 | -- example.com/various@v1.0.0/valid.go -- 17 | -- example.com/various@v1.0.0/not/../clean -- 18 | -- example.com/various@v1.0.0/invalid.go' -- 19 | -- example.com/various@v1.0.0/GO.MOD -- 20 | -- example.com/various@v1.0.0/valid.go -- 21 | duplicate 22 | -------------------------------------------------------------------------------- /zip/testdata/create/bad_file_path.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | wantErr=malformed file path "bad.go'": invalid char '\'' 4 | -- bad.go' -- 5 | package bad 6 | -------------------------------------------------------------------------------- /zip/testdata/create/bad_gomod_case.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | wantErr=GO.MOD: go.mod files must have lowercase names 4 | -- GO.MOD -- 5 | module example.com/m 6 | -------------------------------------------------------------------------------- /zip/testdata/create/bad_mod_path.txt: -------------------------------------------------------------------------------- 1 | path=cache 2 | version=v1.0.0 3 | wantErr=missing dot in first path element 4 | -------------------------------------------------------------------------------- /zip/testdata/create/bad_mod_path_version_suffix.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v2.0.0 3 | wantErr=invalid version: should be v0 or v1, not v2 4 | -------------------------------------------------------------------------------- /zip/testdata/create/bad_version.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0+bad 3 | wantErr=version "v1.0.0+bad" is not canonical (should be "v1.0.0") 4 | -------------------------------------------------------------------------------- /zip/testdata/create/dup_file.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | wantErr=multiple entries for file "dup.go" 4 | -- dup.go -- 5 | package d1 6 | -- dup.go -- 7 | package d2 8 | -------------------------------------------------------------------------------- /zip/testdata/create/dup_file_and_dir.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | wantErr=entry "a.go" is both a file and a directory 4 | -- a.go -- 5 | package a 6 | -- a.go/b.go -- 7 | package b 8 | -------------------------------------------------------------------------------- /zip/testdata/create/empty.txt: -------------------------------------------------------------------------------- 1 | path=example.com/empty 2 | version=v1.0.0 3 | hash=h1:47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU= 4 | -------------------------------------------------------------------------------- /zip/testdata/create/exclude_cap_go_mod_submodule.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | hash=h1:xctQQey8/y7IcBjFZDP/onWLSXhlqcsC3i1fgSdpMHk= 4 | -- a.go -- 5 | package a 6 | -- b/GO.MOD -- 7 | MODULE EXAMPLE.COM/M/B 8 | -- b/b.go -- 9 | package b 10 | -------------------------------------------------------------------------------- /zip/testdata/create/exclude_submodule.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | hash=h1:XduFAgX/GaspZa8Jv4pfzoGEzNaU/r88PiCunijw5ok= 4 | -- go.mod -- 5 | module example.com/m 6 | 7 | go 1.13 8 | -- sub/go.mod -- 9 | module example.com/m/sub 10 | -- sub/x.go' -- 11 | invalid name, but this shouldn't be read 12 | -------------------------------------------------------------------------------- /zip/testdata/create/exclude_vendor.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | hash=h1:5u93LDLN0Me+NGfZtRpA5mHxY8svfykHpq4CMSaBZyc= 4 | -- go.mod -- 5 | module example.com/m 6 | 7 | go 1.13 8 | -- vendor/modules.txt -- 9 | included 10 | see comment in isVendoredPackage and golang.org/issue/31562. 11 | -- vendor/example.com/x/x.go -- 12 | excluded 13 | -- sub/vendor/sub.txt -- 14 | excluded 15 | -- pkg/vendor/vendor.go -- 16 | excluded 17 | see comment in isVendoredPackage and golang.org/issue/37397 18 | -------------------------------------------------------------------------------- /zip/testdata/create/exclude_vendor_go124.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | hash=h1:mJR1q75yiMK6+CDw6DuhMS4v4hNoLXqkyk2Cph4VS8Q= 4 | -- go.mod -- 5 | module example.com/m 6 | 7 | go 1.24 8 | modules.txt is excluded in 1.24+. See golang.org/issue/63395 9 | -- vendor/modules.txt -- 10 | excluded 11 | see comment in isVendoredPackage and golang.org/issue/31562. 12 | -- vendor/example.com/x/x.go -- 13 | excluded 14 | -- sub/vendor/sub.txt -- 15 | excluded 16 | -- pkg/vendor/vendor.go -- 17 | included 18 | see comment in isVendoredPackage and golang.org/issue/37397 19 | -------------------------------------------------------------------------------- /zip/testdata/create/file_case_conflict.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | wantErr=case-insensitive file name collision: "m.go" and "M.GO" 4 | -- m.go -- 5 | package m 6 | -- M.GO -- 7 | package m 8 | -------------------------------------------------------------------------------- /zip/testdata/create/go_mod_dir.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | hash=h1:Mun5l9cBlDnnV6JasTpio2aZJSbFj++h+814mnKC/OM= 4 | -- go.mod/a.go -- 5 | package a 6 | -------------------------------------------------------------------------------- /zip/testdata/create/invalid_utf8_mod_path.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golang/mod/9d3333156f465c85f68264344b5c08fbcf5fcacb/zip/testdata/create/invalid_utf8_mod_path.txt -------------------------------------------------------------------------------- /zip/testdata/create/simple.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | hash=h1:tpqYOOmuilagXzyqoJ3roUjp8gneQeTv5YVpL6BG7/k= 4 | -- go.mod -- 5 | module example.com/m 6 | 7 | go 1.13 8 | -- m.go -- 9 | package m 10 | 11 | func Foo() int { return 42 } 12 | -- cmd/hello/hello.go -- 13 | package main 14 | 15 | import ( 16 | "fmt" 17 | "example.com/m" 18 | ) 19 | 20 | func main() { 21 | fmt.Println(m.Foo()) 22 | } 23 | -------------------------------------------------------------------------------- /zip/testdata/create_from_dir/bad_file_path.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | wantErr=malformed file path "bad.go'": invalid char '\'' 4 | -- bad.go' -- 5 | package bad 6 | -------------------------------------------------------------------------------- /zip/testdata/create_from_dir/bad_gomod_case.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | wantErr=GO.MOD: go.mod files must have lowercase names 4 | -- GO.MOD -- 5 | module example.com/m 6 | -------------------------------------------------------------------------------- /zip/testdata/create_from_dir/bad_mod_path.txt: -------------------------------------------------------------------------------- 1 | path=cache 2 | version=v1.0.0 3 | wantErr=missing dot in first path element 4 | -------------------------------------------------------------------------------- /zip/testdata/create_from_dir/bad_mod_path_version_suffix.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v2.0.0 3 | wantErr=invalid version: should be v0 or v1, not v2 4 | -------------------------------------------------------------------------------- /zip/testdata/create_from_dir/bad_version.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0+bad 3 | wantErr=version "v1.0.0+bad" is not canonical (should be "v1.0.0") 4 | -------------------------------------------------------------------------------- /zip/testdata/create_from_dir/empty.txt: -------------------------------------------------------------------------------- 1 | path=example.com/empty 2 | version=v1.0.0 3 | hash=h1:47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU= 4 | -------------------------------------------------------------------------------- /zip/testdata/create_from_dir/exclude_submodule.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | hash=h1:XduFAgX/GaspZa8Jv4pfzoGEzNaU/r88PiCunijw5ok= 4 | -- go.mod -- 5 | module example.com/m 6 | 7 | go 1.13 8 | -- sub/go.mod -- 9 | module example.com/m/sub 10 | -- sub/x.go' -- 11 | invalid name, but this shouldn't be read 12 | -------------------------------------------------------------------------------- /zip/testdata/create_from_dir/exclude_vcs.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | hash=h1:47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU= 4 | -- .bzr/exclude -- 5 | exclude 6 | -- .git/exclude -- 7 | exclude 8 | -- .hg/exclude -- 9 | exclude 10 | -- .svn/exclude -- 11 | exclude 12 | -------------------------------------------------------------------------------- /zip/testdata/create_from_dir/exclude_vendor.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | hash=h1:5u93LDLN0Me+NGfZtRpA5mHxY8svfykHpq4CMSaBZyc= 4 | -- go.mod -- 5 | module example.com/m 6 | 7 | go 1.13 8 | -- vendor/modules.txt -- 9 | included 10 | see comment in isVendoredPackage and golang.org/issue/31562. 11 | -- vendor/example.com/x/x.go -- 12 | excluded 13 | -- sub/vendor/sub.txt -- 14 | excluded 15 | -- pkg/vendor/vendor.go -- 16 | excluded 17 | see comment in isVendoredPackage and golang.org/issue/37397 18 | -------------------------------------------------------------------------------- /zip/testdata/create_from_dir/exclude_vendor_go124.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | hash=h1:mJR1q75yiMK6+CDw6DuhMS4v4hNoLXqkyk2Cph4VS8Q= 4 | -- go.mod -- 5 | module example.com/m 6 | 7 | go 1.24 8 | modules.txt is excluded in 1.24+. See golang.org/issue/63395 9 | -- vendor/modules.txt -- 10 | excluded 11 | see comment in isVendoredPackage and golang.org/issue/31562. 12 | -- vendor/example.com/x/x.go -- 13 | excluded 14 | -- sub/vendor/sub.txt -- 15 | excluded 16 | -- pkg/vendor/vendor.go -- 17 | included 18 | see comment in isVendoredPackage and golang.org/issue/37397 19 | -------------------------------------------------------------------------------- /zip/testdata/create_from_dir/go_mod_dir.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | hash=h1:Mun5l9cBlDnnV6JasTpio2aZJSbFj++h+814mnKC/OM= 4 | -- go.mod/a.go -- 5 | package a 6 | -------------------------------------------------------------------------------- /zip/testdata/create_from_dir/invalid_utf8_mod_path.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golang/mod/9d3333156f465c85f68264344b5c08fbcf5fcacb/zip/testdata/create_from_dir/invalid_utf8_mod_path.txt -------------------------------------------------------------------------------- /zip/testdata/create_from_dir/simple.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | hash=h1:tpqYOOmuilagXzyqoJ3roUjp8gneQeTv5YVpL6BG7/k= 4 | -- go.mod -- 5 | module example.com/m 6 | 7 | go 1.13 8 | -- m.go -- 9 | package m 10 | 11 | func Foo() int { return 42 } 12 | -- cmd/hello/hello.go -- 13 | package main 14 | 15 | import ( 16 | "fmt" 17 | "example.com/m" 18 | ) 19 | 20 | func main() { 21 | fmt.Println(m.Foo()) 22 | } 23 | -------------------------------------------------------------------------------- /zip/testdata/unzip/bad_file_path.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | wantErr=malformed file path "bad.go'": invalid char '\'' 4 | -- example.com/m@v1.0.0/bad.go' -- 5 | package bad 6 | -------------------------------------------------------------------------------- /zip/testdata/unzip/bad_gomod_case.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | wantErr=go.mod files must have lowercase names 4 | -- example.com/m@v1.0.0/GO.MOD -- 5 | module example.com/m 6 | -------------------------------------------------------------------------------- /zip/testdata/unzip/bad_mod_path.txt: -------------------------------------------------------------------------------- 1 | path=cache 2 | version=v1.0.0 3 | wantErr=missing dot in first path element 4 | -------------------------------------------------------------------------------- /zip/testdata/unzip/bad_mod_path_version_suffix.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v2.0.0 3 | wantErr=invalid version: should be v0 or v1, not v2 4 | -------------------------------------------------------------------------------- /zip/testdata/unzip/bad_submodule.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | wantErr=go.mod file not in module root directory 4 | -- example.com/m@v1.0.0/go.mod -- 5 | module example.com/m 6 | 7 | go 1.13 8 | -- example.com/m@v1.0.0/sub/go.mod -- 9 | module example.com/m/sub 10 | -------------------------------------------------------------------------------- /zip/testdata/unzip/bad_version.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0+bad 3 | wantErr=version "v1.0.0+bad" is not canonical (should be "v1.0.0") 4 | -------------------------------------------------------------------------------- /zip/testdata/unzip/cap_go_mod_not_submodule.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | wantErr=go.mod file not in module root directory 4 | -- example.com/m@v1.0.0/a.go -- 5 | package a 6 | -- example.com/m@v1.0.0/b/GO.MOD -- 7 | MODULE EXAMPLE.COM/M/B 8 | -- example.com/m@v1.0.0/b/b.go -- 9 | package b 10 | -------------------------------------------------------------------------------- /zip/testdata/unzip/dup_file.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | wantErr=multiple entries for file "dup.go" 4 | -- example.com/m@v1.0.0/dup.go -- 5 | package d1 6 | -- example.com/m@v1.0.0/dup.go -- 7 | package d2 8 | -------------------------------------------------------------------------------- /zip/testdata/unzip/dup_file_and_dir.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | wantErr=entry "a.go" is both a file and a directory 4 | -- example.com/m@v1.0.0/a.go -- 5 | package a 6 | -- example.com/m@v1.0.0/a.go/b.go -- 7 | package b 8 | -------------------------------------------------------------------------------- /zip/testdata/unzip/empty.txt: -------------------------------------------------------------------------------- 1 | path=example.com/empty 2 | version=v1.0.0 3 | hash=h1:47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU= 4 | -------------------------------------------------------------------------------- /zip/testdata/unzip/file_case_conflict.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | wantErr=case-insensitive file name collision: "m.go" and "M.GO" 4 | -- example.com/m@v1.0.0/m.go -- 5 | package m 6 | -- example.com/m@v1.0.0/M.GO -- 7 | package m 8 | -------------------------------------------------------------------------------- /zip/testdata/unzip/go_mod_dir.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | hash=h1:Mun5l9cBlDnnV6JasTpio2aZJSbFj++h+814mnKC/OM= 4 | -- example.com/m@v1.0.0/go.mod/a.go -- 5 | package a 6 | -------------------------------------------------------------------------------- /zip/testdata/unzip/invalid_utf8_mod_path.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/golang/mod/9d3333156f465c85f68264344b5c08fbcf5fcacb/zip/testdata/unzip/invalid_utf8_mod_path.txt -------------------------------------------------------------------------------- /zip/testdata/unzip/prefix_only.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | wantErr=example.com/m@v1.0.0: path does not have prefix "example.com/m@v1.0.0/" 4 | -- example.com/m@v1.0.0 -- 5 | -- example.com/m@v1.0.0/go.mod -- 6 | module example.com/m 7 | -------------------------------------------------------------------------------- /zip/testdata/unzip/simple.txt: -------------------------------------------------------------------------------- 1 | path=example.com/m 2 | version=v1.0.0 3 | hash=h1:tpqYOOmuilagXzyqoJ3roUjp8gneQeTv5YVpL6BG7/k= 4 | -- example.com/m@v1.0.0/go.mod -- 5 | module example.com/m 6 | 7 | go 1.13 8 | -- example.com/m@v1.0.0/m.go -- 9 | package m 10 | 11 | func Foo() int { return 42 } 12 | -- example.com/m@v1.0.0/cmd/hello/hello.go -- 13 | package main 14 | 15 | import ( 16 | "fmt" 17 | "example.com/m" 18 | ) 19 | 20 | func main() { 21 | fmt.Println(m.Foo()) 22 | } 23 | -------------------------------------------------------------------------------- /zip/vendor_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package zip 6 | 7 | import "testing" 8 | 9 | var pre124 []string = []string{ 10 | "", 11 | "go1.14", 12 | "go1.21.0", 13 | "go1.22.4", 14 | "go1.23", 15 | "go1.23.1", 16 | "go1.2", 17 | "go1.7", 18 | "go1.9", 19 | } 20 | 21 | var after124 []string = []string{"go1.24.0", "go1.24", "go1.99.0"} 22 | 23 | var allVers []string = append(pre124, after124...) 24 | 25 | func TestIsVendoredPackage(t *testing.T) { 26 | for _, tc := range []struct { 27 | path string 28 | want bool 29 | versions []string 30 | }{ 31 | {path: "vendor/foo/foo.go", want: true, versions: allVers}, 32 | {path: "pkg/vendor/foo/foo.go", want: true, versions: allVers}, 33 | {path: "longpackagename/vendor/foo/foo.go", want: true, versions: allVers}, 34 | {path: "vendor/vendor.go", want: false, versions: allVers}, 35 | {path: "vendor/foo/modules.txt", want: true, versions: allVers}, 36 | {path: "modules.txt", want: false, versions: allVers}, 37 | {path: "vendor/amodules.txt", want: false, versions: allVers}, 38 | 39 | // These test cases were affected by https://golang.org/issue/63395 40 | {path: "vendor/modules.txt", want: false, versions: pre124}, 41 | {path: "vendor/modules.txt", want: true, versions: after124}, 42 | 43 | // These test cases were affected by https://golang.org/issue/37397 44 | {path: "pkg/vendor/vendor.go", want: true, versions: pre124}, 45 | {path: "pkg/vendor/vendor.go", want: false, versions: after124}, 46 | {path: "longpackagename/vendor/vendor.go", want: true, versions: pre124}, 47 | {path: "longpackagename/vendor/vendor.go", want: false, versions: after124}, 48 | } { 49 | for _, v := range tc.versions { 50 | got := isVendoredPackage(tc.path, v) 51 | want := tc.want 52 | if got != want { 53 | t.Errorf("isVendoredPackage(%q, %s) = %t; want %t", tc.path, v, got, tc.want) 54 | } 55 | } 56 | } 57 | } 58 | --------------------------------------------------------------------------------