├── .travis.yml ├── LICENSE ├── MAINTAINERS ├── README.md ├── _testdata └── vendor │ └── vendoredPkg │ └── vendored.go ├── ast.go ├── ast_test.go ├── importer.go └── importer_test.go /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.8 5 | - tip 6 | 7 | matrix: 8 | allow_failures: 9 | - go: tip 10 | 11 | env: 12 | - GOPATH=/tmp/whatever:$GOPATH 13 | 14 | install: 15 | - rm -rf $GOPATH/src/gopkg.in/src-d 16 | - mkdir -p $GOPATH/src/gopkg.in/src-d 17 | - mv $PWD $GOPATH/src/gopkg.in/src-d/go-parse-utils.v1 18 | - cd $GOPATH/src/gopkg.in/src-d/go-parse-utils.v1 19 | - go get -t -v ./... 20 | 21 | script: 22 | - go test -covermode=atomic -coverprofile=coverage.txt -v . 23 | 24 | after_success: 25 | - bash <(curl -s https://codecov.io/bash) -t ac0dc938-196b-4545-bdac-b4d5b5dd045f 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 source{d} 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /MAINTAINERS: -------------------------------------------------------------------------------- 1 | Miguel Molina (@erizocosmico) 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-parse-utils 2 | 3 | `go-parse-utils` is a collection of utilities for parsing code easily. 4 | 5 | [![GoDoc](https://godoc.org/gopkg.in/src-d/go-parse-utils.v1?status.svg)](https://godoc.org/gopkg.in/src-d/go-parse-utils.v1) [![Build Status](https://travis-ci.org/src-d/go-parse-utils.svg?branch=master)](https://travis-ci.org/src-d/go-parse-utils) [![codecov](https://codecov.io/gh/src-d/go-parse-utils/branch/master/graph/badge.svg)](https://codecov.io/gh/src-d/go-parse-utils) [![License](http://img.shields.io/:license-mit-blue.svg)](http://doge.mit-license.org) [![Go Report Card](https://goreportcard.com/badge/gopkg.in/src-d/go-parse-utils.v1)](https://goreportcard.com/report/gopkg.in/src-d/go-parse-utils.v1) 6 | 7 | ### Install 8 | 9 | ``` 10 | go get gopkg.in/src-d/go-parse-utils.v1 11 | ``` 12 | 13 | ### Package AST 14 | 15 | `PackageAST` retrieves the `*ast.Package` of a package in the given path. 16 | 17 | ```go 18 | pkg, err := parseutil.PackageAST("github.com/my/project") 19 | ``` 20 | 21 | ### Source code importer 22 | 23 | The default `importer.Importer` of the Go standard library scans compiled objects, which can be painful to deal with in code generation, as it requires to `go build` before running `go generate`. 24 | 25 | This packages provides an implementation of `importer.Importer` and `importer.ImporterFrom` that reads directly from source code if the package is in the GOPATH, otherwise (the stdlib, for example) falls back to the default importer in the standard library. 26 | 27 | Features: 28 | * It is safe for concurrent use. 29 | * Caches packages after they are first imported. 30 | 31 | ```go 32 | importer := parseutil.NewImporter() 33 | pkg, err := importer.Import("github.com/my/project") 34 | ``` 35 | 36 | ### License 37 | 38 | MIT, see [LICENSE](/LICENSE) 39 | -------------------------------------------------------------------------------- /_testdata/vendor/vendoredPkg/vendored.go: -------------------------------------------------------------------------------- 1 | package vendoredPkg 2 | -------------------------------------------------------------------------------- /ast.go: -------------------------------------------------------------------------------- 1 | package parseutil 2 | 3 | import ( 4 | "errors" 5 | "go/ast" 6 | "go/parser" 7 | "go/token" 8 | "strings" 9 | ) 10 | 11 | // ErrTooManyPackages is returned when there is more than one package in a 12 | // directory where there should only be one Go package. 13 | var ErrTooManyPackages = errors.New("more than one package found in a directory") 14 | 15 | // PackageAST returns the AST of the package at the given path. 16 | func PackageAST(path string) (pkg *ast.Package, err error) { 17 | return parseAndFilterPackages(path, func(k string, v *ast.Package) bool { 18 | return !strings.HasSuffix(k, "_test") 19 | }) 20 | } 21 | 22 | // PackageTestAST returns the AST of the test package at the given path. 23 | func PackageTestAST(path string) (pkg *ast.Package, err error) { 24 | return parseAndFilterPackages(path, func(k string, v *ast.Package) bool { 25 | return strings.HasSuffix(k, "_test") 26 | }) 27 | } 28 | 29 | type packageFilter func(string, *ast.Package) bool 30 | 31 | // filteredPackages filters the parsed packages and then makes sure there is only 32 | // one left. 33 | func parseAndFilterPackages(path string, filter packageFilter) (pkg *ast.Package, err error) { 34 | fset := token.NewFileSet() 35 | srcDir, err := DefaultGoPath.Abs(path) 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | pkgs, err := parser.ParseDir(fset, srcDir, nil, parser.ParseComments) 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | pkgs = filterPkgs(pkgs, filter) 46 | 47 | if len(pkgs) != 1 { 48 | return nil, ErrTooManyPackages 49 | } 50 | 51 | for _, p := range pkgs { 52 | pkg = p 53 | } 54 | 55 | return 56 | } 57 | 58 | func filterPkgs(pkgs map[string]*ast.Package, filter packageFilter) map[string]*ast.Package { 59 | filtered := make(map[string]*ast.Package) 60 | for k, v := range pkgs { 61 | if filter(k, v) { 62 | filtered[k] = v 63 | } 64 | } 65 | 66 | return filtered 67 | } 68 | -------------------------------------------------------------------------------- /ast_test.go: -------------------------------------------------------------------------------- 1 | package parseutil_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "gopkg.in/src-d/go-parse-utils.v1" 8 | ) 9 | 10 | func TestPackageAST(t *testing.T) { 11 | pkg, err := parseutil.PackageAST(project) 12 | require.Nil(t, err) 13 | require.Equal(t, "parseutil", pkg.Name) 14 | } 15 | 16 | func TestPackageTestAST(t *testing.T) { 17 | pkg, err := parseutil.PackageTestAST(project) 18 | require.Nil(t, err) 19 | require.Equal(t, "parseutil_test", pkg.Name) 20 | } 21 | -------------------------------------------------------------------------------- /importer.go: -------------------------------------------------------------------------------- 1 | package parseutil 2 | 3 | import ( 4 | "fmt" 5 | "go/ast" 6 | "go/build" 7 | "go/importer" 8 | "go/parser" 9 | "go/token" 10 | "go/types" 11 | "os" 12 | "path/filepath" 13 | "strings" 14 | "sync" 15 | ) 16 | 17 | // ErrNotInGoPath is an error returned when a package is not in any of the 18 | // possible go paths. 19 | var ErrNotInGoPath = fmt.Errorf("parseutil: package is not in any of the go paths") 20 | 21 | // GoPath is the collection of all go paths. 22 | type GoPath []string 23 | 24 | // Abs returns the absolute path to a package. The go path in the absolute path 25 | // that will be returned is the first that contains the given package. 26 | func (gp GoPath) Abs(pkg string) (string, error) { 27 | path, err := gp.PathOf(pkg) 28 | if err != nil { 29 | return "", err 30 | } 31 | 32 | return filepath.Join(path, "src", pkg), nil 33 | } 34 | 35 | // PathOf returns the first go path that contains the given package. 36 | func (gp GoPath) PathOf(pkg string) (string, error) { 37 | for _, p := range gp { 38 | if _, err := os.Stat(filepath.Join(p, "src", pkg)); err == nil { 39 | return p, nil 40 | } else if !os.IsNotExist(err) { 41 | return "", err 42 | } 43 | } 44 | return "", ErrNotInGoPath 45 | } 46 | 47 | // DefaultGoPath contains the default list of go paths provided either via 48 | // GOPATH environment variable or the default value. 49 | var DefaultGoPath = GoPath(filepath.SplitList(build.Default.GOPATH)) 50 | 51 | // FileFilter returns true if the given file needs to be kept. 52 | type FileFilter func(pkgPath, file string, typ FileType) bool 53 | 54 | // FileFilters represents a colection of FileFilter 55 | type FileFilters []FileFilter 56 | 57 | // KeepFile returns true if and only if the file passes all FileFilters. 58 | func (fs FileFilters) KeepFile(pkgPath, file string, typ FileType) bool { 59 | for _, f := range fs { 60 | if !f(pkgPath, file, typ) { 61 | return false 62 | } 63 | } 64 | 65 | return true 66 | } 67 | 68 | // Filter returns the files passed in files that satisfy all FileFilters. 69 | func (fs FileFilters) Filter(pkgPath string, files []string, typ FileType) (filtered []string) { 70 | for _, f := range files { 71 | if fs.KeepFile(pkgPath, f, typ) { 72 | filtered = append(filtered, f) 73 | } 74 | } 75 | return 76 | } 77 | 78 | // FileType represents the type of go source file type. 79 | type FileType string 80 | 81 | const ( 82 | GoFile FileType = "go" 83 | CgoFile FileType = "cgo" 84 | ) 85 | 86 | // Importer is an implementation of `types.Importer` and `types.ImporterFrom` 87 | // that builds actual source files and not the compiled objects in the pkg 88 | // directory. 89 | // It is safe to use it concurrently. 90 | // A package is cached after building it the first time. 91 | type Importer struct { 92 | mut sync.RWMutex 93 | cache map[string]*types.Package 94 | 95 | defaultImporter types.Importer 96 | } 97 | 98 | // NewImporter creates a new Importer instance with the default importer of 99 | // the runtime assigned as the underlying importer. 100 | func NewImporter() *Importer { 101 | return &Importer{ 102 | cache: make(map[string]*types.Package), 103 | defaultImporter: importer.Default(), 104 | } 105 | } 106 | 107 | // Import returns the imported package for the given import 108 | // path, or an error if the package couldn't be imported. 109 | // Two calls to Import with the same path return the same 110 | // package. 111 | func (i *Importer) Import(path string) (*types.Package, error) { 112 | return i.ImportWithFilters(path, nil) 113 | } 114 | 115 | // ImportWithFilters works like Import but filtering the source files to parse using 116 | // the passed FileFilters. 117 | func (i *Importer) ImportWithFilters(path string, filters FileFilters) (*types.Package, error) { 118 | return i.ImportFromWithFilters(path, "", 0, filters) 119 | } 120 | 121 | // ImportFrom returns the imported package for the given import 122 | // path when imported by the package in srcDir, or an error 123 | // if the package couldn't be imported. The mode value must 124 | // be 0; it is reserved for future use. 125 | // Two calls to ImportFrom with the same path and srcDir return 126 | // the same package. 127 | func (i *Importer) ImportFrom(path, srcDir string, mode types.ImportMode) (*types.Package, error) { 128 | return i.ImportFromWithFilters(path, srcDir, mode, nil) 129 | } 130 | 131 | // ImportFromWithFilters works like ImportFrom but filters the source files using 132 | // the passed FileFilters. 133 | func (i *Importer) ImportFromWithFilters(path, srcDir string, mode types.ImportMode, filters FileFilters) (*types.Package, error) { 134 | i.mut.Lock() 135 | if pkg, ok := i.cache[path]; ok { 136 | i.mut.Unlock() 137 | return pkg, nil 138 | } 139 | i.mut.Unlock() 140 | 141 | root, files, err := i.GetSourceFiles(path, srcDir, filters) 142 | if err != nil { 143 | return nil, err 144 | } 145 | 146 | // If it's not on the GOPATH use the default importer instead 147 | useDefaultImporter := true 148 | for _, p := range DefaultGoPath { 149 | if strings.HasPrefix(root, p) { 150 | useDefaultImporter = false 151 | break 152 | } 153 | } 154 | 155 | if useDefaultImporter { 156 | i.mut.Lock() 157 | defer i.mut.Unlock() 158 | 159 | var pkg *types.Package 160 | var err error 161 | imp, ok := i.defaultImporter.(types.ImporterFrom) 162 | if ok { 163 | pkg, err = imp.ImportFrom(path, srcDir, mode) 164 | } else { 165 | pkg, err = imp.Import(path) 166 | } 167 | 168 | if err != nil { 169 | return nil, err 170 | } 171 | 172 | i.cache[path] = pkg 173 | 174 | return pkg, nil 175 | } 176 | 177 | pkg, err := i.ParseSourceFiles(root, files) 178 | if err != nil { 179 | return nil, err 180 | } 181 | 182 | i.mut.Lock() 183 | i.cache[path] = pkg 184 | i.mut.Unlock() 185 | return pkg, nil 186 | } 187 | 188 | // GetSourceFiles return the go files available in src under path after applying the filters. 189 | func (i *Importer) GetSourceFiles(path, srcDir string, filters FileFilters) (string, []string, error) { 190 | srcDir, err := filepath.Abs(srcDir) 191 | if err != nil { 192 | return "", nil, err 193 | } 194 | pkg, err := build.Import(path, srcDir, 0) 195 | if err != nil { 196 | return "", nil, err 197 | } 198 | 199 | var filenames []string 200 | filenames = append(filenames, filters.Filter(path, pkg.GoFiles, GoFile)...) 201 | filenames = append(filenames, filters.Filter(path, pkg.CgoFiles, CgoFile)...) 202 | 203 | if len(filenames) == 0 { 204 | return "", nil, fmt.Errorf("no go source files in path: %s", path) 205 | } 206 | 207 | var paths []string 208 | for _, f := range filenames { 209 | paths = append(paths, filepath.Join(pkg.Dir, f)) 210 | } 211 | 212 | return pkg.Dir, paths, nil 213 | } 214 | 215 | // ParseSourceFiles parses the files in paths under root returning a types.Package 216 | // and an optional error 217 | func (i *Importer) ParseSourceFiles(root string, paths []string) (*types.Package, error) { 218 | var files []*ast.File 219 | fs := token.NewFileSet() 220 | for _, p := range paths { 221 | f, err := parser.ParseFile(fs, p, nil, 0) 222 | if err != nil { 223 | return nil, err 224 | } 225 | 226 | files = append(files, f) 227 | } 228 | 229 | config := types.Config{ 230 | FakeImportC: true, 231 | Importer: i, 232 | } 233 | 234 | return config.Check(root, fs, files, nil) 235 | } 236 | -------------------------------------------------------------------------------- /importer_test.go: -------------------------------------------------------------------------------- 1 | package parseutil_test 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | 7 | "gopkg.in/src-d/go-parse-utils.v1" 8 | 9 | "github.com/stretchr/testify/require" 10 | _ "google.golang.org/grpc" 11 | ) 12 | 13 | const project = "gopkg.in/src-d/go-parse-utils.v1" 14 | 15 | var projectPath = func() string { 16 | path, err := parseutil.DefaultGoPath.Abs(project) 17 | if err != nil { 18 | panic(err) 19 | } 20 | return path 21 | }() 22 | 23 | func projectFile(path string) string { 24 | return filepath.Join(projectPath, path) 25 | } 26 | 27 | func TestGetSourceFiles(t *testing.T) { 28 | projPath, err := parseutil.DefaultGoPath.PathOf(project) 29 | require.NoError(t, err) 30 | _, paths, err := parseutil.NewImporter(). 31 | GetSourceFiles(project, projPath, parseutil.FileFilters{}) 32 | require.NoError(t, err) 33 | expected := []string{ 34 | projectFile("ast.go"), 35 | projectFile("importer.go"), 36 | } 37 | 38 | require.Equal(t, expected, paths) 39 | } 40 | 41 | func TestParseSourceFiles(t *testing.T) { 42 | paths := []string{ 43 | projectFile("ast.go"), 44 | projectFile("importer.go"), 45 | } 46 | 47 | pkg, err := parseutil.NewImporter().ParseSourceFiles(projectPath, paths) 48 | require.Nil(t, err) 49 | 50 | require.Equal(t, "parseutil", pkg.Name()) 51 | } 52 | 53 | func TestImport(t *testing.T) { 54 | imp := parseutil.NewImporter() 55 | pkg, err := imp.Import(project) 56 | require.Nil(t, err) 57 | require.Equal(t, "parseutil", pkg.Name()) 58 | } 59 | 60 | func TestImportWithFilters(t *testing.T) { 61 | imp := parseutil.NewImporter() 62 | _, err := imp.ImportWithFilters(project, parseutil.FileFilters{ 63 | func(pkgPath, file string, typ parseutil.FileType) bool { 64 | return file != "importer.go" 65 | }, 66 | }) 67 | require.NotNil(t, err, "excluding importer.go makes package unimportable") 68 | } 69 | 70 | func TestImportGoogleGrpc(t *testing.T) { 71 | imp := parseutil.NewImporter() 72 | _, err := imp.Import("google.golang.org/grpc") 73 | require.Nil(t, err, "should be able to import this. Was a bug") 74 | } 75 | 76 | func TestImportFrom(t *testing.T) { 77 | imp := parseutil.NewImporter() 78 | pkg, err := imp.ImportFrom(project, "", 0) 79 | require.Nil(t, err) 80 | require.Equal(t, "parseutil", pkg.Name()) 81 | } 82 | 83 | func TestImportFromVendored(t *testing.T) { 84 | imp := parseutil.NewImporter() 85 | pkg, err := imp.ImportFrom("vendoredPkg", "_testdata", 0) 86 | require.Nil(t, err) 87 | require.Equal(t, "vendoredPkg", pkg.Name()) 88 | } 89 | 90 | func TestFileFilters(t *testing.T) { 91 | fs := parseutil.FileFilters{ 92 | func(pkgPath, file string, typ parseutil.FileType) bool { 93 | return pkgPath == "a" 94 | }, 95 | func(pkgPath, file string, typ parseutil.FileType) bool { 96 | return file == "a" 97 | }, 98 | func(pkgPath, file string, typ parseutil.FileType) bool { 99 | return typ == parseutil.GoFile 100 | }, 101 | } 102 | 103 | require.True(t, fs.KeepFile("a", "a", parseutil.GoFile)) 104 | require.False(t, fs.KeepFile("b", "a", parseutil.GoFile)) 105 | require.False(t, fs.KeepFile("a", "b", parseutil.GoFile)) 106 | require.False(t, fs.KeepFile("a", "a", parseutil.CgoFile)) 107 | } 108 | --------------------------------------------------------------------------------