├── .github └── workflows │ └── ci.yaml ├── .golangci.yml ├── LICENSE ├── Makefile ├── README.md ├── go.mod ├── go.sum ├── internal ├── buildinfo │ ├── parse.go │ └── parse_test.go ├── license │ ├── db │ │ ├── archive.go │ │ ├── gen │ │ │ └── gen.go │ │ ├── licenses.db │ │ └── open.go │ └── resolve.go ├── model │ ├── model.go │ └── model_test.go ├── module │ ├── extract.go │ ├── fetch.go │ └── fetch_test.go └── scan │ ├── conf.go │ ├── result.go │ └── run.go └── main.go /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: [push, pull_request] 3 | jobs: 4 | test: 5 | strategy: 6 | matrix: 7 | go-version: [1.17.x, 1.18.x] 8 | os: [ubuntu-latest, macos-latest, windows-latest] 9 | runs-on: ${{ matrix.os }} 10 | steps: 11 | - name: Install Go 12 | uses: actions/setup-go@v2 13 | with: 14 | go-version: ${{ matrix.go-version }} 15 | - name: Checkout code 16 | uses: actions/checkout@v3 17 | - name: Test 18 | run: go test -race ./... 19 | - name: Build 20 | run: go build 21 | lint: 22 | runs-on: ubuntu-latest 23 | steps: 24 | - name: Install Go 25 | uses: actions/setup-go@v2 26 | with: 27 | go-version: 1.18.x 28 | - name: Checkout 29 | uses: actions/checkout@v3 30 | - name: Lint 31 | uses: golangci/golangci-lint-action@v3 32 | with: 33 | version: latest 34 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters: 2 | disable-all: true 3 | enable: 4 | - bodyclose 5 | - deadcode 6 | - depguard 7 | - dogsled 8 | - dupl 9 | - errcheck 10 | - gochecknoinits 11 | - goconst 12 | - gocritic 13 | - gocyclo 14 | - gofmt 15 | - goimports 16 | - golint 17 | - goprintffuncname 18 | - gosec 19 | - gosimple 20 | - govet 21 | - ineffassign 22 | - interfacer 23 | - lll 24 | - misspell 25 | - nakedret 26 | - nolintlint 27 | - rowserrcheck 28 | - scopelint 29 | - staticcheck 30 | - structcheck 31 | - stylecheck 32 | - typecheck 33 | - unconvert 34 | - unparam 35 | - unused 36 | - varcheck 37 | - whitespace 38 | 39 | linters-settings: 40 | lll: 41 | line-length: 180 -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Utility Warehouse 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | internal/license/db/licenses.db: 2 | go install github.com/google/licenseclassifier/tools/license_serializer 3 | license_serializer -output ./internal/license/db/ 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # lichen 🍃 2 | 3 | Go binary license checker. Extracts module usage information from binaries and analyses their licenses. 4 | 5 | ## Features 6 | 7 | - Accurate module usage extraction (including transitive) from Go compiled binaries. 8 | - License files are resolved from local module storage. 9 | - Licenses are always checked against their respective versions. 10 | - Multi-license usage is covered out the box. 11 | - Local license checking using [google/licenseclassifier](https://github.com/google/licenseclassifier). 12 | - Customisable output via text/template. 13 | - JSON output for further analysis and transforming into CSV, XLSX, etc. 14 | 15 | ### Improvements over existing tooling 16 | 17 | - Some tools attempt to extract module use information from scanning code. This can be flawed, as transitive 18 | dependencies are not well represented (if at all). `lichen` executes `go version -m [exes]` to obtain accurate module 19 | usage information; only those that are required at compile time will be included. Also note that 20 | [rsc/goversion](https://github.com/rsc/goversion) has been avoided due to known issues in relation to binaries compiled 21 | with CGO enabled, and a lack of development activity. 22 | - Existing tools have been known to make requests against the GitHub API for license information. Unfortunately this can 23 | be flawed: the API only returns license details obtained from the HEAD of the `master` branch of a given repository. 24 | This also typically requires a GitHub API token to be available, as rate-limiting will kick in quite quickly. The 25 | GitHub API license detection doesn't offer any significant advantages; it itself simply uses 26 | [licensee/licensee](https://github.com/licensee/licensee) for license checking. `lichen` does not use the GitHub API at 27 | all. 28 | - In some instances, existing tools will clone the repository relating to the module. Often this is suffers from the 29 | same flaws as hitting the GitHub API, as the master branch ends up being inspected. Furthermore, some module URLs do 30 | not easily map to a git repository, resulting in the need for manual mapping in some instances. Finally, this process 31 | has a tendency to be slow. `lichen` takes advantage of Go tooling to retrieve the relevant file(s) in an accurate and 32 | time effective manner - `go mod download` is executed, and the local copy of the module is inspected for license 33 | information. 34 | 35 | ## Install 36 | 37 | ``` 38 | go install github.com/uw-labs/lichen@latest 39 | ``` 40 | 41 | Note that Go must be installed wherever `lichen` is intended to be run, as `lichen` executes various Go commands (as 42 | discussed in the previous section). 43 | 44 | ## Usage 45 | 46 | By default `lichen` simply prints each module with its respective license. A path to at least one Go compiled binary 47 | must be supplied. Permitted licenses can be configured, along with overrides and exceptions (see [Config](#Config)). 48 | 49 | ``` 50 | lichen --config=path/to/lichen.yaml [binary ...] 51 | ``` 52 | 53 | Run ```lichen --help``` for further information on flags. 54 | 55 | Note that the where `lichen` runs the Go executable, the process is created with the same environment as `lichen` 56 | itself - therefore you can set [Go related environment variables](https://pkg.go.dev/cmd/go#hdr-Environment_variables) 57 | (e.g. `GOPRIVATE`) and these will be respected. 58 | 59 | ## Example 60 | 61 | We can run lichen on itself: 62 | 63 | ``` 64 | $ lichen $GOPATH/bin/lichen 65 | github.com/cpuguy83/go-md2man/v2@v2.0.0-20190314233015-f79a8a8ca69d: MIT (allowed) 66 | github.com/google/goterm@v0.0.0-20190703233501-fc88cf888a3f: BSD-3-Clause (allowed) 67 | github.com/google/licenseclassifier@v0.0.0-20200402202327-879cb1424de0: Apache-2.0 (allowed) 68 | github.com/hashicorp/errwrap@v1.0.0: MPL-2.0 (allowed) 69 | github.com/hashicorp/go-multierror@v1.1.0: MPL-2.0 (allowed) 70 | github.com/lucasb-eyer/go-colorful@v1.0.3: MIT (allowed) 71 | github.com/mattn/go-isatty@v0.0.12: MIT (allowed) 72 | github.com/muesli/termenv@v0.5.2: MIT (allowed) 73 | github.com/russross/blackfriday/v2@v2.0.1: BSD-2-Clause (allowed) 74 | github.com/sergi/go-diff@v1.0.0: MIT (allowed) 75 | github.com/shurcooL/sanitized_anchor_name@v1.0.0: MIT (allowed) 76 | github.com/urfave/cli/v2@v2.2.0: MIT (allowed) 77 | golang.org/x/sys@v0.0.0-20200116001909-b77594299b42: BSD-3-Clause (allowed) 78 | gopkg.in/yaml.v2@v2.3.0: Apache-2.0, MIT (allowed) 79 | ``` 80 | 81 | ...and using a custom template: 82 | 83 | ``` 84 | $ lichen --template="{{range .Modules}}{{range .Module.Licenses}}{{.Name | printf \"%s\n\"}}{{end}}{{end}}" $GOPATH/bin/lichen | sort | uniq -c | sort -nr 85 | 8 MIT 86 | 2 MPL-2.0 87 | 2 BSD-3-Clause 88 | 2 Apache-2.0 89 | 1 BSD-2-Clause 90 | ``` 91 | 92 | ## Config 93 | 94 | Configuration is entirely optional. If you wish to use lichen to ensure only permitted licenses are in use, you can 95 | use the configuration to specify these. You can also override certain defaults or force a license if lichen cannot 96 | detect one. 97 | 98 | Example: 99 | 100 | ```yaml 101 | # minimum confidence percentage used during license classification 102 | threshold: .80 103 | 104 | # all permitted licenses - if no list is specified, all licenses are assumed to be allowed 105 | allow: 106 | - "MIT" 107 | - "Apache-2.0" 108 | - "0BSD" 109 | - "BSD-3-Clause" 110 | - "BSD-2-Clause" 111 | - "BSD-2-Clause-FreeBSD" 112 | - "MPL-2.0" 113 | - "ISC" 114 | - "PostgreSQL" 115 | 116 | # overrides for cases where a license cannot be detected, but the software is licensed 117 | override: 118 | - path: "github.com/abc/xyz" 119 | version: "v0.1.0" # version is optional - if specified, the override will only apply for the configured version 120 | licenses: ["MIT"] # specify licenses 121 | 122 | # exceptions for violations 123 | exceptions: 124 | # exceptions for "license not permitted" type violations 125 | licenseNotPermitted: 126 | - path: "github.com/foo/bar" 127 | version: "v0.1.0" # version is optional - if specified, the exception will only apply to the configured version 128 | licenses: ["LGPL-3.0"] # licenses is optional - if specified only violations in relation to the listed licenses will be ignored 129 | - path: "github.com/baz/xyz" 130 | # exceptions for "unresolvable license" type violations 131 | unresolvableLicense: 132 | - path: "github.com/test/foo" 133 | version: "v1.0.1" # version is optional - if unspecified, the exception will apply to all versions 134 | ``` 135 | 136 | ## Credit 137 | 138 | This project was very much inspired by [mitchellh/golicense](https://github.com/mitchellh/golicense) 139 | 140 | ## Caveat emptor 141 | 142 | Just as a linter cannot _guarantee_ working and correct code, this tool cannot guarantee dependencies and their licenses 143 | are determined with absolute correctness. `lichen` is designed to help catch cases that might fall through the net, but 144 | it is by no means a replacement for manual inspection and evaluation of dependencies. 145 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/uw-labs/lichen 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/google/licenseclassifier v0.0.0-20201113175434-78a70215ca36 7 | github.com/hashicorp/go-multierror v1.1.1 8 | github.com/muesli/termenv v0.11.0 9 | github.com/stretchr/testify v1.7.1 10 | github.com/urfave/cli/v2 v2.4.0 11 | gopkg.in/yaml.v2 v2.4.0 12 | ) 13 | 14 | require ( 15 | github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect 16 | github.com/davecgh/go-spew v1.1.0 // indirect 17 | github.com/hashicorp/errwrap v1.0.0 // indirect 18 | github.com/lucasb-eyer/go-colorful v1.2.0 // indirect 19 | github.com/mattn/go-isatty v0.0.14 // indirect 20 | github.com/mattn/go-runewidth v0.0.13 // indirect 21 | github.com/pmezard/go-difflib v1.0.0 // indirect 22 | github.com/rivo/uniseg v0.2.0 // indirect 23 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 24 | github.com/sergi/go-diff v1.0.0 // indirect 25 | golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect 26 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect 27 | ) 28 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 2 | github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= 3 | github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= 4 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 5 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 7 | github.com/google/licenseclassifier v0.0.0-20201113175434-78a70215ca36 h1:YGB3wNLUTvq+lbIwdNRsaMJvoX4mCKkwzHlmlT1V+ow= 8 | github.com/google/licenseclassifier v0.0.0-20201113175434-78a70215ca36/go.mod h1:qsqn2hxC+vURpyBRygGUuinTO42MFRLcsmQ/P8v94+M= 9 | github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= 10 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 11 | github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= 12 | github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= 13 | github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= 14 | github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= 15 | github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= 16 | github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= 17 | github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= 18 | github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 19 | github.com/muesli/termenv v0.11.0 h1:fwNUbu2mfWlgicwG7qYzs06aOI8Z/zKPAv8J4uKbT+o= 20 | github.com/muesli/termenv v0.11.0/go.mod h1:Bd5NYQ7pd+SrtBSrSNoBBmXlcY8+Xj4BMJgh8qcZrvs= 21 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 22 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 23 | github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= 24 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 25 | github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= 26 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 27 | github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= 28 | github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= 29 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 30 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 31 | github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= 32 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 33 | github.com/urfave/cli/v2 v2.4.0 h1:m2pxjjDFgDxSPtO8WSdbndj17Wu2y8vOT86wE/tjr+I= 34 | github.com/urfave/cli/v2 v2.4.0/go.mod h1:NX9W0zmTvedE5oDoOMs2RTC8RvdK98NTYZE5LbaEYPg= 35 | golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= 36 | golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 37 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 38 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 39 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 40 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 41 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 42 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 43 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 44 | -------------------------------------------------------------------------------- /internal/buildinfo/parse.go: -------------------------------------------------------------------------------- 1 | package buildinfo 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strings" 7 | 8 | "github.com/uw-labs/lichen/internal/model" 9 | ) 10 | 11 | var goVersionRgx = regexp.MustCompile(`^(.*?): (?:(?:devel )?go[0-9]+|devel \+[0-9a-f]+)`) 12 | 13 | // Parse parses build info details as returned by `go version -m [bin ...]` 14 | func Parse(info string) ([]model.BuildInfo, error) { 15 | var ( 16 | lines = strings.Split(info, "\n") 17 | results = make([]model.BuildInfo, 0) 18 | current model.BuildInfo 19 | replacement bool 20 | ) 21 | for _, l := range lines { 22 | // ignore blank lines 23 | if l == "" { 24 | continue 25 | } 26 | 27 | // start of new build info output 28 | if !strings.HasPrefix(l, "\t") { 29 | matches := goVersionRgx.FindStringSubmatch(l) 30 | if len(matches) != 2 { 31 | return nil, fmt.Errorf("unrecognised version line: %s", l) 32 | } 33 | if current.Path != "" { 34 | results = append(results, current) 35 | } 36 | current = model.BuildInfo{Path: matches[1]} 37 | continue 38 | } 39 | 40 | // inside build info output 41 | parts := strings.Split(l, "\t") 42 | if len(parts) < 2 { 43 | return nil, fmt.Errorf("invalid build info line: %s", l) 44 | } 45 | if replacement { 46 | if parts[1] != "=>" { 47 | return nil, fmt.Errorf("expected path replacement, received: %s", l) 48 | } 49 | replacement = false 50 | } 51 | switch parts[1] { 52 | case "path": 53 | if len(parts) != 3 { 54 | return nil, fmt.Errorf("invalid path line: %s", l) 55 | } 56 | current.PackagePath = parts[2] 57 | case "mod": 58 | if len(parts) != 5 { 59 | return nil, fmt.Errorf("invalid mod line: %s", l) 60 | } 61 | current.ModulePath = parts[2] 62 | case "dep", "=>": 63 | switch len(parts) { 64 | case 5: 65 | if parts[3] == "(devel)" { 66 | // "mod" line in disguise (Go 1.18) 67 | current.ModulePath = parts[2] 68 | } else { 69 | current.ModuleRefs = append(current.ModuleRefs, model.ModuleReference{ 70 | Path: parts[2], 71 | Version: parts[3], 72 | }) 73 | } 74 | case 4: 75 | replacement = true 76 | default: 77 | return nil, fmt.Errorf("invalid dep line: %s", l) 78 | } 79 | case "build": 80 | // introduced in Go 1.18 - not captured as we aren't using it for anything 81 | case "": 82 | // blank (tab prefixed) lines appear after lines relating to replace directives in Go 1.18 compiled binaries 83 | default: 84 | return nil, fmt.Errorf("unrecognised line: %s", l) 85 | } 86 | } 87 | if current.Path != "" { 88 | results = append(results, current) 89 | } 90 | return results, nil 91 | } 92 | -------------------------------------------------------------------------------- /internal/buildinfo/parse_test.go: -------------------------------------------------------------------------------- 1 | package buildinfo_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | 9 | "github.com/uw-labs/lichen/internal/buildinfo" 10 | "github.com/uw-labs/lichen/internal/model" 11 | ) 12 | 13 | func TestParse(t *testing.T) { 14 | testCases := []struct { 15 | name string 16 | input string 17 | expected []model.BuildInfo 18 | expectedErr string 19 | }{ 20 | { 21 | name: "basic single binary input", 22 | input: `/tmp/lichen: go1.14.4 23 | path github.com/uw-labs/lichen 24 | mod github.com/uw-labs/lichen (devel) 25 | dep github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= 26 | `, 27 | expected: []model.BuildInfo{ 28 | { 29 | Path: "/tmp/lichen", 30 | PackagePath: "github.com/uw-labs/lichen", 31 | ModulePath: "github.com/uw-labs/lichen", 32 | ModuleRefs: []model.ModuleReference{ 33 | { 34 | Path: "github.com/cpuguy83/go-md2man/v2", 35 | Version: "v2.0.0-20190314233015-f79a8a8ca69d", 36 | }, 37 | }, 38 | }, 39 | }, 40 | }, 41 | { 42 | name: "single binary input with dep replace", 43 | input: `/tmp/lichen: go1.14 44 | path github.com/uw-labs/lichen 45 | mod github.com/uw-labs/lichen (devel) 46 | dep github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d 47 | => github.com/uw-labs/go-md2man/v2 v0.4.16-0.20200608113539-44d3cd590db7 h1:7JSMFy7v19QNuP77yBMWawhzb9xD82oPmrlda5yrBkE= 48 | `, 49 | expected: []model.BuildInfo{ 50 | { 51 | Path: "/tmp/lichen", 52 | PackagePath: "github.com/uw-labs/lichen", 53 | ModulePath: "github.com/uw-labs/lichen", 54 | ModuleRefs: []model.ModuleReference{ 55 | { 56 | Path: "github.com/uw-labs/go-md2man/v2", 57 | Version: "v0.4.16-0.20200608113539-44d3cd590db7", 58 | }, 59 | }, 60 | }, 61 | }, 62 | }, 63 | { 64 | name: "basic multi binary input", 65 | input: `/tmp/lichen: go1.14.4 66 | path github.com/uw-labs/lichen 67 | mod github.com/uw-labs/lichen (devel) 68 | dep github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= 69 | /tmp/lichen2: go1.14.4 70 | path github.com/uw-labs/lichen 71 | mod github.com/uw-labs/lichen (devel) 72 | dep github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= 73 | `, 74 | expected: []model.BuildInfo{ 75 | { 76 | Path: "/tmp/lichen", 77 | PackagePath: "github.com/uw-labs/lichen", 78 | ModulePath: "github.com/uw-labs/lichen", 79 | ModuleRefs: []model.ModuleReference{ 80 | { 81 | Path: "github.com/cpuguy83/go-md2man/v2", 82 | Version: "v2.0.0-20190314233015-f79a8a8ca69d", 83 | }, 84 | }, 85 | }, 86 | { 87 | Path: "/tmp/lichen2", 88 | PackagePath: "github.com/uw-labs/lichen", 89 | ModulePath: "github.com/uw-labs/lichen", 90 | ModuleRefs: []model.ModuleReference{ 91 | { 92 | Path: "github.com/google/goterm", 93 | Version: "v0.0.0-20190703233501-fc88cf888a3f", 94 | }, 95 | }, 96 | }, 97 | }, 98 | }, 99 | { 100 | name: "windows basic single binary input", 101 | input: `C:\lichen.exe: go1.14.4 102 | path github.com/uw-labs/lichen 103 | mod github.com/uw-labs/lichen (devel) 104 | dep github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= 105 | `, 106 | expected: []model.BuildInfo{ 107 | { 108 | Path: `C:\lichen.exe`, 109 | PackagePath: "github.com/uw-labs/lichen", 110 | ModulePath: "github.com/uw-labs/lichen", 111 | ModuleRefs: []model.ModuleReference{ 112 | { 113 | Path: "github.com/cpuguy83/go-md2man/v2", 114 | Version: "v2.0.0-20190314233015-f79a8a8ca69d", 115 | }, 116 | }, 117 | }, 118 | }, 119 | }, 120 | { 121 | name: "development version (pre-go1.17)", 122 | input: `/tmp/lichen: devel +01821137c2 Sat Apr 3 01:45:17 2021 +0000`, 123 | expected: []model.BuildInfo{ 124 | { 125 | Path: "/tmp/lichen", 126 | }, 127 | }, 128 | }, 129 | { 130 | name: "development version (current)", 131 | input: `/tmp/lichen: devel go1.18-0c83e01e0c Wed Aug 18 15:11:52 2021 +0000`, 132 | expected: []model.BuildInfo{ 133 | { 134 | Path: "/tmp/lichen", 135 | }, 136 | }, 137 | }, 138 | { 139 | name: "development version (old)", 140 | input: `/tmp/lichen: devel +b7a85e0003 linux/amd64`, 141 | expected: []model.BuildInfo{ 142 | { 143 | Path: "/tmp/lichen", 144 | }, 145 | }, 146 | }, 147 | { 148 | name: "windows development version", 149 | input: `C:\lichen.exe: devel go1.18-0c83e01e0c Wed Aug 18 15:11:52 2021 +0000`, 150 | expected: []model.BuildInfo{ 151 | { 152 | Path: `C:\lichen.exe`, 153 | }, 154 | }, 155 | }, 156 | { 157 | name: "1.18 compiled binary with `build` lines", 158 | input: `/tmp/lichen: go1.18beta2 159 | path github.com/uw-labs/lichen 160 | mod github.com/uw-labs/lichen (devel) 161 | dep github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= 162 | build -compiler=gc 163 | build CGO_ENABLED=1 164 | build vcs=git 165 | `, 166 | expected: []model.BuildInfo{ 167 | { 168 | Path: `/tmp/lichen`, 169 | PackagePath: "github.com/uw-labs/lichen", 170 | ModulePath: "github.com/uw-labs/lichen", 171 | ModuleRefs: []model.ModuleReference{ 172 | { 173 | Path: "github.com/cpuguy83/go-md2man/v2", 174 | Version: "v2.0.0-20190314233015-f79a8a8ca69d", 175 | }, 176 | }, 177 | }, 178 | }, 179 | }, 180 | { 181 | name: "superfluous blank line (observed Go 1.18+)", 182 | input: `/tmp/lichen: go1.18 183 | path github.com/uw-labs/lichen 184 | mod github.com/uw-labs/lichen (devel) 185 | dep github.com/lyft/protoc-gen-star v0.6.0 186 | => github.com/johanbrandhorst/protoc-gen-star v0.4.16-0.20200806111151-9a8e34bf9dea 187 | 188 | dep github.com/spf13/afero v1.8.0 189 | `, 190 | expected: []model.BuildInfo{ 191 | { 192 | Path: `/tmp/lichen`, 193 | PackagePath: "github.com/uw-labs/lichen", 194 | ModulePath: "github.com/uw-labs/lichen", 195 | ModuleRefs: []model.ModuleReference{ 196 | { 197 | Path: "github.com/johanbrandhorst/protoc-gen-star", 198 | Version: "v0.4.16-0.20200806111151-9a8e34bf9dea", 199 | }, 200 | { 201 | Path: "github.com/spf13/afero", 202 | Version: "v1.8.0", 203 | }, 204 | }, 205 | }, 206 | }, 207 | }, 208 | { 209 | name: "mod as dep (observed Go 1.18+)", 210 | input: `lichen: go1.18.1 211 | path command-line-arguments 212 | dep github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= 213 | dep github.com/uw-labs/lichen (devel) 214 | dep golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= 215 | `, 216 | expected: []model.BuildInfo{ 217 | { 218 | Path: `lichen`, 219 | PackagePath: "command-line-arguments", 220 | ModulePath: "github.com/uw-labs/lichen", 221 | ModuleRefs: []model.ModuleReference{ 222 | { 223 | Path: "github.com/cpuguy83/go-md2man/v2", 224 | Version: "v2.0.1", 225 | }, 226 | { 227 | Path: "golang.org/x/sys", 228 | Version: "v0.0.0-20210630005230-0f9fa26af87c", 229 | }, 230 | }, 231 | }, 232 | }, 233 | }, 234 | { 235 | name: "unrecognised line", 236 | input: `/tmp/lichen: invalid`, 237 | expectedErr: "unrecognised version line: /tmp/lichen: invalid", 238 | }, 239 | { 240 | name: "partial path line", 241 | input: `lichen: go1.14.4 242 | path 243 | `, 244 | expectedErr: "invalid path line: \tpath", 245 | }, 246 | { 247 | name: "path line unexpectedly long", 248 | input: `lichen: go1.14.4 249 | path foo bar 250 | `, 251 | expectedErr: "invalid path line: \tpath\tfoo\tbar", 252 | }, 253 | { 254 | name: "partial mod line", 255 | input: `lichen: go1.14.4 256 | mod foo (devel) 257 | `, 258 | expectedErr: "invalid mod line: \tmod\tfoo\t(devel)", 259 | }, 260 | { 261 | name: "mod line unexpectedly long", 262 | input: `lichen: go1.14.4 263 | mod foo (devel) x 264 | `, 265 | expectedErr: "invalid mod line: \tmod\tfoo\t(devel)\tx\t", 266 | }, 267 | { 268 | name: "partial dep line", 269 | input: `lichen: go1.14.4 270 | dep foo 271 | `, 272 | expectedErr: "invalid dep line: \tdep\tfoo", 273 | }, 274 | { 275 | name: "dep line unexpectedly long", 276 | input: `lichen: go1.14.4 277 | dep foo v0 h1:x x 278 | `, 279 | expectedErr: "invalid dep line: \tdep\tfoo\tv0\th1:x\tx", 280 | }, 281 | } 282 | for _, tc := range testCases { 283 | tc := tc 284 | t.Run(tc.name, func(tt *testing.T) { 285 | actual, err := buildinfo.Parse(tc.input) 286 | if tc.expectedErr == "" { 287 | require.NoError(tt, err) 288 | assert.Equal(tt, tc.expected, actual) 289 | } else { 290 | assert.EqualError(tt, err, tc.expectedErr) 291 | } 292 | }) 293 | } 294 | } 295 | -------------------------------------------------------------------------------- /internal/license/db/gen/gen.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "encoding/ascii85" 7 | "fmt" 8 | "io" 9 | "log" 10 | "os" 11 | ) 12 | 13 | func main() { 14 | if err := run(); err != nil { 15 | log.Fatal(err) 16 | } 17 | } 18 | 19 | const tmpl = `// Code generated by lichen/gen; DO NOT EDIT. 20 | 21 | package db 22 | 23 | var archive = []byte(%q) 24 | ` 25 | 26 | func run() error { 27 | f, err := os.Open(os.Args[1]) 28 | if err != nil { 29 | return err 30 | } 31 | defer f.Close() 32 | 33 | w, err := os.Create(os.Args[2]) 34 | if err != nil { 35 | return err 36 | } 37 | defer w.Close() 38 | 39 | buf := &bytes.Buffer{} 40 | encoder := ascii85.NewEncoder(buf) 41 | gz := gzip.NewWriter(encoder) 42 | if _, err := io.Copy(gz, f); err != nil { 43 | return err 44 | } 45 | if err := gz.Flush(); err != nil { 46 | return err 47 | } 48 | if err := gz.Close(); err != nil { 49 | return err 50 | } 51 | if err := encoder.Close(); err != nil { 52 | return err 53 | } 54 | 55 | fmt.Fprintf(w, tmpl, buf.String()) 56 | 57 | return nil 58 | } 59 | -------------------------------------------------------------------------------- /internal/license/db/licenses.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uw-labs/lichen/46b79dff606d9c4931e5435c2c85b71d0ea353ef/internal/license/db/licenses.db -------------------------------------------------------------------------------- /internal/license/db/open.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | //go:generate go run ./gen/ licenses.db archive.go 4 | 5 | import ( 6 | "bytes" 7 | "compress/gzip" 8 | "encoding/ascii85" 9 | "io" 10 | ) 11 | 12 | // Open returns a reader that produces the contents of licenses.db in this directory. This is achieved by reading 13 | // a variable (`archive`) that has been built via code generation. 14 | func Open() (io.ReadCloser, error) { 15 | decoder := ascii85.NewDecoder(bytes.NewReader(archive)) 16 | return gzip.NewReader(decoder) 17 | } 18 | -------------------------------------------------------------------------------- /internal/license/resolve.go: -------------------------------------------------------------------------------- 1 | package license 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "path/filepath" 7 | "regexp" 8 | "strings" 9 | 10 | "github.com/google/licenseclassifier" 11 | "github.com/uw-labs/lichen/internal/license/db" 12 | "github.com/uw-labs/lichen/internal/model" 13 | ) 14 | 15 | // Resolve inspects each module and determines what it is licensed under. The returned slice contains each 16 | // module enriched with license information. 17 | func Resolve(modules []model.Module, threshold float64) ([]model.Module, error) { 18 | archiveFn := licenseclassifier.ArchiveFunc(func() ([]byte, error) { 19 | f, err := db.Open() 20 | if err != nil { 21 | return nil, fmt.Errorf("failed to open license databse: %w", err) 22 | } 23 | defer f.Close() 24 | return ioutil.ReadAll(f) 25 | }) 26 | 27 | lc, err := licenseclassifier.New(threshold, archiveFn) 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | for i, m := range modules { 33 | if m.IsLocal() { 34 | // there is no guarantee we are being run in a location that makes local module references resolvable.. to 35 | // avoid incidental and non-obvious behaviour here, we simply don't touch such references - overrides must 36 | // be provided instead. 37 | continue 38 | } 39 | paths, err := locateLicenses(m.Dir) 40 | if err != nil { 41 | return nil, err 42 | } 43 | licenses, err := classify(lc, paths) 44 | if err != nil { 45 | return nil, err 46 | } 47 | m.Licenses = licenses 48 | modules[i] = m 49 | } 50 | 51 | return modules, nil 52 | } 53 | 54 | var fileRgx = regexp.MustCompile(`(?i)^(li[cs]en[cs]e|copying)`) 55 | 56 | // locateLicenses searches for license files 57 | func locateLicenses(path string) (lp []string, err error) { 58 | files, err := ioutil.ReadDir(path) 59 | if err != nil { 60 | return nil, err 61 | } 62 | for _, f := range files { 63 | if !f.IsDir() && fileRgx.MatchString(f.Name()) && !strings.HasSuffix(f.Name(), ".go") { 64 | lp = append(lp, filepath.Join(path, f.Name())) 65 | } 66 | } 67 | return lp, nil 68 | } 69 | 70 | // classify inspects each license file and classifies it 71 | func classify(lc *licenseclassifier.License, paths []string) ([]model.License, error) { 72 | licenses := make([]model.License, 0) 73 | for _, p := range paths { 74 | content, err := ioutil.ReadFile(p) 75 | if err != nil { 76 | return nil, err 77 | } 78 | hits := make(map[string]float64) 79 | matches := lc.MultipleMatch(string(content), true) 80 | for _, match := range matches { 81 | if conf, found := hits[match.Name]; !found || match.Confidence > conf { 82 | hits[match.Name] = match.Confidence 83 | } 84 | } 85 | for name, confidence := range hits { 86 | licenses = append(licenses, model.License{ 87 | Name: name, 88 | Path: p, 89 | Content: string(content), 90 | Confidence: confidence, 91 | }) 92 | } 93 | } 94 | return licenses, nil 95 | } 96 | -------------------------------------------------------------------------------- /internal/model/model.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | ) 7 | 8 | // BuildInfo encapsulates build info embedded into a Go compile binary 9 | type BuildInfo struct { 10 | Path string // OS level absolute path to the binary this build info relates to 11 | PackagePath string // package path indicated by the build info, e.g. github.com/foo/bar/cmd/baz 12 | ModulePath string // module path indicated by the build info, e.g. github.com/foo/bar 13 | ModuleRefs []ModuleReference // all modules that feature in the build info output 14 | } 15 | 16 | // Module carries details of a Go module 17 | type Module struct { 18 | ModuleReference // reference (path & version) 19 | Dir string // OS level absolute path to where the cached copy of the module is located 20 | Licenses []License // resolved licenses 21 | } 22 | 23 | // ModuleReference is a reference to a particular version of a named module 24 | type ModuleReference struct { 25 | Path string // module path, e.g. github.com/foo/bar 26 | Version string // module version (can take a variety of forms) 27 | } 28 | 29 | // pathRgx covers 30 | // - unix paths: ".", "..", prefixed "./", prefixed "../", prefixed "/" 31 | // - windows paths: ".", "..", prefixed ".\", prefixed "..\", prefixed ":\" 32 | var pathRgx = regexp.MustCompile(`^(\.\.?($|/|\\)|/|[A-Za-z]:\\)`) 33 | 34 | // IsLocal returns true if the module reference points to a local path 35 | func (r ModuleReference) IsLocal() bool { 36 | return r.Version == "" && pathRgx.MatchString(r.Path) 37 | } 38 | 39 | // String returns a typical string representation of a module reference (path@version) 40 | func (r ModuleReference) String() string { 41 | if r.Version == "" { 42 | return r.Path 43 | } 44 | return fmt.Sprintf("%s@%s", r.Path, r.Version) 45 | } 46 | 47 | // License carries license classification details 48 | type License struct { 49 | Path string // OS level absolute path to the license file 50 | Content string // the exact contents of the license file 51 | Name string // SPDX name of the license 52 | Confidence float64 // confidence from license classification 53 | } 54 | -------------------------------------------------------------------------------- /internal/model/model_test.go: -------------------------------------------------------------------------------- 1 | package model_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/uw-labs/lichen/internal/model" 8 | ) 9 | 10 | func TestModuleReference_IsLocal(t *testing.T) { 11 | testCases := []struct { 12 | name string 13 | ref model.ModuleReference 14 | expected bool 15 | }{ 16 | { 17 | name: "with version", 18 | ref: model.ModuleReference{ 19 | Version: "1.0.0", 20 | }, 21 | expected: false, 22 | }, 23 | { 24 | name: "current dir", 25 | ref: model.ModuleReference{ 26 | Path: ".", 27 | }, 28 | expected: true, 29 | }, 30 | { 31 | name: "up one dir", 32 | ref: model.ModuleReference{ 33 | Path: "..", 34 | }, 35 | expected: true, 36 | }, 37 | { 38 | name: "current dir with slash", 39 | ref: model.ModuleReference{ 40 | Path: "./", 41 | }, 42 | expected: true, 43 | }, 44 | { 45 | name: "up one dir with slash", 46 | ref: model.ModuleReference{ 47 | Path: "../", 48 | }, 49 | expected: true, 50 | }, 51 | { 52 | name: "dir relative to current", 53 | ref: model.ModuleReference{ 54 | Path: "./test", 55 | }, 56 | expected: true, 57 | }, 58 | { 59 | name: "dir relative to up one", 60 | ref: model.ModuleReference{ 61 | Path: "../test", 62 | }, 63 | expected: true, 64 | }, 65 | { 66 | name: "dir relative to current, up one", 67 | ref: model.ModuleReference{ 68 | Path: "./../test", 69 | }, 70 | expected: true, 71 | }, 72 | { 73 | name: "absolute path, unix style", 74 | ref: model.ModuleReference{ 75 | Path: "/test/abc", 76 | }, 77 | expected: true, 78 | }, 79 | { 80 | name: "absolute path, windows style", 81 | ref: model.ModuleReference{ 82 | Path: "C:\\test\\abc", 83 | }, 84 | expected: true, 85 | }, 86 | { 87 | name: "github path", 88 | ref: model.ModuleReference{ 89 | Path: "github.com/foo/bar", 90 | }, 91 | expected: false, 92 | }, 93 | { 94 | name: "ambiguous", 95 | ref: model.ModuleReference{ 96 | Path: "github", 97 | }, 98 | expected: false, 99 | }, 100 | } 101 | for _, tc := range testCases { 102 | tc := tc 103 | t.Run(tc.name, func(tt *testing.T) { 104 | actual := tc.ref.IsLocal() 105 | assert.Equal(tt, tc.expected, actual) 106 | }) 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /internal/module/extract.go: -------------------------------------------------------------------------------- 1 | package module 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "os/exec" 9 | 10 | "github.com/hashicorp/go-multierror" 11 | "github.com/uw-labs/lichen/internal/buildinfo" 12 | "github.com/uw-labs/lichen/internal/model" 13 | ) 14 | 15 | // Extract extracts build information from the supplied binaries 16 | func Extract(ctx context.Context, paths ...string) ([]model.BuildInfo, error) { 17 | output, err := goVersion(ctx, paths) 18 | if err != nil { 19 | return nil, err 20 | } 21 | 22 | parsed, err := buildinfo.Parse(output) 23 | if err != nil { 24 | return nil, err 25 | } 26 | if err := verifyExtracted(parsed, paths); err != nil { 27 | return nil, fmt.Errorf("could not extract module information: %w", err) 28 | } 29 | return parsed, nil 30 | } 31 | 32 | // verifyExtracted ensures all paths requests are covered by the parsed output 33 | func verifyExtracted(extracted []model.BuildInfo, requested []string) (err error) { 34 | buildInfos := make(map[string]struct{}, len(extracted)) 35 | for _, binary := range extracted { 36 | buildInfos[binary.Path] = struct{}{} 37 | } 38 | for _, path := range requested { 39 | if _, found := buildInfos[path]; !found { 40 | err = multierror.Append(err, fmt.Errorf("modules could not be obtained from %[1]s (hint: run `go version -m %[1]q`)", path)) 41 | } 42 | } 43 | return 44 | } 45 | 46 | // goVersion runs `go version -m [paths ...]` and returns the output 47 | func goVersion(ctx context.Context, paths []string) (string, error) { 48 | goBin, err := exec.LookPath("go") 49 | if err != nil { 50 | return "", err 51 | } 52 | 53 | tempDir, err := ioutil.TempDir("", "lichen") 54 | if err != nil { 55 | return "", fmt.Errorf("failed to create temp directory: %w", err) 56 | } 57 | defer os.Remove(tempDir) 58 | 59 | args := []string{"version", "-m"} 60 | args = append(args, paths...) 61 | 62 | cmd := exec.CommandContext(ctx, goBin, args...) 63 | cmd.Dir = tempDir 64 | out, err := cmd.Output() 65 | if err != nil { 66 | if exitErr, ok := err.(*exec.ExitError); ok { 67 | return "", fmt.Errorf("error when running 'go version': %w - stderr: %s", err, exitErr.Stderr) 68 | } 69 | return "", fmt.Errorf("error when running 'go version': %w", err) 70 | } 71 | 72 | return string(out), err 73 | } 74 | -------------------------------------------------------------------------------- /internal/module/fetch.go: -------------------------------------------------------------------------------- 1 | package module 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "io/ioutil" 11 | "os" 12 | "os/exec" 13 | 14 | "github.com/hashicorp/go-multierror" 15 | "github.com/uw-labs/lichen/internal/model" 16 | ) 17 | 18 | func Fetch(ctx context.Context, refs []model.ModuleReference) ([]model.Module, error) { 19 | if len(refs) == 0 { 20 | return []model.Module{}, nil 21 | } 22 | 23 | goBin, err := exec.LookPath("go") 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | tempDir, err := ioutil.TempDir("", "lichen") 29 | if err != nil { 30 | return nil, fmt.Errorf("failed to create temp directory: %w", err) 31 | } 32 | defer os.Remove(tempDir) 33 | 34 | args := []string{"mod", "download", "-json"} 35 | for _, ref := range refs { 36 | if !ref.IsLocal() { 37 | args = append(args, ref.String()) 38 | } 39 | } 40 | 41 | cmd := exec.CommandContext(ctx, goBin, args...) 42 | cmd.Dir = tempDir 43 | out, err := cmd.CombinedOutput() 44 | if err != nil { 45 | return nil, fmt.Errorf("failed to fetch: %w (output: %s)", err, string(out)) 46 | } 47 | 48 | // parse JSON output from `go mod download` 49 | modules := make([]model.Module, 0) 50 | dec := json.NewDecoder(bytes.NewReader(out)) 51 | for { 52 | var m model.Module 53 | if err := dec.Decode(&m); err != nil { 54 | if errors.Is(err, io.EOF) { 55 | break 56 | } 57 | return nil, err 58 | } 59 | modules = append(modules, m) 60 | } 61 | 62 | // add local modules, as these won't be included in the set returned by `go mod download` 63 | for _, ref := range refs { 64 | if ref.IsLocal() { 65 | modules = append(modules, model.Module{ 66 | ModuleReference: ref, 67 | }) 68 | } 69 | } 70 | 71 | // sanity check: all modules should have been covered in the output from `go mod download` 72 | if err := verifyFetched(modules, refs); err != nil { 73 | return nil, fmt.Errorf("failed to fetch all modules: %w", err) 74 | } 75 | 76 | return modules, nil 77 | } 78 | 79 | func verifyFetched(fetched []model.Module, requested []model.ModuleReference) (err error) { 80 | fetchedRefs := make(map[model.ModuleReference]struct{}, len(fetched)) 81 | for _, module := range fetched { 82 | fetchedRefs[module.ModuleReference] = struct{}{} 83 | } 84 | for _, ref := range requested { 85 | if _, found := fetchedRefs[ref]; !found { 86 | err = multierror.Append(err, fmt.Errorf("module %s could not be resolved", ref)) 87 | } 88 | } 89 | return 90 | } 91 | -------------------------------------------------------------------------------- /internal/module/fetch_test.go: -------------------------------------------------------------------------------- 1 | package module_test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/uw-labs/lichen/internal/model" 9 | "github.com/uw-labs/lichen/internal/module" 10 | ) 11 | 12 | func TestModuleFetchNoModules(test *testing.T) { 13 | modules, err := module.Fetch(context.Background(), []model.ModuleReference{}) 14 | 15 | assert.NoError(test, err) 16 | assert.Empty(test, modules) 17 | } 18 | -------------------------------------------------------------------------------- /internal/scan/conf.go: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | type Config struct { 4 | Threshold *float64 `yaml:"threshold"` 5 | Allow []string `yaml:"allow"` 6 | Exceptions Exceptions `yaml:"exceptions"` 7 | Overrides []Override `yaml:"override"` 8 | } 9 | 10 | type Exceptions struct { 11 | LicenseNotPermitted []LicenseNotPermitted `yaml:"licenseNotPermitted"` 12 | UnresolvableLicense []UnresolvableLicense `yaml:"unresolvableLicense"` 13 | } 14 | 15 | type LicenseNotPermitted struct { 16 | Path string `yaml:"path"` 17 | Version string `yaml:"version"` 18 | Licenses []string `yaml:"licenses"` 19 | } 20 | 21 | type UnresolvableLicense struct { 22 | Path string `yaml:"path"` 23 | Version string `yaml:"version"` 24 | } 25 | 26 | type Override struct { 27 | Path string `yaml:"path"` 28 | Version string `yaml:"version"` 29 | Licenses []string `yaml:"licenses"` 30 | } 31 | -------------------------------------------------------------------------------- /internal/scan/result.go: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/uw-labs/lichen/internal/model" 7 | ) 8 | 9 | type Summary struct { 10 | Modules []EvaluatedModule 11 | Binaries []model.BuildInfo 12 | } 13 | 14 | type EvaluatedModule struct { 15 | model.Module 16 | Decision Decision 17 | NotPermitted []string `json:",omitempty"` 18 | UsedBy []string 19 | } 20 | 21 | func (r EvaluatedModule) Allowed() bool { 22 | return r.Decision == DecisionAllowed 23 | } 24 | 25 | func (r EvaluatedModule) ExplainDecision() string { 26 | switch r.Decision { 27 | case DecisionAllowed: 28 | return "allowed" 29 | case DecisionNotAllowedUnresolvableLicense: 30 | return "not allowed - unresolvable license" 31 | case DecisionNotAllowedLicenseNotPermitted: 32 | return fmt.Sprintf("not allowed - non-permitted licenses: %v", r.NotPermitted) 33 | default: 34 | panic("unrecognised decision") 35 | } 36 | } 37 | 38 | type Decision int 39 | 40 | const ( 41 | DecisionAllowed Decision = 1 + iota 42 | DecisionNotAllowedUnresolvableLicense 43 | DecisionNotAllowedLicenseNotPermitted 44 | ) 45 | 46 | func (d Decision) MarshalText() ([]byte, error) { 47 | switch d { 48 | case DecisionAllowed: 49 | return []byte("allowed"), nil 50 | case DecisionNotAllowedUnresolvableLicense: 51 | return []byte("unresolvable-license"), nil 52 | case DecisionNotAllowedLicenseNotPermitted: 53 | return []byte("licenses-not-allowed"), nil 54 | default: 55 | panic("unrecognised decision") 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /internal/scan/run.go: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import ( 4 | "context" 5 | "sort" 6 | 7 | "github.com/uw-labs/lichen/internal/license" 8 | "github.com/uw-labs/lichen/internal/model" 9 | "github.com/uw-labs/lichen/internal/module" 10 | ) 11 | 12 | const defaultThreshold = 0.80 13 | 14 | func Run(ctx context.Context, conf Config, binPaths ...string) (Summary, error) { 15 | // extract modules details from each supplied binary 16 | binaries, err := module.Extract(ctx, binPaths...) 17 | if err != nil { 18 | return Summary{}, err 19 | } 20 | 21 | // fetch each module - this returns pertinent details, including the OS path to the module 22 | modules, err := module.Fetch(ctx, uniqueModuleRefs(binaries)) 23 | if err != nil { 24 | return Summary{}, err 25 | } 26 | 27 | // resolve licenses based on a minimum threshold 28 | threshold := defaultThreshold 29 | if conf.Threshold != nil { 30 | threshold = *conf.Threshold 31 | } 32 | modules, err = license.Resolve(modules, threshold) 33 | if err != nil { 34 | return Summary{}, err 35 | } 36 | 37 | // apply any overrides, if configured 38 | if len(conf.Overrides) > 0 { 39 | modules = applyOverrides(modules, conf.Overrides) 40 | } 41 | 42 | // evaluate the modules and sort by path 43 | results := evaluate(conf, binaries, modules) 44 | sort.Slice(results, func(i, j int) bool { 45 | return results[i].Module.Path < results[j].Module.Path 46 | }) 47 | 48 | return Summary{ 49 | Binaries: binaries, 50 | Modules: results, 51 | }, nil 52 | } 53 | 54 | // uniqueModuleRefs returns all unique modules referenced by the supplied binaries 55 | func uniqueModuleRefs(infos []model.BuildInfo) []model.ModuleReference { 56 | unique := make(map[model.ModuleReference]struct{}) 57 | for _, res := range infos { 58 | for _, r := range res.ModuleRefs { 59 | unique[r] = struct{}{} 60 | } 61 | } 62 | 63 | refs := make([]model.ModuleReference, 0, len(unique)) 64 | for r := range unique { 65 | refs = append(refs, r) 66 | } 67 | 68 | return refs 69 | } 70 | 71 | // applyOverrides replaces license information 72 | func applyOverrides(modules []model.Module, overrides []Override) []model.Module { 73 | type replacement struct { 74 | version string 75 | licenses []string 76 | } 77 | replacements := make(map[string]replacement, len(overrides)) 78 | for _, o := range overrides { 79 | replacements[o.Path] = replacement{ 80 | version: o.Version, 81 | licenses: o.Licenses, 82 | } 83 | } 84 | 85 | for i, mod := range modules { 86 | if repl, found := replacements[mod.ModuleReference.Path]; found { 87 | // if an explicit version is configured, only apply the override if the module version matches 88 | if repl.version != "" && repl.version != mod.Version { 89 | continue 90 | } 91 | mod.Licenses = make([]model.License, 0, len(repl.licenses)) 92 | for _, lic := range repl.licenses { 93 | mod.Licenses = append(mod.Licenses, model.License{ 94 | Name: lic, 95 | Confidence: 1, 96 | }) 97 | } 98 | modules[i] = mod 99 | } 100 | } 101 | 102 | return modules 103 | } 104 | 105 | // evaluate inspects each module, checking that (a) license details could be determined, and (b) licenses 106 | // are permitted by the supplied configuration. 107 | func evaluate(conf Config, binaries []model.BuildInfo, modules []model.Module) []EvaluatedModule { 108 | // build a map each module to binaries that reference them 109 | binRefs := make(map[model.ModuleReference][]string, len(modules)) 110 | for _, bin := range binaries { 111 | for _, ref := range bin.ModuleRefs { 112 | binRefs[ref] = append(binRefs[ref], bin.Path) 113 | } 114 | } 115 | 116 | // build a map of permitted licenses 117 | permitted := make(map[string]bool, len(conf.Allow)) 118 | for _, lic := range conf.Allow { 119 | permitted[lic] = true 120 | } 121 | 122 | // check each module 123 | results := make([]EvaluatedModule, 0, len(modules)) 124 | for _, mod := range modules { 125 | res := EvaluatedModule{ 126 | Module: mod, 127 | UsedBy: binRefs[mod.ModuleReference], 128 | Decision: DecisionAllowed, 129 | } 130 | if len(mod.Licenses) == 0 && !ignoreUnresolvable(conf, mod) { 131 | res.Decision = DecisionNotAllowedUnresolvableLicense 132 | } 133 | for _, lic := range mod.Licenses { 134 | if len(permitted) > 0 && !permitted[lic.Name] && !ignoreNotPermitted(conf, mod, lic) { 135 | res.Decision = DecisionNotAllowedLicenseNotPermitted 136 | res.NotPermitted = append(res.NotPermitted, lic.Name) 137 | } 138 | } 139 | results = append(results, res) 140 | } 141 | return results 142 | } 143 | 144 | func ignoreUnresolvable(conf Config, mod model.Module) bool { 145 | for _, exception := range conf.Exceptions.UnresolvableLicense { 146 | if exception.Path == mod.Path && (exception.Version == "" || exception.Version == mod.Version) { 147 | return true 148 | } 149 | } 150 | return false 151 | } 152 | 153 | func ignoreNotPermitted(conf Config, mod model.Module, lic model.License) bool { 154 | for _, exception := range conf.Exceptions.LicenseNotPermitted { 155 | if exception.Path == mod.Path && (exception.Version == "" || exception.Version == mod.Version) { 156 | if len(exception.Licenses) == 0 { 157 | return true 158 | } 159 | for _, exceptionLicense := range exception.Licenses { 160 | if exceptionLicense == lic.Name { 161 | return true 162 | } 163 | } 164 | } 165 | } 166 | return false 167 | } 168 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "io/ioutil" 8 | "log" 9 | "os" 10 | "path/filepath" 11 | "text/template" 12 | 13 | "github.com/hashicorp/go-multierror" 14 | "github.com/muesli/termenv" 15 | "github.com/urfave/cli/v2" 16 | "github.com/uw-labs/lichen/internal/scan" 17 | "gopkg.in/yaml.v2" 18 | ) 19 | 20 | const tmpl = `{{range .Modules}} 21 | {{- .Module}}: {{range $i, $_ := .Module.Licenses}}{{if $i}}, {{end}}{{.Name}}{{end}} 22 | {{- if .Allowed}} ({{ Color "#00ff00" .ExplainDecision}}){{else}} ({{ Color "#ff0000" .ExplainDecision}}){{end}} 23 | {{end}}` 24 | 25 | func main() { 26 | a := &cli.App{ 27 | Name: "lichen", 28 | Usage: "evaluate module dependencies from go compiled binaries", 29 | Flags: []cli.Flag{ 30 | &cli.StringFlag{ 31 | Name: "config", 32 | Aliases: []string{"c"}, 33 | Usage: "path to config file", 34 | }, 35 | &cli.StringFlag{ 36 | Name: "template", 37 | Aliases: []string{"t"}, 38 | Usage: "template for writing out each module and resolved licenses", 39 | Value: tmpl, 40 | }, 41 | &cli.StringFlag{ 42 | Name: "json", 43 | Aliases: []string{"j"}, 44 | Usage: "write JSON results to the supplied file", 45 | }, 46 | }, 47 | Action: run, 48 | } 49 | 50 | if err := a.Run(os.Args); err != nil { 51 | log.Fatal(err) 52 | } 53 | } 54 | 55 | func run(c *cli.Context) error { 56 | if c.NArg() == 0 { 57 | _ = cli.ShowAppHelp(c) 58 | return errors.New("path to at least one binary must be supplied") 59 | } 60 | 61 | f := termenv.TemplateFuncs(termenv.ColorProfile()) 62 | output, err := template.New("output").Funcs(f).Parse(c.String("template")) 63 | if err != nil { 64 | return err 65 | } 66 | 67 | conf, err := parseConfig(c.String("config")) 68 | if err != nil { 69 | return fmt.Errorf("invalid config: %w", err) 70 | } 71 | 72 | paths, err := absolutePaths(c.Args().Slice()) 73 | if err != nil { 74 | return fmt.Errorf("invalid arguments: %w", err) 75 | } 76 | 77 | summary, err := scan.Run(c.Context, conf, paths...) 78 | if err != nil { 79 | return fmt.Errorf("failed to evaluate licenses: %w", err) 80 | } 81 | 82 | if jsonPath := c.String("json"); jsonPath != "" { 83 | if err := writeJSON(jsonPath, summary); err != nil { 84 | return fmt.Errorf("failed to write json: %w", err) 85 | } 86 | } 87 | 88 | if err := output.Execute(os.Stdout, summary); err != nil { 89 | return fmt.Errorf("failed to write results: %w", err) 90 | } 91 | 92 | var rErr error 93 | for _, m := range summary.Modules { 94 | if !m.Allowed() { 95 | rErr = multierror.Append(rErr, fmt.Errorf("%s: %s", m.Module.ModuleReference, m.ExplainDecision())) 96 | } 97 | } 98 | return rErr 99 | } 100 | 101 | func parseConfig(path string) (scan.Config, error) { 102 | var conf scan.Config 103 | if path != "" { 104 | b, err := ioutil.ReadFile(path) 105 | if err != nil { 106 | return scan.Config{}, fmt.Errorf("failed to read file %q: %w", path, err) 107 | } 108 | if err := yaml.Unmarshal(b, &conf); err != nil { 109 | return scan.Config{}, fmt.Errorf("failed to parse yaml: %w", err) 110 | } 111 | } 112 | return conf, nil 113 | } 114 | 115 | func absolutePaths(paths []string) ([]string, error) { 116 | mapped := make([]string, len(paths)) 117 | for i, path := range paths { 118 | abs, err := filepath.Abs(path) 119 | if err != nil { 120 | return nil, fmt.Errorf("failed to get absolute path: %w", err) 121 | } 122 | mapped[i] = abs 123 | } 124 | return mapped, nil 125 | } 126 | 127 | func writeJSON(path string, summary scan.Summary) error { 128 | f, err := os.Create(path) 129 | if err != nil { 130 | return fmt.Errorf("failed to create file for json output: %w", err) 131 | } 132 | defer f.Close() 133 | if err := json.NewEncoder(f).Encode(summary); err != nil { 134 | return fmt.Errorf("json encode failed: %w", err) 135 | } 136 | return nil 137 | } 138 | --------------------------------------------------------------------------------