├── .github
└── workflows
│ ├── lint.yml
│ ├── race.yml
│ ├── release.yml
│ └── test.yml
├── .gitignore
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── FAILURES.md
├── LICENSE.md
├── Makefile
├── README.md
├── cmd
└── license-detector
│ ├── main.go
│ └── main_test.go
├── go.mod
├── go.sum
└── licensedb
├── analysis.go
├── api
└── api.go
├── dataset.projects.gz
├── dataset.zip
├── dataset_test.go
├── filer
├── filer.go
├── filer_test.go
└── test_data
│ ├── git
│ ├── COMMIT_EDITMSG
│ ├── HEAD
│ ├── config
│ ├── description
│ ├── info
│ │ └── exclude
│ ├── logs
│ │ ├── HEAD
│ │ └── refs
│ │ │ └── heads
│ │ │ └── master
│ ├── objects
│ │ ├── 33
│ │ │ └── 4a82b19a7c893d3807ea52ba35ff2170c296cc
│ │ ├── 8a
│ │ │ └── 9b00e3e4f0af606178510c19f40c5a77adb881
│ │ ├── cc
│ │ │ └── 628ccd10742baea8241c5924df992b5c019f71
│ │ ├── ce
│ │ │ └── 013625030ba8dba906f756967f9e9ca394464a
│ │ └── f7
│ │ │ └── 922e986704f99de62ca715d4794324a32e9af2
│ └── refs
│ │ └── heads
│ │ └── master
│ ├── local.zip
│ └── local
│ ├── one
│ └── two
│ └── three
├── internal
├── assets
│ ├── bindata.go
│ ├── extract_names.go
│ └── extract_urls.go
├── db.go
├── fastlog
│ ├── fastlog.go
│ └── fastlog_test.go
├── investigation.go
├── nlp.go
├── nlp_test.go
├── normalize
│ ├── normalize.go
│ └── normalize_test.go
├── processors
│ ├── html2text.go
│ ├── html2text_test.go
│ └── markup.go
└── wmh
│ ├── test_data
│ └── wmh.bin
│ ├── wmh.go
│ └── wmh_test.go
├── licensedb.go
├── licensedb_test.go
└── scan_file_content.go
/.github/workflows/lint.yml:
--------------------------------------------------------------------------------
1 | on: [push, pull_request]
2 | name: Lint
3 | jobs:
4 | lint:
5 | runs-on: ubuntu-latest
6 | steps:
7 | - uses: actions/setup-go@v3
8 | with:
9 | go-version: '>=1.18.0'
10 | - uses: actions/checkout@v3
11 | - uses: golangci/golangci-lint-action@v3
12 | with:
13 | version: latest
14 | only-new-issues: true
15 |
--------------------------------------------------------------------------------
/.github/workflows/race.yml:
--------------------------------------------------------------------------------
1 | on: [push, pull_request]
2 | name: Race
3 | jobs:
4 | race:
5 | runs-on: ubuntu-latest
6 | steps:
7 | - name: Install Go
8 | uses: actions/setup-go@v3
9 | with:
10 | go-version: '1.18'
11 | - name: Checkout code
12 | uses: actions/checkout@v2
13 | - name: Test race
14 | run: go test -v -race -timeout 60m ./...
15 |
--------------------------------------------------------------------------------
/.github/workflows/release.yml:
--------------------------------------------------------------------------------
1 | name: Release
2 | on:
3 | release:
4 | types:
5 | - created
6 |
7 | jobs:
8 | release:
9 | strategy:
10 | fail-fast: false
11 | matrix:
12 | goos: [linux, darwin]
13 | runs-on: ubuntu-latest
14 | steps:
15 | - name: Checkout code
16 | uses: actions/checkout@v2
17 |
18 | - name: Release binaries
19 | uses: mcuadros/go-release-action@master
20 | with:
21 | project_path: cmd/license-detector
22 | binary_name: license-detector
23 | github_token: ${{ secrets.GITHUB_TOKEN }}
24 | goversion: '1.18'
25 | goos: ${{ matrix.goos }}
26 | goarch: amd64
27 |
--------------------------------------------------------------------------------
/.github/workflows/test.yml:
--------------------------------------------------------------------------------
1 | on: [push, pull_request]
2 | name: Test
3 | jobs:
4 | test:
5 | strategy:
6 | fail-fast: false
7 | matrix:
8 | # Oldest supported and latest available. Anything in between should not
9 | # fail in ways that the newest does not also.
10 | go-version: ['1.18', '1.20']
11 | platform: [ubuntu-latest, macos-latest]
12 | runs-on: ${{ matrix.platform }}
13 | steps:
14 | - name: Install Go
15 | uses: actions/setup-go@v3
16 | with:
17 | go-version: ${{ matrix.go-version }}
18 |
19 | - name: Checkout code
20 | uses: actions/checkout@v2
21 |
22 | - name: Test
23 | run: go test ./...
24 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .idea
2 | .vscode
3 | # Binaries for programs and plugins
4 | *.exe
5 | *.dll
6 | *.so
7 | *.dylib
8 |
9 | # Test binary, build with `go test -c`
10 | *.test
11 |
12 | # Output of the go coverage tool, specifically when used with LiteIDE
13 | *.out
14 |
15 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
16 | .glide/
--------------------------------------------------------------------------------
/CODE_OF_CONDUCT.md:
--------------------------------------------------------------------------------
1 | # Contributor Covenant Code of Conduct
2 |
3 | ## Our Pledge
4 |
5 | In the interest of fostering an open and welcoming environment, we as
6 | contributors and maintainers pledge to making participation in our project and
7 | our community a harassment-free experience for everyone, regardless of age, body
8 | size, disability, ethnicity, gender identity and expression, level of experience,
9 | education, socio-economic status, nationality, personal appearance, race,
10 | religion, or sexual identity and orientation.
11 |
12 | ## Our Standards
13 |
14 | Examples of behavior that contributes to creating a positive environment
15 | include:
16 |
17 | * Using welcoming and inclusive language
18 | * Being respectful of differing viewpoints and experiences
19 | * Gracefully accepting constructive criticism
20 | * Focusing on what is best for the community
21 | * Showing empathy towards other community members
22 |
23 | Examples of unacceptable behavior by participants include:
24 |
25 | * The use of sexualized language or imagery and unwelcome sexual attention or
26 | advances
27 | * Trolling, insulting/derogatory comments, and personal or political attacks
28 | * Public or private harassment
29 | * Publishing others' private information, such as a physical or electronic
30 | address, without explicit permission
31 | * Other conduct which could reasonably be considered inappropriate in a
32 | professional setting
33 |
34 | ## Our Responsibilities
35 |
36 | Project maintainers are responsible for clarifying the standards of acceptable
37 | behavior and are expected to take appropriate and fair corrective action in
38 | response to any instances of unacceptable behavior.
39 |
40 | Project maintainers have the right and responsibility to remove, edit, or
41 | reject comments, commits, code, wiki edits, issues, and other contributions
42 | that are not aligned to this Code of Conduct, or to ban temporarily or
43 | permanently any contributor for other behaviors that they deem inappropriate,
44 | threatening, offensive, or harmful.
45 |
46 | ## Scope
47 |
48 | This Code of Conduct applies both within project spaces and in public spaces
49 | when an individual is representing the project or its community. Examples of
50 | representing a project or community include using an official project e-mail
51 | address, posting via an official social media account, or acting as an appointed
52 | representative at an online or offline event. Representation of a project may be
53 | further defined and clarified by project maintainers.
54 |
55 | ## Enforcement
56 |
57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be
58 | reported by contacting the project team at conduct@sourced.tech. All
59 | complaints will be reviewed and investigated and will result in a response that
60 | is deemed necessary and appropriate to the circumstances. The project team is
61 | obligated to maintain confidentiality with regard to the reporter of an incident.
62 | Further details of specific enforcement policies may be posted separately.
63 |
64 | Project maintainers who do not follow or enforce the Code of Conduct in good
65 | faith may face temporary or permanent repercussions as determined by other
66 | members of the project's leadership.
67 |
68 | ## Attribution
69 |
70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
72 |
73 | [homepage]: https://www.contributor-covenant.org
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # Contributing Guidelines
2 |
3 | go-license-detector project is [Apache licensed](LICENSE.md) and accepts
4 | contributions via GitHub pull requests. This document outlines some of the
5 | conventions on development workflow, commit message formatting, contact points,
6 | and other resources to make it easier to get your contribution accepted.
7 |
8 |
9 | ## Support Channels
10 |
11 | The official support channels, for both users and contributors, are:
12 |
13 | - GitHub [issues](https://github.com/go-enry/go-license-detector/issues)*
14 |
15 | *Before opening a new issue or submitting a new pull request, it's helpful to
16 | search the project - it's likely that another user has already reported the
17 | issue you're facing, or it's a known issue that we're already aware of.
18 |
19 |
20 | ## How to Contribute
21 |
22 | Pull Requests (PRs) are the main and exclusive way to contribute to the official go-license-detector project.
23 | In order for a PR to be accepted it needs to pass a list of requirements:
24 |
25 | - All PRs must be written in idiomatic Go, formatted according to [goimports](https://godoc.org/golang.org/x/tools/cmd/goimports), and without any warnings from [go lint](https://github.com/golang/lint) nor [go vet](https://golang.org/cmd/vet/).
26 | - New features should be generally covered with tests.
27 | - The test suite must pass.
28 | - All PRs have to pass the personal evaluation of at least one of the [maintainers](MAINTAINERS.md).
29 |
30 | ### Format of the commit message
31 |
32 | The commit summary must start with a capital letter and with a verb in present tense. No dot in the end.
33 |
34 | ```
35 | Add a feature
36 | Remove unused code
37 | Fix a bug
38 | ```
39 |
40 | Every commit details should describe what was changed, under which context and, if applicable, the GitHub issue it relates to.
--------------------------------------------------------------------------------
/FAILURES.md:
--------------------------------------------------------------------------------
1 | # License detection failures
2 |
3 | This is a list of known license detection failures. Once there appear common patterns, go-license-detector
4 | will be extended to support those.
5 |
6 | ### Clear failures
7 | * [Microsoft/vscode](https://github.com/Microsoft/vscode/blob/master/LICENSE.txt) - license file seems legit, clear failure.
8 | * [adobe/brackets](https://github.com/adobe/brackets/blob/master/LICENSE) - license file seems legit, clear failure.
9 | * [pure-css/pure](https://github.com/pure-css/pure/blob/master/LICENSE.md) - BSD license not detected, clear failure.
10 | * [TryGhost/Ghost](https://github.com/TryGhost/Ghost/blob/master/LICENSE) - MIT license, clear failure.
11 | * [nwjs/nw.js](https://github.com/nwjs/nw.js/blob/nw28/LICENSE) - MIT, clear failure.
12 | * [jenkinsci/jenkins](https://github.com/jenkinsci/jenkins/blob/master/LICENSE.txt) - MIT license, clear failure.
13 | * [kahun/awesome-sysadmin](https://github.com/kahun/awesome-sysadmin/blob/master/LICENSE.txt) - CC-BY-SA 4.0, clear failure.
14 | * [AFNetworking/AFNetworking](https://github.com/AFNetworking/AFNetworking/blob/master/LICENSE) - MIT, clear failure.
15 | * [eslint/eslint](https://github.com/eslint/eslint/blob/master/LICENSE) - MIT, clear failure.
16 | * [CocoaLumberjack/CocoaLumberjack](https://github.com/CocoaLumberjack/CocoaLumberjack/blob/master/LICENSE.txt) - BSD, clear failure.
17 | * [linnovate/mean](https://github.com/linnovate/mean/blob/master/LICENSE) - MIT with broken lines, clear failure.
18 | * [bcit-ci/CodeIgniter](https://github.com/bcit-ci/CodeIgniter/blob/develop/license.txt) - MIT, clear failure.
19 | * [moment/moment](https://github.com/moment/moment/blob/develop/LICENSE) - MIT, clear failure.
20 | * [webpack/webpack](https://github.com/webpack/webpack/blob/master/LICENSE) - MIT, clear failure.
21 | * [Alamofire/Alamofire](https://github.com/Alamofire/Alamofire/blob/master/LICENSE) - MIT, clear failure.
22 |
23 | ### Custom file names
24 | * [atech/postal](https://github.com/atech/postal) - the file is `MIT-LICENCE`.
25 | * [rust-lang/rust](https://github.com/rust-lang/rust) - license files: `LICENSE-APACHE` and `LICENSE-MIT`; also the `README` tells about them.
26 | * [ariya/phantomjs](https://github.com/ariya/phantomjs/blob/master/LICENSE.BSD) - the file is `LICENSE.BSD`.
27 | * [mpv-player/mpv](https://github.com/mpv-player/mpv) - `LICENSE.GPL` and `LICENSE.LGPL`.
28 | * [v8/v8](https://github.com/v8/v8) - `LICENSE.v8` and 4 other custom suffixes; `LICENSE` is a concatenation.
29 | * [philc/vimium](https://github.com/philc/vimium/blob/master/MIT-LICENSE.txt) - `MIT-LICENSE.txt`.
30 | * [torch/torch7](https://github.com/torch/torch7/blob/master/COPYRIGHT.txt) - `COPYRIGHT.txt` with custom header.
31 | * [mathiasbynens/dotfiles](https://github.com/mathiasbynens/dotfiles/blob/master/LICENSE-MIT.txt) - the file is `LICENSE-MIT.txt`.
32 | * [Marak/faker.js](https://github.com/Marak/faker.js/blob/master/MIT-LICENSE.txt) - file is `MIT-LICENSE.txt`, custom header.
33 | * [gionkunz/chartist-js](https://github.com/gionkunz/chartist-js/blob/develop/LICENSE-MIT) - the file is `LICENSE-MIT`.
34 | * [php-fig/fig-standards](https://github.com/php-fig/fig-standards) - the file names are `LICENSE-CC.md` and `LICENSE-MIT.md`.
35 | * [VundleVim/Vundle.vim](https://github.com/VundleVim/Vundle.vim/blob/master/LICENSE-MIT.txt) - the file name is `LICENSE-MIT.txt`.
36 | * [carhartl/jquery-cookie](https://github.com/carhartl/jquery-cookie/blob/master/MIT-LICENSE.txt) - the file name is `MIT-LICENSE.txt`.
37 | * [JetBrains/kotlin](https://github.com/JetBrains/kotlin/tree/master/license) - `license` directory, the standard file there points to Apache.
38 |
39 | ### Pointers
40 | * [akullpp/awesome-java](https://github.com/akullpp/awesome-java/blob/master/LICENSE.md) - file consists of the single reference to CC-BY-SA-4.0.
41 | * [Unitech/pm2](https://github.com/Unitech/pm2/blob/master/LICENSE) - the whole content is the name of the real license file.
42 | * [ruanyf/es6tutorial](https://github.com/ruanyf/es6tutorial/blob/gh-pages/LICENSE) - human-readable summary of CC-BY-NC 4.0.
43 | * [lukasz-madon/awesome-remote-job](https://github.com/lukasz-madon/awesome-remote-job) - `README` contains a link to CC0.
44 | * [sindresorhus/quick-look-plugins](https://github.com/sindresorhus/quick-look-plugins) - `README` points to CC0.
45 | * [MaximAbramchuck/awesome-interview-questions](https://github.com/MaximAbramchuck/awesome-interview-questions) - `README` states CC0.
46 | * [enaqx/awesome-react](https://github.com/enaqx/awesome-react#license) - `README` states CC0.
47 | * [sorrycc/awesome-javascript](https://github.com/sorrycc/awesome-javascript#license) - `README` points to CC0.
48 | * [vuejs/awesome-vue](https://github.com/vuejs/awesome-vue) - CC0 is appended to the end of `README`.
49 | * [terryum/awesome-deep-learning-papers](https://github.com/terryum/awesome-deep-learning-papers) - `README` points to CC0.
50 | * [gztchan/awesome-design](https://github.com/gztchan/awesome-design) - `README` points to CC0.
51 | * [sindresorhus/awesome-electron](https://github.com/sindresorhus/awesome-electron) - `README` points to CC0.
52 | * [donnemartin/system-design-primer](https://github.com/donnemartin/system-design-primer/blob/master/LICENSE.txt) - custom text, link to CC-BY 4.0.
53 | * [vinta/awesome-python](https://github.com/vinta/awesome-python/blob/master/LICENSE) - points to CC-BY 4.0.
54 | * [EbookFoundation/free-programming-books](https://github.com/EbookFoundation/free-programming-books/blob/master/LICENSE) - points to CC-BY 4.0.
55 | * [kamranahmedse/design-patterns-for-humans](https://github.com/kamranahmedse/design-patterns-for-humans#license) - `README` points to CC-BY 4.0.
56 | * [interagent/http-api-design](https://github.com/interagent/http-api-design/blob/master/LICENSE.md) - license file points to CC-BY 3.0.
57 | * [kamranahmedse/developer-roadmap](https://github.com/kamranahmedse/developer-roadmap) - `README` points to CC-BY 3.0.
58 | * [saltstack/salt](https://github.com/saltstack/salt/blob/develop/LICENSE) - custom text, link to Apache.
59 | * [inconshreveable/ngrok](https://github.com/inconshreveable/ngrok/blob/master/LICENSE) - points to Apache.
60 | * [google/python-fire](https://github.com/google/python-fire/blob/master/LICENSE) - points to Apache.
61 | * [swagger-api/swagger-ui](https://github.com/swagger-api/swagger-ui/blob/master/LICENSE) - points to Apache.
62 | * [SignalR/SignalR](https://github.com/SignalR/SignalR/blob/dev/LICENSE.txt) - points to Apache.
63 | * [facebookarchive/three20](https://github.com/facebookarchive/three20/blob/master/LICENSE) - points to Apache.
64 | * [atlassian/localstack](https://github.com/atlassian/localstack/blob/master/LICENSE.txt) - points to Apache.
65 | * [dmlc/xgboost](https://github.com/dmlc/xgboost/blob/master/LICENSE) - points to Apache.
66 | * [videojs/video.js](https://github.com/videojs/video.js/blob/master/LICENSE) - points to Apache.
67 | * [alibaba/druid](https://github.com/alibaba/druid/blob/master/license.txt) - points to Apache.
68 | * [astaxie/beego](https://github.com/astaxie/beego/blob/master/LICENSE) - points to Apache.
69 | * [requests/requests](https://github.com/requests/requests/blob/master/LICENSE) - points to Apache.
70 | * [alibaba/fastjson](https://github.com/alibaba/fastjson/blob/master/license.txt) - file points to Apache.
71 | * [Reactive-Extensions/RxJS](https://github.com/Reactive-Extensions/RxJS/blob/master/license.txt) - points to Apache.
72 | * [lord/slate](https://github.com/lord/slate/blob/master/LICENSE) - points to Apache.
73 | * [donnemartin/data-science-ipython-notebooks](https://github.com/donnemartin/data-science-ipython-notebooks/blob/master/LICENSE) - points to Apache.
74 | * [donnemartin/interactive-coding-challenges](https://github.com/donnemartin/interactive-coding-challenges/blob/master/LICENSE) - points to Apache.
75 | * [SFTtech/openage](https://github.com/SFTtech/openage) - `README` mentions GNU GPLv3.
76 | * [FFmpeg/FFmpeg](https://github.com/FFmpeg/FFmpeg/blob/master/LICENSE.md) - custom license text, points to LGPL and GPL.
77 | * [androidannotations/androidannotations](https://github.com/androidannotations/androidannotations/blob/develop/LICENSE.txt) - license points to Apache and CDDL.
78 | * [Theano/Theano](https://github.com/Theano/Theano/blob/master/LICENSE.txt) - symlink to `doc/LICENSE.txt`.
79 | * [mxgmn/WaveFunctionCollapse](https://github.com/mxgmn/WaveFunctionCollapse/blob/master/LICENSE.md) - custom text, points to MIT.
80 | * [marionettejs/backbone.marionette](https://github.com/marionettejs/backbone.marionette/blob/master/license.txt) - custom text, mentions MIT, links to http://mutedsolutions.mit-license.org/.
81 | * [date-fns/date-fns](https://github.com/date-fns/date-fns/blob/master/LICENSE.md) - custom text, points to http://kossnocorp.mit-license.org/.
82 | * [shadowsocks/shadowsocks-android](https://github.com/shadowsocks/shadowsocks-android/blob/master/LICENSE) - shortened GPL.
83 | * [mozilla/BrowserQuest](https://github.com/mozilla/BrowserQuest/blob/master/LICENSE) - file points to MPL and CC-BY-SA 3.0.
84 |
85 | ### Headers and footers
86 | * [Carthage/Carthage](https://github.com/Carthage/Carthage/blob/master/LICENSE.md) - extra content with copyright at the bottom of MIT license.
87 | * [serverless/serverless](https://github.com/serverless/serverless/blob/master/LICENSE.txt) - MIT with extra notice at the bottom.
88 | * [gitlabhq/gitlabhq](https://github.com/gitlabhq/gitlabhq/blob/master/LICENSE) - extra content at the end of MIT or BSD license.
89 | * [akveo/blur-admin](https://github.com/akveo/blur-admin/blob/master/LICENSE.txt) - MIT license with garbage in the end.
90 | * [keras-team/keras](https://github.com/keras-team/keras/blob/master/LICENSE) - MIT with many copyright notices in the beginning.
91 | * [meteor/meteor](https://github.com/meteor/meteor/blob/devel/LICENSE) - custom notice appended in the end of MIT license.
92 | * [celery/celery](https://github.com/celery/celery/blob/master/LICENSE) - BSD with custom header and footer.
93 | * [parse-community/parse-server](https://github.com/parse-community/parse-server/blob/master/LICENSE) - custom footer after BSD.
94 | * [ccgus/fmdb](https://github.com/ccgus/fmdb/blob/master/LICENSE.txt) - custom header before MIT.
95 | * [BVLC/caffe](https://github.com/BVLC/caffe/blob/master/LICENSE) - custom header and footer.
96 | * [JuliaLang/julia](https://github.com/JuliaLang/julia/blob/master/LICENSE.md) - custom header and huge footer with dependencies description.
97 | * [google/deepdream](https://github.com/google/deepdream/blob/master/LICENSE) - Apache with custom header.
98 | * [pytorch/pytorch](https://github.com/pytorch/pytorch/blob/master/LICENSE) - custom header before BSD.
99 | * [strongloop/loopback](https://github.com/strongloop/loopback/blob/master/LICENSE) - MIT with custom header.
100 | * [aosabook/500lines](https://github.com/aosabook/500lines/blob/master/LICENSE.md) - custom format, BSD in the end.
101 | * [janpaepke/ScrollMagic](https://github.com/janpaepke/ScrollMagic/blob/master/LICENSE.md) - MIT with custom header, mentions GPL.
102 | * [google/protobuf](https://github.com/google/protobuf/blob/master/LICENSE) - BSD with custom header.
103 | * [rapid7/metasploit-framework](https://github.com/rapid7/metasploit-framework/blob/master/COPYING) - BSD with custom footer.
104 | * [ipython/ipython](https://github.com/ipython/ipython/blob/master/COPYING.rst) - custom format, huge footer.
105 | * [jquery/jquery](https://github.com/jquery/jquery/blob/master/LICENSE.txt) - JQuery with custom footer (he-he).
106 | * [hapijs/hapi](https://github.com/hapijs/hapi/blob/master/LICENSE) - BSD with custom header and especially footer.
107 | * [sqlmapproject/sqlmap](https://github.com/sqlmapproject/sqlmap/blob/master/LICENSE) - GPL with custom header.
108 |
109 | ### Concatenations
110 | * [nodejs/node](https://github.com/nodejs/node/blob/master/LICENSE) - multiple licenses in the same file.
111 | * [chrissimpkins/Hack](https://github.com/source-foundry/Hack/blob/master/LICENSE.md) - multiple licenses in the same file.
112 | * [shadowsocks/shadowsocks-windows](https://github.com/shadowsocks/shadowsocks-windows/blob/master/LICENSE.txt) - multiple licenses concatenated together.
113 | * [Tencent/mars](https://github.com/Tencent/mars/blob/master/LICENSE) - many licenses concatenated, custom header.
114 | * [lodash/lodash](https://github.com/lodash/lodash/blob/master/LICENSE) - several licenses concatenated.
115 | * [libuv/libuv](https://github.com/libuv/libuv/blob/v1.x/LICENSE) - several licenses concatenated together.
116 | * [iview/iview](https://github.com/iview/iview/blob/2.0/LICENSE) - several MIT licenses concatenated together.
117 | * [fatih/vim-go](https://github.com/fatih/vim-go/blob/master/LICENSE) - two BSDs concatenated together, custom header, middle and bottom.
118 | * [bumptech/glide](https://github.com/bumptech/glide/blob/master/LICENSE) - concatenated licenses.
119 | * [sqlitebrowser/sqlitebrowser](https://github.com/sqlitebrowser/sqlitebrowser/blob/master/LICENSE) - GPL and MPL concatenated.
120 | * [Mantle/Mantle](https://github.com/Mantle/Mantle/blob/master/LICENSE.md) - two licenses concatenated.
121 | * [kripken/emscripten](https://github.com/kripken/emscripten/blob/incoming/LICENSE) - several licenses concatenated, custom header and footer.
122 | * [browserify/browserify](https://github.com/browserify/browserify/blob/master/LICENSE) - several licenses concatenated.
123 | * [aFarkas/html5shiv](https://github.com/aFarkas/html5shiv/blob/master/MIT%20and%20GPL2%20licenses.md) - the file is `MIT and GPL2 licenses.md` and it is a concatenation.
124 | * [Microsoft/CNTK](https://github.com/Microsoft/CNTK/blob/master/LICENSE.md) - custom format, concatenation.
125 | * [stedolan/jq](https://github.com/stedolan/jq/blob/master/COPYING) - concatenation of several licenses.
126 | * [jquery/jquery-ui](https://github.com/jquery/jquery-ui/blob/master/LICENSE.txt) - concatenation of JQuery and CC0.
127 | * [realm/realm-cocoa](https://github.com/realm/realm-cocoa/blob/master/LICENSE) - Apache concatenated with other content.
128 | * [robbiehanson/CocoaAsyncSocket](https://github.com/robbiehanson/CocoaAsyncSocket/blob/master/LICENSE.txt) - public domain and BSD, custom format.
129 | * [jquery/jquery-mobile](https://github.com/jquery/jquery-mobile/blob/master/LICENSE.txt) - concatenation of JQuery and CC0.
130 |
131 | ### Die hards
132 | * [RubaXa/Sortable](https://github.com/RubaXa/Sortable) - license is appended to the end of `README.md`.
133 | * [Automattic/mongoose](https://github.com/Automattic/mongoose) - BSD license is appended to the end of `README`.
134 | * [mperham/sidekiq](https://github.com/mperham/sidekiq/blob/master/LICENSE) - license file is completely custom, mentions LGPL.
135 | * [opencv/opencv](https://github.com/opencv/opencv/blob/master/LICENSE) - license is completely custom format but resembles a BSD.
136 | * [python/cpython](https://github.com/python/cpython/blob/master/LICENSE) - license is PSF 2.
137 | * [tmux/tmux](https://github.com/tmux/tmux) - custom `COPYING`, BSD is mentioned in `README`.
138 | * [facebook/nuclide](https://github.com/facebook/nuclide/blob/master/LICENSE) - custom license? resembles BSD.
139 | * [realm/realm-java](https://github.com/realm/realm-java/blob/master/LICENSE) - custom format, Apache concatenated with various notices.
140 | * [phanan/htaccess](https://github.com/phanan/htaccess/blob/master/LICENSE) - custom, says something about public domain and unlicense.
141 | * [cockroachdb/cockroach](https://github.com/cockroachdb/cockroach/blob/master/LICENSE) - custom text, mentions Apache, CCL, MIT and BSD,
142 | * [fbsamples/f8app](https://github.com/fbsamples/f8app/blob/master/LICENSE) - customized MIT.
143 | * [FortAwesome/Font-Awesome](https://github.com/FortAwesome/Font-Awesome/blob/master/LICENSE.txt) - custom format, mentions several licenses.
144 | * [Microsoft/api-guidelines](https://github.com/Microsoft/api-guidelines/blob/master/Guidelines.md#44-license) - pointer to CC-BY 4.0 in `Guidelines.md`.
145 | * [neovim/neovim](https://github.com/neovim/neovim/blob/master/LICENSE) - Apache with custom header and notes.
146 | * [moklick/frontend-stuff](https://github.com/moklick/frontend-stuff/blob/master/LICENSE) - summarized CC0.
147 |
148 | ### Miscellaneous
149 | * [IanLunn/Hover](https://github.com/IanLunn/Hover/blob/master/license.txt) - custom license, nothing can be done.
150 | * [xdissent/ievms](https://github.com/xdissent/ievms) - "none" license stated in the `README`.
151 | * [isocpp/CppCoreGuidelines](https://github.com/isocpp/CppCoreGuidelines/blob/master/LICENSE) - custom license.
152 | * [Modernizr/Modernizr](https://github.com/Modernizr/Modernizr) - license file is a joke, `README` mentions MIT.
153 | * [froala/design-blocks](https://github.com/froala/design-blocks/blob/dev/LICENSE) - FROALA OPEN WEB DESIGN LICENSE.
154 | * [Swordfish90/cool-retro-term](https://github.com/Swordfish90/cool-retro-term) - `gpl-3.0.txt` and `gpl-2.0.txt` files.
155 |
--------------------------------------------------------------------------------
/LICENSE.md:
--------------------------------------------------------------------------------
1 | Apache License
2 | ==============
3 |
4 | _Version 2.0, January 2004_
5 | _<>_
6 |
7 | ### Terms and Conditions for use, reproduction, and distribution
8 |
9 | #### 1. Definitions
10 |
11 | “License” shall mean the terms and conditions for use, reproduction, and
12 | distribution as defined by Sections 1 through 9 of this document.
13 |
14 | “Licensor” shall mean the copyright owner or entity authorized by the copyright
15 | owner that is granting the License.
16 |
17 | “Legal Entity” shall mean the union of the acting entity and all other entities
18 | that control, are controlled by, or are under common control with that entity.
19 | For the purposes of this definition, “control” means **(i)** the power, direct or
20 | indirect, to cause the direction or management of such entity, whether by
21 | contract or otherwise, or **(ii)** ownership of fifty percent (50%) or more of the
22 | outstanding shares, or **(iii)** beneficial ownership of such entity.
23 |
24 | “You” (or “Your”) shall mean an individual or Legal Entity exercising
25 | permissions granted by this License.
26 |
27 | “Source” form shall mean the preferred form for making modifications, including
28 | but not limited to software source code, documentation source, and configuration
29 | files.
30 |
31 | “Object” form shall mean any form resulting from mechanical transformation or
32 | translation of a Source form, including but not limited to compiled object code,
33 | generated documentation, and conversions to other media types.
34 |
35 | “Work” shall mean the work of authorship, whether in Source or Object form, made
36 | available under the License, as indicated by a copyright notice that is included
37 | in or attached to the work (an example is provided in the Appendix below).
38 |
39 | “Derivative Works” shall mean any work, whether in Source or Object form, that
40 | is based on (or derived from) the Work and for which the editorial revisions,
41 | annotations, elaborations, or other modifications represent, as a whole, an
42 | original work of authorship. For the purposes of this License, Derivative Works
43 | shall not include works that remain separable from, or merely link (or bind by
44 | name) to the interfaces of, the Work and Derivative Works thereof.
45 |
46 | “Contribution” shall mean any work of authorship, including the original version
47 | of the Work and any modifications or additions to that Work or Derivative Works
48 | thereof, that is intentionally submitted to Licensor for inclusion in the Work
49 | by the copyright owner or by an individual or Legal Entity authorized to submit
50 | on behalf of the copyright owner. For the purposes of this definition,
51 | “submitted” means any form of electronic, verbal, or written communication sent
52 | to the Licensor or its representatives, including but not limited to
53 | communication on electronic mailing lists, source code control systems, and
54 | issue tracking systems that are managed by, or on behalf of, the Licensor for
55 | the purpose of discussing and improving the Work, but excluding communication
56 | that is conspicuously marked or otherwise designated in writing by the copyright
57 | owner as “Not a Contribution.”
58 |
59 | “Contributor” shall mean Licensor and any individual or Legal Entity on behalf
60 | of whom a Contribution has been received by Licensor and subsequently
61 | incorporated within the Work.
62 |
63 | #### 2. Grant of Copyright License
64 |
65 | Subject to the terms and conditions of this License, each Contributor hereby
66 | grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
67 | irrevocable copyright license to reproduce, prepare Derivative Works of,
68 | publicly display, publicly perform, sublicense, and distribute the Work and such
69 | Derivative Works in Source or Object form.
70 |
71 | #### 3. Grant of Patent License
72 |
73 | Subject to the terms and conditions of this License, each Contributor hereby
74 | grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
75 | irrevocable (except as stated in this section) patent license to make, have
76 | made, use, offer to sell, sell, import, and otherwise transfer the Work, where
77 | such license applies only to those patent claims licensable by such Contributor
78 | that are necessarily infringed by their Contribution(s) alone or by combination
79 | of their Contribution(s) with the Work to which such Contribution(s) was
80 | submitted. If You institute patent litigation against any entity (including a
81 | cross-claim or counterclaim in a lawsuit) alleging that the Work or a
82 | Contribution incorporated within the Work constitutes direct or contributory
83 | patent infringement, then any patent licenses granted to You under this License
84 | for that Work shall terminate as of the date such litigation is filed.
85 |
86 | #### 4. Redistribution
87 |
88 | You may reproduce and distribute copies of the Work or Derivative Works thereof
89 | in any medium, with or without modifications, and in Source or Object form,
90 | provided that You meet the following conditions:
91 |
92 | * **(a)** You must give any other recipients of the Work or Derivative Works a copy of
93 | this License; and
94 | * **(b)** You must cause any modified files to carry prominent notices stating that You
95 | changed the files; and
96 | * **(c)** You must retain, in the Source form of any Derivative Works that You distribute,
97 | all copyright, patent, trademark, and attribution notices from the Source form
98 | of the Work, excluding those notices that do not pertain to any part of the
99 | Derivative Works; and
100 | * **(d)** If the Work includes a “NOTICE” text file as part of its distribution, then any
101 | Derivative Works that You distribute must include a readable copy of the
102 | attribution notices contained within such NOTICE file, excluding those notices
103 | that do not pertain to any part of the Derivative Works, in at least one of the
104 | following places: within a NOTICE text file distributed as part of the
105 | Derivative Works; within the Source form or documentation, if provided along
106 | with the Derivative Works; or, within a display generated by the Derivative
107 | Works, if and wherever such third-party notices normally appear. The contents of
108 | the NOTICE file are for informational purposes only and do not modify the
109 | License. You may add Your own attribution notices within Derivative Works that
110 | You distribute, alongside or as an addendum to the NOTICE text from the Work,
111 | provided that such additional attribution notices cannot be construed as
112 | modifying the License.
113 |
114 | You may add Your own copyright statement to Your modifications and may provide
115 | additional or different license terms and conditions for use, reproduction, or
116 | distribution of Your modifications, or for any such Derivative Works as a whole,
117 | provided Your use, reproduction, and distribution of the Work otherwise complies
118 | with the conditions stated in this License.
119 |
120 | #### 5. Submission of Contributions
121 |
122 | Unless You explicitly state otherwise, any Contribution intentionally submitted
123 | for inclusion in the Work by You to the Licensor shall be under the terms and
124 | conditions of this License, without any additional terms or conditions.
125 | Notwithstanding the above, nothing herein shall supersede or modify the terms of
126 | any separate license agreement you may have executed with Licensor regarding
127 | such Contributions.
128 |
129 | #### 6. Trademarks
130 |
131 | This License does not grant permission to use the trade names, trademarks,
132 | service marks, or product names of the Licensor, except as required for
133 | reasonable and customary use in describing the origin of the Work and
134 | reproducing the content of the NOTICE file.
135 |
136 | #### 7. Disclaimer of Warranty
137 |
138 | Unless required by applicable law or agreed to in writing, Licensor provides the
139 | Work (and each Contributor provides its Contributions) on an “AS IS” BASIS,
140 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
141 | including, without limitation, any warranties or conditions of TITLE,
142 | NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are
143 | solely responsible for determining the appropriateness of using or
144 | redistributing the Work and assume any risks associated with Your exercise of
145 | permissions under this License.
146 |
147 | #### 8. Limitation of Liability
148 |
149 | In no event and under no legal theory, whether in tort (including negligence),
150 | contract, or otherwise, unless required by applicable law (such as deliberate
151 | and grossly negligent acts) or agreed to in writing, shall any Contributor be
152 | liable to You for damages, including any direct, indirect, special, incidental,
153 | or consequential damages of any character arising as a result of this License or
154 | out of the use or inability to use the Work (including but not limited to
155 | damages for loss of goodwill, work stoppage, computer failure or malfunction, or
156 | any and all other commercial damages or losses), even if such Contributor has
157 | been advised of the possibility of such damages.
158 |
159 | #### 9. Accepting Warranty or Additional Liability
160 |
161 | While redistributing the Work or Derivative Works thereof, You may choose to
162 | offer, and charge a fee for, acceptance of support, warranty, indemnity, or
163 | other liability obligations and/or rights consistent with this License. However,
164 | in accepting such obligations, You may act only on Your own behalf and on Your
165 | sole responsibility, not on behalf of any other Contributor, and only if You
166 | agree to indemnify, defend, and hold each Contributor harmless for any liability
167 | incurred by, or claims asserted against, such Contributor by reason of your
168 | accepting any such warranty or additional liability.
169 |
170 | _END OF TERMS AND CONDITIONS_
171 |
172 | ### APPENDIX: How to apply the Apache License to your work
173 |
174 | To apply the Apache License to your work, attach the following boilerplate
175 | notice, with the fields enclosed by brackets `[]` replaced with your own
176 | identifying information. (Don't include the brackets!) The text should be
177 | enclosed in the appropriate comment syntax for the file format. We also
178 | recommend that a file or class name and description of purpose be included on
179 | the same “printed page” as the copyright notice for easier identification within
180 | third-party archives.
181 |
182 | Copyright [yyyy] [name of copyright owner]
183 |
184 | Licensed under the Apache License, Version 2.0 (the "License");
185 | you may not use this file except in compliance with the License.
186 | You may obtain a copy of the License at
187 |
188 | http://www.apache.org/licenses/LICENSE-2.0
189 |
190 | Unless required by applicable law or agreed to in writing, software
191 | distributed under the License is distributed on an "AS IS" BASIS,
192 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
193 | See the License for the specific language governing permissions and
194 | limitations under the License.
195 |
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | GOPATH ?= $(shell go env GOPATH)
2 | SPDX_DATA_VERSION ?= 3.17
3 |
4 | licensedb/internal/assets/bindata.go: licenses.tar urls.csv names.csv $(GOPATH)/bin/go-bindata
5 | rm -rf license-list-data-$(SPDX_DATA_VERSION)
6 | rm -f license-list-data.tar.gz
7 | $(GOPATH)/bin/go-bindata -nometadata -pkg assets -o licensedb/internal/assets/bindata.go licenses.tar urls.csv names.csv
8 | rm licenses.tar urls.csv names.csv
9 |
10 | licenses.tar: license-list-data.tar.gz
11 | tar -xf license-list-data.tar.gz license-list-data-$(SPDX_DATA_VERSION)/text
12 | tar -cf licenses.tar -C license-list-data-$(SPDX_DATA_VERSION)/text .
13 |
14 | license-list-data-$(SPDX_DATA_VERSION)/json/details: license-list-data.tar.gz
15 | tar -xf license-list-data.tar.gz license-list-data-$(SPDX_DATA_VERSION)/json/details
16 |
17 | urls.csv: license-list-data-$(SPDX_DATA_VERSION)/json/details
18 | go run licensedb/internal/assets/extract_urls.go license-list-data-$(SPDX_DATA_VERSION)/json/details > urls.csv
19 |
20 | names.csv: license-list-data-$(SPDX_DATA_VERSION)/json/details
21 | go run licensedb/internal/assets/extract_names.go license-list-data-$(SPDX_DATA_VERSION)/json/details > names.csv
22 |
23 | license-list-data.tar.gz:
24 | curl -SLk -o license-list-data.tar.gz https://github.com/spdx/license-list-data/archive/v$(SPDX_DATA_VERSION).tar.gz
25 |
26 | $(GOPATH)/bin/go-bindata:
27 | go get -v github.com/jteeuwen/go-bindata/go-bindata@6025e8de665b31fa74ab1a66f2cddd8c0abf887e
28 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # go-license-detector [](https://pkg.go.dev/github.com/go-enry/go-license-detector/v4@v4.0.0/licensedb) [](https://github.com/go-enry/go-license-detector/actions) [](https://goreportcard.com/badge/github.com/go-enry/go-license-detector)
2 |
3 | Project license detector - a command line application and a library, written in Go.
4 | It scans the given directory for license files, normalizes and hashes them and outputs
5 | all the fuzzy matches with the list of reference texts.
6 | The returned names follow [SPDX](https://spdx.org/licenses/) standard.
7 | Read the [blog post](https://blog.sourced.tech/post/gld/).
8 |
9 | Why? There are no similar projects which can be compiled into a native binary without
10 | dependencies and also support the whole SPDX license database (≈400 items).
11 | This implementation is also fast, requires little memory, and the API is easy to use.
12 |
13 | The license texts are taken directly from [license-list-data](https://github.com/spdx/license-list-data)
14 | repository. The detection algorithm is **not template matching**;
15 | this directly implies that go-license-detector does not provide any legal guarantees.
16 | The intended area of it's usage is data mining.
17 |
18 | ## Installation
19 |
20 | ```
21 | go get github.com/go-enry/go-license-detector/v4/licensedb
22 | ```
23 |
24 | The CLI is available for download at the [release](https://github.com/go-enry/go-license-detector/releases/latest) page.
25 |
26 | ## Algorithm
27 |
28 | 1. Find files in the root directory which may represent a license. E.g. `LICENSE` or `license.md`.
29 | 2. If the file is Markdown or reStructuredText, render to HTML and then convert to plain text. Original HTML files are also converted.
30 | 3. Normalize the text according to [SPDX recommendations](https://spdx.org/spdx-license-list/matching-guidelines).
31 | 4. Split the text into unigrams and build the weighted bag of words.
32 | 5. Calculate [Weighted MinHash](https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/36928.pdf).
33 | 6. Apply Locality Sensitive Hashing and pick the reference licenses which are close.
34 | 7. For each of the candidate, calculate the [Levenshtein distance](https://en.wikipedia.org/wiki/Levenshtein_distance) - `D`.
35 | the corresponding text is the single line with each unigram represented by a single rune (character).
36 | 8. Set the similarity as `1 - D / L` where `L` is the number of unigrams in the quieried license.
37 |
38 | This pipeline guarantees constant time queries, though requires some initialization to preprocess
39 | the reference licenses.
40 |
41 | If there are not license files found:
42 |
43 | 1. Look for README files.
44 | 2. If the file is Markdown or reStructuredText, render to HTML and then convert to plain text. Original HTML files are also converted.
45 | 3. Scan for words like "copyright", "license" and "released under". Take the neighborhood.
46 | 4. Run Named Entity Recognition (NER) over that surrounding context and extract the possible license name.
47 | 5. Match it against the list of license names from SPDX.
48 |
49 | ## Usage
50 |
51 | Command line:
52 |
53 | ```bash
54 | license-detector /path/to/project
55 | license-detector https://github.com/go-git/go-git
56 | ```
57 |
58 | Library (for a single license detection):
59 |
60 | ```go
61 | import (
62 | "github.com/go-enry/go-license-detector/v4/licensedb"
63 | "github.com/go-enry/go-license-detector/v4/licensedb/filer"
64 | )
65 |
66 | func main() {
67 | licenses, err := licensedb.Detect(filer.FromDirectory("/path/to/project"))
68 | }
69 | ```
70 |
71 | Library (for a convenient data structure that can be formatted as JSON):
72 |
73 | ```go
74 | import (
75 | "encoding/json"
76 | "fmt"
77 |
78 | "github.com/go-enry/go-license-detector/v4/licensedb"
79 | )
80 |
81 | func main() {
82 | results := licensedb.Analyse("/path/to/project1", "/path/to/project2")
83 | bytes, err := json.MarshalIndent(results, "", "\t")
84 | if err != nil {
85 | fmt.Printf("could not encode result to JSON: %v\n", err)
86 | }
87 | fmt.Println(string(bytes))
88 | }
89 | ```
90 |
91 |
92 | ## Quality
93 |
94 | On the [dataset](licensedb/dataset.zip) of ~1000 most starred repositories on GitHub as of early February 2018
95 | ([list](licensedb/dataset.projects.gz)), **99%** of the licenses are detected.
96 | The analysis of detection failures is going in [FAILURES.md](FAILURES.md).
97 |
98 | Comparison to other projects on that dataset:
99 |
100 | |Detector|Detection rate|Time to scan, sec|
101 | |:-------|:----------------------------------------:|:-----------------------------------------|
102 | |[go-license-detector](https://github.com/go-enry/go-license-detector)| 99% (897/902) | 13.5 |
103 | |[benbalter/licensee](https://github.com/benbalter/licensee)| 75% (673/902) | 111 |
104 | |[google/licenseclassifier](https://github.com/google/licenseclassifier)| 76% (682/902) | 907 |
105 | |[boyter/lc](https://github.com/boyter/lc)| 88% (797/902) | 548 |
106 | |[amzn/askalono](https://github.com/amzn/askalono)| 87% (785/902) | 165 |
107 | |[LiD](https://source.codeaurora.org/external/qostg/lid)| 94% (847/902) | 3660 |
108 |
109 | How this was measured
110 | $ cd $(go env GOPATH)/src/github.com/go-enry/go-license-detector/v4/licensedb
111 | $ mkdir dataset && cd dataset
112 | $ unzip ../dataset.zip
113 | $ # go-enry/go-license-detector
114 | $ time license-detector * \
115 | | grep -Pzo '\n[-0-9a-zA-Z]+\n\tno license' | grep -Pa '\tno ' | wc -l
116 | $ # benbalter/licensee
117 | $ time ls -1 | xargs -n1 -P4 licensee \
118 | | grep -E "^License: Other" | wc -l
119 | $ # google/licenseclassifier
120 | $ time find -type f -print | xargs -n1 -P4 identify_license \
121 | | cut -d/ -f2 | sort | uniq | wc -l
122 | $ # boyter/lc
123 | $ time lc . \
124 | | grep -vE 'NOASSERTION|----|Directory' | cut -d" " -f1 | sort | uniq | wc -l
125 | $ # amzn/askalono
126 | $ echo '#!/bin/sh
127 | result=$(askalono id "$1")
128 | echo "$1
129 | $result"' > ../askalono.wrapper
130 | $ time find -type f -print | xargs -n1 -P4 sh ../askalono.wrapper | grep -Pzo '.*\nLicense: .*\n' askalono.txt | grep -av "License: " | cut -d/ -f 2 | sort | uniq | wc -l
131 | $ # LiD
132 | $ time license-identifier -I dataset -F csv -O lid
133 | $ cat lid_*.csv | cut -d, -f1 | cut -d"'" -f 2 | grep / | cut -d/ -f2 | sort | uniq | wc -l
134 |
135 | Title
11 |
some text
12 | Another title
13 |
14 | And a third one.
15 | > < & " ' ¢ £ ¥ € © ®
16 | “ ” ‘ ’ ‚ &rbquo; „ – — •
17 | … ′ ‹ › ™ − » « ° §
18 | ¡
19 | > < & " ' ¢ £ ¥ € © ®
20 | “ ” ‘ ’ ‚ &rbquo; „ – — •
21 | … ′ ‹ › ™ − » « ° §
22 | ¡
23 | blah blah LINK text proceeds
24 | hello
25 |
26 | http://foo`
27 | assert.Equal(t, `Title
28 | some text
29 | Another title.
30 | ---
31 | And a third one.
32 | > < & " ' ¢ £ ¥ € © ®
33 | “ ” ‘ ’ ‚ " „ – — •
34 | … ′ ‹ › ™ − » « ° §
35 | ¡
36 | > < & " ' ¢ £ ¥ € © ®
37 | “ ” ‘ ’ ‚ " „ – — •
38 | … ′ ‹ › ™ − » « ° §
39 | ¡
40 | blah blah wow:// LINK text proceeds
41 |
42 | hello
43 |
44 | http://foo`, string(HTML([]byte(text))))
45 | }
46 |
--------------------------------------------------------------------------------
/licensedb/internal/processors/markup.go:
--------------------------------------------------------------------------------
1 | package processors
2 |
3 | import (
4 | "bytes"
5 | "sync"
6 |
7 | rst "github.com/hhatto/gorst"
8 | "github.com/russross/blackfriday/v2"
9 | )
10 |
11 | var (
12 | parserLock sync.Mutex
13 | )
14 |
15 | // Markdown converts Markdown to plain text. It tries to revert all the decorations.
16 | func Markdown(text []byte) []byte {
17 | html := blackfriday.Run(text)
18 | // Repeat to times to heal broken HTML
19 | return HTML(html)
20 | }
21 |
22 | // RestructuredText converts ReStructuredText to plain text.
23 | // It tries to revert all the decorations.
24 | func RestructuredText(text []byte) []byte {
25 | parserLock.Lock()
26 | defer parserLock.Unlock()
27 | parser := rst.NewParser(nil)
28 | input := bytes.NewBuffer(text)
29 | output := &bytes.Buffer{}
30 | parser.ReStructuredText(input, rst.ToHTML(output))
31 | // Repeat to times to heal broken HTML
32 | return HTML(output.Bytes())
33 | }
34 |
--------------------------------------------------------------------------------
/licensedb/internal/wmh/test_data/wmh.bin:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/go-enry/go-license-detector/e0d6f0187f3a3aaeb8236f9860337ffb92438723/licensedb/internal/wmh/test_data/wmh.bin
--------------------------------------------------------------------------------
/licensedb/internal/wmh/wmh.go:
--------------------------------------------------------------------------------
1 | package wmh
2 |
3 | import (
4 | "encoding/binary"
5 | "errors"
6 | "log"
7 | "math"
8 | "reflect"
9 | "unsafe"
10 |
11 | "github.com/go-enry/go-license-detector/v4/licensedb/internal/fastlog"
12 | "golang.org/x/exp/rand"
13 | "gonum.org/v1/gonum/stat/distuv"
14 | )
15 |
16 | const maxUint16 = 65535
17 |
18 | // WeightedMinHasher calculates Weighted MinHash-es.
19 | // https://ekzhu.github.io/datasketch/weightedminhash.html
20 | type WeightedMinHasher struct {
21 | // Size of each hash element in bits. Supported values are 16, 32 and 64.
22 | Bitness int
23 |
24 | dim int
25 | sampleSize int
26 | rs [][]float32
27 | lnCs [][]float32
28 | betas [][]uint16 // attempt to save some memory - [0, 1] is scaled to maxUint16
29 | }
30 |
31 | // NewWeightedMinHasher initializes a new instance of WeightedMinHasher.
32 | // `dim` is the bag size.
33 | // `sampleSize` is the hash length.
34 | // `seed` is the random generator seed, as Weighted MinHash is probabilistic.
35 | func NewWeightedMinHasher(dim int, sampleSize int, seed int64) *WeightedMinHasher {
36 | randSrc := rand.New(rand.NewSource(uint64(seed)))
37 | gammaGen := distuv.Gamma{Alpha: 2, Beta: 1, Src: randSrc}
38 | hasher := &WeightedMinHasher{Bitness: 64, dim: dim, sampleSize: sampleSize}
39 | hasher.rs = make([][]float32, sampleSize)
40 | for y := 0; y < sampleSize; y++ {
41 | arr := make([]float32, dim)
42 | hasher.rs[y] = arr
43 | for x := 0; x < dim; x++ {
44 | arr[x] = float32(gammaGen.Rand())
45 | }
46 | }
47 | hasher.lnCs = make([][]float32, sampleSize)
48 | for y := 0; y < sampleSize; y++ {
49 | arr := make([]float32, dim)
50 | hasher.lnCs[y] = arr
51 | for x := 0; x < dim; x++ {
52 | arr[x] = fastlog.Log(float32(gammaGen.Rand()))
53 | }
54 | }
55 | uniformGen := distuv.Uniform{Min: 0, Max: 1, Src: randSrc}
56 | hasher.betas = make([][]uint16, sampleSize)
57 | for y := 0; y < sampleSize; y++ {
58 | arr := make([]uint16, dim)
59 | hasher.betas[y] = arr
60 | for x := 0; x < dim; x++ {
61 | arr[x] = uint16(uniformGen.Rand() * maxUint16)
62 | }
63 | }
64 | return hasher
65 | }
66 |
67 | // MarshalBinary serializes the WeightedMinHasher.
68 | func (wmh *WeightedMinHasher) MarshalBinary() (data []byte, err error) {
69 | data = make([]byte, 9+wmh.sampleSize*wmh.dim*(4*2+2))
70 | data[0] = byte(wmh.Bitness)
71 | binary.LittleEndian.PutUint32(data[1:5], uint32(wmh.dim))
72 | binary.LittleEndian.PutUint32(data[5:9], uint32(wmh.sampleSize))
73 | offset := 9
74 | writeFloat32Slice := func(arr []float32) {
75 | header := (*reflect.SliceHeader)(unsafe.Pointer(&arr))
76 | header.Len *= 4
77 | header.Cap *= 4
78 | buffer := *(*[]byte)(unsafe.Pointer(header))
79 | copy(data[offset:], buffer)
80 | offset += len(buffer)
81 | }
82 | for _, arr := range wmh.rs {
83 | writeFloat32Slice(arr)
84 | }
85 | for _, arr := range wmh.lnCs {
86 | writeFloat32Slice(arr)
87 | }
88 | for _, arr := range wmh.betas {
89 | header := (*reflect.SliceHeader)(unsafe.Pointer(&arr))
90 | header.Len *= 2
91 | header.Cap *= 2
92 | buffer := *(*[]byte)(unsafe.Pointer(header))
93 | copy(data[offset:], buffer)
94 | offset += len(buffer)
95 | }
96 | return data, nil
97 | }
98 |
99 | // UnmarshalBinary reads a WeightedMinHasher previously serialized with MarshalBinary().
100 | func (wmh *WeightedMinHasher) UnmarshalBinary(data []byte) error {
101 | if len(data) < 9 {
102 | return errors.New("invalid binary format: no header")
103 | }
104 | wmh.Bitness = int(data[0])
105 | wmh.dim = int(binary.LittleEndian.Uint32(data[1:5]))
106 | wmh.sampleSize = int(binary.LittleEndian.Uint32(data[5:9]))
107 | if len(data)-9 != wmh.sampleSize*wmh.dim*(4*2+2) {
108 | return errors.New("invalid binary format: body size mismatch")
109 | }
110 | wmh.rs = make([][]float32, wmh.sampleSize)
111 | wmh.lnCs = make([][]float32, wmh.sampleSize)
112 | wmh.betas = make([][]uint16, wmh.sampleSize)
113 | readFloat32Slice := func(dest []float32, src []byte) {
114 | header := (*reflect.SliceHeader)(unsafe.Pointer(&src))
115 | header.Len /= 4
116 | header.Cap /= 4
117 | buffer := *(*[]float32)(unsafe.Pointer(header))
118 | copy(dest, buffer)
119 | }
120 | offset := 9
121 | for i := range wmh.rs {
122 | wmh.rs[i] = make([]float32, wmh.dim)
123 | nextOffset := offset + wmh.dim*4
124 | readFloat32Slice(wmh.rs[i], data[offset:nextOffset])
125 | offset = nextOffset
126 | }
127 | for i := range wmh.lnCs {
128 | wmh.lnCs[i] = make([]float32, wmh.dim)
129 | nextOffset := offset + wmh.dim*4
130 | readFloat32Slice(wmh.lnCs[i], data[offset:nextOffset])
131 | offset = nextOffset
132 | }
133 | for i := range wmh.betas {
134 | wmh.betas[i] = make([]uint16, wmh.dim)
135 | nextOffset := offset + wmh.dim*2
136 | slice := data[offset:nextOffset]
137 | header := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
138 | header.Len /= 2
139 | header.Cap /= 2
140 | buffer := *(*[]uint16)(unsafe.Pointer(header))
141 | copy(wmh.betas[i], buffer)
142 | offset = nextOffset
143 | }
144 | return nil
145 | }
146 |
147 | // Hash calculates the Weighted MinHash from the weighted bag of features.
148 | // Each feature has an index and a value.
149 | func (wmh *WeightedMinHasher) Hash(values []float32, indices []int) []uint64 {
150 | if len(values) != len(indices) {
151 | log.Panicf("len(values)=%d is not equal to len(indices)=%d", len(values), len(indices))
152 | }
153 | for i, v := range values {
154 | if v < 0 {
155 | log.Panicf("negative value in the vector: %f @ %d", v, i)
156 | }
157 | }
158 | for vi, j := range indices {
159 | if j >= wmh.dim {
160 | log.Panicf("index is out of range: %d @ %d", j, vi)
161 | }
162 | }
163 | hashvalues := make([]uint64, wmh.sampleSize)
164 | for s := 0; s < wmh.sampleSize; s++ {
165 | minLnA := float32(math.MaxFloat32)
166 | var k int
167 | var minT float32
168 | for vi, j := range indices {
169 | vlog := fastlog.Log(values[vi])
170 | beta := float32(wmh.betas[s][j]) / float32(maxUint16)
171 | // t = np.floor((vlog / self.rs[i]) + self.betas[i])
172 | t := float32(math.Floor(float64(vlog/wmh.rs[s][j] + beta)))
173 | // ln_y = (t - self.betas[i]) * self.rs[i]
174 | lnY := (t - beta) * wmh.rs[s][j]
175 | // ln_a = self.ln_cs[i] - ln_y - self.rs[i]
176 | lnA := wmh.lnCs[s][j] - lnY - wmh.rs[s][j]
177 | // k = np.nanargmin(ln_a)
178 | if lnA < minLnA {
179 | minLnA = lnA
180 | k = j
181 | minT = t
182 | }
183 | }
184 | // hashvalues[i][0], hashvalues[i][1] = k, int(t[k])
185 | switch wmh.Bitness {
186 | case 64:
187 | hashvalues[s] = uint64(uint64(k) | (uint64(minT) << 32))
188 | case 32:
189 | hashvalues[s] = uint64(uint32(k) | (uint32(minT) << 16))
190 | case 16:
191 | hashvalues[s] = uint64(uint16(k) | (uint16(minT) << 8))
192 | default:
193 | log.Fatalf("unsupported bitness value: %d", wmh.Bitness)
194 | }
195 | }
196 | return hashvalues
197 | }
198 |
--------------------------------------------------------------------------------
/licensedb/internal/wmh/wmh_test.go:
--------------------------------------------------------------------------------
1 | package wmh
2 |
3 | import (
4 | "testing"
5 |
6 | "github.com/stretchr/testify/assert"
7 | )
8 |
9 | func TestWMHSerialize(t *testing.T) {
10 | hasher := NewWeightedMinHasher(100, 50, 7)
11 | bytes, err := hasher.MarshalBinary()
12 | assert.Nil(t, err)
13 | newHasher := &WeightedMinHasher{}
14 | err = newHasher.UnmarshalBinary(bytes)
15 | assert.Nil(t, err)
16 | assert.Equal(t, hasher.Bitness, newHasher.Bitness)
17 | assert.Equal(t, hasher.dim, newHasher.dim)
18 | assert.Equal(t, hasher.sampleSize, newHasher.sampleSize)
19 | assert.Equal(t, hasher.rs, newHasher.rs)
20 | assert.Equal(t, hasher.lnCs, newHasher.lnCs)
21 | assert.Equal(t, hasher.betas, newHasher.betas)
22 | }
23 |
24 | func TestWMHHash(t *testing.T) {
25 | hasher := NewWeightedMinHasher(100, 50, 7)
26 | assert.NotNil(t, hasher)
27 | hasher.Bitness = 32
28 | hash := hasher.Hash([]float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
29 | []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90})
30 | /*
31 | import numpy, datasketch
32 | gen = datasketch.WeightedMinHashGenerator(100, 50, 7)
33 | with open("test_data/wmh.bin", "rb") as fin:
34 | fin.read(9)
35 | gen.rs = numpy.frombuffer(fin.read(100*50*4), dtype=numpy.float32).reshape(50, 100)
36 | gen.ln_cs = numpy.frombuffer(fin.read(100*50*4), dtype=numpy.float32).reshape(50, 100)
37 | betas = numpy.frombuffer(fin.read(100*50*2), dtype=numpy.uint16)
38 | gen.betas = (betas / ((1 << 16) - 1)).astype(numpy.float32).reshape(50, 100)
39 | v = numpy.zeros(100, numpy.float32)
40 | for i, ii in enumerate([0, 10, 20, 30, 40, 50, 60, 70, 80, 90]):
41 | v[ii] = i + 1
42 | mh = gen.minhash(v)
43 | for h in mh.hashvalues:
44 | print("%d," % (h[0] | (h[1] << 16)))
45 | */
46 | truth := []uint64{
47 | 65586,
48 | 0,
49 | 65626,
50 | 65616,
51 | 65626,
52 | 30,
53 | 65616,
54 | 90,
55 | 40,
56 | 65576,
57 | 65596,
58 | 65586,
59 | 65626,
60 | 65626,
61 | 589884,
62 | 20,
63 | 65616,
64 | 65626,
65 | 65596,
66 | 65626,
67 | 262234,
68 | 131152,
69 | 65596,
70 | 65596,
71 | 65556,
72 | 65626,
73 | 65576,
74 | 65606,
75 | 65626,
76 | 65606,
77 | 10,
78 | 90,
79 | 65596,
80 | 65586,
81 | 65626,
82 | 65606,
83 | 65626,
84 | 0,
85 | 131162,
86 | 65626,
87 | 65576,
88 | 65626,
89 | 65616,
90 | 65606,
91 | 65606,
92 | 131152,
93 | 65566,
94 | 65626,
95 | 65586,
96 | 65626,
97 | }
98 | assert.Equal(t, truth, hash)
99 | }
100 |
101 | func TestWMHTrash(t *testing.T) {
102 | hasher := NewWeightedMinHasher(100, 50, 7)
103 | assert.Panics(t, func() {
104 | hasher.Hash([]float32{1, 2, 3, 4, 5, 6, 7, 8, 9},
105 | []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90})
106 | })
107 | assert.Panics(t, func() {
108 | hasher.Hash([]float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
109 | []int{0, 10, 20, 30, 40, 50, 60, 70, 80})
110 | })
111 | assert.Panics(t, func() {
112 | hasher.Hash([]float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
113 | []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 100})
114 | })
115 | }
116 |
--------------------------------------------------------------------------------
/licensedb/licensedb.go:
--------------------------------------------------------------------------------
1 | package licensedb
2 |
3 | import (
4 | "errors"
5 | paths "path"
6 |
7 | "github.com/go-enry/go-license-detector/v4/licensedb/api"
8 | "github.com/go-enry/go-license-detector/v4/licensedb/filer"
9 | "github.com/go-enry/go-license-detector/v4/licensedb/internal"
10 | )
11 |
12 | var (
13 | // ErrNoLicenseFound is raised if no license files were found.
14 | ErrNoLicenseFound = errors.New("no license file was found")
15 | // ErrUnknownLicenseID is raised if license identifier is not known.
16 | // Probably you need to upgrade version of the SPDX.
17 | ErrUnknownLicenseID = errors.New("license id is not known")
18 | )
19 |
20 | // Detect returns the most probable reference licenses matched for the given
21 | // file tree. Each match has the confidence assigned, from 0 to 1, 1 means 100% confident.
22 | func Detect(fs filer.Filer) (map[string]api.Match, error) {
23 | files, err := fs.ReadDir("")
24 | if err != nil {
25 | return nil, err
26 | }
27 | fileNames := []string{}
28 | for _, file := range files {
29 | if !file.IsDir {
30 | fileNames = append(fileNames, file.Name)
31 | } else if internal.IsLicenseDirectory(file.Name) {
32 | // "license" directory, let's look inside
33 | subfiles, err := fs.ReadDir(file.Name)
34 | if err == nil {
35 | for _, subfile := range subfiles {
36 | if !subfile.IsDir {
37 | fileNames = append(fileNames, paths.Join(file.Name, subfile.Name))
38 | }
39 | }
40 | }
41 | }
42 | }
43 | candidates := internal.ExtractLicenseFiles(fileNames, fs)
44 | licenses := internal.InvestigateLicenseTexts(candidates)
45 | if len(licenses) > 0 {
46 | return licenses, nil
47 | }
48 | // Plan B: take the README, find the section about the license and apply NER
49 | candidates = internal.ExtractReadmeFiles(fileNames, fs)
50 | if len(candidates) == 0 {
51 | return nil, ErrNoLicenseFound
52 | }
53 | licenses = internal.InvestigateReadmeTexts(candidates, fs)
54 | if len(licenses) == 0 {
55 | return nil, ErrNoLicenseFound
56 | }
57 | return licenses, nil
58 | }
59 |
60 | // Preload database with licenses - load internal database from assets into memory.
61 | // This method is an optimization for cases when the `Detect` method should return fast,
62 | // e.g. in HTTP web servers where connection timeout can occur during detect
63 | // `Preload` method could be called before server startup.
64 | // This method os optional and it's not required to be called, other APIs loads license database
65 | // lazily on first invocation.
66 | func Preload() {
67 | internal.Preload()
68 | }
69 |
70 | // LicenseURLs returns the list of the URLs for the given license identifier
71 | func LicenseURLs(id string) ([]string, error) {
72 | urls, err := internal.LookupURLs(id)
73 | if err != nil {
74 | if errors.Is(err, internal.ErrUnknownLicenseID) {
75 | return nil, ErrUnknownLicenseID
76 | }
77 | return nil, err
78 | }
79 | return urls, nil
80 | }
81 |
82 | // LicenseName returns the name for the given license identifier
83 | func LicenseName(id string) (string, error) {
84 | name, err := internal.LookupName(id)
85 | if err != nil {
86 | if errors.Is(err, internal.ErrUnknownLicenseID) {
87 | return "", ErrUnknownLicenseID
88 | }
89 | return "", err
90 | }
91 | return name, nil
92 | }
93 |
--------------------------------------------------------------------------------
/licensedb/licensedb_test.go:
--------------------------------------------------------------------------------
1 | package licensedb
2 |
3 | import (
4 | "os"
5 | "path/filepath"
6 | "testing"
7 |
8 | "github.com/stretchr/testify/assert"
9 | "github.com/stretchr/testify/require"
10 |
11 | "github.com/go-enry/go-license-detector/v4/licensedb/filer"
12 | )
13 |
14 | func BenchmarkDetect(b *testing.B) {
15 | f := pwdFiler()
16 | b.ResetTimer()
17 | for i := 0; i < b.N; i++ {
18 | _, err := Detect(f)
19 | if err != nil {
20 | panic(err)
21 | }
22 | }
23 | }
24 |
25 | func BenchmarkDetectWithPreload(b *testing.B) {
26 | f := pwdFiler()
27 | Preload()
28 | b.ResetTimer()
29 | for i := 0; i < b.N; i++ {
30 | _, err := Detect(f)
31 | if err != nil {
32 | panic(err)
33 | }
34 | }
35 | }
36 |
37 | func pwdFiler() filer.Filer {
38 | pwd, err := os.Getwd()
39 | if err != nil {
40 | panic(err)
41 | }
42 | root := filepath.Dir(pwd)
43 | f, err := filer.FromDirectory(root)
44 | if err != nil {
45 | panic(err)
46 | }
47 | return f
48 | }
49 |
50 | func TestLicenseURLs(t *testing.T) {
51 | t.Run("existing license", func(t *testing.T) {
52 | res, err := LicenseURLs("ODbL-1.0")
53 | require.NoError(t, err)
54 | assert.Equal(t, []string{"http://www.opendatacommons.org/licenses/odbl/1.0/", "https://opendatacommons.org/licenses/odbl/1-0/"}, res)
55 | })
56 |
57 | t.Run("not existing license", func(t *testing.T) {
58 | _, err := LicenseURLs("bad-license-key")
59 | require.Equal(t, ErrUnknownLicenseID, err)
60 | })
61 | }
62 |
63 | func TestLicenseName(t *testing.T) {
64 | t.Run("existing license", func(t *testing.T) {
65 | res, err := LicenseName("ODbL-1.0")
66 | require.NoError(t, err)
67 | assert.Equal(t, "Open Data Commons Open Database License v1.0", res)
68 | })
69 |
70 | t.Run("not existing license", func(t *testing.T) {
71 | _, err := LicenseName("bad-license-key")
72 | require.Equal(t, ErrUnknownLicenseID, err)
73 | })
74 | }
75 |
--------------------------------------------------------------------------------
/licensedb/scan_file_content.go:
--------------------------------------------------------------------------------
1 | package licensedb
2 |
3 | import (
4 | "github.com/go-enry/go-license-detector/v4/licensedb/internal"
5 | )
6 |
7 | // InvestigateLicenseText takes the license text and returns the most probable reference licenses matched.
8 | // Each match has the confidence assigned, from 0 to 1, 1 means 100% confident.
9 | func InvestigateLicenseText(text []byte) map[string]float32 {
10 | return internal.InvestigateLicenseText(text)
11 | }
12 |
--------------------------------------------------------------------------------