├── Makefile ├── .dockerignore ├── go.mod ├── Dockerfile ├── .gitignore ├── go.sum ├── LICENSE ├── .goreleaser.yaml ├── README.md ├── .github └── workflows │ └── codeql-analysis.yml └── cmd └── vl └── main.go /Makefile: -------------------------------------------------------------------------------- 1 | vl: 2 | @go build ./cmd/vl 3 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | .cache 3 | .gitignore 4 | 5 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ellisonleao/vl 2 | 3 | go 1.19 4 | 5 | require github.com/hashicorp/go-retryablehttp v0.7.1 6 | 7 | require github.com/hashicorp/go-cleanhttp v0.5.1 // indirect 8 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.18.3-stretch AS build 2 | WORKDIR / 3 | COPY . . 4 | RUN CGO_ENABLED=0 go build -ldflags='-s -w' -o /vl ./cmd/vl/main.go 5 | 6 | FROM scratch 7 | WORKDIR /bin 8 | COPY --from=build /vl /bin/vl 9 | ENTRYPOINT ["./vl"] 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # binary 15 | vl 16 | 17 | dist/ 18 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= 3 | github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= 4 | github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= 5 | github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= 6 | github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ= 7 | github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= 8 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 9 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 NpX 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | # This is an example .goreleaser.yml file with some sensible defaults. 2 | # Make sure to check the documentation at https://goreleaser.com 3 | before: 4 | hooks: 5 | # You may remove this if you don't use go modules. 6 | - go mod tidy 7 | builds: 8 | - env: 9 | - CGO_ENABLED=0 10 | goos: 11 | - linux 12 | - windows 13 | - darwin 14 | main: ./cmd/vl 15 | 16 | archives: 17 | - format: tar.gz 18 | # this name template makes the OS and Arch compatible with the results of uname. 19 | name_template: >- 20 | {{ .ProjectName }}_ 21 | {{- title .Os }}_ 22 | {{- if eq .Arch "amd64" }}x86_64 23 | {{- else if eq .Arch "386" }}i386 24 | {{- else }}{{ .Arch }}{{ end }} 25 | {{- if .Arm }}v{{ .Arm }}{{ end }} 26 | # use zip for windows archives 27 | format_overrides: 28 | - goos: windows 29 | format: zip 30 | checksum: 31 | name_template: 'checksums.txt' 32 | snapshot: 33 | name_template: "{{ incpatch .Version }}-next" 34 | changelog: 35 | sort: asc 36 | filters: 37 | exclude: 38 | - '^docs:' 39 | - '^test:' 40 | 41 | # The lines beneath this are called `modelines`. See `:help modeline` 42 | # Feel free to remove those if you don't want/use them. 43 | # yaml-language-server: $schema=https://goreleaser.com/static/schema.json 44 | # vim: set ts=2 sw=2 tw=0 fo=cnqoj 45 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # verify-links 2 | 3 | CLI tool that helps verify current status of URIs in text files 4 | 5 | # Table of Contents 6 | 7 | - [Prerequisites](#prerequisites) 8 | - [Installing](#installing) 9 | - [Usage](#usage) 10 | - [Common usage](#common-usage) 11 | - [Flags](#flags) 12 | - [Running with docker](#running-with-docker) 13 | - [Screenshots](#screenshots) 14 | - [Terminal output](#terminal-output) 15 | - [Github Action](#github-action) 16 | - [Running in a Github Action](#running-in-a-github-action) 17 | 18 | # Prerequisites 19 | 20 | - Golang 1.16 21 | 22 | # Installing 23 | 24 | ``` 25 | go install github.com/ellisonleao/vl/cmd/vl@latest 26 | ``` 27 | 28 | # Usage 29 | 30 | ## Common usage 31 | 32 | ``` 33 | $ vl FILE 34 | ``` 35 | 36 | ## Flags 37 | 38 | ``` 39 | -a (Skip status codes list) 40 | ``` 41 | 42 | Example: 43 | 44 | ```sh 45 | $ vl README.md -a 500,404 46 | ``` 47 | 48 | All `500` and `404` errors will be ignored and not considered as errors 49 | 50 | ``` 51 | -t (timeout for each request) 52 | ``` 53 | 54 | Example: 55 | 56 | ```sh 57 | $ vl README.md -t 30s 58 | ``` 59 | 60 | Each request that takes more than 30s will be considered as an error. The values 61 | accepted must be under the durations values. Some examples 62 | [here](https://golang.org/pkg/time/#ParseDuration) 63 | 64 | ``` 65 | -w Whitelist URIs 66 | ``` 67 | 68 | Example: 69 | 70 | ```sh 71 | $ vl README.md -w server1.com,server2.com 72 | ``` 73 | 74 | Adds a list of whitelisted links that shouldn't be verified. Links must be exactly 75 | passed as they are in the text file 76 | 77 | ## Running with docker 78 | 79 | ``` 80 | $ docker run -it --rm -v $PWD:/vl ellisonleao/vl /vl/yourfile.md 81 | ``` 82 | 83 | # Screenshots 84 | 85 | ## Terminal output 86 | 87 | _terminal colors are only working in linux_ 88 | 89 | ![](https://i.postimg.cc/xqD8YDfz/Screenshot-from-2021-03-18-17-42-31.png) 90 | 91 | ## Github Action 92 | 93 | ![](https://i.postimg.cc/VNpd4bxg/Screenshot-from-2021-03-18-18-29-21.png) 94 | 95 | # Running in a Github Action 96 | 97 | An example of a workflow file: 98 | 99 | ```yaml 100 | --- 101 | name: CI 102 | on: [push, pull_request] 103 | 104 | jobs: 105 | test: 106 | runs-on: ubuntu-latest 107 | steps: 108 | - uses: actions/setup-go@v3 109 | id: go 110 | with: 111 | go-version: '>=1.17.0' 112 | 113 | - uses: actions/checkout@v3 114 | 115 | - run: go install github.com/ellisonleao/vl/cmd/vl@latest 116 | - run: vl README.md 117 | ``` 118 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ main ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ main ] 20 | schedule: 21 | - cron: '30 20 * * 6' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'go' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://git.io/codeql-language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v2 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v1 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 52 | 53 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 54 | # If this step fails, then you should remove it and run the build manually (see below) 55 | - name: Autobuild 56 | uses: github/codeql-action/autobuild@v1 57 | 58 | # ℹ️ Command-line programs to run using the OS shell. 59 | # 📚 https://git.io/JvXDl 60 | 61 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 62 | # and modify them (or add more) to build your code if your project 63 | # uses a compiled language 64 | 65 | #- run: | 66 | # make bootstrap 67 | # make release 68 | 69 | - name: Perform CodeQL Analysis 70 | uses: github/codeql-action/analyze@v1 71 | -------------------------------------------------------------------------------- /cmd/vl/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto/tls" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "math/rand" 9 | "net/http" 10 | "os" 11 | "regexp" 12 | "strconv" 13 | "strings" 14 | "time" 15 | 16 | "github.com/hashicorp/go-retryablehttp" 17 | ) 18 | 19 | var ( 20 | urlRE = regexp.MustCompile(`https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9]{1,6}\b([-a-zA-Z0-9!@:%_\+.~#?&\/\/=$]*)`) 21 | skipStatus = flag.String("a", "", "-a 500,400") 22 | timeout = flag.Duration("t", 10*time.Second, "-t 10s or -t 1h") 23 | whitelist = flag.String("w", "", "-w server1.com,server2.com") 24 | s = rand.NewSource(time.Now().Unix()) 25 | ) 26 | 27 | var ( 28 | errorColor = "\033[1;31m%d\033[0m" 29 | errorStrColor = "\033[1;31m%s\033[0m" 30 | okColor = "\033[1;32m%d\033[0m" 31 | okStrColor = "\033[1;32m%s\033[0m" 32 | debugColor = "\033[1;36m%d\033[0m" 33 | ) 34 | 35 | type response struct { 36 | URL string 37 | Response *http.Response 38 | Err error 39 | } 40 | 41 | func main() { 42 | flag.Parse() 43 | 44 | args := flag.Args() 45 | if len(args) == 0 { 46 | log.Fatal("filename is required") 47 | } 48 | 49 | // read file 50 | file, err := os.ReadFile(args[0]) 51 | if err != nil { 52 | log.Fatalf("error on reading file: %v", err) 53 | } 54 | 55 | // validate skipStatus 56 | var ( 57 | skipped []int 58 | skippedURIs []string 59 | ) 60 | if len(*skipStatus) > 0 { 61 | splitted := strings.Split(*skipStatus, ",") 62 | for _, item := range splitted { 63 | val, err := strconv.Atoi(item) 64 | if err != nil { 65 | log.Fatalf("could not parse skip status value: %v \n", err) 66 | } 67 | skipped = append(skipped, val) 68 | } 69 | } 70 | 71 | // validate whitelist 72 | var whitelisted []string 73 | if len(*whitelist) > 0 { 74 | whitelisted = strings.Split(*whitelist, ",") 75 | } 76 | 77 | tr := &http.Transport{ 78 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 79 | } 80 | 81 | matches := urlRE.FindAllString(string(file), -1) 82 | client := retryablehttp.NewClient() 83 | client.RetryMax = 10 84 | client.RetryWaitMax = 10 * time.Second 85 | client.HTTPClient.Timeout = *timeout 86 | client.Logger = nil 87 | client.HTTPClient.Transport = tr 88 | 89 | results := make(chan *response) 90 | 91 | // producer 92 | counter := 0 93 | for _, url := range matches { 94 | u := url 95 | if matchWhitelisted(u, whitelisted) { 96 | continue 97 | } 98 | counter++ 99 | go worker(u, results, client) 100 | } 101 | fmt.Printf("Found %d URIs\n", len(matches)) 102 | 103 | totalErrors := 0 104 | for counter > 0 { 105 | resp := <-results 106 | counter-- 107 | if resp.Err != nil && resp.Response == nil { 108 | fmt.Printf("[%s] %s\n", fmt.Sprintf(errorStrColor, "ERROR"), resp.Err.Error()) 109 | totalErrors++ 110 | continue 111 | } 112 | 113 | shouldSkipURL := len(skipped) > 0 && isIn(resp.Response.StatusCode, skipped) 114 | statusColor := okColor 115 | if resp.Response.StatusCode > http.StatusBadRequest && !shouldSkipURL { 116 | statusColor = errorColor 117 | totalErrors++ 118 | } else if shouldSkipURL { 119 | statusColor = debugColor 120 | skippedURIs = append(skippedURIs, resp.URL) 121 | } 122 | 123 | fmt.Printf("[%s] %s \n", fmt.Sprintf(statusColor, resp.Response.StatusCode), resp.URL) 124 | } 125 | 126 | if len(whitelisted) > 0 { 127 | fmt.Println("Whitelisted URIs:") 128 | for _, wl := range whitelisted { 129 | fmt.Printf("- %s \n", fmt.Sprintf(okStrColor, wl)) 130 | } 131 | } 132 | 133 | if len(skippedURIs) > 0 { 134 | fmt.Printf("Skipped URIs with status %v: \n", skipped) 135 | for _, sk := range skippedURIs { 136 | fmt.Printf("- %s \n", fmt.Sprintf(okStrColor, sk)) 137 | } 138 | } 139 | 140 | if totalErrors > 0 { 141 | fmt.Printf("Total Errors: %s \n", fmt.Sprintf(errorColor, totalErrors)) 142 | os.Exit(1) 143 | } 144 | } 145 | 146 | func newRequest(url string) (*retryablehttp.Request, error) { 147 | userAgents := []string{ 148 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15", 149 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0", 150 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36", 151 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:77.0) Gecko/20100101 Firefox/77.0", 152 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36", 153 | } 154 | 155 | req, err := retryablehttp.NewRequest("GET", url, nil) 156 | if err != nil { 157 | return nil, err 158 | } 159 | 160 | userAgent := userAgents[rand.Intn(len(userAgents))] 161 | 162 | req.Header.Add("User-Agent", userAgent) 163 | 164 | return req, err 165 | } 166 | 167 | func worker(url string, results chan<- *response, client *retryablehttp.Client) { 168 | var err error 169 | 170 | response := &response{ 171 | URL: url, 172 | } 173 | 174 | req, err := newRequest(url) 175 | if err != nil { 176 | response.Err = err 177 | return 178 | } 179 | 180 | resp, err := client.Do(req) 181 | response.Response = resp 182 | response.Err = err 183 | results <- response 184 | } 185 | 186 | func isIn(val int, values []int) bool { 187 | for _, i := range values { 188 | if i == val { 189 | return true 190 | } 191 | } 192 | return false 193 | } 194 | 195 | func matchWhitelisted(uri string, urls []string) bool { 196 | for _, url := range urls { 197 | if strings.Contains(uri, url) { 198 | return true 199 | } 200 | } 201 | return false 202 | } 203 | --------------------------------------------------------------------------------