├── .dockerignore
├── README.md
├── .gitignore
├── Dockerfile
├── linereader
    └── linereader.go
├── data
    └── data.go
├── checkers
    ├── links_test.go
    ├── frontmatter_test.go
    ├── frontmatter.go
    └── links.go
├── Makefile
├── main.go
└── LICENSE


/.dockerignore:
--------------------------------------------------------------------------------
 1 | Makefile
 2 | Dockerfile
 3 | .dockerignore
 4 | markdownlink.zip
 5 | markdownlink
 6 | markdownlink.app
 7 | markdownlink.exe
 8 | README.md
 9 | LICENSE
10 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | 
2 | # markdownlint
3 | 
4 | Docker documentation source checker tool
5 | 
6 | This tool ensures the markdown files its given are free from common errors.
7 | 
8 | Its used by the https://docs.docker.com project for testing PR's.
9 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects)
 2 | *.o
 3 | *.a
 4 | *.so
 5 | 
 6 | # Folders
 7 | _obj
 8 | _test
 9 | 
10 | # Architecture specific extensions/prefixes
11 | *.[568vq]
12 | [568vq].out
13 | 
14 | *.cgo1.go
15 | *.cgo2.c
16 | _cgo_defun.c
17 | _cgo_gotypes.go
18 | _cgo_export.*
19 | 
20 | _testmain.go
21 | 
22 | markdownlint
23 | *.exe
24 | *.app
25 | *.zip
26 | *.test
27 | *.prof
28 | 


--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM golang
 2 | 
 3 | # Simplify making releases
 4 | RUN apt-get update \
 5 | 	&& apt-get install -yq zip bzip2
 6 | RUN wget -O github-release.bz2 https://github.com/aktau/github-release/releases/download/v0.6.2/linux-amd64-github-release.tar.bz2 \
 7 |         && tar jxvf github-release.bz2 \
 8 |         && mv bin/linux/amd64/github-release /usr/local/bin/ \
 9 |         && rm github-release.bz2
10 | 
11 | ENV GOPATH /go
12 | ENV USER root
13 | 
14 | WORKDIR /go/src/github.com/docker/markdownlint
15 | 
16 | RUN go get github.com/russross/blackfriday
17 | RUN go get github.com/miekg/mmark
18 | 
19 | ADD . /go/src/github.com/docker/markdownlint
20 | RUN go get -d -v
21 | RUN go test -v ./...
22 | 
23 | RUN go build -o markdownlint main.go \
24 | 	&& GOOS=windows GOARCH=amd64 go build -o markdownlint.exe main.go \
25 | 	&& GOOS=darwin GOARCH=amd64 go build -o markdownlint.app main.go \
26 | 	&& zip markdownlint.zip markdownlint markdownlint.exe markdownlint.app
27 | 


--------------------------------------------------------------------------------
/linereader/linereader.go:
--------------------------------------------------------------------------------
 1 | package linereader
 2 | 
 3 | import (
 4 | 	"bufio"
 5 | 	"fmt"
 6 | 	"io/ioutil"
 7 | 	"os"
 8 | 	"strings"
 9 | )
10 | 
11 | // Fake Reader that can 'unread' a complete line
12 | type LineReader struct {
13 | 	file       *os.File
14 | 	reader     *bufio.Reader
15 | 	unreadLine string
16 | }
17 | 
18 | // For testing
19 | func ByteReader(str string) *LineReader {
20 | 	reader := strings.NewReader(str)
21 | 	r := new(LineReader)
22 | 	r.reader = bufio.NewReader(reader)
23 | 	return r
24 | }
25 | 
26 | func OpenReader(filename string) (*LineReader, error) {
27 | 	f, err := os.Open(filename)
28 | 	if err != nil {
29 | 		return nil, err
30 | 	}
31 | 	reader := bufio.NewReader(f)
32 | 	r := new(LineReader)
33 | 	r.file = f
34 | 	r.reader = reader
35 | 	return r, nil
36 | }
37 | 
38 | func (r *LineReader) ReadLine() (line []byte, isPrefix bool, err error) {
39 | 	if r.unreadLine == "" {
40 | 		return r.reader.ReadLine()
41 | 	}
42 | 	lines := strings.SplitN(r.unreadLine, "\n", 2)
43 | 	r.unreadLine = lines[1]
44 | 	return []byte(lines[0]), false, nil
45 | }
46 | 
47 | func (r *LineReader) UnreadLine(str string) {
48 | 	r.unreadLine = strings.Join([]string{str, r.unreadLine}, "\n")
49 | }
50 | 
51 | func (r *LineReader) Read(p []byte) (l int, err error) {
52 | 	offset := 0
53 | 	if r.unreadLine != "" {
54 | 		copy(p, r.unreadLine)
55 | 		offset = len(r.unreadLine)
56 | 		p[offset] = '\n'
57 | 		offset++
58 | 	}
59 | 	buff, err := ioutil.ReadAll(r.reader)
60 | 	cl := copy(p[offset:], buff[0:])
61 | 	if cl != len(buff) && err == nil {
62 | 		err = fmt.Errorf("supplied buffer (%d) too small to fit remainder of file (%d), only copied %d", len(p), offset+len(buff), cl)
63 | 	}
64 | 	l = offset + cl
65 | 	return l, err
66 | }
67 | 
68 | func (r *LineReader) Close() {
69 | 	if r.file != nil {
70 | 		r.file.Close()
71 | 	}
72 | }
73 | 


--------------------------------------------------------------------------------
/data/data.go:
--------------------------------------------------------------------------------
 1 | package data
 2 | 
 3 | import (
 4 | 	"flag"
 5 | 	"fmt"
 6 | )
 7 | 
 8 | var verbose = flag.Bool("v", false, "verbose log output")
 9 | 
10 | func ErrorLog(format string, a ...interface{}) (n int, err error) {
11 | 	return fmt.Printf("ERROR: "+format, a...)
12 | }
13 | 
14 | func VerboseLog(format string, a ...interface{}) (n int, err error) {
15 | 	if !*verbose {
16 | 		return 0, nil
17 | 	}
18 | 	return fmt.Printf(format, a...)
19 | }
20 | 
21 | // allFiles a lookup table of all the files in the 'docs' dir
22 | // also takes advantage of the random order to avoid testing markdown files in the same order.
23 | type FileDetails struct {
24 | 	FullPath         string
25 | 	Meta             map[string]string
26 | 	FormatErrors     string
27 | 	FormatErrorCount int
28 | }
29 | 
30 | func NewFileDetails(file, path string) *FileDetails {
31 | 	detail := new(FileDetails)
32 | 	detail.FullPath = path
33 | 	detail.Meta = make(map[string]string)
34 | 	detail.FormatErrors = ""
35 | 	detail.FormatErrorCount = 0
36 | 	return detail
37 | }
38 | 
39 | type BigMap map[string]*FileDetails
40 | 
41 | var AllFiles BigMap
42 | 
43 | func AddFile(file, path string) {
44 | 	if _, ok := AllFiles[file]; !ok {
45 | 		AllFiles[file] = NewFileDetails(file, path)
46 | 	}
47 | }
48 | 
49 | type LinkDetails struct {
50 | 	Count      int
51 | 	LinksFrom  map[int]string
52 | 	ActualLink map[int]string
53 | 	Response   int
54 | }
55 | 
56 | var ResponseCode = map[int]string{
57 | 	999:  "failed to parse",
58 | 	888:  "failed to crawl, ignoring",
59 | 	2900: "local file path - ok",
60 | 	900:  "mail/irc link, not checked",
61 | 	200:  "ok",
62 | 	777:  "source type path, but no match found",
63 | 	290:  "local file path, but missing `.md`",
64 | 	404:  "external url, but failed",
65 | 	666:  "Don't link to docs.docker.com",
66 | 	299:  "Skipped due to filter",
67 | }
68 | 
69 | var AllLinks map[string]*LinkDetails = make(map[string]*LinkDetails)
70 | 


--------------------------------------------------------------------------------
/checkers/links_test.go:
--------------------------------------------------------------------------------
 1 | package checkers
 2 | 
 3 | import (
 4 | 	"bytes"
 5 | 	"fmt"
 6 | 	"testing"
 7 | 
 8 | 	"github.com/docker/markdownlint/data"
 9 | 	"github.com/miekg/mmark"
10 | )
11 | 
12 | func TestMarkdownLinks(t *testing.T) {
13 | 	file := "test/index.md"
14 | 	data.AllFiles = make(map[string]*data.FileDetails)
15 | 	data.AddFile(file, file)
16 | 
17 | 	htmlFlags := 0
18 | 	renderParameters := mmark.HtmlRendererParameters{}
19 | 	renderer := &TestRenderer{
20 | 		LinkFrom: file,
21 | 		//Html:     mmark.HtmlRenderer(htmlFlags, "", "").(*mmark.Html),
22 | 		Renderer: mmark.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters),
23 | 	}
24 | 	out := bytes.NewBuffer(make([]byte, 1024))
25 | 
26 | 	tests := map[string]string{
27 | 		"../first.md":            "first.md",
28 | 		"second.md":              "test/second.md",
29 | 		"./second.md":            "test/second.md",
30 | 		"banana/second.md":       "test/banana/second.md",
31 | 		"/test/banana/second.md": "test/banana/second.md",
32 | 		"twice.md":               "test/twice.md",
33 | 		"banana/twice.md":        "test/banana/twice.md",
34 | 	}
35 | 
36 | 	for _, path := range tests {
37 | 		data.AddFile(path, path)
38 | 	}
39 | 	for link, _ := range tests {
40 | 		renderer.Link(out, []byte(link), []byte("title"), []byte("content"))
41 | 	}
42 | 
43 | 	for link, details := range data.AllLinks {
44 | 		data.AllLinks[link].Response = testUrl(link, "", true)
45 | 		//fmt.Printf("\t\t(%d) %d links to %s\n", data.AllLinks[link].Response, details.Count, link)
46 | 		fmt.Printf("%s links to %s\n", details.ActualLink[0], link)
47 | 		if _, ok := data.AllFiles[link]; !ok {
48 | 			t.Errorf("ERROR(%d): not found %s links to %s\n", details.Response, details.ActualLink[0], link)
49 | 		}
50 | 		if tests[details.ActualLink[0]] != link {
51 | 			t.Errorf("ERROR(%d): %s links to %s, should link to %s\n", details.Response, details.ActualLink[0], link, tests[details.ActualLink[0]])
52 | 		}
53 | 	}
54 | }
55 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | 
 2 | # Adds build information from git repo
 3 | #
 4 | # as suggested by tatsushid in
 5 | # https://github.com/spf13/hugo/issues/540
 6 | 
 7 | COMMIT_HASH=`git rev-parse --short HEAD 2>/dev/null`
 8 | BUILD_DATE=`date +%FT%T%z`
 9 | LDFLAGS=-ldflags "-X github.com/spf13/hugo/hugolib.CommitHash=${COMMIT_HASH} -X github.com/spf13/hugo/hugolib.BuildDate=${BUILD_DATE}"
10 | 
11 | build:
12 | 	go build -o markdownlint main.go
13 | 
14 | shell: docker-build
15 | 	docker run --rm -it -v $(CURDIR):/go/src/github.com/docker/markdownlint markdownlint bash
16 | 
17 | docker-build:
18 | 	rm -f markdownlint.zip
19 | 	docker build -t markdownlint .
20 | 
21 | docker: docker-build
22 | 	docker rm markdownlint-build || true
23 | 	docker run --name markdownlint-build markdownlint ls
24 | 	docker cp markdownlint-build:/go/src/github.com/docker/markdownlint/markdownlint.zip .
25 | 	rm -f markdownlint
26 | 	unzip -o markdownlint.zip
27 | 
28 | run:
29 | 	./markdownlint .
30 | 
31 | validate:
32 | 	docker run \
33 | 		-v $(CURDIR)/markdownlint:/usr/bin/markdownlint \
34 | 		--volumes-from docsdockercom_data_1 \
35 | 		--rm -it \
36 | 			debian /usr/bin/markdownlint /docs/content/
37 | 
38 | AWSTOKENSFILE ?= ../aws.env
39 | -include $(AWSTOKENSFILE)
40 | export GITHUB_USERNAME GITHUB_TOKEN
41 | 
42 | RELEASE_DATE=`date +%F`
43 | 
44 | release: docker
45 | 	# TODO: check that we have upstream master, bail if not
46 | 	docker run --rm -it -e GITHUB_TOKEN markdownlint \
47 | 		github-release release --user docker --repo markdownlint --tag $(RELEASE_DATE)
48 | 	docker run --rm -it -e GITHUB_TOKEN markdownlint \
49 | 		github-release upload --user docker --repo markdownlint --tag $(RELEASE_DATE) \
50 | 			--name markdownlint \
51 | 			--file markdownlint
52 | 	docker run --rm -it -e GITHUB_TOKEN markdownlint \
53 | 		github-release upload --user docker --repo markdownlint --tag $(RELEASE_DATE) \
54 | 			--name markdownlint-osx \
55 | 			--file markdownlint.app
56 | 	docker run --rm -it -e GITHUB_TOKEN markdownlint \
57 | 		github-release upload --user docker --repo markdownlint --tag $(RELEASE_DATE) \
58 | 			--name markdownlint.exe \
59 | 			--file markdownlint.exe
60 | 


--------------------------------------------------------------------------------
/checkers/frontmatter_test.go:
--------------------------------------------------------------------------------
 1 | package checkers
 2 | 
 3 | import (
 4 | 	"testing"
 5 | 
 6 | 	"github.com/docker/markdownlint/data"
 7 | 	"github.com/docker/markdownlint/linereader"
 8 | )
 9 | 
10 | // NOTE: this has some spaces and tabs as well as newlines at the start. this is intentional
11 | const OK_TOPIC = `
12 | 
13 | 
14 | 
15 | <!--[metadata]>
16 | +++
17 | title = "Dockerfile reference"
18 | description = "Dockerfiles use a simple DSL which allows you to automate the steps you would normally manually take to create an image."
19 | keywords = ["builder, docker, Dockerfile, automation, image creation"]
20 | [menu.main]
21 | parent = "mn_reference"
22 | +++
23 | <![end-metadata]-->
24 | # Dockerfile reference
25 | `
26 | const MISSING_COMMENT_START_TOPIC = `
27 | 
28 | 
29 | 
30 | +++
31 | title = "Dockerfile reference"
32 | description = "Dockerfiles use a simple DSL which allows you to automate the steps you would normally manually take to create an image."
33 | keywords = ["builder, docker, Dockerfile, automation, image creation"]
34 | [menu.main]
35 | parent = "mn_reference"
36 | +++
37 | <![end-metadata]-->
38 | # Dockerfile reference
39 | `
40 | const MISSING_COMMENT_END_TOPIC = `
41 | 
42 | 
43 | 
44 | <!--[metadata]>
45 | +++
46 | title = "Dockerfile reference end comment missing"
47 | description = "Dockerfiles use a simple DSL which allows you to automate the steps you would normally manually take to create an image."
48 | keywords = ["builder, docker, Dockerfile, automation, image creation"]
49 | [menu.main]
50 | parent = "mn_reference"
51 | +++
52 | # Dockerfile reference
53 | `
54 | 
55 | func TestFrontmatterFound(t *testing.T) {
56 | 	file := "ok"
57 | 	data.AllFiles = make(map[string]*data.FileDetails)
58 | 	data.AddFile(file, file)
59 | 
60 | 	err := CheckHugoFrontmatter(linereader.ByteReader(OK_TOPIC), file)
61 | 
62 | 	if err != nil {
63 | 		t.Errorf("ERROR parsing: %v", err)
64 | 	}
65 | }
66 | 
67 | func TestFrontmatterError(t *testing.T) {
68 | 	file := "missing"
69 | 	data.AllFiles = make(map[string]*data.FileDetails)
70 | 	data.AddFile(file, file)
71 | 
72 | 	err := CheckHugoFrontmatter(linereader.ByteReader(MISSING_COMMENT_END_TOPIC), file)
73 | 
74 | 	if err == nil {
75 | 		t.Errorf("Expected error")
76 | 	} else {
77 | 		if err.Error() != "Did not find expected close metadata comment" {
78 | 			t.Errorf("unexpected ERROR parsing: %v", err)
79 | 		}
80 | 	}
81 | }
82 | 


--------------------------------------------------------------------------------
/checkers/frontmatter.go:
--------------------------------------------------------------------------------
  1 | package checkers
  2 | 
  3 | import (
  4 | 	"fmt"
  5 | 	"strings"
  6 | 	"unicode"
  7 | 	"unicode/utf8"
  8 | 
  9 | 	"github.com/docker/markdownlint/data"
 10 | 	"github.com/docker/markdownlint/linereader"
 11 | )
 12 | 
 13 | // https://gohugo.io/content/front-matter/
 14 | func CheckHugoFrontmatter(reader *linereader.LineReader, file string) (err error) {
 15 | 	err = doCheckHugoFrontmatter(reader, file)
 16 | 	if err != nil {
 17 | 		data.AllFiles[file].FormatErrors = fmt.Sprintf("%s* frontmatter: (%s) %s\n", data.AllFiles[file].FormatErrors, file, err)
 18 | 		data.AllFiles[file].FormatErrorCount++
 19 | 
 20 | 	}
 21 | 	return err
 22 | }
 23 | 
 24 | func doCheckHugoFrontmatter(reader *linereader.LineReader, file string) (err error) {
 25 | 	foundComment := false
 26 | 	for err == nil {
 27 | 		byteBuff, _, err := reader.ReadLine()
 28 | 		if err != nil {
 29 | 			return err
 30 | 		}
 31 | 		buff := string(byteBuff)
 32 | 		if buff == "+++" {
 33 | 			data.VerboseLog("Found TOML start")
 34 | 			break
 35 | 		}
 36 | 		if strings.HasPrefix(buff, "<!--") {
 37 | 			if !strings.HasSuffix(buff, "-->") {
 38 | 				data.VerboseLog("found comment start")
 39 | 				foundComment = true
 40 | 				continue
 41 | 			}
 42 | 		}
 43 | 		//data.VerboseLog("ReadLine: %s, %v, %s\n", string(byteBuff), isPrefix, err)
 44 | 		for i := 0; i < len(buff); {
 45 | 			runeValue, width := utf8.DecodeRuneInString(buff[i:])
 46 | 			if unicode.IsSpace(runeValue) {
 47 | 				i += width
 48 | 			} else {
 49 | 				data.VerboseLog("Unexpected non-whitespace char: %s", buff)
 50 | 				return fmt.Errorf("Unexpected non-whitespace char: %s", buff)
 51 | 			}
 52 | 		}
 53 | 	}
 54 | 
 55 | 	data.AllFiles[file].Meta = make(map[string]string)
 56 | 
 57 | 	// read lines until `+++` ending
 58 | 	for err == nil {
 59 | 		byteBuff, _, err := reader.ReadLine()
 60 | 		if err != nil {
 61 | 			return err
 62 | 		}
 63 | 		buff := string(byteBuff)
 64 | 		if buff == "+++" {
 65 | 			data.VerboseLog("Found TOML end")
 66 | 			break
 67 | 		}
 68 | 		data.VerboseLog("\t%s\n", buff)
 69 | 
 70 | 		meta := strings.SplitN(buff, "=", 2)
 71 | 		data.VerboseLog("\t%d\t%v\n", len(meta), meta)
 72 | 		if len(meta) == 2 {
 73 | 			data.VerboseLog("\t\t%s: %s\n", meta[0], meta[1])
 74 | 			data.AllFiles[file].Meta[strings.Trim(meta[0], " ")] = strings.Trim(meta[1], " ")
 75 | 		}
 76 | 	}
 77 | 	// remove trailing close comment
 78 | 	if foundComment {
 79 | 		byteBuff, _, err := reader.ReadLine()
 80 | 		if err != nil {
 81 | 			return err
 82 | 		}
 83 | 		buff := string(byteBuff)
 84 | 		data.VerboseLog("is this a comment? (%s)\n", buff)
 85 | 		if strings.HasSuffix(buff, "-->") {
 86 | 			if !strings.HasPrefix(buff, "<!--") {
 87 | 				data.VerboseLog("found comment end\n")
 88 | 				foundComment = false
 89 | 			}
 90 | 		}
 91 | 		if foundComment {
 92 | 			reader.UnreadLine(buff)
 93 | 			return fmt.Errorf("Did not find expected close metadata comment")
 94 | 		}
 95 | 	}
 96 | 
 97 | 	// ensure that the minimum metadata keys are set
 98 | 	// ignore draft files
 99 | 	if draft, ok := data.AllFiles[file].Meta["draft"]; !ok || draft != "true" {
100 | 		if _, ok := data.AllFiles[file].Meta["title"]; !ok {
101 | 			return fmt.Errorf("Did not find `title` metadata element")
102 | 		}
103 | 		if _, ok := data.AllFiles[file].Meta["description"]; !ok {
104 | 			return fmt.Errorf("Did not find `description` metadata element")
105 | 		}
106 | 		if _, ok := data.AllFiles[file].Meta["keywords"]; !ok {
107 | 			return fmt.Errorf("Did not find `keywords` metadata element")
108 | 		}
109 | 	}
110 | 	return nil
111 | }
112 | 
113 | func FrontSummary(filter string) (int, string) {
114 | 	errorCount := 0
115 | 	errorString := ""
116 | 	for file, details := range data.AllFiles {
117 | 		if strings.HasPrefix(file, filter) {
118 | 			errorCount += details.FormatErrorCount
119 | 			errorString = fmt.Sprintf("%s%s", errorString, details.FormatErrors)
120 | 		}
121 | 	}
122 | 	return errorCount, errorString
123 | }
124 | 


--------------------------------------------------------------------------------
/main.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"flag"
  5 | 	"fmt"
  6 | 	"os"
  7 | 	"path/filepath"
  8 | 	"strings"
  9 | 
 10 | 	"github.com/docker/markdownlint/checkers"
 11 | 	"github.com/docker/markdownlint/data"
 12 | 	"github.com/docker/markdownlint/linereader"
 13 | )
 14 | 
 15 | func main() {
 16 | 	flag.Parse()
 17 | 	args := flag.Args()
 18 | 	if len(args) < 1 {
 19 | 		printUsage()
 20 | 		os.Exit(-1)
 21 | 	}
 22 | 	dir := args[0]
 23 | 	filter := ""
 24 | 	if len(args) >= 2 {
 25 | 		filter = args[1]
 26 | 	}
 27 | 
 28 | 	data.AllFiles = make(map[string]*data.FileDetails)
 29 | 
 30 | 	fmt.Println("Finding files")
 31 | 	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 32 | 		if err != nil {
 33 | 			data.ErrorLog("%s\n", err)
 34 | 			return err
 35 | 		}
 36 | 		data.VerboseLog("FOUND: %s\n", path)
 37 | 		if info.IsDir() {
 38 | 			return nil
 39 | 		}
 40 | 		file, err := filepath.Rel(dir, path)
 41 | 		if err != nil {
 42 | 			data.ErrorLog("%s\n", err)
 43 | 			return err
 44 | 		}
 45 | 		// verboseLog("\t walked to %s\n", file)
 46 | 		data.AddFile(file, path)
 47 | 		return nil
 48 | 	})
 49 | 	if err != nil {
 50 | 		data.ErrorLog("%s\n", err)
 51 | 		os.Exit(-1)
 52 | 	}
 53 | 
 54 | 	count := 0
 55 | 	for file, details := range data.AllFiles {
 56 | 		if !strings.HasPrefix(file, filter) {
 57 | 			data.VerboseLog("FILTERED: %s\n", file)
 58 | 			continue
 59 | 		}
 60 | 		if !strings.HasSuffix(file, ".md") {
 61 | 			data.VerboseLog("SKIPPING: %s\n", file)
 62 | 			continue
 63 | 		}
 64 | 		// fmt.Printf("opening: %s\n", file)
 65 | 		count++
 66 | 		if count%100 == 0 {
 67 | 			fmt.Printf("\topened %d files so far\n", count)
 68 | 		}
 69 | 
 70 | 		reader, err := linereader.OpenReader(details.FullPath)
 71 | 		if err != nil {
 72 | 			data.ErrorLog("%s\n", err)
 73 | 			data.AllFiles[file].FormatErrorCount++
 74 | 		}
 75 | 
 76 | 		err = checkers.CheckHugoFrontmatter(reader, file)
 77 | 		if err != nil {
 78 | 			data.ErrorLog("(%s) frontmatter: %s\n", file, err)
 79 | 		}
 80 | 
 81 | 		if draft, ok := data.AllFiles[file].Meta["draft"]; ok || draft == "true" {
 82 | 			data.VerboseLog("Draft=%s: SKIPPING %s link check.\n", draft, file)
 83 | 		} else {
 84 | 			//fmt.Printf("Draft=%s: %s link check.\n", draft, file)
 85 | 			err = checkers.CheckMarkdownLinks(reader, file)
 86 | 			if err != nil {
 87 | 				// this only errors if there is a fatal issue
 88 | 				data.ErrorLog("(%s) links: %s\n", file, err)
 89 | 				data.AllFiles[file].FormatErrorCount++
 90 | 			}
 91 | 		}
 92 | 		reader.Close()
 93 | 	}
 94 | 	fmt.Printf("Starting to test links (Filter = %s)\n", filter)
 95 | 	checkers.TestLinks(filter, true)
 96 | 
 97 | 	// TODO (JIRA: DOCS-181): Title, unique across products if not, file should include an {identifier}
 98 | 
 99 | 	summaryFileName := "markdownlint.summary.txt"
100 | 	f, err := os.Create(summaryFileName)
101 | 	if err == nil {
102 | 		fmt.Printf("Also writing summary to %s :\n\n", summaryFileName)
103 | 		defer f.Close()
104 | 	}
105 | 
106 | 	if filter != "" {
107 | 		Printf(f, "# Filtered (%s) Summary:\n\n", filter)
108 | 	} else {
109 | 		Printf(f, "# Summary:\n\n")
110 | 	}
111 | 	errorCount, errorString := checkers.FrontSummary(filter)
112 | 	Printf(f, errorString)
113 | 	count, errorString = checkers.LinkSummary(filter)
114 | 	errorCount += count
115 | 	//Printf(f, errorString)
116 | 	Printf(f, "\n\tFound: %d files\n", len(data.AllFiles))
117 | 	Printf(f, "\tFound: %d errors\n", errorCount)
118 | 	// return the number of 404's to show that there are things to be fixed
119 | 	os.Exit(errorCount)
120 | }
121 | 
122 | func Printf(f *os.File, format string, a ...interface{}) {
123 | 	str := fmt.Sprintf(format, a...)
124 | 	fmt.Print(str)
125 | 	if f != nil {
126 | 		// Don't reall want to know we can't write..
127 | 		f.WriteString(str)
128 | 	}
129 | }
130 | 
131 | func printUsage() {
132 | 	fmt.Println("Please specify a directory to check")
133 | 	fmt.Println("\tfor example: markdownlint . [filter]")
134 | 	fmt.Println("\t [filter] can be any string prefix inside the dir specified")
135 | 	flag.PrintDefaults()
136 | }
137 | 


--------------------------------------------------------------------------------
/checkers/links.go:
--------------------------------------------------------------------------------
  1 | package checkers
  2 | 
  3 | import (
  4 | 	"bytes"
  5 | 	"fmt"
  6 | 	"net/http"
  7 | 	"net/url"
  8 | 	"os"
  9 | 	"path/filepath"
 10 | 	"strings"
 11 | 	"time"
 12 | 
 13 | 	"github.com/docker/markdownlint/data"
 14 | 	"github.com/docker/markdownlint/linereader"
 15 | 
 16 | 	"github.com/miekg/mmark"
 17 | )
 18 | 
 19 | var skipUrls = map[string]int{
 20 | 	"https://build.opensuse.org/project/show/Virtualization:containers":            1,
 21 | 	"https://build.opensuse.org/":                                                  1,
 22 | 	"https://linux.oracle.com":                                                     1,
 23 | 	"http://supervisord.org/":                                                      1,
 24 | 	"http://goo.gl/HSz8UT":                                                         1,
 25 | 	"https://www.linkedin.com/company/docker":                                      1,
 26 | 	"https://cloud.docker.com/stack/deploy/":                                       1,
 27 | 	"https://cloud.docker.com/account/":                                            1,
 28 | 	"https://reddit.com/r/docker":                                                  1,
 29 | 	"https://www.reddit.com/r/docker":                                              1,
 30 | 	"https://godoc.org/golang.org/x/crypto/ssh":                                    1,
 31 | 	"https://letsencrypt.org/how-it-works/":                                        1,
 32 | 	"https://cloud.google.com/compute/docs/disks/persistent-disks":                 1,
 33 | 	"https://godoc.org/github.com/docker/distribution/notifications#RequestRecord": 1,
 34 | 	"https://support.docker.com":                                                   1,
 35 | 	"https://www.weave.works/docs/net/latest/introducing-weave/":                   1,
 36 | }
 37 | 
 38 | func CheckMarkdownLinks(reader *linereader.LineReader, file string) (err error) {
 39 | 	// mmark.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters)
 40 | 	htmlFlags := 0
 41 | 	htmlFlags |= mmark.HTML_FOOTNOTE_RETURN_LINKS
 42 | 
 43 | 	renderParameters := mmark.HtmlRendererParameters{
 44 | 	//		FootnoteAnchorPrefix:       viper.GetString("FootnoteAnchorPrefix"),
 45 | 	//		FootnoteReturnLinkContents: viper.GetString("FootnoteReturnLinkContents"),
 46 | 	}
 47 | 
 48 | 	renderer := &TestRenderer{
 49 | 		LinkFrom: file,
 50 | 		Renderer: mmark.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters),
 51 | 	}
 52 | 
 53 | 	extensions := 0 |
 54 | 		//mmark.EXTENSION_NO_INTRA_EMPHASIS |
 55 | 		mmark.EXTENSION_TABLES | mmark.EXTENSION_FENCED_CODE |
 56 | 		mmark.EXTENSION_AUTOLINK |
 57 | 		//mmark.EXTENSION_STRIKETHROUGH |
 58 | 		mmark.EXTENSION_SPACE_HEADERS | mmark.EXTENSION_FOOTNOTES |
 59 | 		mmark.EXTENSION_HEADER_IDS | mmark.EXTENSION_AUTO_HEADER_IDS //|
 60 | 	//	mmark.EXTENSION_DEFINITION_LISTS
 61 | 
 62 | 	//var output []byte
 63 | 	buf := make([]byte, 1024*1024)
 64 | 	length, err := reader.Read(buf)
 65 | 	if length == 0 || err != nil {
 66 | 		return err
 67 | 	}
 68 | 	data.VerboseLog("RUNNING Markdown on %s length(%d) - not counting frontmater\n", file, length)
 69 | 	_ = mmark.Parse(buf, renderer, extensions)
 70 | 	data.VerboseLog("FINISHED Markdown on %s\n", file)
 71 | 
 72 | 	return nil
 73 | }
 74 | 
 75 | var statusCount = make(map[int]int)
 76 | 
 77 | func LinkSummary(filter string) (int, string) {
 78 | 	okCount := 0
 79 | 	errorCount := 0
 80 | 	errorString := ""
 81 | 	for link, details := range data.AllLinks {
 82 | 		if details.Response == 200 ||
 83 | 			details.Response == 900 ||
 84 | 			details.Response == 888 ||
 85 | 			details.Response == 299 ||
 86 | 			details.Response == 2900 ||
 87 | 			details.Response == 666 {
 88 | 			okCount++
 89 | 		} else {
 90 | 			for i, file := range data.AllLinks[link].LinksFrom {
 91 | 				if strings.HasPrefix(file, filter) {
 92 | 					errorCount++
 93 | 					errorString = fmt.Sprintf("%s* link error: (in page %s) %s\n", errorString, file, data.AllLinks[link].ActualLink[i])
 94 | 				}
 95 | 			}
 96 | 		}
 97 | 	}
 98 | 	return errorCount, errorString
 99 | }
100 | 
101 | func TestLinks(filter string, checkExternalLinks bool) {
102 | 	linkCount := 0
103 | 	for link, details := range data.AllLinks {
104 | 		linkCount++
105 | 		status := testUrl(link, filter, checkExternalLinks)
106 | 		data.AllLinks[link].Response = status
107 | 		statusCount[status]++
108 | 		if status == 200 ||
109 | 			status == 900 ||
110 | 			status == 888 ||
111 | 			status == 299 ||
112 | 			status == 2900 ||
113 | 			status == 666 {
114 | 			// These are not counted to the error count
115 | 			data.VerboseLog("\t\t(%d) %d links to %s\n", status, details.Count, link)
116 | 		} else {
117 | 			data.ErrorLog("(%d) %d links to (%s)\n", status, details.Count, link)
118 | 			for i, file := range data.AllLinks[link].LinksFrom {
119 | 				fmt.Printf("\t\t\t link %s on page %s\n", data.AllLinks[link].ActualLink[i], file)
120 | 			}
121 | 		}
122 | 	}
123 | 	for status, count := range statusCount {
124 | 		fmt.Printf("\t%d: %d times (%s)\n", status, count, data.ResponseCode[status])
125 | 	}
126 | 	fmt.Printf("\tTotal Links: %d\n", linkCount)
127 | }
128 | 
129 | func testUrl(link, filter string, checkExternalLinks bool) int {
130 | 	if _, ok := skipUrls[link]; ok {
131 | 		fmt.Printf("Skipping: %s\n", link)
132 | 		return 299
133 | 	}
134 | 	base, err := url.Parse(link)
135 | 	if err != nil {
136 | 		fmt.Println("ERROR: failed to Parse \"" + link + "\"")
137 | 		return 999
138 | 	}
139 | 	switch base.Scheme {
140 | 	case "":
141 | 		// Internal markdown link
142 | 		// otherwuse, look in data.AllFiles
143 | 		if strings.HasPrefix(link, "#") {
144 | 			// internal link to an anchor
145 | 			//TODO: need to look for anchor
146 | 			return 200
147 | 		} else {
148 | 			path := strings.Split(link, "#")
149 | 			relUrl := path[0]
150 | 			if !strings.HasPrefix(relUrl, filter) {
151 | 				//fmt.Printf("Filtered(%s): %s\n", filter, link)
152 | 				return 299
153 | 			}
154 | 			// TODO: need to test for path[1] anchor
155 | 			if _, ok := data.AllFiles[relUrl]; ok {
156 | 				return 2900
157 | 			}
158 | 			if _, ok := data.AllFiles[relUrl+".md"]; ok {
159 | 				return 290
160 | 			}
161 | 			fmt.Printf("\t\tERROR: failed to find %s or %s.md\n", relUrl, relUrl)
162 | 		}
163 | 		ok := 777
164 | 		return ok
165 | 	case "mailto", "irc":
166 | 		err = fmt.Errorf("%s", base.Scheme)
167 | 		return 900
168 | 	}
169 | 	// http / https
170 | 	if base.Host == "docs.docker.com" {
171 | 		err = fmt.Errorf("avoid linking directly to %s", base.Host)
172 | 		return 666
173 | 	}
174 | 	if !checkExternalLinks {
175 | 		data.VerboseLog("External link checking disabled: %s\n", link)
176 | 		return 299
177 | 	}
178 | 	httpClient := &http.Client{
179 | 		Timeout: 10 * time.Second,
180 | 	}
181 | 	resp, err := httpClient.Get(link)
182 | 	if err != nil {
183 | 		fmt.Println("Warning: Failed to crawl \"" + link + "\"  " + err.Error())
184 | 		return 888
185 | 	}
186 | 
187 | 	loc, err := resp.Location()
188 | 	if err == nil && link != loc.String() {
189 | 		fmt.Printf("\t crawled \"%s\"", link)
190 | 		fmt.Printf("\t\t to \"%s\"", loc)
191 | 	}
192 | 
193 | 	b := resp.Body
194 | 	defer b.Close() // close Body when the function returns
195 | 
196 | 	return resp.StatusCode
197 | }
198 | 
199 | type TestRenderer struct {
200 | 	LinkFrom string
201 | 	mmark.Renderer
202 | }
203 | 
204 | func (renderer *TestRenderer) Image(out *bytes.Buffer, link []byte, title []byte, alt []byte, subFigure bool) {
205 | 	renderer.Link(out, link, title, alt)
206 | }
207 | 
208 | func (renderer *TestRenderer) Link(out *bytes.Buffer, linkB []byte, title []byte, content []byte) {
209 | 	actualLink := string(linkB)
210 | 	data.VerboseLog("Link [%s](%s) in file %s\n", string(content), actualLink, renderer.LinkFrom)
211 | 
212 | 	var link string
213 | 
214 | 	base, err := url.Parse(actualLink)
215 | 	if err == nil && base.Scheme == "" {
216 | 		if strings.HasPrefix(actualLink, "#") {
217 | 			link = actualLink
218 | 		} else if strings.HasPrefix(actualLink, "/") {
219 | 			link = strings.TrimLeft(actualLink, "/")
220 | 		} else {
221 | 			// TODO: fix for relative paths.
222 | 			// TODO: need to check the from links are all the same dir too
223 | 			link = filepath.Clean(filepath.FromSlash(actualLink))
224 | 
225 | 			if strings.IndexRune(link, os.PathSeparator) == 0 { // filepath.IsAbs fails to me.
226 | 				link = link[1:]
227 | 			} else {
228 | 				// TODO: need to check all the LinksFrom
229 | 				link = filepath.Join(filepath.Dir(renderer.LinkFrom), link)
230 | 			}
231 | 			data.VerboseLog("---- converted %s (on page %s, in %s) into %s\n", actualLink, renderer.LinkFrom, filepath.Dir(renderer.LinkFrom), link)
232 | 		}
233 | 	} else {
234 | 		link = actualLink
235 | 	}
236 | 
237 | 	_, ok := data.AllLinks[link]
238 | 	if !ok {
239 | 		data.AllLinks[link] = new(data.LinkDetails)
240 | 		data.AllLinks[link].LinksFrom = make(map[int]string)
241 | 		data.AllLinks[link].ActualLink = make(map[int]string)
242 | 	}
243 | 	data.AllLinks[link].LinksFrom[data.AllLinks[link].Count] = renderer.LinkFrom
244 | 	data.AllLinks[link].ActualLink[data.AllLinks[link].Count] = actualLink
245 | 	data.AllLinks[link].Count++
246 | }
247 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                                  Apache License
  2 |                            Version 2.0, January 2004
  3 |                         http://www.apache.org/licenses/
  4 | 
  5 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 |    1. Definitions.
  8 | 
  9 |       "License" shall mean the terms and conditions for use, reproduction,
 10 |       and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |       "Licensor" shall mean the copyright owner or entity authorized by
 13 |       the copyright owner that is granting the License.
 14 | 
 15 |       "Legal Entity" shall mean the union of the acting entity and all
 16 |       other entities that control, are controlled by, or are under common
 17 |       control with that entity. For the purposes of this definition,
 18 |       "control" means (i) the power, direct or indirect, to cause the
 19 |       direction or management of such entity, whether by contract or
 20 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |       outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |       "You" (or "Your") shall mean an individual or Legal Entity
 24 |       exercising permissions granted by this License.
 25 | 
 26 |       "Source" form shall mean the preferred form for making modifications,
 27 |       including but not limited to software source code, documentation
 28 |       source, and configuration files.
 29 | 
 30 |       "Object" form shall mean any form resulting from mechanical
 31 |       transformation or translation of a Source form, including but
 32 |       not limited to compiled object code, generated documentation,
 33 |       and conversions to other media types.
 34 | 
 35 |       "Work" shall mean the work of authorship, whether in Source or
 36 |       Object form, made available under the License, as indicated by a
 37 |       copyright notice that is included in or attached to the work
 38 |       (an example is provided in the Appendix below).
 39 | 
 40 |       "Derivative Works" shall mean any work, whether in Source or Object
 41 |       form, that is based on (or derived from) the Work and for which the
 42 |       editorial revisions, annotations, elaborations, or other modifications
 43 |       represent, as a whole, an original work of authorship. For the purposes
 44 |       of this License, Derivative Works shall not include works that remain
 45 |       separable from, or merely link (or bind by name) to the interfaces of,
 46 |       the Work and Derivative Works thereof.
 47 | 
 48 |       "Contribution" shall mean any work of authorship, including
 49 |       the original version of the Work and any modifications or additions
 50 |       to that Work or Derivative Works thereof, that is intentionally
 51 |       submitted to Licensor for inclusion in the Work by the copyright owner
 52 |       or by an individual or Legal Entity authorized to submit on behalf of
 53 |       the copyright owner. For the purposes of this definition, "submitted"
 54 |       means any form of electronic, verbal, or written communication sent
 55 |       to the Licensor or its representatives, including but not limited to
 56 |       communication on electronic mailing lists, source code control systems,
 57 |       and issue tracking systems that are managed by, or on behalf of, the
 58 |       Licensor for the purpose of discussing and improving the Work, but
 59 |       excluding communication that is conspicuously marked or otherwise
 60 |       designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |       on behalf of whom a Contribution has been received by Licensor and
 64 |       subsequently incorporated within the Work.
 65 | 
 66 |    2. Grant of Copyright License. Subject to the terms and conditions of
 67 |       this License, each Contributor hereby grants to You a perpetual,
 68 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |       copyright license to reproduce, prepare Derivative Works of,
 70 |       publicly display, publicly perform, sublicense, and distribute the
 71 |       Work and such Derivative Works in Source or Object form.
 72 | 
 73 |    3. Grant of Patent License. Subject to the terms and conditions of
 74 |       this License, each Contributor hereby grants to You a perpetual,
 75 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |       (except as stated in this section) patent license to make, have made,
 77 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |       where such license applies only to those patent claims licensable
 79 |       by such Contributor that are necessarily infringed by their
 80 |       Contribution(s) alone or by combination of their Contribution(s)
 81 |       with the Work to which such Contribution(s) was submitted. If You
 82 |       institute patent litigation against any entity (including a
 83 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |       or a Contribution incorporated within the Work constitutes direct
 85 |       or contributory patent infringement, then any patent licenses
 86 |       granted to You under this License for that Work shall terminate
 87 |       as of the date such litigation is filed.
 88 | 
 89 |    4. Redistribution. You may reproduce and distribute copies of the
 90 |       Work or Derivative Works thereof in any medium, with or without
 91 |       modifications, and in Source or Object form, provided that You
 92 |       meet the following conditions:
 93 | 
 94 |       (a) You must give any other recipients of the Work or
 95 |           Derivative Works a copy of this License; and
 96 | 
 97 |       (b) You must cause any modified files to carry prominent notices
 98 |           stating that You changed the files; and
 99 | 
100 |       (c) You must retain, in the Source form of any Derivative Works
101 |           that You distribute, all copyright, patent, trademark, and
102 |           attribution notices from the Source form of the Work,
103 |           excluding those notices that do not pertain to any part of
104 |           the Derivative Works; and
105 | 
106 |       (d) If the Work includes a "NOTICE" text file as part of its
107 |           distribution, then any Derivative Works that You distribute must
108 |           include a readable copy of the attribution notices contained
109 |           within such NOTICE file, excluding those notices that do not
110 |           pertain to any part of the Derivative Works, in at least one
111 |           of the following places: within a NOTICE text file distributed
112 |           as part of the Derivative Works; within the Source form or
113 |           documentation, if provided along with the Derivative Works; or,
114 |           within a display generated by the Derivative Works, if and
115 |           wherever such third-party notices normally appear. The contents
116 |           of the NOTICE file are for informational purposes only and
117 |           do not modify the License. You may add Your own attribution
118 |           notices within Derivative Works that You distribute, alongside
119 |           or as an addendum to the NOTICE text from the Work, provided
120 |           that such additional attribution notices cannot be construed
121 |           as modifying the License.
122 | 
123 |       You may add Your own copyright statement to Your modifications and
124 |       may provide additional or different license terms and conditions
125 |       for use, reproduction, or distribution of Your modifications, or
126 |       for any such Derivative Works as a whole, provided Your use,
127 |       reproduction, and distribution of the Work otherwise complies with
128 |       the conditions stated in this License.
129 | 
130 |    5. Submission of Contributions. Unless You explicitly state otherwise,
131 |       any Contribution intentionally submitted for inclusion in the Work
132 |       by You to the Licensor shall be under the terms and conditions of
133 |       this License, without any additional terms or conditions.
134 |       Notwithstanding the above, nothing herein shall supersede or modify
135 |       the terms of any separate license agreement you may have executed
136 |       with Licensor regarding such Contributions.
137 | 
138 |    6. Trademarks. This License does not grant permission to use the trade
139 |       names, trademarks, service marks, or product names of the Licensor,
140 |       except as required for reasonable and customary use in describing the
141 |       origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 |    7. Disclaimer of Warranty. Unless required by applicable law or
144 |       agreed to in writing, Licensor provides the Work (and each
145 |       Contributor provides its Contributions) on an "AS IS" BASIS,
146 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |       implied, including, without limitation, any warranties or conditions
148 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |       PARTICULAR PURPOSE. You are solely responsible for determining the
150 |       appropriateness of using or redistributing the Work and assume any
151 |       risks associated with Your exercise of permissions under this License.
152 | 
153 |    8. Limitation of Liability. In no event and under no legal theory,
154 |       whether in tort (including negligence), contract, or otherwise,
155 |       unless required by applicable law (such as deliberate and grossly
156 |       negligent acts) or agreed to in writing, shall any Contributor be
157 |       liable to You for damages, including any direct, indirect, special,
158 |       incidental, or consequential damages of any character arising as a
159 |       result of this License or out of the use or inability to use the
160 |       Work (including but not limited to damages for loss of goodwill,
161 |       work stoppage, computer failure or malfunction, or any and all
162 |       other commercial damages or losses), even if such Contributor
163 |       has been advised of the possibility of such damages.
164 | 
165 |    9. Accepting Warranty or Additional Liability. While redistributing
166 |       the Work or Derivative Works thereof, You may choose to offer,
167 |       and charge a fee for, acceptance of support, warranty, indemnity,
168 |       or other liability obligations and/or rights consistent with this
169 |       License. However, in accepting such obligations, You may act only
170 |       on Your own behalf and on Your sole responsibility, not on behalf
171 |       of any other Contributor, and only if You agree to indemnify,
172 |       defend, and hold each Contributor harmless for any liability
173 |       incurred by, or claims asserted against, such Contributor by reason
174 |       of your accepting any such warranty or additional liability.
175 | 
176 |    END OF TERMS AND CONDITIONS
177 | 
178 |    APPENDIX: How to apply the Apache License to your work.
179 | 
180 |       To apply the Apache License to your work, attach the following
181 |       boilerplate notice, with the fields enclosed by brackets "{}"
182 |       replaced with your own identifying information. (Don't include
183 |       the brackets!)  The text should be enclosed in the appropriate
184 |       comment syntax for the file format. We also recommend that a
185 |       file or class name and description of purpose be included on the
186 |       same "printed page" as the copyright notice for easier
187 |       identification within third-party archives.
188 | 
189 |    Copyright {yyyy} {name of copyright owner}
190 | 
191 |    Licensed under the Apache License, Version 2.0 (the "License");
192 |    you may not use this file except in compliance with the License.
193 |    You may obtain a copy of the License at
194 | 
195 |        http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 |    Unless required by applicable law or agreed to in writing, software
198 |    distributed under the License is distributed on an "AS IS" BASIS,
199 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 |    See the License for the specific language governing permissions and
201 |    limitations under the License.
202 | 
203 | 


--------------------------------------------------------------------------------