├── .gitignore
├── AnalysisApi
├── ArgOverwritten
│ ├── .gitignore
│ ├── LICENSE
│ ├── README.md
│ ├── cmd
│ │ └── argoverwritten
│ │ │ └── main.go
│ ├── go.mod
│ ├── go.sum
│ └── passes
│ │ └── ArgOverwritten
│ │ ├── analyzer.go
│ │ ├── analyzer_test.go
│ │ └── testdata
│ │ ├── AnonymousFunction
│ │ └── main.go
│ │ ├── AssigningParamToAVariableFirst
│ │ └── main.go
│ │ ├── DecrementOperator
│ │ └── main.go
│ │ ├── EmptyBodyFunction
│ │ ├── add_amd64.s
│ │ └── main.go
│ │ ├── MultipleParamsOfSameType
│ │ └── main.go
│ │ ├── NoWarnings
│ │ └── main.go
│ │ ├── OverwritingParamFromOuterScope
│ │ └── main.go
│ │ ├── ShadowingVariable
│ │ └── main.go
│ │ └── SimpleOverwriting
│ │ └── main.go
└── README.md
├── CompilerFrontEndASTInGo
├── ArgsOverwriteAnalyzer.go
├── ArgsOverwriteAnalyzer_test.go
├── CodeExamples
│ ├── BadExpr.txt
│ ├── DifferentSpecsAndDecls.go
│ └── SimpleForLoop.go
├── README.md
├── result
│ └── ArgsOverwriteAnalyzer.go
└── testdata
│ ├── AnonymousFunction
│ └── main.go
│ ├── AssigningParamToAVariableFirst
│ └── main.go
│ ├── DecrementOperator
│ └── main.go
│ ├── EmptyBodyFunction
│ ├── add_amd64.s
│ └── main.go
│ ├── LhsMultipleValues
│ └── main.go
│ ├── MultipleParamsOfSameType
│ └── main.go
│ ├── NoWarnings
│ └── main.go
│ ├── OverwritingParamFromOuterScope
│ └── main.go
│ ├── ShadowingVariable
│ └── main.go
│ └── SimpleOverwriting
│ └── main.go
├── CompilerMiddleEndSSAInGo
├── CodeExamples
│ ├── ElseIf
│ │ └── ElseIf.go
│ └── Map
│ │ └── Map.go
├── README.md
├── TerminationInsideGoroutine.go
├── TerminationInsideGoroutine_test.go
└── testdata
│ ├── FatalInsideGoroutineAnonymousFlag
│ └── main.go
│ ├── FatalInsideGoroutineSimpleFlag
│ └── main.go
│ ├── FatalInsideRegularFunctionNoFlag
│ └── main.go
│ └── SkipInsideGoroutineSimpleFlag
│ └── main.go
├── Conclusion
└── README.md
├── Intro
└── README.md
├── README.md
├── go.mod
└── go.sum
/.gitignore:
--------------------------------------------------------------------------------
1 | *.o
2 | *.a
3 | *.so
4 | .idea
5 | .vscode
6 |
--------------------------------------------------------------------------------
/AnalysisApi/ArgOverwritten/.gitignore:
--------------------------------------------------------------------------------
1 | *.o
2 | *.a
3 | *.so
4 | .idea
5 | .vscode
6 |
--------------------------------------------------------------------------------
/AnalysisApi/ArgOverwritten/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2020 Amit Davidson
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/AnalysisApi/ArgOverwritten/README.md:
--------------------------------------------------------------------------------
1 | # ArgOverwritten
2 |
3 | [](http://golang.org)
4 | [](http://golang.org)
5 | [](https://lbesson.mit-license.org/)
6 | [](http://makeapullrequest.com)
7 |
8 | ArgOverwritten finds function arguments being overwritten
9 |
10 | ## Quick Start:
11 |
12 | Download the package
13 |
14 | ``` bash
15 | go get github.com/amit-davidson/ArgOverwritten/cmd/argoverwritten
16 | ```
17 |
18 | Pass the entry point
19 |
20 | ``` bash
21 | go vet -vettool=$(which argoverwritten) ${path_to_file}
22 | ```
23 |
24 | ## Example:
25 | ``` go
26 | package testdata
27 |
28 | func body(a int) {
29 | _ = func() {
30 | a = 5
31 | }
32 | }
33 |
34 | func main() {
35 | body(5)
36 | }
37 | ```
38 | ``` bash
39 | go vet -vettool=$(which argoverwritten) /testdata/OverwritingParamFromOuterScope
40 | ```
41 |
42 | Output
43 | ```
44 | /testdata/OverwritingParamFromOuterScope/main.go:5:3: "a" overwrites func parameter
45 | ```
--------------------------------------------------------------------------------
/AnalysisApi/ArgOverwritten/cmd/argoverwritten/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/amit-davidson/ArgOverwritten/passes/ArgOverwritten"
5 | "golang.org/x/tools/go/analysis/singlechecker"
6 | )
7 |
8 | func main() {
9 | singlechecker.Main(ArgOverwritten.Analyzer)
10 | }
11 |
--------------------------------------------------------------------------------
/AnalysisApi/ArgOverwritten/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/amit-davidson/ArgOverwritten
2 |
3 | go 1.15
4 |
5 | require golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e
6 |
--------------------------------------------------------------------------------
/AnalysisApi/ArgOverwritten/go.sum:
--------------------------------------------------------------------------------
1 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
2 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
3 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
4 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
5 | golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
6 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
7 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
8 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
9 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
10 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
11 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
12 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
13 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
14 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
15 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
16 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
17 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
18 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
19 | golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e h1:4nW4NLDYnU28ojHaHO8OVxFHk/aQ33U01a9cjED+pzE=
20 | golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
21 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
22 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
23 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
24 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
25 |
--------------------------------------------------------------------------------
/AnalysisApi/ArgOverwritten/passes/ArgOverwritten/analyzer.go:
--------------------------------------------------------------------------------
1 | package ArgOverwritten
2 |
3 | import (
4 | "fmt"
5 | "go/ast"
6 | "golang.org/x/tools/go/analysis"
7 | )
8 |
9 | var Analyzer = &analysis.Analyzer{
10 | Name: "ArgOverwritten",
11 | Doc: doc,
12 | Run: run,
13 | }
14 |
15 | const (
16 | doc = "ArgOverwritten finds function arguments being overwritten"
17 | )
18 |
19 | func report(pass *analysis.Pass, ident *ast.Ident) {
20 | message := fmt.Sprintf("\"%s\" overwrites func parameter", ident.Name)
21 | pass.Report(analysis.Diagnostic{
22 | Pos: ident.Pos(),
23 | Message: message,
24 | })
25 | }
26 |
27 | func run(pass *analysis.Pass) (interface{}, error) {
28 | visitor := func(node ast.Node) bool {
29 | var typ *ast.FuncType
30 | var body *ast.BlockStmt
31 | switch fn := node.(type) {
32 | case *ast.FuncDecl: // Regular function
33 | typ = fn.Type
34 | body = fn.Body
35 | case *ast.FuncLit: // Anonymous function
36 | typ = fn.Type
37 | body = fn.Body
38 | }
39 | if typ == nil || body == nil { // Exclude other types but also external functions with missing body
40 | return true
41 | }
42 | if len(typ.Params.List) == 0 {
43 | return true
44 | }
45 |
46 | for _, field := range typ.Params.List {
47 | for _, arg := range field.Names {
48 | obj := pass.TypesInfo.ObjectOf(arg)
49 | ast.Inspect(body, func(node ast.Node) bool {
50 | switch stmt := node.(type) {
51 | case *ast.AssignStmt:
52 | for _, lhs := range stmt.Lhs {
53 | ident, ok := lhs.(*ast.Ident)
54 | if ok && pass.TypesInfo.ObjectOf(ident) == obj {
55 | report(pass, ident)
56 | }
57 | }
58 | case *ast.IncDecStmt:
59 | ident, ok := stmt.X.(*ast.Ident)
60 | if ok && pass.TypesInfo.ObjectOf(ident) == obj {
61 | report(pass, ident)
62 | }
63 | }
64 | return true
65 | })
66 | }
67 | }
68 | return true
69 | }
70 | for _, f := range pass.Files {
71 | ast.Inspect(f, visitor)
72 | }
73 | return nil, nil
74 | }
75 |
--------------------------------------------------------------------------------
/AnalysisApi/ArgOverwritten/passes/ArgOverwritten/analyzer_test.go:
--------------------------------------------------------------------------------
1 | package ArgOverwritten
2 |
3 | import (
4 | "fmt"
5 | "golang.org/x/tools/go/analysis/analysistest"
6 | "os"
7 | "testing"
8 | )
9 |
10 | func TestAnalyzer(t *testing.T) {
11 | var testCases = []struct {
12 | name string
13 | }{
14 | {name: "SimpleOverwriting"},
15 | {name: "AnonymousFunction"},
16 | {name: "OverwritingParamFromOuterScope"},
17 | {name: "AssigningParamToAVariableFirst"},
18 | {name: "MultipleParamsOfSameType"},
19 | {name: "ShadowingVariable"},
20 | {name: "EmptyBodyFunction"},
21 | {name: "NoWarnings"},
22 | {name: "DecrementOperator"},
23 | }
24 | for _, tc := range testCases {
25 | t.Run(tc.name, func(t *testing.T) {
26 | analysistest.Run(t, fmt.Sprintf("%s%s%s", analysistest.TestData(), string(os.PathSeparator), tc.name), Analyzer)
27 | })
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/AnalysisApi/ArgOverwritten/passes/ArgOverwritten/testdata/AnonymousFunction/main.go:
--------------------------------------------------------------------------------
1 | package testdata
2 |
3 | func main() {
4 | _ = func(a int) {
5 | a = 5 // want `"a" overwrites func parameter`
6 | }
7 | }
--------------------------------------------------------------------------------
/AnalysisApi/ArgOverwritten/passes/ArgOverwritten/testdata/AssigningParamToAVariableFirst/main.go:
--------------------------------------------------------------------------------
1 | package testdata
2 |
3 | func body(a int) {
4 | b := a
5 | b = 5
6 | _ = b
7 | }
8 |
9 | func main() {
10 | body(5)
11 | }
--------------------------------------------------------------------------------
/AnalysisApi/ArgOverwritten/passes/ArgOverwritten/testdata/DecrementOperator/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | func retry(fn func() error, retries int) error {
4 | for {
5 | if err := fn(); err != nil {
6 | if retries < 1 {
7 | return err
8 | }
9 | retries-- // want `"retries" overwrites func parameter`
10 | continue
11 | }
12 | return nil
13 | }
14 | }
15 | func main() {
16 | _ = retry(func() error {
17 | return nil
18 | }, 5)
19 | }
20 |
--------------------------------------------------------------------------------
/AnalysisApi/ArgOverwritten/passes/ArgOverwritten/testdata/EmptyBodyFunction/add_amd64.s:
--------------------------------------------------------------------------------
1 | TEXT ·Add(SB),$0-24
2 | MOVQ a+0(FP), AX
3 | ADDQ b+8(FP), AX
4 | MOVQ AX, ret+16(FP)
5 | RET
6 |
--------------------------------------------------------------------------------
/AnalysisApi/ArgOverwritten/passes/ArgOverwritten/testdata/EmptyBodyFunction/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | // Add returns the sum of a and b.
4 | func Add(a int64, b int64) int64
5 |
6 | func body(a int) {
7 | b := a
8 | b = 5
9 | _ = b
10 | }
11 |
12 | func main() {
13 | body(5)
14 | _ = Add(3, 4)
15 | }
16 |
--------------------------------------------------------------------------------
/AnalysisApi/ArgOverwritten/passes/ArgOverwritten/testdata/MultipleParamsOfSameType/main.go:
--------------------------------------------------------------------------------
1 | package testdata
2 |
3 | func body(a, b int, c int) {
4 | a = 5 // want `"a" overwrites func parameter`
5 | _ = b
6 | c = 3 // want `"c" overwrites func parameter`
7 | }
8 |
9 | func main() {
10 | body(1, 2, 3)
11 | }
12 |
--------------------------------------------------------------------------------
/AnalysisApi/ArgOverwritten/passes/ArgOverwritten/testdata/NoWarnings/main.go:
--------------------------------------------------------------------------------
1 | package testdata
2 |
3 | import "fmt"
4 |
5 | func closeBody(body int) {
6 | fmt.Print(body)
7 | }
8 |
9 | func main() {
10 | closeBody(1)
11 | }
12 |
--------------------------------------------------------------------------------
/AnalysisApi/ArgOverwritten/passes/ArgOverwritten/testdata/OverwritingParamFromOuterScope/main.go:
--------------------------------------------------------------------------------
1 | package testdata
2 |
3 | func body(a int) {
4 | _ = func() {
5 | a = 5 // want `"a" overwrites func parameter`
6 | }
7 | }
8 |
9 | func main() {
10 | body(5)
11 | }
--------------------------------------------------------------------------------
/AnalysisApi/ArgOverwritten/passes/ArgOverwritten/testdata/ShadowingVariable/main.go:
--------------------------------------------------------------------------------
1 | package testdata
2 |
3 | func body(a, b int, c int) {
4 | f := func(a int, b int) {
5 | a = 5 // want `"a" overwrites func parameter`
6 | }
7 | _ = f
8 | }
9 |
10 | func main() {
11 | body(1, 2, 3)
12 | }
13 |
--------------------------------------------------------------------------------
/AnalysisApi/ArgOverwritten/passes/ArgOverwritten/testdata/SimpleOverwriting/main.go:
--------------------------------------------------------------------------------
1 | package testdata
2 |
3 | func closeBody(body int) {
4 | body = 1 // want `"body" overwrites func parameter`
5 | }
6 |
7 | func main() {
8 | closeBody(1)
9 | }
10 |
--------------------------------------------------------------------------------
/AnalysisApi/README.md:
--------------------------------------------------------------------------------
1 | ## 4 Analysis API:
2 | ### 4.1 (tools/go/analysis]((https://pkg.go.dev/golang.org/x/tools/go/analysis))
3 | The package defines an [API](https://pkg.go.dev/golang.org/x/tools/go/analysis) for modular static analysis tools. In other words, it's a common interface for all static
4 | code analyzers.
5 |
6 | Why should you care? What's the difference between using the analysis API, and the way we wrote so far?
7 | The analysis API makes writing analyses easier by taking care of:
8 | - file parsing
9 | - testing
10 | - integration with go vet
11 | - many more
12 |
13 | Also, it enforces a single pattern for all the static analysis tools such as how analysis are structured and how
14 | warnings are reported
15 |
16 | ### 4.2 Analysis API members
17 | The primary type in the API is [`analysis.Analyzer`](https://pkg.go.dev/golang.org/x/tools/go/analysis#hdr-Analyzer).
18 | It describes an analysis function: its name, documentation, flags, relationship to other analyzers, and of course, it's logic.
19 |
20 | ``` go
21 | type Analyzer struct {
22 | Name string
23 | Doc string
24 | Requires []*Analyzer
25 | Run func(*Pass) (interface{}, error)
26 |
27 | ...
28 | }
29 | ```
30 |
31 | The `Name` and `Doc` are obvious. They are used to describe what the tool does.
32 |
33 | Another interesting is the `Requires` field. It specifies a list of analyses upon which this one depends and whose
34 | results it may access, and it constrains the order in which a driver may run analyses.
35 |
36 | > To use SSA in the analysis api, we would have to require the [SSA builder](https://pkg.go.dev/golang.org/x/tools/go/analysis/passes/buildssa)
37 | > `Requires: []*analysis.Analyzer {
38 | buildssa.Analyzer
39 | }`
40 |
41 |
42 | The most important one is the `Run` function. It contains the logic that should is executed upon a single package.
43 | It takes as an argument [`*analysis.Pass`](https://pkg.go.dev/golang.org/x/tools/go/analysis#hdr-Pass) and returns a
44 | value to be used by other analyzers and an error.
45 |
46 | ``` go
47 | type Pass struct {
48 | Fset *token.FileSet
49 | Files []*ast.File
50 | Pkg *types.Package
51 | TypesInfo *types.Info
52 | Report func(Diagnostic)
53 |
54 | ...
55 | }
56 | ```
57 |
58 | A `Pass` describes a single unit of work: the application of a particular `Analyzer` to a particular package of Go code. The `Pass` provides information to the Analyzer's `Run` function about the analyzed package and provides operations to the `Run` function for reporting diagnostics and other information back to the driver. It also provides `Fset`, `Files`, `Pkg`, and `TypesInfo` that we know from earlier, so we don't have to take care of ourselves.
59 |
60 | The [`Report`](https://pkg.go.dev/golang.org/x/tools/go/analysis#Pass.Reportf) function emits a diagnostic, a message associated with a source position. For most analyses, diagnostics are their primary result. For convenience, `Pass` provides a helper method, `Reportf`, to report a new diagnostic by formatting a string. Diagnostic is defined as:
61 |
62 | ``` go
63 | type Diagnostic struct {
64 | Pos token.Pos
65 | Category string // optional
66 | Message string
67 | }
68 | ```
69 |
70 | ### 4.3 Project structure
71 | First let's define the project structure:
72 | ```
73 | │── README.md
74 | │── cmd
75 | │ └── analyzerName
76 | │ └── main.go
77 | │── go.mod
78 | │── go.sum
79 | └── passes
80 | └── passName
81 | │── pass.go
82 | │── pass_test.go
83 | └── testdata
84 | ```
85 |
86 | We create a directory where all of our passes reside in named `passes`. Each pass lives in its package, including its logic and tests.
87 | Then we define the usual `cmd` for our executables that contains all the analyzers the module has.
88 |
89 | Regarding our analyzers we wrote previously, we had to handle both the logic of the analyzer and the instrumentation around it.
90 | When converting our code to the analysis API, the AST traversal part (the logic) will sit under `passes` and loading the code
91 | part will be taken care of by the analysis API so we can ignore it.
92 |
93 | Next , we need a way to run the Analyzer and to test it.
94 |
95 | ### 4.4 Running our code
96 | inside `main.go`, we'll add the following code.
97 |
98 | ``` go
99 | package main
100 |
101 | import (
102 | "path/to/our/pass"
103 | "golang.org/x/tools/go/analysis/singlechecker"
104 | )
105 |
106 | func main() { singlechecker.Main(passName.Analyzer) }
107 | ```
108 | Analyzers are provided in the form of packages that a driver program is expected to import.
109 | The [`singlechecker`](https://pkg.go.dev/golang.org/x/tools/go/analysis/singlechecker) package provides the `main` function for a command that runs one Analyzer. By convention, each Analyzer should be accompanied by a singlechecker-based command defined in its entirety as: This code calls our Analyzer.
110 | If we wanted our command to run multiple analyzers, we would have to use [`multichecker`](https://pkg.go.dev/golang.org/x/tools/go/analysis/multichecker).
111 |
112 | Now we can run it using
113 | ``` bash
114 | go install path/to/analyzer
115 | go vet -vettool=$(which analyzername) path/to/files
116 | ```
117 |
118 |
119 | ### 4.5 Testing our code
120 | The [`analysistest`](https://godoc.org/golang.org/x/tools/go/analysis/analysistes) subpackage provides utilities for testing an Analyzer. Using `analysistest.Run`, it is possible to run an analyzer on a package of `testdata` files and check that it reported all the expected diagnostics.
121 | Expectations are expressed using "// want ..." comments in the input code, such as the following:
122 |
123 | ``` go
124 | package testdata
125 |
126 | func main() {
127 | _ = func(a int) {
128 | a = 5 // want `"a" overwrites func parameter`
129 | }
130 | }
131 | ```
132 |
133 | ### 4.6 Overviewing an analyzer!
134 | In this section, we'll convert our [`ArgsOverwrite`](https://github.com/amit-davidson/GopherCon2021IsraelStaticAnalysisWorkshop/blob/master/CompilerFrontEndASTInGo/result/ArgsOverwriteAnalyzer.gohttps://github.com/amit-davidson/GopherCon2021IsraelStaticAnalysisWorkshop/blob/master/CompilerFrontEndASTInGo/result/ArgsOverwriteAnalyzer.go)
135 | Analyzer from earlier to the analysis API
136 |
137 | ### 4.7 Congratulations
138 | You have a good understanding of what the analysis API is and how to use it help us in writing analyses in the future.
139 |
140 | In the [next section](https://github.com/amit-davidson/GopherCon2021IsraelStaticAnalysisWorkshop/blob/master/Conclusion/README.md)
141 | we'll conclude this workshop by touching a point regarding static code analyzers in general and take a look at other code
142 | analyzers written by the Go community.
--------------------------------------------------------------------------------
/CompilerFrontEndASTInGo/ArgsOverwriteAnalyzer.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "go/ast"
6 | "go/importer"
7 | "go/parser"
8 | "go/token"
9 | "go/types"
10 | "log"
11 | "os"
12 | )
13 |
14 | func main() {
15 | path := os.Args[1]
16 | fset, pkgs, err := loadProgram(path)
17 | if err != nil {
18 | log.Fatalf("Failed to parse dir %s: %s", path, err)
19 | }
20 |
21 | for _, pkg := range pkgs {
22 | outputs := analyzePackage(pkg, fset)
23 | for _, message := range outputs {
24 | fmt.Println(message)
25 | }
26 | }
27 | }
28 |
29 | func loadProgram(path string) (*token.FileSet, map[string]*ast.Package, error) {
30 | fset := token.NewFileSet()
31 | pkgs, err := parser.ParseDir(fset, path, nil, 0)
32 | if err != nil {
33 | return nil, nil, err
34 | }
35 | return fset, pkgs, nil
36 | }
37 |
38 | func populateTypes(conf types.Config, fset *token.FileSet, f *ast.File) (*types.Info, error) {
39 | info := &types.Info{
40 | Defs: make(map[*ast.Ident]types.Object),
41 | Uses: make(map[*ast.Ident]types.Object),
42 | }
43 | _, err := conf.Check("", fset, []*ast.File{f}, info)
44 | if err != nil {
45 | return nil, err
46 | }
47 | return info, nil
48 | }
49 |
50 | func analyzePackage(p *ast.Package, fset *token.FileSet) []string {
51 | var info *types.Info
52 | var err error
53 | outputs := make([]string, 0)
54 |
55 | visitor := func(node ast.Node) bool {
56 | var typ *ast.FuncType
57 | var body *ast.BlockStmt
58 | switch fn := node.(type) {
59 | case *ast.FuncDecl: // Regular function
60 | typ = fn.Type
61 | body = fn.Body
62 | case *ast.FuncLit: // Anonymous function
63 | typ = fn.Type
64 | body = fn.Body
65 | }
66 | if typ == nil || body == nil { // Exclude other types but also external functions with missing body
67 | return true
68 | }
69 | if len(typ.Params.List) == 0 {
70 | return true
71 | }
72 |
73 | for _, field := range typ.Params.List {
74 | for _, arg := range field.Names {
75 | obj := info.ObjectOf(arg)
76 | ast.Inspect(body, func(node ast.Node) bool {
77 | switch stmt := node.(type) {
78 | case *ast.AssignStmt:
79 | var ident *ast.Ident
80 | // -------------------------------------------------------------------------
81 | // info.ObjectOf requires *ast.ident. How can I extract *ast.ident from *ast.AssignStmt.Lhs?
82 | // The flow should be similar to how *ast.Ident is taken in the *ast.IncDecStmt case
83 | // -------------------------------------------------------------------------
84 | if info.ObjectOf(ident) == obj {
85 | outputs = append(outputs, output(ident, fset.Position(ident.Pos())))
86 | }
87 |
88 | case *ast.IncDecStmt:
89 | ident, ok := stmt.X.(*ast.Ident)
90 | if ok && info.ObjectOf(ident) == obj {
91 | outputs = append(outputs, output(ident, fset.Position(ident.Pos())))
92 | }
93 | }
94 | return true
95 | })
96 | }
97 | }
98 | return true
99 | }
100 |
101 | conf := types.Config{Importer: importer.Default()}
102 | for _, f := range p.Files {
103 | info, err = populateTypes(conf, fset, f)
104 | if err != nil {
105 | log.Fatal(err)
106 | }
107 | ast.Inspect(f, visitor)
108 | }
109 | return outputs
110 | }
111 |
112 | func output(ident *ast.Ident, pos token.Position) string {
113 | return fmt.Sprintf("\"%s\" overwrites func parameter in pos: %s:%d:%d", ident.Name, pos.Filename, pos.Line, pos.Column)
114 | }
115 |
--------------------------------------------------------------------------------
/CompilerFrontEndASTInGo/ArgsOverwriteAnalyzer_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/stretchr/testify/assert"
5 | "github.com/stretchr/testify/require"
6 | "go/ast"
7 | "go/parser"
8 | "go/token"
9 | "path/filepath"
10 | "testing"
11 | )
12 |
13 | func Test_analyzePackage(t *testing.T) {
14 | var testCases = []struct {
15 | name string
16 | result []string
17 | }{
18 | {name: "SimpleOverwriting", result: []string{"\"body\" overwrites func parameter in pos"}},
19 | {name: "AnonymousFunction", result: []string{"\"a\" overwrites func parameter in pos"}},
20 | {name: "OverwritingParamFromOuterScope", result: []string{"\"a\" overwrites func parameter in pos"}},
21 | {name: "AssigningParamToAVariableFirst", result: []string{}},
22 | {name: "MultipleParamsOfSameType", result: []string{"\"a\" overwrites func parameter in pos", "\"c\" overwrites func parameter in pos"}},
23 | {name: "ShadowingVariable", result: []string{"\"a\" overwrites func parameter in pos"}},
24 | {name: "EmptyBodyFunction", result: []string{}},
25 | {name: "NoWarnings", result: []string{}},
26 | {name: "DecrementOperator", result: []string{"\"retries\" overwrites func parameter"}},
27 | {name: "LhsMultipleValues", result: []string{"\"a\" overwrites func parameter", "\"b\" overwrites func parameter"}},
28 | }
29 | for _, tc := range testCases {
30 | t.Run(tc.name, func(t *testing.T) {
31 | path := filepath.Join(".", "testdata", tc.name)
32 | fset := token.NewFileSet()
33 | pkgs, err := parser.ParseDir(fset, path, nil, 0)
34 | require.NoError(t, err)
35 | require.Len(t, pkgs, 1)
36 | var testPkg *ast.Package
37 | for _, pkg := range pkgs {
38 | testPkg = pkg
39 | }
40 |
41 | outputs := analyzePackage(testPkg, fset)
42 | assert.Len(t, outputs, len(tc.result))
43 | for i := range outputs {
44 | assert.Contains(t, outputs[i], tc.result[i])
45 | }
46 | })
47 | }
48 | }
49 |
--------------------------------------------------------------------------------
/CompilerFrontEndASTInGo/CodeExamples/BadExpr.txt:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | func main() {
4 | if {
5 | println("Hello, World!")
6 | }
7 | }
8 |
9 | // focus on the missing condition if and how it's represented in the AST
--------------------------------------------------------------------------------
/CompilerFrontEndASTInGo/CodeExamples/DifferentSpecsAndDecls.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | const (
4 | a = iota
5 | b
6 | c = 3
7 | )
8 |
9 | type d int
10 |
11 | var (
12 | e = 7
13 | )
14 |
15 | func main() {
16 | const name = 8
17 | name2 = 7
18 | }
19 |
20 | // 1. See how each declaration is represented in the AST and the difference between them. Specifically look which type
21 | // implements each interface (spec or decl) by looking at the docs for go/ast: https://golang.org/pkg/go/ast/.
22 |
23 | // You can also look at the code, it gives a good overview: https://golang.org/src/go/ast/ast.go?s=26852:26891#L851
24 | // and here:https://golang.org/src/go/ast/ast.go?s=29093:29170#L929
25 |
26 | //2. Look at the difference between the "name" and "name2".
27 |
--------------------------------------------------------------------------------
/CompilerFrontEndASTInGo/CodeExamples/SimpleForLoop.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | func main() {
4 | for n := 0; n <= 5; n++ {
5 | if n%2 == 0 {
6 | continue
7 | }
8 | }
9 | }
10 |
11 | //1. See the properties of the for loop
12 |
13 | //2. See the properties of the if condition
14 |
15 | //3. See how continue is represented in the code
16 |
17 | // You can go to https://golang.org/pkg/go/ast/ for the types and their documentation. It'll give you an explanation
18 | // about the properties of each type.
19 |
--------------------------------------------------------------------------------
/CompilerFrontEndASTInGo/README.md:
--------------------------------------------------------------------------------
1 | ## Compiler front end, AST, and analysis introduction
2 | ### 2.1 Go packages overview
3 | There are six relevant packages regarding the compiler front end when talking about static analysis:
4 |
5 | - [token](https://golang.org/pkg/go/token/) - Package `token` defines constants representing the lexical tokens of Go
6 | - [scanner](https://golang.org/pkg/go/scanner/) - Package `scanner` implements a scanner for Go source text. It takes a `[]byte` as the source, which can then be tokenized through repeated calls to the `Scan` method.
7 | - [parser](https://golang.org/pkg/go/parser/) - Package `parser` implements a parser for Go source files. The output is an abstract syntax tree (AST) representing the Go source
8 | - [AST](https://golang.org/pkg/go/ast/) - Package `AST` declares the types used to represent syntax trees for Go packages.
9 | - [constant](https://golang.org/pkg/go/constant/) - Package `constant` implements Values representing untyped Go constants and their corresponding operations.
10 | - [types](https://golang.org/pkg/go/types/) - Package `types` declares the data types and implements the algorithms for type-checking of Go packages
11 |
12 | The `scanner` package is fed with `[]byte` representing the source code. Its output is a list of tokens defined by the
13 | `token` package, and the parser package uses them to create the `AST` tree. After the tree is constructed,
14 | the parser runs type-checking algorithms run over the tree, validates its correctness, and evaluates constants.
15 |
16 | 
17 |
18 | ### 2.2 AST in Go
19 | An abstract syntax tree (AST) is a way of representing the syntax of a programming language as a hierarchical tree-like structure. Let's take a look at the following program for an explanation.
20 |
21 | ``` go
22 | package main
23 | import "fmt"
24 |
25 | func main() {
26 | fmt.Println("hello world")
27 | }
28 | ```
29 |
30 | We can use this [AST visualizer](http://goast.yuroyoro.net/) to view it's AST.
31 | ```
32 | 0 *ast.File {
33 | 1 . Package: 1:1
34 | 2 . Name: *ast.Ident {
35 | 3 . . NamePos: 1:9
36 | 4 . . Name: "main"
37 | 5 . }
38 | 6 . Decls: []ast.Decl (len = 2) {
39 | 7 . . 0: *ast.GenDecl {
40 | 8 . . . TokPos: 3:1
41 | 9 . . . Tok: import
42 | 10 . . . Lparen: -
43 | 11 . . . Specs: []ast.Spec (len = 1) {
44 | 12 . . . . 0: *ast.ImportSpec {
45 | 13 . . . . . Path: *ast.BasicLit {
46 | 14 . . . . . . ValuePos: 3:8
47 | 15 . . . . . . Kind: STRING
48 | 16 . . . . . . Value: "\"fmt\""
49 | 17 . . . . . }
50 | 18 . . . . . EndPos: -
51 | 19 . . . . }
52 | 20 . . . }
53 | 21 . . . Rparen: -
54 | 22 . . }
55 | 23 . . 1: *ast.FuncDecl {
56 | 24 . . . Name: *ast.Ident {
57 | 25 . . . . NamePos: 5:6
58 | 26 . . . . Name: "main"
59 | 27 . . . . Obj: *ast.Object {
60 | 28 . . . . . Kind: func
61 | 29 . . . . . Name: "main"
62 | 30 . . . . . Decl: *(obj @ 23)
63 | 31 . . . . }
64 | 32 . . . }
65 | 33 . . . Type: *ast.FuncType {
66 | 34 . . . . Func: 5:1
67 | 35 . . . . Params: *ast.FieldList {
68 | 36 . . . . . Opening: 5:10
69 | 37 . . . . . Closing: 5:11
70 | 38 . . . . }
71 | 39 . . . }
72 | 40 . . . Body: *ast.BlockStmt {
73 | 41 . . . . Lbrace: 5:13
74 | 42 . . . . List: []ast.Stmt (len = 1) {
75 | 43 . . . . . 0: *ast.ExprStmt {
76 | 44 . . . . . . X: *ast.CallExpr {
77 | 45 . . . . . . . Fun: *ast.SelectorExpr {
78 | 46 . . . . . . . . X: *ast.Ident {
79 | 47 . . . . . . . . . NamePos: 6:3
80 | 48 . . . . . . . . . Name: "fmt"
81 | 49 . . . . . . . . }
82 | 50 . . . . . . . . Sel: *ast.Ident {
83 | 51 . . . . . . . . . NamePos: 6:7
84 | 52 . . . . . . . . . Name: "Println"
85 | 53 . . . . . . . . }
86 | 54 . . . . . . . }
87 | 55 . . . . . . . Lparen: 6:14
88 | 56 . . . . . . . Args: []ast.Expr (len = 1) {
89 | 57 . . . . . . . . 0: *ast.BasicLit {
90 | 58 . . . . . . . . . ValuePos: 6:15
91 | 59 . . . . . . . . . Kind: STRING
92 | 60 . . . . . . . . . Value: "\"hello world\""
93 | 61 . . . . . . . . }
94 | 62 . . . . . . . }
95 | 63 . . . . . . . Ellipsis: -
96 | 64 . . . . . . . Rparen: 6:28
97 | 65 . . . . . . }
98 | 66 . . . . . }
99 | 67 . . . . }
100 | 68 . . . . Rbrace: 7:1
101 | 69 . . . }
102 | 70 . . }
103 | 71 . }
104 | 72 . Scope: *ast.Scope {
105 | 73 . . Objects: map[string]*ast.Object (len = 1) {
106 | 74 . . . "main": *(obj @ 27)
107 | 75 . . }
108 | 76 . }
109 | 77 . Imports: []*ast.ImportSpec (len = 1) {
110 | 78 . . 0: *(obj @ 12)
111 | 79 . }
112 | 80 . Unresolved: []*ast.Ident (len = 1) {
113 | 81 . . 0: *(obj @ 46)
114 | 82 . }
115 | 83 }
116 | ```
117 |
118 | Let's focus on the JSON under [`*ast.File`](https://golang.org/pkg/go/ast/#File) representing a Go source file. The file is the root node, and it contains all
119 | the top-level declarations in the file - the import and the main function. Under `mains'` body, we have a
120 | [`*ast.blockStmt`](https://golang.org/pkg/go/ast/#BlockStmt) containing a list of the function statements. Similar to HTML, the dependency of the nodes create a
121 | tree-like structure.
122 |
123 | The syntax is "abstract" in the sense that it does not represent every detail appearing in the real syntax, but rather
124 | just the structural or content-related details. For instance, grouping parentheses are implicit in the tree structure,
125 | so these are not represented as separate nodes.
126 |
127 | ### 2.3 go/ast Package members
128 | The AST package contains the types used to represent syntax trees in Go. We can divide the members into three categories:
129 | Interfaces, concrete types, and others.
130 |
131 | - Concrete Types: The full list is [long](https://golang.org/pkg/go/ast/#ArrayType). Those are the tree nodes, and they contain values such as: `FuncDecl`, `IncDecStmt`, `Ident`, `Comment`, and so on.
132 | - Interfaces: `Node`, `Decl`, `Spec`, `Stmt`, `Expr`
133 | - Others: `Package`, `File`, `Scope`, `Object`
134 |
135 | Well take a look at `AssignStmt`
136 | ```go
137 | type AssignStmt struct {
138 | Lhs []Expr // All the variables to left side of assign operator
139 | TokPos token.Pos // position of operator
140 | Tok token.Token // assignment token. `=`, `:=`, `+=`, `<<=` and so on...
141 | Rhs []Expr // Expressions to right of the assignment operator
142 | }
143 | ```
144 | For example, in the expression: `a := 5`,
145 | - Lhs is [a]
146 | - Rhs is [5]
147 | - TokPos [3] (the position of the ":" character)
148 | - Tok [:=]
149 |
150 | Pretty straight forward. Now we'll look at `Expr` which `AssignStmt` implements. `Expr` is common interface for
151 | everything that returns a value. As you can see, it only contains the node interface (which is implemented by all the nodes on the AST graph).
152 |
153 | ```go
154 | type Expr interface {
155 | Node
156 | // contains filtered or unexported methods
157 | }
158 | ```
159 |
160 | From the other's group we'll look at `File`.
161 | ```go
162 | type File struct {
163 | Doc *CommentGroup // associated documentation; or nil
164 | Package token.Pos // position of "package" keyword
165 | Name *Ident // package name
166 | Decls []Decl // top-level declarations; or nil
167 | Scope *Scope // package scope (this file only)
168 | Imports []*ImportSpec // imports in this file
169 | Unresolved []*Ident // unresolved identifiers in this file
170 | Comments []*CommentGroup // list of all comments in the source file
171 | }
172 |
173 | ```
174 |
175 | An `ast.File` is the root node of each of the files we analyze. When analyzing a program, we'll iterate over the files
176 | and for each we'll pass it to the `ast.Inspect` function for iteration.
177 |
178 | It's worth mentioning again that `ast` package contains only the "abstract" parts so it ignores parentheses, colon, etc...
179 |
180 | ### 2.4 Exercise:
181 | In the folder [`CompilerFrontEndASTInGo/CodeExamples`](https://github.com/amit-davidson/GopherCon2021IsraelStaticAnalysisWorkshop/tree/master/CompilerFrontEndASTInGo/CodeExamples)
182 | there are some interesting programs (well... AST-wise). Using our [AST visualizer](http://goast.yuroyoro.net/)
183 | from earlier, take each of the program and look at their AST. I added comments explaining the important points.
184 |
185 | You should start with [`SimpleForLoop`](https://github.com/amit-davidson/GopherCon2021IsraelStaticAnalysisWorkshop/blob/master/CompilerFrontEndASTInGo/CodeExamples/SimpleForLoop.go),
186 | move to [`BadExpr`](https://github.com/amit-davidson/GopherCon2021IsraelStaticAnalysisWorkshop/blob/master/CompilerFrontEndASTInGo/CodeExamples/BadExpr.txt)
187 | and finish with [`DifferentSpecsAndDecls`](https://github.com/amit-davidson/GopherCon2021IsraelStaticAnalysisWorkshop/blob/master/CompilerFrontEndASTInGo/CodeExamples/DifferentSpecsAndDecls.go)
188 |
189 | ### 2.5 Loading a program using the parser
190 | To load the program, we need to parse it first
191 | ``` go
192 | package main
193 |
194 | import (
195 | "fmt"
196 | "go/ast"
197 | "go/parser"
198 | "go/token"
199 | )
200 |
201 | func main() {
202 |
203 | src := `package main
204 |
205 | import "fmt"
206 |
207 | func main() {
208 | fmt.Println("hello world")}
209 | `
210 |
211 | fset := token.NewFileSet()
212 | f, err := parser.ParseFile(fset, "", src, 0)
213 | if err != nil {
214 | fmt.Println(err)
215 | return
216 | }
217 | visitor := func(node ast.Node) bool {
218 | strLit, ok := (node).(*ast.BasicLit)
219 | if ok && strLit.Value == "\"hello world\"" {
220 | pos := fset.Position(strLit.Pos())
221 | fmt.Printf("We found hello world in pos:%d:%d", pos.Line, pos.Column)
222 | return false
223 | }
224 | return true
225 | }
226 | ast.Inspect(f, visitor)
227 | }
228 | ```
229 |
230 |
231 | We first create a [`fileSet`](https://golang.org/pkg/go/token/#FileSet), representing a set of source files. `FileSet` has the properties `files` and `base`
232 | recording the used files and all the files' total size. Using the size property of `token.File`, we can easily determine
233 | in which file a statement is, given its position.
234 |
235 | ```go
236 | fset := token.NewFileSet()
237 | ```
238 |
239 | 
240 |
241 | Then, we call the [`parser.ParseFile`](https://golang.org/pkg/go/parser/#ParseFile) function, providing it our `fileSet` to populate it, an empty path, a string as the
242 | source so the parser will use it instead of loading from a file, and a build mode - 0. In this example, we used 0 to
243 | fully load the program, but any other [mode](https://golang.org/pkg/go/parser/#Mode) can be used.
244 |
245 | ```go
246 | f, err := parser.ParseFile(fset, "", src, 0)
247 | if err != nil {
248 | fmt.Println(err)
249 | return
250 | }
251 | ```
252 | > Tip: Instead of iterating file by file, you can load an entire directory using [`parser.ParseDir`](https://golang.org/pkg/go/parser/#ParseDir)
253 |
254 | Finally, we define a visitor function that will be called with each node inside the AST. We pass our function to
255 | [`ast.Inspect`](https://golang.org/pkg/go/ast/#Inspect) to iterate over all the nodes in depth-first order and print a message when we reach the
256 | `Hello World` string literal with it's position in the code. We return `true` each iteration to keep traversing the tree until we found the desired
257 | node. Then, we print our message and return false to indicate we're done searching and to exit the traverse function.
258 |
259 | ```go
260 | visitor := func(node ast.Node) bool {
261 | strLit, ok := (node).(*ast.BasicLit)
262 | if ok && strLit.Value == "\"hello world\"" {
263 | pos := fset.Position(strLit.Pos())
264 | fmt.Printf("We found hello world in pos:%d:%d", pos.Line, pos.Column)
265 | return false
266 | }
267 | return true
268 | }
269 | ast.Inspect(f, visitor)
270 | ```
271 |
272 | ### 2.6 Exercise 2!
273 | In the file [`CompilerFrontEndASTInGo/ArgsOverwriteAnalyzer.go`](https://github.com/amit-davidson/GopherCon2021IsraelStaticAnalysisWorkshop/blob/master/CompilerFrontEndASTInGo/ArgsOverwriteAnalyzer.go)
274 | we have an analyzer that checks if function arguments were modified as in the example below. The problem is that there
275 | are some parts missing from it. There are comments in the places where you should add your code according to the comment.
276 | You can run the tests [`CompilerFrontEndASTInGo/ArgsOverwriteAnalyzer_test.go`](https://github.com/amit-davidson/GopherCon2021IsraelStaticAnalysisWorkshop/blob/master/CompilerFrontEndASTInGo/ArgsOverwriteAnalyzer_test.go)
277 | to make sure your tests pass. You can also debug using the test to inspect the AST graph of this program.
278 |
279 | If you give up, you can see the result in [`CompilerFrontEndASTInGo/result/ArgsOverwriteAnalyzer.go`](https://github.com/amit-davidson/GopherCon2021IsraelStaticAnalysisWorkshop/blob/master/CompilerFrontEndASTInGo/result/ArgsOverwriteAnalyzer.go) :)
280 |
281 |
282 | ### 2.7 Congratulations
283 | You have a good understanding of what AST is, the different Go packages used to create static code analyzers that
284 | interact with it and how to write such analyzers.
285 |
286 | In the [next section](https://github.com/amit-davidson/GopherCon2021IsraelStaticAnalysisWorkshop/blob/master/CompilerMiddleEndSSAInGo)
287 | we'll focus on the middle end level, and see how analyzer "operating" in this level work.
288 |
--------------------------------------------------------------------------------
/CompilerFrontEndASTInGo/result/ArgsOverwriteAnalyzer.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "go/ast"
6 | "go/importer"
7 | "go/parser"
8 | "go/token"
9 | "go/types"
10 | "log"
11 | "os"
12 | )
13 |
14 | func main() {
15 | path := os.Args[1]
16 | fset, pkgs, err := loadProgram(path)
17 | if err != nil {
18 | log.Fatalf("Failed to parse dir %s: %s", path, err)
19 | }
20 |
21 | for _, pkg := range pkgs {
22 | outputs := analyzePackage(pkg, fset)
23 | for _, message := range outputs {
24 | fmt.Println(message)
25 | }
26 | }
27 | }
28 |
29 | func loadProgram(path string) (*token.FileSet, map[string]*ast.Package, error) {
30 | fset := token.NewFileSet()
31 | pkgs, err := parser.ParseDir(fset, path, nil, 0)
32 | if err != nil {
33 | return nil, nil, err
34 | }
35 | return fset, pkgs, nil
36 | }
37 |
38 | func populateTypes(conf types.Config, fset *token.FileSet, f *ast.File) (*types.Info, error) {
39 | info := &types.Info{
40 | Defs: make(map[*ast.Ident]types.Object),
41 | Uses: make(map[*ast.Ident]types.Object),
42 | }
43 | _, err := conf.Check("", fset, []*ast.File{f}, info)
44 | if err != nil {
45 | return nil, err
46 | }
47 | return info, nil
48 | }
49 |
50 | func analyzePackage(p *ast.Package, fset *token.FileSet) []string {
51 | var info *types.Info
52 | var err error
53 | outputs := make([]string, 0)
54 |
55 | visitor := func(node ast.Node) bool {
56 | var typ *ast.FuncType
57 | var body *ast.BlockStmt
58 | switch fn := node.(type) {
59 | case *ast.FuncDecl: // Regular function
60 | typ = fn.Type
61 | body = fn.Body
62 | case *ast.FuncLit: // Anonymous function
63 | typ = fn.Type
64 | body = fn.Body
65 | }
66 | if typ == nil || body == nil { // Exclude other types but also external functions with missing body
67 | return true
68 | }
69 | if len(typ.Params.List) == 0 {
70 | return true
71 | }
72 |
73 | for _, field := range typ.Params.List {
74 | for _, arg := range field.Names {
75 | obj := info.ObjectOf(arg)
76 | ast.Inspect(body, func(node ast.Node) bool {
77 | switch stmt := node.(type) {
78 | case *ast.AssignStmt:
79 | for _, lhs := range stmt.Lhs {
80 | ident, ok := lhs.(*ast.Ident)
81 | if ok && info.ObjectOf(ident) == obj {
82 | outputs = append(outputs, output(ident, fset.Position(ident.Pos())))
83 | }
84 | }
85 | case *ast.IncDecStmt:
86 | ident, ok := stmt.X.(*ast.Ident)
87 | if ok && info.ObjectOf(ident) == obj {
88 | outputs = append(outputs, output(ident, fset.Position(ident.Pos())))
89 | }
90 | }
91 | return true
92 | })
93 | }
94 | }
95 | return true
96 | }
97 |
98 | conf := types.Config{Importer: importer.Default()}
99 | for _, f := range p.Files {
100 | info, err = populateTypes(conf, fset, f)
101 | if err != nil {
102 | log.Fatal(err)
103 | }
104 | ast.Inspect(f, visitor)
105 | }
106 | return outputs
107 | }
108 |
109 | func output(ident *ast.Ident, pos token.Position) string {
110 | return fmt.Sprintf("\"%s\" overwrites func parameter in pos: %s:%d:%d", ident.Name, pos.Filename, pos.Line, pos.Column)
111 | }
112 |
--------------------------------------------------------------------------------
/CompilerFrontEndASTInGo/testdata/AnonymousFunction/main.go:
--------------------------------------------------------------------------------
1 | package testdata
2 |
3 | func main() {
4 | _ = func(a int) {
5 | a = 5
6 | }
7 | }
--------------------------------------------------------------------------------
/CompilerFrontEndASTInGo/testdata/AssigningParamToAVariableFirst/main.go:
--------------------------------------------------------------------------------
1 | package testdata
2 |
3 | func body(a int) {
4 | b := a
5 | b = 5
6 | _ = b
7 | }
8 |
9 | func main() {
10 | body(5)
11 | }
--------------------------------------------------------------------------------
/CompilerFrontEndASTInGo/testdata/DecrementOperator/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | func retry(fn func() error, retries int) error {
4 | for {
5 | if err := fn(); err != nil {
6 | if retries < 1 {
7 | return err
8 | }
9 | retries--
10 | continue
11 | }
12 | return nil
13 | }
14 | }
15 |
16 | func main() {
17 | _ = retry(func() error {
18 | return nil
19 | }, 5)
20 | }
21 |
--------------------------------------------------------------------------------
/CompilerFrontEndASTInGo/testdata/EmptyBodyFunction/add_amd64.s:
--------------------------------------------------------------------------------
1 | TEXT ·Add(SB),$0-24
2 | MOVQ a+0(FP), AX
3 | ADDQ b+8(FP), AX
4 | MOVQ AX, ret+16(FP)
5 | RET
6 |
--------------------------------------------------------------------------------
/CompilerFrontEndASTInGo/testdata/EmptyBodyFunction/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | // Add returns the sum of a and b.
4 | func Add(a int64, b int64) int64
5 |
6 | func body(a int) {
7 | b := a
8 | b = 5
9 | _ = b
10 | }
11 |
12 | func main() {
13 | body(5)
14 | _ = Add(3, 4)
15 | }
16 |
--------------------------------------------------------------------------------
/CompilerFrontEndASTInGo/testdata/LhsMultipleValues/main.go:
--------------------------------------------------------------------------------
1 | package testdata
2 |
3 | func main() {
4 | _ = func(a, b int) {
5 | a, b = 5, 7
6 | }
7 | }
8 |
--------------------------------------------------------------------------------
/CompilerFrontEndASTInGo/testdata/MultipleParamsOfSameType/main.go:
--------------------------------------------------------------------------------
1 | package testdata
2 |
3 | func body(a, b int, c int) {
4 | a = 5
5 | _ = b
6 | c = 3
7 | }
8 |
9 | func main() {
10 | body(1, 2, 3)
11 | }
12 |
--------------------------------------------------------------------------------
/CompilerFrontEndASTInGo/testdata/NoWarnings/main.go:
--------------------------------------------------------------------------------
1 | package testdata
2 |
3 | import "fmt"
4 |
5 | func closeBody(body int) {
6 | fmt.Print(body)
7 | }
8 |
9 | func main() {
10 | closeBody(1)
11 | }
12 |
--------------------------------------------------------------------------------
/CompilerFrontEndASTInGo/testdata/OverwritingParamFromOuterScope/main.go:
--------------------------------------------------------------------------------
1 | package testdata
2 |
3 | func body(a int) {
4 | _ = func() {
5 | a = 5
6 | }
7 | }
8 |
9 | func main() {
10 | body(5)
11 | }
--------------------------------------------------------------------------------
/CompilerFrontEndASTInGo/testdata/ShadowingVariable/main.go:
--------------------------------------------------------------------------------
1 | package testdata
2 |
3 | func body(a, b int, c int) {
4 | f := func(a int, b int) {
5 | a = 5
6 | }
7 | _ = f
8 | }
9 |
10 | func main() {
11 | body(1, 2, 3)
12 | }
13 |
--------------------------------------------------------------------------------
/CompilerFrontEndASTInGo/testdata/SimpleOverwriting/main.go:
--------------------------------------------------------------------------------
1 | package testdata
2 |
3 | func closeBody(body int) {
4 | body = 1
5 | }
6 |
7 | func main() {
8 | closeBody(1)
9 | }
10 |
--------------------------------------------------------------------------------
/CompilerMiddleEndSSAInGo/CodeExamples/ElseIf/ElseIf.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "os"
5 | "strconv"
6 | )
7 |
8 | var a int
9 |
10 | func main() {
11 | a, _ = strconv.Atoi(os.Args[1])
12 | if a > 0 {
13 | a = 3
14 | } else if a == 0 {
15 | a = 4
16 | } else {
17 | a = 5
18 | }
19 | }
20 |
21 | // 1. In the source code, the condition can go to the if, the else if or the else blocks. Here, each block can have up to 2
22 | // successors. Notice that the first block can only jump to 1 (a=3) or 3 (a==0) and 3 evaluates the condition again and can go to
23 | // 4 (a=4) or 5(a=5)
24 |
25 | // 2. `a` is declared in the global scope (and not under the function scope). Because it's in the global scope, it might
26 | // be used in other places, so the compiler isn't sure if it could have side effects, (not considered dead) so it can't
27 | // optimize away it using dead code elimination.
28 |
--------------------------------------------------------------------------------
/CompilerMiddleEndSSAInGo/CodeExamples/Map/Map.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | func main() {
4 |
5 | m := make(map[string]int)
6 |
7 | m["k1"] = 7
8 | m["k2"] = 13
9 |
10 | v1 := m["k1"]
11 | _ = v1
12 |
13 | _, prs := m["k2"]
14 | _ = prs
15 |
16 | delete(m, "k2")
17 | }
18 |
19 | // See how different map operations are carried out in SSA:
20 | // 1. Creating a map
21 | // 2. Assigning a value to a map
22 | // 3. Reading from a map
23 | // 4. Reading from a map into multiple variables
24 | // 5. Deleting a key from a map
25 |
--------------------------------------------------------------------------------
/CompilerMiddleEndSSAInGo/README.md:
--------------------------------------------------------------------------------
1 | ## 3. Compiler middle end and static analysis with SSA In Go
2 | ### 3.1 What is SSA?
3 | [SSA](https://en.wikipedia.org/wiki/Static_single_assignment_form) stands for static single assignment. It's a property of an IR **that requires each variable to be assigned exactly once**, and every variable be defined before it is used.
4 | The primary usefulness of SSA comes from how it simplifies the properties of variables and improves compilers optimizations.
5 |
6 | For example, consider this piece of code:
7 | ```
8 | y := 1
9 | y := 2
10 | x := y
11 | ```
12 |
13 | In an SSA form, it'll be translated to:
14 | ```
15 | y1 := 1
16 | y2 := 2
17 | x1 := y2
18 | ```
19 | Humans can see that the first assignment is unnecessary and that the value of `y` used in the third line comes from the
20 | second assignment of `y`. In SSA form, both of these are immediate
21 |
22 | ### 3.2 go/ssa package members
23 | The package [`tools/go/ssa`](https://pkg.go.dev/golang.org/x/tools/go/ssa) defines the representation of elements of Go programs in SSA format.
24 | The key types form a hierarchical structure.
25 |
26 | #### [Program](https://pkg.go.dev/golang.org/x/tools/go/ssa#Program)
27 | A Program is a partial or complete Go program converted to an SSA form.
28 |
29 | 
30 |
31 | #### [Package](https://pkg.go.dev/golang.org/x/tools/go/ssa#Package)
32 | A Package is a single analyzed Go package containing Members for all package-level functions, variables, constants, and types it declares.
33 |
34 | 
35 |
36 | #### [Function](https://pkg.go.dev/golang.org/x/tools/go/ssa#Function)
37 | Function represents the parameters, results, and code of a function or method.
38 |
39 | 
40 |
41 | #### [Basic Block](https://pkg.go.dev/golang.org/x/tools/go/ssa#BasicBlock)
42 | BasicBlock represents an SSA basic block. A set of instructions that are executed and can't jump somewhere else. Basic blocks are connected using conditions and goto statements.
43 |
44 | 
45 |
46 | Control Flow Graph (CFG) - In a control-flow graph, each node in the graph represents a basic block.
47 | Together, they compose all paths that might be traversed through a program during its execution.
48 |
49 | 
50 |
51 | #### [Instruction](https://pkg.go.dev/golang.org/x/tools/go/ssa#Instruction)
52 | a statement that consumes values and performs computation. For example, `Call`, `Return`, `TypeAssert`, etc
53 |
54 | 
55 |
56 | #### [Value](https://pkg.go.dev/golang.org/x/tools/go/ssa#Value)
57 | an expression that yields a value. For example, function calls are both `Instruction` and `Value` since they both consume values and yield a value.
58 |
59 | 
60 |
61 | And when combined:
62 |
63 | 
64 |
65 | The package contains other [types](https://pkg.go.dev/golang.org/x/tools/go/ssa#pkg-overview) - Include language keywords such as `Defer`, `If` but also lower level primitives like `MakeChan` and `Alloc`.
66 |
67 |
68 | ### 3.3 Overviewing an analyzer!
69 | In this section we'll look over an analyzer that warns when `t.Fatal` is used inside a goroutine as described here:
70 | https://github.com/ipfs/go-ipfs/issues/2043
71 |
72 | ### 3.4 SSA vs AST
73 | The most important difference is that AST reasons about the structure of the code, where SSA reasons about how data
74 | flows in the code. Why do need both? Each "level" suits for a different problem. You can think of it as satellite vs
75 | terrain modes on maps. They both represent the same source map, but each mode solves a different problem.
76 |
77 | We can summarize the differences using the following table:
78 | | | SSA | AST |
79 | |----------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
80 | | Why to Choose? | - Better when need to handle how data flows through the code.
- Does optimizations to the code such as inlining or constant propagation so some functions or variables might be missing
- Package types are closer to the language
| - Better when when analyzing the code itself, or you don’t want to reason about the control flow graph.
- Runs over the source code, so optimizations don’t happen yet.
|
81 | | Examples | - Checking a function for infinite recursion
- Checking if all flows after “mutex.Lock” are covered with “mutex.unlock”
| - Passing the correct types to string format
- Shifts that equal or exceed the width of the integer
- Modifying B.n when benchmarking
- Validate the order of imports according to a convention
|
82 |
83 | ### 3.5 Congratulations
84 | You have a good understanding of what IR and SSA are, the SSA package used to create static code analyzers that
85 | use it and how to write such analyzers.
86 |
87 | In the [next section](https://github.com/amit-davidson/GopherCon2021IsraelStaticAnalysisWorkshop/tree/master/AnalysisApi)
88 | we'll focus on the analysis API. A package used define a common API for all code analyzers and to make writing analyses easier.
89 | It also provides us an infrastructure that helps us with all the non-logic code such as loading, testing and running our
90 | analysis.
91 |
92 | ### Extra:
93 |
94 | ### 3.6 Viewing SSA
95 | We can [`ssadump`](https://pkg.go.dev/golang.org/x/tools/cmd/ssadump) to view the SSA form of programs.
96 | > You can also use this [SSA visualizer](http://golang-ssaview.herokuapp.com/) in view SSA in your CLI. For this example,
97 | > I chose not to, since it uses a different [build mode](https://pkg.go.dev/golang.org/x/tools/go/ssa#BuilderMode) than
98 | > the one we need.
99 |
100 | Let's consider this program:
101 | ``` go
102 | package main
103 |
104 | import (
105 | "fmt"
106 | "math"
107 | "os"
108 | )
109 |
110 | func main() {
111 | shapeType := os.Args[1]
112 | squareOrCircleArea(shapeType)
113 | }
114 |
115 | func squareOrCircleArea(shapeType string) {
116 | r := 2.0
117 | area := r * r
118 | if shapeType == "circle" {
119 | area *= math.Pi
120 | }
121 | fmt.Printf("Total area is: %g", area)
122 | }
123 | ```
124 |
125 | I'll focus on the `squareOrCircleArea` function.
126 | ```go
127 | func squareOrCircleArea(shapeType string):
128 | 0: entry P:0 S:2
129 | t0 = 2:float64 * 2:float64 float64
130 | t1 = shapeType == "circle":string bool
131 | if t1 goto 1 else 2
132 | 1: if.then P:1 S:1
133 | t2 = t0 * 3.14159:float64 float64
134 | jump 2
135 | 2: if.done P:2 S:0
136 | t3 = phi [0: t0, 1: t2] #area float64
137 | t4 = new [1]interface{} (varargs) *[1]interface{}
138 | t5 = &t4[0:int] *interface{}
139 | t6 = make interface{} <- float64 (t3) interface{}
140 | *t5 = t6
141 | t7 = slice t4[:] []interface{}
142 | t8 = fmt.Printf("Total area is: %g":string, t7...) (n int, err error)
143 | return
144 | ```
145 |
146 | Looking at the first basic block (0) we can see straight away that the variable names were replaced with `t` followed by a number.
147 | Also, the assignment to `r` is missing and it's values are already used in the assignment to `area` (`t0`) in the first
148 | line. This is the result of constant propagation and dead code elimination indicating this code is already optimized.
149 |
150 | In the end of the block, we can see a conditional goto (as opposed to the conventional if structure) to the correct
151 | basic block, according to the shape type.
152 |
153 | ```go
154 | 0: entry P:0 S:2
155 | t0 = 2:float64 * 2:float64 float64
156 | t1 = shapeType == "circle":string bool
157 | if t1 goto 1 else 2
158 | ```
159 | In the source code, we multiply the value of area with PI and assign it back to the area. In SSA form, each variable is
160 | assigned once. We can see that `t0` is no longer used and instead `t2` is declared, even though in high level they point
161 | to the same variable.
162 | ```go
163 | 1:
164 | t2 = t0 * 3.14159:float64 float64
165 | jump 2
166 | ```
167 |
168 | In the last block we see an instruction called `phi`. This instruction represents an SSA φ-node which combines values
169 | that differ across incoming control-flow edges and yields a new value. We won't delve deeper, but in short, it says
170 | the value can be either `t0` or `t2`, depending on the control flow.
171 |
172 | At that point, we're ready to print the variable, but there are many instructions between `t3` and the `fmt.Printf` function.
173 | IR is much more verbose and includes instructions that may by represented with single "action" in the source code. In
174 | this case, `fmt.Printf` uses variadic parameters. Behind the scenes, we have to declare a list of interfaces, convert
175 | our `float64` to the `interface{}` type and only then pass it to the function.
176 | ```go
177 | 2: if.done P:2 S:0
178 | t3 = phi [0: t0, 1: t2] #area float64
179 | t4 = new [1]interface{} (varargs) *[1]interface{}
180 | t5 = &t4[0:int] *interface{}
181 | t6 = make interface{} <- float64 (t3) interface{}
182 | *t5 = t6
183 | t7 = slice t4[:] []interface{}
184 | t8 = fmt.Printf("Total area is: %g":string, t7...) (n int, err error)
185 | return
186 | ```
187 |
188 | ### 3.7 Exercise
189 | In the folder [`CompilerMiddleEndSSAInGo/CodeExamples`](https://github.com/amit-davidson/GopherCon2021IsraelStaticAnalysisWorkshop/tree/master/CompilerMiddleEndSSAInGo/CodeExamples)
190 | there are some interesting programs. Using our [`ssadump`](https://pkg.go.dev/golang.org/x/tools/cmd/ssadump) from earlier, take each of the program and look at their SSA.
191 | I added comments with notes with explaining the important points. You should start first with [`CompilerMiddleEndSSAInGo/CodeExamples/Map`](https://github.com/amit-davidson/GopherCon2021IsraelStaticAnalysisWorkshop/blob/master/CompilerMiddleEndSSAInGo/CodeExamples/Map/Map.go)
192 | and then [`CompilerMiddleEndSSAInGo/CodeExamples/ElseIf`](https://github.com/amit-davidson/GopherCon2021IsraelStaticAnalysisWorkshop/blob/master/CompilerMiddleEndSSAInGo/CodeExamples/ElseIf/ElseIf.go)
193 |
194 | ```bash
195 | go get -u golang.org/x/tools/cmd/ssadump
196 | ssadump -build=FI ./CompilerMiddleEndSSAInGo/CodeExamples/Map/
197 | ssadump -build=FI ./CompilerMiddleEndSSAInGo/CodeExamples/ElseIf/
198 | ```
199 | We use the `F` to print the SSA code, and `I` to ignore `init` function.
200 |
201 |
202 |
--------------------------------------------------------------------------------
/CompilerMiddleEndSSAInGo/TerminationInsideGoroutine.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "errors"
5 | "fmt"
6 | "go/token"
7 | "go/types"
8 | "golang.org/x/tools/go/packages"
9 | "golang.org/x/tools/go/ssa"
10 | "golang.org/x/tools/go/ssa/ssautil"
11 | "log"
12 | "os"
13 | )
14 |
15 | var ErrNoPackages = errors.New("no packages in the path")
16 | var ErrLoadPackages = errors.New("loading the following file contained errors")
17 | var errMessage = "is called in a separate goroutine, but it must be called in the same goroutine as the test"
18 | var loadMode = packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles | packages.NeedImports | packages.NeedExportsFile | packages.NeedTypes | packages.NeedSyntax | packages.NeedTypesInfo | packages.NeedTypesSizes
19 |
20 | func main() {
21 | path := os.Args[1]
22 | prog, pkg, err := loadPackage(path)
23 | if err != nil {
24 | log.Fatalf("Failed to parse dssa %s: %s", path, err)
25 | }
26 | funcs := getAllFunctions(pkg)
27 | for _, fn := range funcs {
28 | output := checkTerminationInsideGoroutine(fn, prog.Fset)
29 | fmt.Println(output)
30 | }
31 |
32 | }
33 |
34 | func output(name string, pos token.Position) string {
35 | return fmt.Sprintf("T.%s %s: %s:%d:%d", name, errMessage, pos.Filename, pos.Line, pos.Column)
36 | }
37 |
38 | func checkTerminationInsideGoroutine(fn *ssa.Function, fset *token.FileSet) string {
39 | for _, block := range fn.Blocks {
40 | for _, ins := range block.Instrs {
41 | gostmt, ok := ins.(*ssa.Go)
42 | if !ok {
43 | continue
44 | }
45 | gofn := gostmt.Call.StaticCallee()
46 | if gofn == nil {
47 | continue
48 | }
49 | if gofn.Blocks == nil {
50 | continue
51 | }
52 | for _, block := range gofn.Blocks {
53 | for _, ins := range block.Instrs {
54 | call, ok := ins.(*ssa.Call)
55 | if !ok {
56 | continue
57 | }
58 | callee := call.Call.StaticCallee()
59 | if callee == nil {
60 | continue
61 | }
62 | recv := callee.Signature.Recv()
63 | if recv == nil {
64 | continue
65 | }
66 | if types.TypeString(recv.Type(), nil) != "*testing.common" {
67 | continue
68 | }
69 | terminateFn, ok := callee.Object().(*types.Func)
70 | if !ok {
71 | continue
72 | }
73 | name := terminateFn.Name()
74 | switch name {
75 | case "FailNow", "Fatal", "Fatalf", "SkipNow", "Skip", "Skipf":
76 | default:
77 | continue
78 | }
79 | return output(name, fset.Position(call.Pos()))
80 | }
81 | }
82 | }
83 | }
84 | return ""
85 | }
86 |
87 | func getAllFunctions(pkg *ssa.Package) []*ssa.Function {
88 | funcs := make([]*ssa.Function, 0)
89 | for _, mem := range pkg.Members {
90 | if fdecl, ok := mem.(*ssa.Function); ok {
91 | var addAnons func(f *ssa.Function)
92 | addAnons = func(f *ssa.Function) {
93 | if f.Name() == "init" {
94 | return
95 | }
96 | funcs = append(funcs, f)
97 | for _, anon := range f.AnonFuncs {
98 | addAnons(anon)
99 | }
100 | }
101 | addAnons(fdecl)
102 | }
103 | }
104 | return funcs
105 | }
106 |
107 | func loadPackage(path string) (*ssa.Program, *ssa.Package, error) {
108 | conf1 := packages.Config{
109 | Mode: loadMode,
110 | }
111 | loadQuery := fmt.Sprintf("file=%s", path)
112 | pkgs, err := packages.Load(&conf1, loadQuery)
113 | if err != nil {
114 | return nil, nil, err
115 | }
116 |
117 | if len(pkgs) == 0 {
118 | return nil, nil, fmt.Errorf("%s: %w", path, ErrNoPackages)
119 | }
120 |
121 | if len(pkgs[0].Errors) > 0 {
122 | return nil, nil, fmt.Errorf("%w %s: %s", ErrLoadPackages, path, pkgs[0].Errors[0].Msg)
123 | }
124 |
125 | ssaProg, builtPkgs := ssautil.Packages(pkgs, 0)
126 | ssaProg.Build()
127 |
128 | return ssaProg, builtPkgs[0], nil
129 | }
130 |
--------------------------------------------------------------------------------
/CompilerMiddleEndSSAInGo/TerminationInsideGoroutine_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/stretchr/testify/assert"
5 | "github.com/stretchr/testify/require"
6 | "path/filepath"
7 | "testing"
8 | )
9 |
10 | func Test_analyzePackage(t *testing.T) {
11 | var testCases = []struct {
12 | name string
13 |
14 | result string
15 | }{
16 | {name: "FatalInsideGoroutineSimpleFlag", result: errMessage},
17 | {name: "SkipInsideGoroutineSimpleFlag", result: errMessage},
18 | {name: "FatalInsideGoroutineAnonymousFlag", result: errMessage},
19 | {name: "FatalInsideRegularFunctionNoFlag", result: ""},
20 | }
21 | for _, tc := range testCases {
22 | t.Run(tc.name, func(t *testing.T) {
23 | path := filepath.Join(".", "testdata", tc.name, "main.go")
24 | prog, pkg, err := loadPackage(path)
25 | require.NoError(t, err)
26 | funcs := getAllFunctions(pkg)
27 | for _, fn := range funcs {
28 | res := checkTerminationInsideGoroutine(fn, prog.Fset)
29 | if fn.Name() == "main" {
30 | assert.Contains(t, res, tc.result)
31 | }
32 | }
33 | })
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/CompilerMiddleEndSSAInGo/testdata/FatalInsideGoroutineAnonymousFlag/main.go:
--------------------------------------------------------------------------------
1 | package testdata
2 |
3 | import "testing"
4 |
5 | func main() {
6 | var t *testing.T
7 | fn := func() {
8 | t.Fatal()
9 | }
10 | go fn()
11 | }
12 |
--------------------------------------------------------------------------------
/CompilerMiddleEndSSAInGo/testdata/FatalInsideGoroutineSimpleFlag/main.go:
--------------------------------------------------------------------------------
1 | package testdata
2 |
3 | import "testing"
4 |
5 | func main() {
6 | var t *testing.T
7 | go func() {
8 | t.Fatal()
9 | }()
10 | }
11 |
--------------------------------------------------------------------------------
/CompilerMiddleEndSSAInGo/testdata/FatalInsideRegularFunctionNoFlag/main.go:
--------------------------------------------------------------------------------
1 | package testdata
2 |
3 | import "testing"
4 |
5 | func main() {
6 | var t *testing.T
7 | fn := func() {
8 | t.Fatal()
9 | }
10 | fn()
11 | }
12 |
--------------------------------------------------------------------------------
/CompilerMiddleEndSSAInGo/testdata/SkipInsideGoroutineSimpleFlag/main.go:
--------------------------------------------------------------------------------
1 | package testdata
2 |
3 | import "testing"
4 |
5 | func main() {
6 | var t *testing.T
7 | go func() {
8 | t.Skip()
9 | }()
10 | }
11 |
--------------------------------------------------------------------------------
/Conclusion/README.md:
--------------------------------------------------------------------------------
1 | ## 5. Conclusion and further discussion
2 | ### 5.1 Precision and Soundness
3 | Soundness and precision are two properties used to measure static analysis tools.
4 |
5 | A static analysis tool is sound if it finds all of its occurrences in the code for a specific property to check. In other words, the fewer false negatives the static tool has, the more sound it is.
6 |
7 | Given the following program:
8 | ``` go
9 | r := 1
10 | if rand.Int() > 0.5 {
11 | r = 2
12 | }
13 | ```
14 | An unsound analysis will result in `r=1` ignoring the possible value of `2`, whereas a sound analysis will result in `r=1` or `r=2`.
15 |
16 | Precision is the ability of a static analysis tool to flag only the property we're interested in. As previously, the fewer false positives, the more the tool is precise.
17 | If we return the example from above, an imprecise analysis might say `r` is between `0` and `100`, where a precise analysis will say `r` is only `1` or `2`.
18 |
19 | Precision, Soundness and computing resources are tradeoff. Having the ability to flag more cases makes the program more
20 | sound but might result in false positives. On the other way around, limiting the number of cases to cover makes the
21 | analysis more precise. And choosing to go in either direction results in an increased amount of computing resources such
22 | as memory and time.
23 |
24 | When you'll write static analysis tools, you might encounter where this trade off comes into play, especially on IR level.
25 | Usually, it's easier to implement a more sounder analysis then a more precise one, so in reality, most of the tools go
26 | in this direction.
27 |
28 | ## 5.2 Other tools
29 | There are famous built-in tools such as `go vet` and `go fmt`, but there are many more others:
30 |
31 | - `go fmt` - Gofmt formats Go programs. It uses tabs for indentation and blanks for alignment. Alignment assumes that an editor is using a fixed-width font.
32 | - `go vet` - Vet examines Go source code and reports suspicious constructs, such as `Printf` calls whose arguments do not align with the format string
33 | - `go imports` - The command go imports updates your Go import lines, adding missing ones and removing unreferenced ones.
34 | - `go fix` - Fix finds Go programs that use old APIs and rewrites them to use newer ones. After you update to a new Go release, fix helps make the necessary changes to your programs.
35 |
36 | There's also an awesome (pun intended) [list](https://github.com/golangci/awesome-go-linters) of Go analysis tools written by the Go community.
37 | It contains tools you can integrate into your toolchain.
38 |
39 | - [staticcheck](https://github.com/dominikh/go-tools) is similar to `go vet` but applies many more checks such as forgetting to `unlock` a `mutex` using the defer statement, validating JSON tags correctness, and so on.
40 |
41 | - [golanglint-ci](https://github.com/golangci/golangci-lint) is a fast Go linters runner. It runs linters in parallel, uses caching, supports `yaml` config, has integrations with all major IDE, and has dozens of linters included. You can look at the full list of linters [here](https://golangci-lint.run/usage/linters/).
42 |
43 | - [Chronos](https://github.com/amit-davidson/Chronos) a static race detector I built
44 |
45 | ### 5.3 Further reading:
46 | - A deeper dive into the topics of the AST part in Go - https://github.com/golang/example/tree/master/gotypes
47 | - A deeper dive into SSA in GO- https://www.youtube.com/watch?v=uTMvKVma5ms&ab_channel=GopherAcademy
48 | - A viewer used to look at the code at different phases of the compilation process - https://golang.design/gossa
49 | - More builtin package about static code analysis we didn't cover - https://pkg.go.dev/golang.org/x/tools
50 |
51 | ### 5.4 Discussion
52 | Any questions?
53 |
--------------------------------------------------------------------------------
/Intro/README.md:
--------------------------------------------------------------------------------
1 | ## 1.1 program analysis
2 | ### 1.1.1 What is program analysis?
3 |
4 | From Wikipedia:
5 |
6 | > Program analysis is the process of automatically analyzing the behavior of computer programs regarding a property such as correctness, robustness, safety and liveness. Program analysis focuses on two major areas: program optimization and program correctness. The first focuses on improving the program's performance while reducing the resource usage while the latter focuses on ensuring that the program does what it is supposed to do.
7 |
8 |
9 |
10 | ### 1.1.2 Benefits of program analysis
11 | Program analysis can help detect bugs before they reach production. You've probably used them already and didn't even
12 | thought about it. A linter is used to flag styling errors, a profiler to find performance issues and aid in optimizing
13 | them, and even tests to validate program correctness.
14 |
15 | I've gathered a list of issues that could be solved using static analysis tools:
16 |
17 | - https://github.com/ipfs/go-ipfs/issues/2043 - Using `t.Fatal` inside a goroutine
18 | - https://github.com/golang/go/issues/23842 - Calling `waitgroup.Add` inside the calling a goroutine
19 | - https://github.com/cockroachdb/cockroach/issues/7972 - Deadlock
20 | - https://github.com/instana/go-sensor/issues/51 - race condition
21 |
22 | ### 1.1.3 Static analysis vs. Dynamic analysis
23 |
24 | > Program analysis can be performed without executing the program (static program analysis), during runtime (dynamic program analysis) or in a combination of both.
25 |
26 | The obvious difference between static and dynamic analysis is that dynamic is performed at runtime, whereas static works without running it.
27 |
28 | In practice, the difference is much more apparent. The main benefit of dynamic code analysis is that it finds bugs that can actually occur. Also, they are usually easier to write and require fewer resources to run. The problem with dynamic code analysis is that they make the code run slower (I'm looking at you `go race`), which might be intolerant in production.
29 |
30 | On the other hand, static analysis can also find bugs **that may**/**before they** occur. The reason is that static
31 | analysis can evaluate paths of the code or workloads that don't often happen in production.
32 | Evaluating all the possibilities comes at the expanse of the time and resources required to perform analysis and
33 | inaccuracy of the final result.
34 |
35 | ## 1.2 Compilers
36 | ### 1.2.1 Overview
37 | From Wikipedia:
38 | > In computing, a compiler is a computer program that translates computer code written in one programming language (the source language) into another language (the target language). The name "compiler" is primarily used for programs that translate source code from a high-level programming language to a lower level language (e.g., assembly language, object code, or machine code) to create an executable program.
39 |
40 | Compilers are divided into three stages - front end, middle end, and the back end.
41 |
42 | - Front end - The front end scans the input and verifies syntax and semantics according to a specific source language.
43 | For example, it makes sure the code contains only familiar characters; it validates that an if statement has a condition
44 | and is not wrapped with parentheses. It also does type checking to make sure the correct types are passed accordingly around the program.
45 | The front end transforms the input program into an [intermediate representation (IR)]((https://en.wikipedia.org/wiki/Intermediate_representation)) for further processing by the middle end.
46 |
47 | - Middle end - The middle end performs optimizations on the intermediate representation to improve the performance and the quality of the produced machine code. Popular optimizations include: [dead code elimination](https://en.wikipedia.org/wiki/Dead_code_elimination), [constant propagation](https://en.wikipedia.org/wiki/Constant_folding) and [inlining](https://en.wikipedia.org/wiki/Inline_expansion).
48 |
49 | - Back end - The back end is responsible for the CPU architecture specific optimizations and code generation - converting IR to machine code.
50 |
51 | 
52 |
53 | ### What is IR?
54 | An [**intermediate representation** (**IR**)](https://en.wikipedia.org/wiki/Intermediate_representation) is the code
55 | used internally by a compiler to represent source code. An IR is designed to be conducive for further processing,
56 | such as optimization and translation. A "good" IR must be _accurate_ – capable of representing the source code without
57 | loss of information – and _independent_ of any particular source or target language.
58 |
59 | ### 1.2.2 Frontend
60 | Most commonly today, the frontend is broken into three phases: lexical analysis (also known as tokenizing or scanning),
61 | syntax analysis (also known as scanning or parsing), and semantic analysis.
62 |
63 | #### Tokenizing
64 | converts a sequence of characters into a sequence of tokens. A token is a pair consisting of a token name and token value.
65 |
66 | | Token name | Sample token values |
67 | |------------|----------------------------------|
68 | | identifier | x, color, UP |
69 | | keyword | if, while, return |
70 | | separator | }, (, ; |
71 | | operator | +, <, = |
72 | | literal | true, 6.02e23, "music" |
73 | | comment | // can't happen in production |
74 |
75 |
76 | #### Parsing
77 | This involves parsing the token sequence to identify the program. This phase builds a parse tree or an abstract syntax tree, which replaces the linear sequence of tokens with a tree structure.
78 |
79 | #### Semantic Analysis
80 | This phase performs checks such as type checking and rejecting incorrect programs. It also constructs the symbol table
81 | used to map between identifiers and information relating to their declaration or appearance in the source.
82 |
83 | For the given code, the type of the variable `n` is inferred in the semantic analysis phase, marking it as `int`
84 | ``` go
85 | n := 50 + 70
86 | ```
87 |
88 |
89 |
90 | By looking at all the steps, we can see how they come together, and a tree representation of the code is built from source.
91 |
92 | 
93 |
94 | ### 1.2.3 Middle end
95 | The middle end part performs optimizations regardless of the source code language and the target machine.
96 | As opposed to the front end phase, the middle end analysis is more complex. By estimating how the data will flow through
97 | the code, the compiler does optimizations ranging from the scope of a function to the entire program (interprocedural).
98 |
99 | I'll demonstrate an optimization using Constant Propagation mentioned earlier. Constant propagation is the process of substituting the values of known
100 | constants in expressions. Constant propagation eliminates cases in which values are copied from one location or
101 | variable to another to assign their value to another variable.
102 |
103 | The algorithm works by doing iterations from top to bottom and and each pass see if it can improve current state. If no
104 | more improvements can be made anymore, the algorithm terminates.
105 |
106 | Propagating X:
107 | ```
108 | int x = 14; <-
109 | int y = 7 - x / 2;
110 | return y * (28 / x + 2);
111 | ```
112 | ```
113 | int x = 14;
114 | int y = 7 - 14 / 2; <-
115 | return y * (28 / x + 2);
116 | ```
117 |
118 | ```
119 | int x = 14;
120 | int y = 7 - 14 / 2;
121 | return y * (28 / 14 + 2); <-
122 | ```
123 | Propagating Y:
124 | ```
125 | int x = 14; <-
126 | int y = 7 - 14 / 2;
127 | return y * (28 / 14 + 2);
128 | ```
129 | ```
130 | int x = 14;
131 | int y = 0; <-
132 | return y * (28 / 14 + 2);
133 | ```
134 |
135 | ```
136 | int x = 14;
137 | int y = 0;
138 | return 0; <-
139 | ```
140 |
141 | We can further optimize this code using dead code elimination - The process of removing code that does not affect
142 | the program results. We can optimize away `x` and `y' which results in
143 | ```
144 | return 0;
145 | ```
146 |
147 | ### 1.3 Congratulations
148 | You now have a grasp about program analyzers and a good understanding of how compilers are built and work.
149 |
150 | In the [next section](https://github.com/amit-davidson/GopherCon2021IsraelStaticAnalysisWorkshop/blob/master/CompilerFrontEndASTInGo)
151 | we'll focus on the front end level, and see how analyzer "operating" in this level work
152 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Static Analysis with Go - A Practitioner's Guide
2 | [](http://golang.org)
3 | [](http://makeapullrequest.com)
4 |
5 | Hi, and welcome to Static Analysis with Go - A Practitioner's Guide.
6 | This is a workshop about writing static code analyzers in Go for Go.
7 | In this lecture you will learn about how to write a static code analysis in Go, and implement one yourself.
8 |
9 | By the end of this workshop, you'll have a better understanding of the Go packages related to writing static code
10 | analyzers and you'll also know how to write a code analyzer yourself.
11 |
12 | I will start the lecture by giving an overview of static analyzers and how compilers work. Afterwards, you will learn
13 | about 2 different representations of the code (AST and SSA), and write an analyzer in each of those.
14 | You will also learn about the analysis API making writing analyzers easier and then we'll finish with a discussion.
15 |
16 | ### Requirements:
17 | Install the repo:
18 | ```bash
19 | git clone https://github.com/amit-davidson/GopherCon2021IsraelStaticAnalysisWorkshop.git
20 | ```
21 |
22 | Install [Go tools](https://github.com/golang/tools):
23 | ```bash
24 | go get -u golang.org/x/tools/...
25 | ```
26 |
27 | Validate the installation by running
28 | ```bash
29 | ssadump -h
30 | ```
31 | and making sure you get a help message that starts with: `Usage of ssadump:`
32 |
33 | ### Contents:
34 | 1. [Introduction to compilers and program analysis](https://github.com/amit-davidson/GopherCon2021IsraelStaticAnalysisWorkshop/tree/master/Intro)
35 | 2. [Compiler front end and static analysis with AST In Go](https://github.com/amit-davidson/GopherCon2021IsraelStaticAnalysisWorkshop/blob/master/CompilerFrontEndASTInGo)
36 | 3. [Compiler middle end and static analysis with SSA In Go](https://github.com/amit-davidson/GopherCon2021IsraelStaticAnalysisWorkshop/blob/master/CompilerMiddleEndSSAInGo)
37 | 3. [The analysis API](https://github.com/amit-davidson/GopherCon2021IsraelStaticAnalysisWorkshop/tree/master/AnalysisApi)
38 | 3. [Conclusion](https://github.com/amit-davidson/GopherCon2021IsraelStaticAnalysisWorkshop/tree/master/Conclusion)
--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/amit-davidson/GopherCon2021IsraelStaticAnalysisWorkshop
2 |
3 | go 1.15
4 |
5 | require (
6 | github.com/stretchr/testify v1.6.1
7 | golang.org/x/mod v0.4.1 // indirect
8 | golang.org/x/sys v0.1.0 // indirect
9 | golang.org/x/tools v0.1.0
10 | )
11 |
--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
1 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
3 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
4 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
5 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
6 | github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
7 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
8 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
9 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
10 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
11 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
12 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
13 | golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY=
14 | golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
15 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
16 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
17 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
18 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
19 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
20 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
21 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
22 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
23 | golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
24 | golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
25 | golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
26 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
27 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
28 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
29 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
30 | golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
31 | golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
32 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
33 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
34 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
35 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
36 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
37 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
38 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
39 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
40 |
--------------------------------------------------------------------------------