├── Procfile ├── app ├── analyze │ ├── processors │ │ ├── test │ │ │ ├── main0.go │ │ │ ├── main1.go │ │ │ └── 1.patch │ │ ├── processor.go │ │ ├── nop_processor.go │ │ ├── def.go │ │ ├── factory.go │ │ ├── errors.go │ │ ├── executor.go │ │ ├── result.go │ │ ├── repo_processor_factory.go │ │ ├── repo_processor.go │ │ ├── github_go_repo.go │ │ ├── github_go_pr_test.go │ │ └── github_go_pr.go │ ├── linters │ │ ├── result │ │ │ ├── result.go │ │ │ └── issue.go │ │ ├── linter.go │ │ ├── runner.go │ │ ├── linter_mock.go │ │ └── golinters │ │ │ └── golangci_lint.go │ ├── analyzequeue │ │ ├── task │ │ │ └── task.go │ │ ├── consumers │ │ │ ├── analyze_pr_test.go │ │ │ ├── base_consumer.go │ │ │ ├── analyze_pr.go │ │ │ └── analyze_repo.go │ │ ├── consume.go │ │ ├── produce.go │ │ └── consume_test.go │ ├── reporters │ │ ├── reporter.go │ │ ├── reporter_mock.go │ │ └── github_reviewer.go │ ├── repostate │ │ ├── storage.go │ │ ├── api_storage.go │ │ └── storage_mock.go │ ├── prstate │ │ ├── storage.go │ │ ├── api_storage.go │ │ └── storage_mock.go │ └── repoinfo │ │ ├── fetcher_mock.go │ │ └── fetcher.go ├── scripts │ └── cleanup.sh ├── lib │ ├── fetchers │ │ ├── repo.go │ │ ├── fetcher.go │ │ ├── git_test.go │ │ ├── git.go │ │ └── fetcher_mock.go │ ├── goutils │ │ ├── environments │ │ │ ├── environment.go │ │ │ └── golang.go │ │ └── workspaces │ │ │ ├── workspace.go │ │ │ ├── go2.go │ │ │ └── go.go │ ├── runmode │ │ └── runmode.go │ ├── fsutils │ │ ├── fsutils.go │ │ ├── path_resolver.go │ │ └── path_resolver_test.go │ ├── timeutils │ │ └── track.go │ ├── executors │ │ ├── env.go │ │ ├── executor.go │ │ ├── temp_dir_shell_test.go │ │ ├── remote_shell_test.go │ │ ├── temp_dir_shell.go │ │ ├── remote_shell.go │ │ ├── shell.go │ │ ├── executor_mock.go │ │ └── container.go │ ├── errorutils │ │ └── errors.go │ ├── queue │ │ └── queue.go │ ├── github │ │ ├── context.go │ │ ├── client_mock.go │ │ └── client.go │ ├── httputils │ │ ├── client.go │ │ └── client_mock.go │ └── experiments │ │ └── checker.go ├── test │ ├── linters.go │ └── env.go ├── cmd │ └── golangci-worker │ │ └── golangci-worker.go ├── analytics │ ├── mixpanel.go │ ├── amplitude.go │ ├── errors.go │ ├── context.go │ ├── logger.go │ └── track.go └── docker │ └── Dockerfile ├── .gitignore ├── Makefile ├── .golangci.yml ├── go.mod ├── .circleci └── config.yml ├── README.md ├── CONTRIBUTING.md └── go.sum /Procfile: -------------------------------------------------------------------------------- 1 | worker: golangci-worker 2 | -------------------------------------------------------------------------------- /app/analyze/processors/test/main0.go: -------------------------------------------------------------------------------- 1 | package p 2 | 3 | func F0() error { 4 | return nil 5 | } 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.env 2 | /.env.* 3 | /vendor 4 | /golangci-worker 5 | /gometalinter_bin 6 | /bin 7 | /*.log 8 | /logs/ 9 | -------------------------------------------------------------------------------- /app/scripts/cleanup.sh: -------------------------------------------------------------------------------- 1 | go clean -cache 2 | rm -rf /tmp/glide-vendor* 3 | rm -rf /tmp/go-build* 4 | rm -rf $HOME/.glide/cache -------------------------------------------------------------------------------- /app/lib/fetchers/repo.go: -------------------------------------------------------------------------------- 1 | package fetchers 2 | 3 | type Repo struct { 4 | CloneURL string 5 | Ref string 6 | FullPath string 7 | } 8 | -------------------------------------------------------------------------------- /app/lib/goutils/environments/environment.go: -------------------------------------------------------------------------------- 1 | package environments 2 | 3 | type EnvSettable interface { 4 | SetEnv(key, value string) 5 | } 6 | -------------------------------------------------------------------------------- /app/analyze/processors/processor.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | import "context" 4 | 5 | type Processor interface { 6 | Process(ctx context.Context) error 7 | } 8 | -------------------------------------------------------------------------------- /app/analyze/processors/test/main1.go: -------------------------------------------------------------------------------- 1 | package p 2 | 3 | import "fmt" 4 | 5 | func F0New() error { 6 | return nil 7 | } 8 | 9 | func F1() error { 10 | return fmt.Errorf("error") 11 | } 12 | -------------------------------------------------------------------------------- /app/analyze/processors/nop_processor.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | import "context" 4 | 5 | type NopProcessor struct{} 6 | 7 | func (p NopProcessor) Process(ctx context.Context) error { 8 | return nil 9 | } 10 | -------------------------------------------------------------------------------- /app/analyze/linters/result/result.go: -------------------------------------------------------------------------------- 1 | package result 2 | 3 | type Result struct { 4 | Issues []Issue 5 | MaxIssuesPerFile int // Needed for gofmt and goimports where it is 1 6 | ResultJSON interface{} 7 | } 8 | -------------------------------------------------------------------------------- /app/lib/runmode/runmode.go: -------------------------------------------------------------------------------- 1 | package runmode 2 | 3 | import "os" 4 | 5 | func IsProduction() bool { 6 | return os.Getenv("GO_ENV") == "prod" 7 | } 8 | 9 | func IsDebug() bool { 10 | return os.Getenv("DEBUG") == "1" 11 | } 12 | -------------------------------------------------------------------------------- /app/lib/fsutils/fsutils.go: -------------------------------------------------------------------------------- 1 | package fsutils 2 | 3 | import ( 4 | "go/build" 5 | "path" 6 | ) 7 | 8 | func GetProjectRoot() string { 9 | return path.Join(build.Default.GOPATH, "src", "github.com", "golangci", "golangci-worker") 10 | } 11 | -------------------------------------------------------------------------------- /app/analyze/processors/test/1.patch: -------------------------------------------------------------------------------- 1 | --- a/main.go 2 | +++ a/main.go 3 | @@ -1,5 +1,11 @@ 4 | package p 5 | 6 | -func F0() error { 7 | +import "fmt" 8 | + 9 | +func F0New() error { 10 | return nil 11 | } 12 | + 13 | +func F1() error { 14 | + return fmt.Errorf("error") 15 | +} 16 | -------------------------------------------------------------------------------- /app/lib/timeutils/track.go: -------------------------------------------------------------------------------- 1 | package timeutils 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/sirupsen/logrus" 8 | ) 9 | 10 | func Track(now time.Time, format string, args ...interface{}) { 11 | logrus.Infof("[timing] %s took %s", fmt.Sprintf(format, args...), time.Since(now)) 12 | } 13 | -------------------------------------------------------------------------------- /app/lib/goutils/environments/golang.go: -------------------------------------------------------------------------------- 1 | package environments 2 | 3 | type Golang struct { 4 | gopath string 5 | } 6 | 7 | func NewGolang(gopath string) *Golang { 8 | return &Golang{ 9 | gopath: gopath, 10 | } 11 | } 12 | 13 | func (g Golang) Setup(es EnvSettable) { 14 | es.SetEnv("GOPATH", g.gopath) 15 | } 16 | -------------------------------------------------------------------------------- /app/test/linters.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "github.com/golangci/golangci-worker/app/analyze/linters/result" 5 | ) 6 | 7 | func NewIssue(linter, message string, line int) result.Issue { 8 | return result.Issue{ 9 | FromLinter: linter, 10 | Text: message, 11 | File: "p/f.go", 12 | LineNumber: line, 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /app/lib/executors/env.go: -------------------------------------------------------------------------------- 1 | package executors 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | ) 7 | 8 | type envStore struct { 9 | env []string 10 | } 11 | 12 | func newEnvStore() *envStore { 13 | return &envStore{ 14 | env: os.Environ(), 15 | } 16 | } 17 | 18 | func (e *envStore) SetEnv(k, v string) { 19 | e.env = append(e.env, fmt.Sprintf("%s=%s", k, v)) 20 | } 21 | -------------------------------------------------------------------------------- /app/lib/fetchers/fetcher.go: -------------------------------------------------------------------------------- 1 | package fetchers 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/golangci/golangci-worker/app/lib/executors" 7 | ) 8 | 9 | //go:generate mockgen -package fetchers -source fetcher.go -destination fetcher_mock.go 10 | 11 | type Fetcher interface { 12 | Fetch(ctx context.Context, repo *Repo, exec executors.Executor) error 13 | } 14 | -------------------------------------------------------------------------------- /app/lib/errorutils/errors.go: -------------------------------------------------------------------------------- 1 | package errorutils 2 | 3 | type InternalError struct { 4 | PublicDesc string 5 | PrivateDesc string 6 | } 7 | 8 | func (e InternalError) Error() string { 9 | return e.PrivateDesc 10 | } 11 | 12 | type BadInputError struct { 13 | PublicDesc string 14 | } 15 | 16 | func (e BadInputError) Error() string { 17 | return e.PublicDesc 18 | } 19 | -------------------------------------------------------------------------------- /app/analyze/analyzequeue/task/task.go: -------------------------------------------------------------------------------- 1 | package task 2 | 3 | import "github.com/golangci/golangci-worker/app/lib/github" 4 | 5 | type PRAnalysis struct { 6 | github.Context 7 | APIRequestID string 8 | UserID uint 9 | AnalysisGUID string 10 | } 11 | 12 | type RepoAnalysis struct { 13 | Name string 14 | AnalysisGUID string 15 | Branch string 16 | } 17 | -------------------------------------------------------------------------------- /app/analyze/reporters/reporter.go: -------------------------------------------------------------------------------- 1 | package reporters 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/golangci/golangci-worker/app/analyze/linters/result" 7 | ) 8 | 9 | //go:generate mockgen -package reporters -source reporter.go -destination reporter_mock.go 10 | 11 | type Reporter interface { 12 | Report(ctx context.Context, ref string, issues []result.Issue) error 13 | } 14 | -------------------------------------------------------------------------------- /app/analyze/processors/def.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | const ( 4 | internalError = "Internal error" 5 | 6 | statusSentToQueue = "sent_to_queue" 7 | statusProcessing = "processing" 8 | statusProcessed = "processed" 9 | statusNotFound = "not_found" 10 | 11 | noGoFilesToAnalyzeMessage = "No Go files to analyze" 12 | noGoFilesToAnalyzeErr = "no go files to analyze" 13 | ) 14 | -------------------------------------------------------------------------------- /app/cmd/golangci-worker/golangci-worker.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/golangci/golangci-worker/app/analyze/analyzequeue" 5 | "github.com/golangci/golangci-worker/app/lib/queue" 6 | "github.com/sirupsen/logrus" 7 | ) 8 | 9 | func main() { 10 | queue.Init() 11 | analyzequeue.RegisterTasks() 12 | if err := analyzequeue.RunWorker(); err != nil { 13 | logrus.Fatalf("Can't run analyze worker: %s", err) 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /app/lib/goutils/workspaces/workspace.go: -------------------------------------------------------------------------------- 1 | package workspaces 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/golangci/golangci-api/pkg/goenv/result" 7 | "github.com/golangci/golangci-worker/app/lib/executors" 8 | "github.com/golangci/golangci-worker/app/lib/fetchers" 9 | ) 10 | 11 | type Installer interface { 12 | Setup(ctx context.Context, repo *fetchers.Repo, projectPathParts ...string) (executors.Executor, *result.Log, error) 13 | } 14 | -------------------------------------------------------------------------------- /app/analyze/linters/linter.go: -------------------------------------------------------------------------------- 1 | package linters 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/golangci/golangci-worker/app/analyze/linters/result" 7 | "github.com/golangci/golangci-worker/app/lib/executors" 8 | ) 9 | 10 | //go:generate mockgen -package linters -source linter.go -destination linter_mock.go 11 | 12 | type Linter interface { 13 | Run(ctx context.Context, exec executors.Executor) (*result.Result, error) 14 | Name() string 15 | } 16 | -------------------------------------------------------------------------------- /app/analyze/linters/result/issue.go: -------------------------------------------------------------------------------- 1 | package result 2 | 3 | type Issue struct { 4 | FromLinter string 5 | Text string 6 | File string 7 | LineNumber int 8 | HunkPos int 9 | } 10 | 11 | func NewIssue(fromLinter, text, file string, lineNumber, hunkPos int) Issue { 12 | return Issue{ 13 | FromLinter: fromLinter, 14 | Text: text, 15 | File: file, 16 | LineNumber: lineNumber, 17 | HunkPos: hunkPos, 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /app/lib/executors/executor.go: -------------------------------------------------------------------------------- 1 | package executors 2 | 3 | import "context" 4 | 5 | //go:generate mockgen -package executors -source executor.go -destination executor_mock.go 6 | 7 | type Executor interface { 8 | Run(ctx context.Context, name string, args ...string) (string, error) 9 | 10 | WithEnv(k, v string) Executor 11 | SetEnv(k, v string) 12 | 13 | WorkDir() string 14 | WithWorkDir(wd string) Executor 15 | 16 | CopyFile(ctx context.Context, dst, src string) error 17 | 18 | Clean() 19 | } 20 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | run_dev: 2 | godotenv go run app/cmd/golangci-worker/golangci-worker.go 3 | 4 | gen: 5 | go generate ./... 6 | 7 | build: 8 | go build ./app/cmd/... 9 | 10 | test: 11 | go test -v -count 1 ./... 12 | golangci-lint run -v 13 | 14 | test_repo: 15 | # set env vars PR, REPO 16 | SLOW_TESTS_ENABLED=1 go test -v ./app/analyze -run TestAnalyzeRepo 17 | 18 | test_repo_fake_github: 19 | # set env vars PR, REPO 20 | SLOW_TESTS_ENABLED=1 go test -v ./app/analyze/processors -count=1 -run TestProcessRepoWithFakeGithub -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters-settings: 2 | govet: 3 | check-shadowing: true 4 | golint: 5 | min-confidence: 0 6 | gocyclo: 7 | min-complexity: 10 8 | maligned: 9 | suggest-new: true 10 | dupl: 11 | threshold: 100 12 | goconst: 13 | min-len: 2 14 | min-occurrences: 2 15 | 16 | linters: 17 | enable-all: true 18 | disable: 19 | - maligned 20 | - lll 21 | - prealloc 22 | - dupl # temporary 23 | - gosec 24 | - scopelint 25 | - gochecknoglobals 26 | - gochecknoinits 27 | -------------------------------------------------------------------------------- /app/analyze/repostate/storage.go: -------------------------------------------------------------------------------- 1 | package repostate 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | //go:generate mockgen -package repostate -source storage.go -destination storage_mock.go 9 | 10 | type State struct { 11 | CreatedAt time.Time 12 | Status string 13 | ResultJSON interface{} 14 | } 15 | 16 | type Storage interface { 17 | UpdateState(ctx context.Context, owner, name, analysisID string, state *State) error 18 | GetState(ctx context.Context, owner, name, analysisID string) (*State, error) 19 | } 20 | -------------------------------------------------------------------------------- /app/analytics/mixpanel.go: -------------------------------------------------------------------------------- 1 | package analytics 2 | 3 | import ( 4 | "os" 5 | "sync" 6 | 7 | "github.com/dukex/mixpanel" 8 | "github.com/golangci/golangci-worker/app/lib/runmode" 9 | ) 10 | 11 | var mixpanelClient mixpanel.Mixpanel 12 | var mixpanelClientOnce sync.Once 13 | 14 | func getMixpanelClient() mixpanel.Mixpanel { 15 | mixpanelClientOnce.Do(func() { 16 | if runmode.IsProduction() { 17 | apiKey := os.Getenv("MIXPANEL_API_KEY") 18 | mixpanelClient = mixpanel.New(apiKey, "") 19 | } 20 | }) 21 | 22 | return mixpanelClient 23 | } 24 | -------------------------------------------------------------------------------- /app/analytics/amplitude.go: -------------------------------------------------------------------------------- 1 | package analytics 2 | 3 | import ( 4 | "os" 5 | "sync" 6 | 7 | "github.com/golangci/golangci-worker/app/lib/runmode" 8 | "github.com/savaki/amplitude-go" 9 | ) 10 | 11 | var amplitudeClient *amplitude.Client 12 | var amplitudeClientOnce sync.Once 13 | 14 | func getAmplitudeClient() *amplitude.Client { 15 | amplitudeClientOnce.Do(func() { 16 | if runmode.IsProduction() { 17 | apiKey := os.Getenv("AMPLITUDE_API_KEY") 18 | amplitudeClient = amplitude.New(apiKey) 19 | } 20 | }) 21 | 22 | return amplitudeClient 23 | } 24 | -------------------------------------------------------------------------------- /app/analyze/prstate/storage.go: -------------------------------------------------------------------------------- 1 | package prstate 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | //go:generate mockgen -package prstate -source storage.go -destination storage_mock.go 9 | 10 | type State struct { 11 | CreatedAt time.Time 12 | Status string 13 | ReportedIssuesCount int 14 | ResultJSON interface{} 15 | } 16 | 17 | type Storage interface { 18 | UpdateState(ctx context.Context, owner, name, analysisID string, state *State) error 19 | GetState(ctx context.Context, owner, name, analysisID string) (*State, error) 20 | } 21 | -------------------------------------------------------------------------------- /app/analytics/errors.go: -------------------------------------------------------------------------------- 1 | package analytics 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/golangci/golangci-shared/pkg/config" 7 | "github.com/golangci/golangci-shared/pkg/logutil" 8 | 9 | "github.com/golangci/golangci-shared/pkg/apperrors" 10 | "github.com/golangci/golangci-worker/app/lib/runmode" 11 | ) 12 | 13 | func trackError(ctx context.Context, err error, level apperrors.Level) { 14 | if !runmode.IsProduction() { 15 | return 16 | } 17 | 18 | log := logutil.NewStderrLog("trackError") 19 | cfg := config.NewEnvConfig(log) 20 | et := apperrors.GetTracker(cfg, log, "worker") 21 | et.Track(level, err.Error(), getTrackingProps(ctx)) 22 | } 23 | -------------------------------------------------------------------------------- /app/test/env.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "path" 7 | "sync" 8 | "testing" 9 | 10 | "github.com/golangci/golangci-worker/app/lib/fsutils" 11 | "github.com/joho/godotenv" 12 | ) 13 | 14 | var initOnce sync.Once 15 | 16 | func LoadEnv() { 17 | envNames := []string{".env"} 18 | for _, envName := range envNames { 19 | fpath := path.Join(fsutils.GetProjectRoot(), envName) 20 | err := godotenv.Overload(fpath) 21 | if err != nil { 22 | log.Fatalf("Can't load %s: %s", envName, err) 23 | } 24 | } 25 | } 26 | 27 | func Init() { 28 | initOnce.Do(func() { 29 | LoadEnv() 30 | }) 31 | } 32 | 33 | func MarkAsSlow(t *testing.T) { 34 | if os.Getenv("SLOW_TESTS_ENABLED") != "1" { 35 | t.SkipNow() 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /app/lib/fetchers/git_test.go: -------------------------------------------------------------------------------- 1 | package fetchers 2 | 3 | import ( 4 | "context" 5 | "io/ioutil" 6 | "testing" 7 | 8 | "github.com/golangci/golangci-worker/app/lib/executors" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestGitOnTestRepo(t *testing.T) { 13 | exec, err := executors.NewTempDirShell("test.git") 14 | assert.NoError(t, err) 15 | defer exec.Clean() 16 | g := NewGit() 17 | 18 | repo := &Repo{ 19 | Ref: "test-branch", 20 | CloneURL: "git@github.com:golangci/test.git", 21 | } 22 | 23 | err = g.Fetch(context.Background(), repo, exec) 24 | assert.NoError(t, err) 25 | 26 | files, err := ioutil.ReadDir(exec.WorkDir()) 27 | assert.NoError(t, err) 28 | assert.Len(t, files, 3) 29 | assert.Equal(t, ".git", files[0].Name()) 30 | assert.Equal(t, "README.md", files[1].Name()) 31 | assert.Equal(t, "main.go", files[2].Name()) 32 | } 33 | -------------------------------------------------------------------------------- /app/analyze/analyzequeue/consumers/analyze_pr_test.go: -------------------------------------------------------------------------------- 1 | package consumers 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "strconv" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/golangci/golangci-worker/app/test" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestAnalyzeRepo(t *testing.T) { 15 | test.MarkAsSlow(t) 16 | test.Init() 17 | 18 | prNumber := 1 19 | if pr := os.Getenv("PR"); pr != "" { 20 | var err error 21 | prNumber, err = strconv.Atoi(pr) 22 | assert.NoError(t, err) 23 | } 24 | const userID = 1 25 | 26 | repoOwner := "golangci" 27 | repoName := "golangci-worker" 28 | if r := os.Getenv("REPO"); r != "" { 29 | parts := strings.SplitN(r, "/", 2) 30 | repoOwner, repoName = parts[0], parts[1] 31 | } 32 | 33 | err := NewAnalyzePR().Consume(context.Background(), repoOwner, repoName, 34 | os.Getenv("TEST_GITHUB_TOKEN"), prNumber, "", userID, "test-guid") 35 | assert.NoError(t, err) 36 | } 37 | -------------------------------------------------------------------------------- /app/analyze/processors/factory.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/golangci/golangci-worker/app/analytics" 8 | "github.com/golangci/golangci-worker/app/analyze/analyzequeue/task" 9 | "github.com/golangci/golangci-worker/app/lib/github" 10 | ) 11 | 12 | type Factory interface { 13 | BuildProcessor(ctx context.Context, t *task.PRAnalysis) (Processor, error) 14 | } 15 | 16 | type githubFactory struct{} 17 | 18 | func NewGithubFactory() Factory { 19 | return githubFactory{} 20 | } 21 | 22 | func (gf githubFactory) BuildProcessor(ctx context.Context, t *task.PRAnalysis) (Processor, error) { 23 | p, err := newGithubGoPR(ctx, &t.Context, githubGoPRConfig{}, t.AnalysisGUID) 24 | if err != nil { 25 | if !github.IsRecoverableError(err) { 26 | analytics.Log(ctx).Warnf("%s: skip current task: use nop processor", err) 27 | return NopProcessor{}, nil 28 | } 29 | return nil, fmt.Errorf("can't make github go processor: %s", err) 30 | } 31 | 32 | return p, nil 33 | } 34 | -------------------------------------------------------------------------------- /app/lib/queue/queue.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "sync" 8 | "time" 9 | 10 | "github.com/RichardKnop/machinery/v1" 11 | "github.com/RichardKnop/machinery/v1/config" 12 | "github.com/sirupsen/logrus" 13 | ) 14 | 15 | var server *machinery.Server 16 | var initOnce sync.Once 17 | 18 | func initServer() { 19 | redisURL := fmt.Sprintf("%s/1", os.Getenv("REDIS_URL")) // use separate DB #1 for queue 20 | logrus.Infof("REDIS_URL=%q", redisURL) 21 | 22 | cnf := &config.Config{ 23 | Broker: redisURL, 24 | DefaultQueue: "machinery_tasks", 25 | ResultBackend: redisURL, 26 | ResultsExpireIn: int((7 * 24 * time.Hour).Seconds()), // store results for 1 week 27 | } 28 | 29 | var err error 30 | server, err = machinery.NewServer(cnf) 31 | if err != nil { 32 | log.Fatalf("Can't init machinery queue server: %s", err) 33 | } 34 | } 35 | 36 | func Init() { 37 | initOnce.Do(initServer) 38 | } 39 | 40 | func GetServer() *machinery.Server { 41 | return server 42 | } 43 | -------------------------------------------------------------------------------- /app/analyze/processors/errors.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | import ( 4 | "errors" 5 | "os" 6 | "strings" 7 | 8 | "github.com/golangci/golangci-worker/app/lib/github" 9 | ) 10 | 11 | var ( 12 | errNothingToAnalyze = errors.New("nothing to analyze") 13 | ) 14 | 15 | type IgnoredError struct { 16 | Status github.Status 17 | StatusDesc string 18 | IsRecoverable bool 19 | } 20 | 21 | func (e IgnoredError) Error() string { 22 | return e.StatusDesc 23 | } 24 | 25 | func escapeErrorText(text string, secrets map[string]string) string { 26 | ret := text 27 | for secret, replacement := range secrets { 28 | ret = strings.Replace(ret, secret, replacement, -1) 29 | } 30 | 31 | return ret 32 | } 33 | 34 | func buildSecrets() map[string]string { 35 | const hidden = "{hidden}" 36 | ret := map[string]string{} 37 | 38 | for _, kv := range os.Environ() { 39 | parts := strings.Split(kv, "=") 40 | if len(parts) != 2 { 41 | continue 42 | } 43 | 44 | v := parts[1] 45 | if len(v) >= 6 { 46 | ret[v] = hidden 47 | } 48 | } 49 | 50 | return ret 51 | } 52 | -------------------------------------------------------------------------------- /app/lib/executors/temp_dir_shell_test.go: -------------------------------------------------------------------------------- 1 | package executors 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestTempDirShellWithEnv(t *testing.T) { 11 | ts, err := NewTempDirShell(t.Name()) 12 | assert.NoError(t, err) 13 | assert.NotEmpty(t, ts.wd) 14 | assert.Equal(t, os.Environ(), ts.env) 15 | 16 | defer ts.Clean() 17 | 18 | tse := ts.WithEnv("k", "v").(*TempDirShell) 19 | assert.NotEmpty(t, ts.wd) 20 | assert.Equal(t, ts.wd, tse.wd) // check was saved 21 | 22 | assert.Equal(t, os.Environ(), ts.env) // check didn't change 23 | assert.Equal(t, append(os.Environ(), "k=v"), tse.env) 24 | } 25 | 26 | func exists(t *testing.T, path string) bool { 27 | _, err := os.Stat(path) 28 | if err == nil { 29 | return true 30 | } 31 | 32 | if os.IsNotExist(err) { 33 | return false 34 | } 35 | 36 | assert.NoError(t, err) 37 | return true 38 | } 39 | 40 | func TestTempDirShellClean(t *testing.T) { 41 | ts, err := NewTempDirShell(t.Name()) 42 | assert.NoError(t, err) 43 | 44 | assert.True(t, exists(t, ts.WorkDir())) 45 | ts.Clean() 46 | assert.False(t, exists(t, ts.WorkDir())) 47 | } 48 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/golangci/golangci-worker 2 | 3 | // +heroku goVersion go1.11 4 | // +heroku install ./app/cmd/... 5 | 6 | require ( 7 | github.com/RichardKnop/machinery v0.0.0-20180221144734-c5e057032f00 8 | github.com/cenkalti/backoff v2.0.0+incompatible 9 | github.com/dukex/mixpanel v0.0.0-20170510165255-53bfdf679eec 10 | github.com/golang/mock v1.1.1 11 | github.com/golangci/getrepoinfo v0.0.0-20180818083854-2a0c71df2c85 12 | github.com/golangci/golangci-api v0.0.0-20181118193359-820cf3a69851 13 | github.com/golangci/golangci-lint v0.0.0-20181114200623-a84578d603c7 14 | github.com/golangci/golangci-shared v0.0.0-20181003182622-9200811537b3 15 | github.com/google/go-github v0.0.0-20180123235826-b1f138353a62 16 | github.com/joho/godotenv v0.0.0-20180115024921-6bb08516677f 17 | github.com/levigross/grequests v0.0.0-20180717012718-3f841d606c5a 18 | github.com/pkg/errors v0.8.0 19 | github.com/savaki/amplitude-go v0.0.0-20160610055645-f62e3b57c0e4 20 | github.com/shirou/gopsutil v0.0.0-20180801053943-8048a2e9c577 21 | github.com/sirupsen/logrus v1.0.5 22 | github.com/stretchr/testify v1.2.1 23 | golang.org/x/oauth2 v0.0.0-20180118004544-b28fcf2b08a1 24 | ) 25 | -------------------------------------------------------------------------------- /app/analyze/linters/runner.go: -------------------------------------------------------------------------------- 1 | package linters 2 | 3 | import ( 4 | "context" 5 | "log" 6 | 7 | "github.com/golangci/golangci-worker/app/analyze/linters/result" 8 | "github.com/golangci/golangci-worker/app/lib/executors" 9 | ) 10 | 11 | type Runner interface { 12 | Run(ctx context.Context, linters []Linter, exec executors.Executor) (*result.Result, error) 13 | } 14 | 15 | type SimpleRunner struct { 16 | } 17 | 18 | func (r SimpleRunner) Run(ctx context.Context, linters []Linter, exec executors.Executor) (*result.Result, error) { 19 | results := []result.Result{} 20 | for _, linter := range linters { 21 | res, err := linter.Run(ctx, exec) 22 | if err != nil { 23 | return nil, err // don't wrap error here, need to save original error 24 | } 25 | 26 | results = append(results, *res) 27 | } 28 | 29 | return r.mergeResults(results), nil 30 | } 31 | 32 | func (r SimpleRunner) mergeResults(results []result.Result) *result.Result { 33 | if len(results) == 0 { 34 | return nil 35 | } 36 | 37 | if len(results) > 1 { 38 | log.Fatalf("len(results) can't be more than 1: %+v", results) 39 | } 40 | 41 | // TODO: support for multiple linters, not only golangci-lint 42 | return &results[0] 43 | } 44 | -------------------------------------------------------------------------------- /app/lib/github/context.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/google/go-github/github" 8 | gh "github.com/google/go-github/github" 9 | "golang.org/x/oauth2" 10 | ) 11 | 12 | type Repo struct { 13 | Owner, Name string 14 | } 15 | 16 | func (r Repo) FullName() string { 17 | return fmt.Sprintf("%s/%s", r.Owner, r.Name) 18 | } 19 | 20 | type Context struct { 21 | Repo Repo 22 | GithubAccessToken string 23 | PullRequestNumber int 24 | } 25 | 26 | func (c Context) GetClient(ctx context.Context) *github.Client { 27 | ts := oauth2.StaticTokenSource( 28 | &oauth2.Token{AccessToken: c.GithubAccessToken}, 29 | ) 30 | tc := oauth2.NewClient(ctx, ts) 31 | return github.NewClient(tc) 32 | } 33 | 34 | func (c Context) GetCloneURL(repo *gh.Repository) string { 35 | if repo.GetPrivate() { 36 | return fmt.Sprintf("https://%s@github.com/%s/%s.git", 37 | c.GithubAccessToken, // it's already the private token 38 | c.Repo.Owner, c.Repo.Name) 39 | } 40 | 41 | return repo.GetCloneURL() 42 | } 43 | 44 | var FakeContext = Context{ 45 | Repo: Repo{ 46 | Owner: "owner", 47 | Name: "name", 48 | }, 49 | GithubAccessToken: "access_token", 50 | PullRequestNumber: 1, 51 | } 52 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Golang CircleCI 2.0 configuration file 2 | # 3 | # Check https://circleci.com/docs/2.0/language-go/ for more details 4 | version: 2 5 | jobs: 6 | build: 7 | docker: 8 | # specify the version 9 | - image: circleci/golang:1.11 10 | - image: redis 11 | 12 | # Specify service dependencies here if necessary 13 | # CircleCI maintains a library of pre-built images 14 | # documented at https://circleci.com/docs/2.0/circleci-images/ 15 | # - image: circleci/postgres:9.4 16 | 17 | #### TEMPLATE_NOTE: go expects specific checkout path representing url 18 | #### expecting it in the form of 19 | #### /go/src/github.com/circleci/go-tool 20 | #### /go/src/bitbucket.org/circleci/go-tool 21 | working_directory: /go/src/github.com/golangci/golangci-worker 22 | steps: 23 | - checkout 24 | 25 | # # specify any bash command here prefixed with `run: ` 26 | - run: GO111MODULE=on go mod vendor 27 | - run: make build 28 | - run: go get -u github.com/golangci/golangci-lint/cmd/golangci-lint 29 | - run: echo 'REDIS_URL="redis://localhost:6379"' >.env 30 | - run: echo 'WEB_ROOT="https://golangci.com"' >>.env 31 | - run: make test 32 | -------------------------------------------------------------------------------- /app/analytics/context.go: -------------------------------------------------------------------------------- 1 | package analytics 2 | 3 | import "context" 4 | 5 | func GetTracker(_ context.Context) Tracker { 6 | return amplitudeMixpanelTracker{} 7 | } 8 | 9 | type trackingContextKeyType string 10 | 11 | const trackingContextKey trackingContextKeyType = "tracking context" 12 | 13 | func ContextWithTrackingProps(ctx context.Context, props map[string]interface{}) context.Context { 14 | return context.WithValue(ctx, trackingContextKey, props) 15 | } 16 | 17 | func getTrackingProps(ctx context.Context) map[string]interface{} { 18 | tp := ctx.Value(trackingContextKey) 19 | if tp == nil { 20 | return map[string]interface{}{} 21 | } 22 | 23 | return tp.(map[string]interface{}) 24 | } 25 | 26 | func ContextWithEventPropsCollector(ctx context.Context, name EventName) context.Context { 27 | return context.WithValue(ctx, name, map[string]interface{}{}) 28 | } 29 | 30 | func SaveEventProp(ctx context.Context, name EventName, key string, value interface{}) { 31 | ec := ctx.Value(name).(map[string]interface{}) 32 | ec[key] = value 33 | } 34 | 35 | func SaveEventProps(ctx context.Context, name EventName, props map[string]interface{}) { 36 | ec := ctx.Value(name).(map[string]interface{}) 37 | 38 | for k, v := range props { 39 | ec[k] = v 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /app/analyze/prstate/api_storage.go: -------------------------------------------------------------------------------- 1 | //nolint:dupl 2 | package prstate 3 | 4 | import ( 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "os" 9 | 10 | "github.com/golangci/golangci-worker/app/lib/httputils" 11 | ) 12 | 13 | type APIStorage struct { 14 | host string 15 | client httputils.Client 16 | } 17 | 18 | func NewAPIStorage(client httputils.Client) *APIStorage { 19 | return &APIStorage{ 20 | client: client, 21 | host: os.Getenv("API_URL"), 22 | } 23 | } 24 | 25 | func (s APIStorage) getStatusURL(owner, name, analysisID string) string { 26 | return fmt.Sprintf("%s/v1/repos/github.com/%s/%s/analyzes/%s/state", s.host, owner, name, analysisID) 27 | } 28 | 29 | func (s APIStorage) UpdateState(ctx context.Context, owner, name, analysisID string, state *State) error { 30 | return s.client.Put(ctx, s.getStatusURL(owner, name, analysisID), state) 31 | } 32 | 33 | func (s APIStorage) GetState(ctx context.Context, owner, name, analysisID string) (*State, error) { 34 | bodyReader, err := s.client.Get(ctx, s.getStatusURL(owner, name, analysisID)) 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | defer bodyReader.Close() 40 | 41 | var state State 42 | if err = json.NewDecoder(bodyReader).Decode(&state); err != nil { 43 | return nil, fmt.Errorf("can't read json body: %s", err) 44 | } 45 | 46 | return &state, nil 47 | } 48 | -------------------------------------------------------------------------------- /app/analyze/repostate/api_storage.go: -------------------------------------------------------------------------------- 1 | //nolint:dupl 2 | package repostate 3 | 4 | import ( 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "os" 9 | 10 | "github.com/golangci/golangci-worker/app/lib/httputils" 11 | ) 12 | 13 | type APIStorage struct { 14 | host string 15 | client httputils.Client 16 | } 17 | 18 | func NewAPIStorage(client httputils.Client) *APIStorage { 19 | return &APIStorage{ 20 | client: client, 21 | host: os.Getenv("API_URL"), 22 | } 23 | } 24 | 25 | func (s APIStorage) getAnalysisURL(owner, name, analysisID string) string { 26 | return fmt.Sprintf("%s/v1/repos/github.com/%s/%s/repoanalyzes/%s", s.host, owner, name, analysisID) 27 | } 28 | 29 | func (s APIStorage) UpdateState(ctx context.Context, owner, name, analysisID string, state *State) error { 30 | return s.client.Put(ctx, s.getAnalysisURL(owner, name, analysisID), state) 31 | } 32 | 33 | func (s APIStorage) GetState(ctx context.Context, owner, name, analysisID string) (*State, error) { 34 | bodyReader, err := s.client.Get(ctx, s.getAnalysisURL(owner, name, analysisID)) 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | defer bodyReader.Close() 40 | 41 | var state State 42 | if err = json.NewDecoder(bodyReader).Decode(&state); err != nil { 43 | return nil, fmt.Errorf("can't read json body: %s", err) 44 | } 45 | 46 | return &state, nil 47 | } 48 | -------------------------------------------------------------------------------- /app/lib/fetchers/git.go: -------------------------------------------------------------------------------- 1 | package fetchers 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | 7 | "github.com/golangci/golangci-worker/app/analytics" 8 | 9 | "github.com/golangci/golangci-worker/app/lib/executors" 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | var ErrNoBranchOrRepo = errors.New("repo or branch not found") 14 | 15 | type Git struct{} 16 | 17 | func NewGit() *Git { 18 | return &Git{} 19 | } 20 | 21 | func (gf Git) Fetch(ctx context.Context, repo *Repo, exec executors.Executor) error { 22 | args := []string{"clone", "-q", "--depth", "1", "--branch", 23 | repo.Ref, repo.CloneURL, "."} 24 | if out, err := exec.Run(ctx, "git", args...); err != nil { 25 | noBranchOrRepo := strings.Contains(err.Error(), "could not read Username for") || 26 | strings.Contains(err.Error(), "Could not find remote branch") 27 | if noBranchOrRepo { 28 | return errors.Wrap(ErrNoBranchOrRepo, err.Error()) 29 | } 30 | 31 | return errors.Wrapf(err, "can't run git cmd %v: %s", args, out) 32 | } 33 | 34 | // some repos have deps in submodules, e.g. https://github.com/orbs-network/orbs-network-go 35 | if out, err := exec.Run(ctx, "git", "submodule", "init"); err != nil { 36 | analytics.Log(ctx).Warnf("Failed to init git submodule: %s, %s", err, out) 37 | return nil 38 | } 39 | if out, err := exec.Run(ctx, "git", "submodule", "update", "--init", "--recursive"); err != nil { 40 | analytics.Log(ctx).Warnf("Failed to update git submodule: %s, %s", err, out) 41 | return nil 42 | } 43 | 44 | return nil 45 | } 46 | -------------------------------------------------------------------------------- /app/analyze/analyzequeue/consume.go: -------------------------------------------------------------------------------- 1 | package analyzequeue 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/golangci/golangci-shared/pkg/apperrors" 7 | "github.com/golangci/golangci-shared/pkg/config" 8 | "github.com/golangci/golangci-shared/pkg/logutil" 9 | "github.com/golangci/golangci-worker/app/analyze/analyzequeue/consumers" 10 | "github.com/golangci/golangci-worker/app/analyze/processors" 11 | "github.com/golangci/golangci-worker/app/lib/experiments" 12 | "github.com/golangci/golangci-worker/app/lib/queue" 13 | ) 14 | 15 | func RegisterTasks() { 16 | log := logutil.NewStderrLog("repo analysis") 17 | log.SetLevel(logutil.LogLevelInfo) 18 | cfg := config.NewEnvConfig(log) 19 | et := apperrors.GetTracker(cfg, log, "worker") 20 | 21 | trackedLog := apperrors.WrapLogWithTracker(log, nil, et) 22 | ec := experiments.NewChecker(cfg, trackedLog) 23 | 24 | rpf := processors.NewRepoProcessorFactory(&processors.StaticRepoConfig{}, trackedLog) 25 | repoAnalyzer := consumers.NewAnalyzeRepo(ec, rpf) 26 | 27 | server := queue.GetServer() 28 | err := server.RegisterTasks(map[string]interface{}{ 29 | "analyzeV2": consumers.NewAnalyzePR().Consume, 30 | "analyzeRepo": repoAnalyzer.Consume, 31 | }) 32 | if err != nil { 33 | log.Fatalf("Can't register queue tasks: %s", err) 34 | } 35 | } 36 | 37 | func RunWorker() error { 38 | server := queue.GetServer() 39 | worker := server.NewWorker("worker_name", 1) 40 | err := worker.Launch() 41 | if err != nil { 42 | return fmt.Errorf("can't launch worker: %s", err) 43 | } 44 | 45 | return nil 46 | } 47 | -------------------------------------------------------------------------------- /app/lib/fetchers/fetcher_mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: fetcher.go 3 | 4 | package fetchers 5 | 6 | import ( 7 | context "context" 8 | gomock "github.com/golang/mock/gomock" 9 | executors "github.com/golangci/golangci-worker/app/lib/executors" 10 | reflect "reflect" 11 | ) 12 | 13 | // MockFetcher is a mock of Fetcher interface 14 | type MockFetcher struct { 15 | ctrl *gomock.Controller 16 | recorder *MockFetcherMockRecorder 17 | } 18 | 19 | // MockFetcherMockRecorder is the mock recorder for MockFetcher 20 | type MockFetcherMockRecorder struct { 21 | mock *MockFetcher 22 | } 23 | 24 | // NewMockFetcher creates a new mock instance 25 | func NewMockFetcher(ctrl *gomock.Controller) *MockFetcher { 26 | mock := &MockFetcher{ctrl: ctrl} 27 | mock.recorder = &MockFetcherMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (_m *MockFetcher) EXPECT() *MockFetcherMockRecorder { 33 | return _m.recorder 34 | } 35 | 36 | // Fetch mocks base method 37 | func (_m *MockFetcher) Fetch(ctx context.Context, repo *Repo, exec executors.Executor) error { 38 | ret := _m.ctrl.Call(_m, "Fetch", ctx, repo, exec) 39 | ret0, _ := ret[0].(error) 40 | return ret0 41 | } 42 | 43 | // Fetch indicates an expected call of Fetch 44 | func (_mr *MockFetcherMockRecorder) Fetch(arg0, arg1, arg2 interface{}) *gomock.Call { 45 | return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Fetch", reflect.TypeOf((*MockFetcher)(nil).Fetch), arg0, arg1, arg2) 46 | } 47 | -------------------------------------------------------------------------------- /app/analyze/processors/executor.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/golangci/golangci-shared/pkg/config" 9 | "github.com/golangci/golangci-shared/pkg/logutil" 10 | "github.com/golangci/golangci-worker/app/lib/executors" 11 | "github.com/golangci/golangci-worker/app/lib/experiments" 12 | "github.com/golangci/golangci-worker/app/lib/github" 13 | "github.com/pkg/errors" 14 | ) 15 | 16 | func makeExecutor(ctx context.Context, repo *github.Repo, forPull bool, log logutil.Log, ec *experiments.Checker) (executors.Executor, error) { 17 | if log == nil { // TODO: remove 18 | log = logutil.NewStderrLog("executor") 19 | log.SetLevel(logutil.LogLevelInfo) 20 | } 21 | if ec == nil { // TODO: remove 22 | cfg := config.NewEnvConfig(log) 23 | ec = experiments.NewChecker(cfg, log) 24 | } 25 | 26 | if ec.IsActiveForAnalysis("use_container_executor", repo, forPull) { 27 | ce, err := executors.NewContainer(log) 28 | if err != nil { 29 | return nil, errors.Wrap(err, "can't build container executor") 30 | } 31 | 32 | if err = ce.Setup(ctx); err != nil { 33 | return nil, errors.Wrap(err, "failed to setup container executor") 34 | } 35 | return ce.WithWorkDir("/goapp"), nil 36 | } 37 | 38 | s := executors.NewRemoteShell( 39 | os.Getenv("REMOTE_SHELL_USER"), 40 | os.Getenv("REMOTE_SHELL_HOST"), 41 | os.Getenv("REMOTE_SHELL_KEY_FILE_PATH"), 42 | ) 43 | if err := s.SetupTempWorkDir(ctx); err != nil { 44 | return nil, fmt.Errorf("can't setup temp work dir: %s", err) 45 | } 46 | 47 | return s, nil 48 | } 49 | -------------------------------------------------------------------------------- /app/analyze/reporters/reporter_mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: reporter.go 3 | 4 | package reporters 5 | 6 | import ( 7 | context "context" 8 | gomock "github.com/golang/mock/gomock" 9 | result "github.com/golangci/golangci-worker/app/analyze/linters/result" 10 | reflect "reflect" 11 | ) 12 | 13 | // MockReporter is a mock of Reporter interface 14 | type MockReporter struct { 15 | ctrl *gomock.Controller 16 | recorder *MockReporterMockRecorder 17 | } 18 | 19 | // MockReporterMockRecorder is the mock recorder for MockReporter 20 | type MockReporterMockRecorder struct { 21 | mock *MockReporter 22 | } 23 | 24 | // NewMockReporter creates a new mock instance 25 | func NewMockReporter(ctrl *gomock.Controller) *MockReporter { 26 | mock := &MockReporter{ctrl: ctrl} 27 | mock.recorder = &MockReporterMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (_m *MockReporter) EXPECT() *MockReporterMockRecorder { 33 | return _m.recorder 34 | } 35 | 36 | // Report mocks base method 37 | func (_m *MockReporter) Report(ctx context.Context, ref string, issues []result.Issue) error { 38 | ret := _m.ctrl.Call(_m, "Report", ctx, ref, issues) 39 | ret0, _ := ret[0].(error) 40 | return ret0 41 | } 42 | 43 | // Report indicates an expected call of Report 44 | func (_mr *MockReporterMockRecorder) Report(arg0, arg1, arg2 interface{}) *gomock.Call { 45 | return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Report", reflect.TypeOf((*MockReporter)(nil).Report), arg0, arg1, arg2) 46 | } 47 | -------------------------------------------------------------------------------- /app/analytics/logger.go: -------------------------------------------------------------------------------- 1 | package analytics 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | 8 | "github.com/golangci/golangci-shared/pkg/apperrors" 9 | "github.com/golangci/golangci-worker/app/lib/runmode" 10 | "github.com/sirupsen/logrus" 11 | ) 12 | 13 | var initLogrusOnce sync.Once 14 | 15 | func initLogrus() { 16 | level := logrus.InfoLevel 17 | if runmode.IsDebug() { 18 | level = logrus.DebugLevel 19 | } 20 | logrus.SetLevel(level) 21 | } 22 | 23 | type Logger interface { 24 | Warnf(format string, args ...interface{}) 25 | Errorf(format string, args ...interface{}) 26 | Infof(format string, args ...interface{}) 27 | Debugf(format string, args ...interface{}) 28 | } 29 | 30 | type logger struct { 31 | ctx context.Context 32 | } 33 | 34 | func (log logger) le() *logrus.Entry { 35 | return logrus.WithFields(getTrackingProps(log.ctx)) 36 | } 37 | 38 | func (log logger) Warnf(format string, args ...interface{}) { 39 | err := fmt.Errorf(format, args...) 40 | log.le().Warn(err.Error()) 41 | trackError(log.ctx, err, apperrors.LevelWarn) 42 | } 43 | 44 | func (log logger) Errorf(format string, args ...interface{}) { 45 | err := fmt.Errorf(format, args...) 46 | log.le().Error(err.Error()) 47 | trackError(log.ctx, err, apperrors.LevelError) 48 | } 49 | 50 | func (log logger) Infof(format string, args ...interface{}) { 51 | log.le().Infof(format, args...) 52 | } 53 | 54 | func (log logger) Debugf(format string, args ...interface{}) { 55 | log.le().Debugf(format, args...) 56 | } 57 | 58 | func Log(ctx context.Context) Logger { 59 | initLogrusOnce.Do(initLogrus) 60 | 61 | return logger{ 62 | ctx: ctx, 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /app/analyze/repoinfo/fetcher_mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: fetcher.go 3 | 4 | package repoinfo 5 | 6 | import ( 7 | context "context" 8 | gomock "github.com/golang/mock/gomock" 9 | executors "github.com/golangci/golangci-worker/app/lib/executors" 10 | fetchers "github.com/golangci/golangci-worker/app/lib/fetchers" 11 | reflect "reflect" 12 | ) 13 | 14 | // MockFetcher is a mock of Fetcher interface 15 | type MockFetcher struct { 16 | ctrl *gomock.Controller 17 | recorder *MockFetcherMockRecorder 18 | } 19 | 20 | // MockFetcherMockRecorder is the mock recorder for MockFetcher 21 | type MockFetcherMockRecorder struct { 22 | mock *MockFetcher 23 | } 24 | 25 | // NewMockFetcher creates a new mock instance 26 | func NewMockFetcher(ctrl *gomock.Controller) *MockFetcher { 27 | mock := &MockFetcher{ctrl: ctrl} 28 | mock.recorder = &MockFetcherMockRecorder{mock} 29 | return mock 30 | } 31 | 32 | // EXPECT returns an object that allows the caller to indicate expected use 33 | func (_m *MockFetcher) EXPECT() *MockFetcherMockRecorder { 34 | return _m.recorder 35 | } 36 | 37 | // Fetch mocks base method 38 | func (_m *MockFetcher) Fetch(ctx context.Context, repo *fetchers.Repo, exec executors.Executor) (*Info, error) { 39 | ret := _m.ctrl.Call(_m, "Fetch", ctx, repo, exec) 40 | ret0, _ := ret[0].(*Info) 41 | ret1, _ := ret[1].(error) 42 | return ret0, ret1 43 | } 44 | 45 | // Fetch indicates an expected call of Fetch 46 | func (_mr *MockFetcherMockRecorder) Fetch(arg0, arg1, arg2 interface{}) *gomock.Call { 47 | return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Fetch", reflect.TypeOf((*MockFetcher)(nil).Fetch), arg0, arg1, arg2) 48 | } 49 | -------------------------------------------------------------------------------- /app/lib/httputils/client.go: -------------------------------------------------------------------------------- 1 | package httputils 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | 8 | "github.com/golangci/golangci-worker/app/analytics" 9 | "github.com/levigross/grequests" 10 | ) 11 | 12 | //go:generate mockgen -package httputils -source client.go -destination client_mock.go 13 | 14 | type Client interface { 15 | Get(ctx context.Context, url string) (io.ReadCloser, error) 16 | Put(ctx context.Context, url string, jsonObj interface{}) error 17 | } 18 | 19 | type GrequestsClient struct{} 20 | 21 | func (c GrequestsClient) Get(ctx context.Context, url string) (io.ReadCloser, error) { 22 | resp, err := grequests.Get(url, &grequests.RequestOptions{ 23 | Context: ctx, 24 | }) 25 | if err != nil { 26 | return nil, fmt.Errorf("unable to make GET http request %q: %s", url, err) 27 | } 28 | 29 | if !resp.Ok { 30 | if cerr := resp.Close(); cerr != nil { 31 | analytics.Log(ctx).Warnf("Can't close %q response: %s", url, cerr) 32 | } 33 | 34 | return nil, fmt.Errorf("got error code from %q: %d", url, resp.StatusCode) 35 | } 36 | 37 | return resp, nil 38 | } 39 | 40 | func (c GrequestsClient) Put(ctx context.Context, url string, jsonObj interface{}) error { 41 | resp, err := grequests.Put(url, &grequests.RequestOptions{ 42 | Context: ctx, 43 | JSON: jsonObj, 44 | }) 45 | if err != nil { 46 | return fmt.Errorf("unable to make PUT http request %q: %s", url, err) 47 | } 48 | 49 | if !resp.Ok { 50 | if cerr := resp.Close(); cerr != nil { 51 | analytics.Log(ctx).Warnf("Can't close %q response: %s", url, cerr) 52 | } 53 | 54 | return fmt.Errorf("got error code from %q: %d", url, resp.StatusCode) 55 | } 56 | 57 | return nil 58 | } 59 | -------------------------------------------------------------------------------- /app/analyze/repoinfo/fetcher.go: -------------------------------------------------------------------------------- 1 | package repoinfo 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | 7 | info "github.com/golangci/getrepoinfo/pkg/repoinfo" 8 | "github.com/golangci/golangci-worker/app/analytics" 9 | "github.com/golangci/golangci-worker/app/lib/executors" 10 | "github.com/golangci/golangci-worker/app/lib/fetchers" 11 | "github.com/pkg/errors" 12 | ) 13 | 14 | //go:generate mockgen -package repoinfo -source fetcher.go -destination fetcher_mock.go 15 | 16 | type Info struct { 17 | info.Info 18 | Error string 19 | } 20 | 21 | type Fetcher interface { 22 | Fetch(ctx context.Context, repo *fetchers.Repo, exec executors.Executor) (*Info, error) 23 | } 24 | 25 | type CloningFetcher struct { 26 | repoFetcher fetchers.Fetcher 27 | } 28 | 29 | func NewCloningFetcher(repoFetcher fetchers.Fetcher) *CloningFetcher { 30 | return &CloningFetcher{ 31 | repoFetcher: repoFetcher, 32 | } 33 | } 34 | 35 | func (f CloningFetcher) Fetch(ctx context.Context, repo *fetchers.Repo, exec executors.Executor) (*Info, error) { 36 | // fetch into the current dir 37 | if err := f.repoFetcher.Fetch(ctx, repo, exec); err != nil { 38 | return nil, errors.Wrapf(err, "failed to fetch repo ref %q by url %q", repo.Ref, repo.CloneURL) 39 | } 40 | 41 | out, err := exec.Run(ctx, "getrepoinfo", "--repo", repo.FullPath) 42 | if err != nil { 43 | return nil, errors.Wrapf(err, "failed to run 'getrepoinfo --repo %s'", repo.FullPath) 44 | } 45 | 46 | var ret Info 47 | if err = json.Unmarshal([]byte(out), &ret); err != nil { 48 | return nil, errors.Wrap(err, "json unmarshal failed") 49 | } 50 | 51 | if ret.Error != "" { 52 | analytics.Log(ctx).Warnf("Got getrepoinfo error in json: %s", ret.Error) 53 | } 54 | 55 | return &ret, nil 56 | } 57 | -------------------------------------------------------------------------------- /app/analyze/processors/result.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | import ( 4 | "strconv" 5 | "time" 6 | ) 7 | 8 | type JSONDuration time.Duration 9 | 10 | func (d JSONDuration) MarshalJSON() ([]byte, error) { 11 | return []byte(strconv.Itoa(int(time.Duration(d) / time.Millisecond))), nil 12 | } 13 | 14 | func (d JSONDuration) String() string { 15 | return time.Duration(d).String() 16 | } 17 | 18 | type Timing struct { 19 | Name string 20 | Duration JSONDuration `json:"DurationMs"` 21 | } 22 | 23 | type Warning struct { 24 | Tag string 25 | Text string 26 | } 27 | 28 | type resultCollector struct { 29 | timings []Timing 30 | warnings []Warning 31 | } 32 | 33 | func (r *resultCollector) trackTiming(name string, f func()) { 34 | startedAt := time.Now() 35 | f() 36 | r.timings = append(r.timings, Timing{ 37 | Name: name, 38 | Duration: JSONDuration(time.Since(startedAt)), 39 | }) 40 | } 41 | 42 | func (r *resultCollector) addTimingFrom(name string, from time.Time) { 43 | r.timings = append(r.timings, Timing{ 44 | Name: name, 45 | Duration: JSONDuration(time.Since(from)), 46 | }) 47 | } 48 | 49 | func (r *resultCollector) publicWarn(tag string, text string) { 50 | r.warnings = append(r.warnings, Warning{ 51 | Tag: tag, 52 | Text: text, 53 | }) 54 | } 55 | 56 | type workerRes struct { 57 | Timings []Timing `json:",omitempty"` 58 | Warnings []Warning `json:",omitempty"` 59 | Error string `json:",omitempty"` 60 | } 61 | 62 | type resultJSON struct { 63 | Version int 64 | GolangciLintRes interface{} 65 | WorkerRes workerRes 66 | } 67 | 68 | func fromDBTime(t time.Time) time.Time { 69 | return time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), time.Local) 70 | } 71 | -------------------------------------------------------------------------------- /app/analytics/track.go: -------------------------------------------------------------------------------- 1 | package analytics 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/dukex/mixpanel" 7 | "github.com/savaki/amplitude-go" 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | type EventName string 12 | 13 | const EventPRChecked EventName = "PR checked" 14 | const EventRepoAnalyzed EventName = "Repo analyzed" 15 | 16 | type Tracker interface { 17 | Track(ctx context.Context, event EventName) 18 | } 19 | 20 | type amplitudeMixpanelTracker struct{} 21 | 22 | func (t amplitudeMixpanelTracker) Track(ctx context.Context, eventName EventName) { 23 | trackingProps := getTrackingProps(ctx) 24 | userID := trackingProps["userIDString"].(string) 25 | 26 | eventProps := map[string]interface{}{} 27 | for k, v := range trackingProps { 28 | if k != "userIDString" { 29 | eventProps[k] = v 30 | } 31 | } 32 | 33 | addedEventProps := ctx.Value(eventName).(map[string]interface{}) 34 | for k, v := range addedEventProps { 35 | eventProps[k] = v 36 | } 37 | log.Infof("track event %s with props %+v", eventName, eventProps) 38 | 39 | ac := getAmplitudeClient() 40 | if ac != nil { 41 | ev := amplitude.Event{ 42 | UserId: userID, 43 | EventType: string(eventName), 44 | EventProperties: eventProps, 45 | } 46 | if err := ac.Publish(ev); err != nil { 47 | Log(ctx).Warnf("Can't publish %+v to amplitude: %s", ev, err) 48 | } 49 | } 50 | 51 | mp := getMixpanelClient() 52 | if mp != nil { 53 | const ip = "0" // don't auto-detect 54 | ev := &mixpanel.Event{ 55 | IP: ip, 56 | Properties: eventProps, 57 | } 58 | if err := mp.Track(userID, string(eventName), ev); err != nil { 59 | Log(ctx).Warnf("Can't publish event %s (%+v) to mixpanel: %s", string(eventName), ev, err) 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /app/lib/executors/remote_shell_test.go: -------------------------------------------------------------------------------- 1 | package executors 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/golangci/golangci-worker/app/test" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func testNewRemoteShell() *RemoteShell { 15 | return NewRemoteShell( 16 | os.Getenv("REMOTE_SHELL_USER"), 17 | os.Getenv("REMOTE_SHELL_HOST"), 18 | os.Getenv("REMOTE_SHELL_KEY_FILE_PATH"), 19 | ) 20 | } 21 | 22 | func TestRemoteShellEnv(t *testing.T) { 23 | t.SkipNow() 24 | 25 | test.Init() 26 | s := testNewRemoteShell().WithEnv("TEST_KEY", "TEST_VALUE") 27 | 28 | out, err := s.Run(context.Background(), "printenv", "TEST_KEY") 29 | assert.NoError(t, err) 30 | assert.Equal(t, "TEST_VALUE", strings.TrimSpace(out)) 31 | } 32 | 33 | func TestRemoteShellClean(t *testing.T) { 34 | t.SkipNow() 35 | 36 | test.Init() 37 | s := testNewRemoteShell() 38 | assert.NoError(t, s.SetupTempWorkDir(context.Background())) 39 | 40 | s.Clean() // must remove temp work dir 41 | 42 | _, err := s.Run(context.Background(), "test", "!", "-e", s.tempWorkDir) // returns 0 only if dir doesn't exist 43 | assert.NoError(t, err) 44 | } 45 | 46 | func TestRemoteShellInWorkDir(t *testing.T) { 47 | t.SkipNow() 48 | 49 | test.Init() 50 | s := testNewRemoteShell() 51 | assert.NoError(t, s.SetupTempWorkDir(context.Background())) 52 | defer s.Clean() 53 | 54 | testSubdir := "testdir" 55 | _, err := s.Run(context.Background(), "mkdir", filepath.Join(s.WorkDir(), testSubdir)) 56 | assert.NoError(t, err) 57 | 58 | wd := filepath.Join(s.tempWorkDir, testSubdir) 59 | out, err := s.WithWorkDir(wd).Run(context.Background(), "pwd") 60 | assert.NoError(t, err) 61 | assert.Equal(t, wd, strings.TrimSpace(out)) 62 | } 63 | -------------------------------------------------------------------------------- /app/lib/goutils/workspaces/go2.go: -------------------------------------------------------------------------------- 1 | package workspaces 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "path" 8 | 9 | "github.com/golangci/golangci-api/pkg/goenv/result" 10 | "github.com/golangci/golangci-shared/pkg/logutil" 11 | "github.com/golangci/golangci-worker/app/lib/executors" 12 | "github.com/golangci/golangci-worker/app/lib/fetchers" 13 | "github.com/pkg/errors" 14 | ) 15 | 16 | type Go2 struct { 17 | exec executors.Executor 18 | log logutil.Log 19 | repoFetcher fetchers.Fetcher 20 | } 21 | 22 | var _ Installer = &Go2{} 23 | 24 | func NewGo2(exec executors.Executor, log logutil.Log, repoFetcher fetchers.Fetcher) *Go2 { 25 | return &Go2{ 26 | exec: exec, 27 | log: log, 28 | repoFetcher: repoFetcher, 29 | } 30 | } 31 | 32 | func (w *Go2) Setup(ctx context.Context, repo *fetchers.Repo, projectPathParts ...string) (executors.Executor, *result.Log, error) { 33 | if err := w.repoFetcher.Fetch(ctx, repo, w.exec); err != nil { 34 | return nil, nil, errors.Wrap(err, "failed to fetch repo") 35 | } 36 | 37 | exec := w.exec.WithEnv("REPO", path.Join(projectPathParts...)).WithEnv("FORMAT_JSON", "1") 38 | out, err := exec.Run(ctx, "goenvbuild") 39 | if err != nil { 40 | return nil, nil, errors.Wrap(err, "goenvbuild failed") 41 | } 42 | 43 | var envbuildResult result.Result 44 | if err = json.Unmarshal([]byte(out), &envbuildResult); err != nil { 45 | return nil, nil, errors.Wrap(err, "failed to unmarshal goenvbuild result json") 46 | } 47 | 48 | w.log.Infof("Got envbuild result %s", out) 49 | if envbuildResult.Error != "" { 50 | return nil, nil, fmt.Errorf("goenvbuild internal error: %s", envbuildResult.Error) 51 | } 52 | 53 | retExec := w.exec.WithWorkDir(envbuildResult.WorkDir) 54 | for k, v := range envbuildResult.Environment { 55 | retExec = retExec.WithEnv(k, v) 56 | } 57 | 58 | return retExec, envbuildResult.Log, nil 59 | } 60 | -------------------------------------------------------------------------------- /app/analyze/analyzequeue/produce.go: -------------------------------------------------------------------------------- 1 | package analyzequeue 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/RichardKnop/machinery/v1/tasks" 7 | "github.com/golangci/golangci-worker/app/analyze/analyzequeue/task" 8 | "github.com/golangci/golangci-worker/app/lib/queue" 9 | ) 10 | 11 | func SchedulePRAnalysis(t *task.PRAnalysis) error { 12 | args := []tasks.Arg{ 13 | { 14 | Type: "string", 15 | Value: t.Repo.Owner, 16 | }, 17 | { 18 | Type: "string", 19 | Value: t.Repo.Name, 20 | }, 21 | { 22 | Type: "string", 23 | Value: t.GithubAccessToken, 24 | }, 25 | { 26 | Type: "int", 27 | Value: t.PullRequestNumber, 28 | }, 29 | { 30 | Type: "string", 31 | Value: t.APIRequestID, 32 | }, 33 | { 34 | Type: "uint", 35 | Value: t.UserID, 36 | }, 37 | { 38 | Type: "string", 39 | Value: t.AnalysisGUID, 40 | }, 41 | } 42 | signature := &tasks.Signature{ 43 | Name: "analyzeV2", 44 | Args: args, 45 | RetryCount: 3, 46 | RetryTimeout: 600, // 600 sec 47 | } 48 | 49 | _, err := queue.GetServer().SendTask(signature) 50 | if err != nil { 51 | return fmt.Errorf("failed to send the pr analysis task %v to analyze queue: %s", t, err) 52 | } 53 | 54 | return nil 55 | } 56 | 57 | func ScheduleRepoAnalysis(t *task.RepoAnalysis) error { 58 | args := []tasks.Arg{ 59 | { 60 | Type: "string", 61 | Value: t.Name, 62 | }, 63 | { 64 | Type: "string", 65 | Value: t.AnalysisGUID, 66 | }, 67 | { 68 | Type: "string", 69 | Value: t.Branch, 70 | }, 71 | } 72 | signature := &tasks.Signature{ 73 | Name: "analyzeRepo", 74 | Args: args, 75 | RetryCount: 3, 76 | RetryTimeout: 600, // 600 sec 77 | } 78 | 79 | _, err := queue.GetServer().SendTask(signature) 80 | if err != nil { 81 | return fmt.Errorf("failed to send the repo analysis task %v to analyze queue: %s", t, err) 82 | } 83 | 84 | return nil 85 | } 86 | -------------------------------------------------------------------------------- /app/lib/httputils/client_mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: client.go 3 | 4 | package httputils 5 | 6 | import ( 7 | context "context" 8 | gomock "github.com/golang/mock/gomock" 9 | io "io" 10 | reflect "reflect" 11 | ) 12 | 13 | // MockClient is a mock of Client interface 14 | type MockClient struct { 15 | ctrl *gomock.Controller 16 | recorder *MockClientMockRecorder 17 | } 18 | 19 | // MockClientMockRecorder is the mock recorder for MockClient 20 | type MockClientMockRecorder struct { 21 | mock *MockClient 22 | } 23 | 24 | // NewMockClient creates a new mock instance 25 | func NewMockClient(ctrl *gomock.Controller) *MockClient { 26 | mock := &MockClient{ctrl: ctrl} 27 | mock.recorder = &MockClientMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (_m *MockClient) EXPECT() *MockClientMockRecorder { 33 | return _m.recorder 34 | } 35 | 36 | // Get mocks base method 37 | func (_m *MockClient) Get(ctx context.Context, url string) (io.ReadCloser, error) { 38 | ret := _m.ctrl.Call(_m, "Get", ctx, url) 39 | ret0, _ := ret[0].(io.ReadCloser) 40 | ret1, _ := ret[1].(error) 41 | return ret0, ret1 42 | } 43 | 44 | // Get indicates an expected call of Get 45 | func (_mr *MockClientMockRecorder) Get(arg0, arg1 interface{}) *gomock.Call { 46 | return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Get", reflect.TypeOf((*MockClient)(nil).Get), arg0, arg1) 47 | } 48 | 49 | // Put mocks base method 50 | func (_m *MockClient) Put(ctx context.Context, url string, jsonObj interface{}) error { 51 | ret := _m.ctrl.Call(_m, "Put", ctx, url, jsonObj) 52 | ret0, _ := ret[0].(error) 53 | return ret0 54 | } 55 | 56 | // Put indicates an expected call of Put 57 | func (_mr *MockClientMockRecorder) Put(arg0, arg1, arg2 interface{}) *gomock.Call { 58 | return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Put", reflect.TypeOf((*MockClient)(nil).Put), arg0, arg1, arg2) 59 | } 60 | -------------------------------------------------------------------------------- /app/analyze/analyzequeue/consumers/base_consumer.go: -------------------------------------------------------------------------------- 1 | package consumers 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "runtime/debug" 7 | "time" 8 | 9 | "github.com/golangci/golangci-worker/app/analytics" 10 | ) 11 | 12 | type baseConsumer struct { 13 | eventName analytics.EventName 14 | needSendToAnalytics bool 15 | } 16 | 17 | const statusOk = "ok" 18 | const statusFail = "fail" 19 | 20 | func (c baseConsumer) prepareContext(ctx context.Context, trackingProps map[string]interface{}) context.Context { 21 | ctx = analytics.ContextWithEventPropsCollector(ctx, c.eventName) 22 | ctx = analytics.ContextWithTrackingProps(ctx, trackingProps) 23 | return ctx 24 | } 25 | 26 | func (c baseConsumer) wrapConsuming(ctx context.Context, f func() error) (err error) { 27 | defer func() { 28 | if r := recover(); r != nil { 29 | err = fmt.Errorf("panic recovered: %v, %s, source is %s", r, debug.Stack(), err) 30 | analytics.Log(ctx).Errorf("processing of %q task failed: %s", c.eventName, err) 31 | } 32 | }() 33 | 34 | analytics.Log(ctx).Infof("Starting consuming of %s...", c.eventName) 35 | 36 | startedAt := time.Now() 37 | err = f() 38 | duration := time.Since(startedAt) 39 | analytics.Log(ctx).Infof("Finished consuming of %s for %s", c.eventName, duration) 40 | 41 | if err != nil { 42 | analytics.Log(ctx).Errorf("processing of %q task failed: %s", c.eventName, err) 43 | } 44 | 45 | if c.needSendToAnalytics { 46 | c.sendAnalytics(ctx, duration, err) 47 | } 48 | 49 | return err 50 | } 51 | 52 | func (c baseConsumer) sendAnalytics(ctx context.Context, duration time.Duration, err error) { 53 | props := map[string]interface{}{ 54 | "durationSeconds": int(duration / time.Second), 55 | } 56 | if err == nil { 57 | props["status"] = statusOk 58 | } else { 59 | props["status"] = statusFail 60 | props["error"] = err.Error() 61 | } 62 | analytics.SaveEventProps(ctx, c.eventName, props) 63 | 64 | tracker := analytics.GetTracker(ctx) 65 | tracker.Track(ctx, c.eventName) 66 | } 67 | -------------------------------------------------------------------------------- /app/analyze/linters/linter_mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: linter.go 3 | 4 | package linters 5 | 6 | import ( 7 | context "context" 8 | gomock "github.com/golang/mock/gomock" 9 | result "github.com/golangci/golangci-worker/app/analyze/linters/result" 10 | executors "github.com/golangci/golangci-worker/app/lib/executors" 11 | reflect "reflect" 12 | ) 13 | 14 | // MockLinter is a mock of Linter interface 15 | type MockLinter struct { 16 | ctrl *gomock.Controller 17 | recorder *MockLinterMockRecorder 18 | } 19 | 20 | // MockLinterMockRecorder is the mock recorder for MockLinter 21 | type MockLinterMockRecorder struct { 22 | mock *MockLinter 23 | } 24 | 25 | // NewMockLinter creates a new mock instance 26 | func NewMockLinter(ctrl *gomock.Controller) *MockLinter { 27 | mock := &MockLinter{ctrl: ctrl} 28 | mock.recorder = &MockLinterMockRecorder{mock} 29 | return mock 30 | } 31 | 32 | // EXPECT returns an object that allows the caller to indicate expected use 33 | func (_m *MockLinter) EXPECT() *MockLinterMockRecorder { 34 | return _m.recorder 35 | } 36 | 37 | // Run mocks base method 38 | func (_m *MockLinter) Run(ctx context.Context, exec executors.Executor) (*result.Result, error) { 39 | ret := _m.ctrl.Call(_m, "Run", ctx, exec) 40 | ret0, _ := ret[0].(*result.Result) 41 | ret1, _ := ret[1].(error) 42 | return ret0, ret1 43 | } 44 | 45 | // Run indicates an expected call of Run 46 | func (_mr *MockLinterMockRecorder) Run(arg0, arg1 interface{}) *gomock.Call { 47 | return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Run", reflect.TypeOf((*MockLinter)(nil).Run), arg0, arg1) 48 | } 49 | 50 | // Name mocks base method 51 | func (_m *MockLinter) Name() string { 52 | ret := _m.ctrl.Call(_m, "Name") 53 | ret0, _ := ret[0].(string) 54 | return ret0 55 | } 56 | 57 | // Name indicates an expected call of Name 58 | func (_mr *MockLinterMockRecorder) Name() *gomock.Call { 59 | return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Name", reflect.TypeOf((*MockLinter)(nil).Name)) 60 | } 61 | -------------------------------------------------------------------------------- /app/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | # (cd app/docker && docker build -t golangci/build-runner .) 2 | FROM golang:1.11 as builder 3 | 4 | ENV GOPATH=/go 5 | ENV GOBINPATH=$GOPATH/bin 6 | 7 | WORKDIR ${GOPATH} 8 | 9 | ENV DEP_RELEASE_TAG=v0.5.0 10 | RUN curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh 11 | 12 | ENV GLIDE_RELEASE_TAG=v0.13.2 13 | RUN (wget -O - https://github.com/Masterminds/glide/releases/download/${GLIDE_RELEASE_TAG}/glide-${GLIDE_RELEASE_TAG}-linux-amd64.tar.gz | tar -zxvf -) && \ 14 | mv linux-amd64/glide ${GOBINPATH}/ 15 | 16 | ENV GOVENDOR_VERSION=v1.0.8 17 | RUN wget https://github.com/kardianos/govendor/releases/download/${GOVENDOR_VERSION}/govendor_linux_amd64 -O $GOBINPATH/govendor && \ 18 | chmod a+x $GOBINPATH/govendor 19 | 20 | ENV GODEP_VERSION=v80 21 | RUN wget https://github.com/tools/godep/releases/download/${GODEP_VERSION}/godep_linux_amd64 -O $GOBINPATH/godep && \ 22 | chmod a+x $GOBINPATH/godep 23 | 24 | WORKDIR ${GOPATH}/src/github.com/golangci/getrepoinfo 25 | RUN git clone https://github.com/golangci/getrepoinfo.git . && \ 26 | git checkout dba22f1e4de557d0afe6970cb31e413bbe450cbd && \ 27 | go install ./cmd/getrepoinfo 28 | 29 | RUN curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s -- -b $GOPATH/bin v1.12.3 30 | 31 | WORKDIR ${GOPATH}/src/github.com/golangci/golangci-api 32 | RUN git clone https://github.com/golangci/golangci-api.git . && \ 33 | git checkout e545e490e0c7a973a2b761ea6e2d4e45a4f489b9 && \ 34 | GO111MODULE=on go mod vendor && \ 35 | go install ./cmd/ensuredeps && \ 36 | go install ./cmd/buildrunner && \ 37 | go install ./cmd/goenvbuild 38 | 39 | RUN mkdir /app && echo 'echo TODO: remove' >/app/cleanup.sh 40 | 41 | FROM golang:1.11 42 | 43 | ENV GOPATH=/go 44 | ENV GOBINPATH=$GOPATH/bin 45 | ENV PATH=$PATH:/usr/local/go/bin:$GOBINPATH 46 | 47 | COPY --from=builder ${GOPATH}/bin/* ${GOPATH}/bin/ 48 | COPY --from=builder /app/cleanup.sh /app/ 49 | 50 | WORKDIR /goapp 51 | 52 | ENV PORT=7000 MAX_LIFETIME=30m 53 | 54 | CMD ["buildrunner"] -------------------------------------------------------------------------------- /app/lib/executors/temp_dir_shell.go: -------------------------------------------------------------------------------- 1 | package executors 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "log" 9 | "os" 10 | "path/filepath" 11 | 12 | "github.com/golangci/golangci-worker/app/analytics" 13 | ) 14 | 15 | type TempDirShell struct { 16 | shell 17 | } 18 | 19 | var _ Executor = &TempDirShell{} 20 | 21 | var tmpRoot string 22 | 23 | func init() { 24 | var err error 25 | tmpRoot, err = filepath.EvalSymlinks("/tmp") 26 | if err != nil { 27 | log.Fatalf("can't eval symlinks on /tmp: %s", err) 28 | } 29 | } 30 | 31 | func NewTempDirShell(tag string) (*TempDirShell, error) { 32 | wd, err := ioutil.TempDir(tmpRoot, fmt.Sprintf("golangci.%s", tag)) 33 | if err != nil { 34 | return nil, fmt.Errorf("can't make temp dir: %s", err) 35 | } 36 | 37 | return &TempDirShell{ 38 | shell: *newShell(wd), 39 | }, nil 40 | } 41 | 42 | func (s TempDirShell) WorkDir() string { 43 | return s.wd 44 | } 45 | 46 | func (s *TempDirShell) SetWorkDir(wd string) { 47 | s.wd = wd 48 | } 49 | 50 | func (s TempDirShell) Clean() { 51 | if err := os.RemoveAll(s.wd); err != nil { 52 | analytics.Log(context.TODO()).Warnf("Can't remove temp dir %s: %s", s.wd, err) 53 | } 54 | } 55 | 56 | func (s TempDirShell) WithEnv(k, v string) Executor { 57 | eCopy := s 58 | eCopy.SetEnv(k, v) 59 | return &eCopy 60 | } 61 | 62 | func (s TempDirShell) WithWorkDir(wd string) Executor { 63 | eCopy := s 64 | eCopy.wd = wd 65 | return &eCopy 66 | } 67 | 68 | func (s TempDirShell) CopyFile(ctx context.Context, dst, src string) error { 69 | dst = filepath.Join(s.WorkDir(), dst) 70 | 71 | from, err := os.Open(src) 72 | if err != nil { 73 | return fmt.Errorf("can't open %s: %s", src, err) 74 | } 75 | defer from.Close() 76 | 77 | to, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE, 0666) 78 | if err != nil { 79 | return fmt.Errorf("can't open %s: %s", dst, err) 80 | } 81 | defer to.Close() 82 | 83 | _, err = io.Copy(to, from) 84 | if err != nil { 85 | return fmt.Errorf("can't copy from %s to %s: %s", src, dst, err) 86 | } 87 | 88 | return nil 89 | } 90 | -------------------------------------------------------------------------------- /app/analyze/analyzequeue/consumers/analyze_pr.go: -------------------------------------------------------------------------------- 1 | package consumers 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strconv" 7 | "time" 8 | 9 | "github.com/golangci/golangci-worker/app/analytics" 10 | "github.com/golangci/golangci-worker/app/analyze/analyzequeue/task" 11 | "github.com/golangci/golangci-worker/app/analyze/processors" 12 | "github.com/golangci/golangci-worker/app/lib/github" 13 | ) 14 | 15 | var ProcessorFactory = processors.NewGithubFactory() 16 | 17 | type AnalyzePR struct { 18 | baseConsumer 19 | } 20 | 21 | func NewAnalyzePR() *AnalyzePR { 22 | return &AnalyzePR{ 23 | baseConsumer: baseConsumer{ 24 | eventName: analytics.EventPRChecked, 25 | needSendToAnalytics: true, 26 | }, 27 | } 28 | } 29 | 30 | func (c AnalyzePR) Consume(ctx context.Context, repoOwner, repoName, githubAccessToken string, 31 | pullRequestNumber int, APIRequestID string, userID uint, analysisGUID string) error { 32 | 33 | t := &task.PRAnalysis{ 34 | Context: github.Context{ 35 | Repo: github.Repo{ 36 | Owner: repoOwner, 37 | Name: repoName, 38 | }, 39 | GithubAccessToken: githubAccessToken, 40 | PullRequestNumber: pullRequestNumber, 41 | }, 42 | APIRequestID: APIRequestID, 43 | UserID: userID, 44 | AnalysisGUID: analysisGUID, 45 | } 46 | 47 | ctx = c.prepareContext(ctx, map[string]interface{}{ 48 | "repoName": fmt.Sprintf("%s/%s", repoOwner, repoName), 49 | "provider": "github", 50 | "prNumber": pullRequestNumber, 51 | "userIDString": strconv.Itoa(int(userID)), 52 | "analysisGUID": analysisGUID, 53 | }) 54 | 55 | return c.wrapConsuming(ctx, func() error { 56 | var cancel context.CancelFunc 57 | // If you change timeout value don't forget to change it 58 | // in golangci-api stale analyzes checker 59 | ctx, cancel = context.WithTimeout(ctx, 10*time.Minute) 60 | defer cancel() 61 | 62 | p, err := ProcessorFactory.BuildProcessor(ctx, t) 63 | if err != nil { 64 | return fmt.Errorf("can't build processor for task %+v: %s", t, err) 65 | } 66 | 67 | if err = p.Process(ctx); err != nil { 68 | return fmt.Errorf("can't process pr analysis of %+v: %s", t, err) 69 | } 70 | 71 | return nil 72 | }) 73 | } 74 | -------------------------------------------------------------------------------- /app/analyze/prstate/storage_mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: storage.go 3 | 4 | package prstate 5 | 6 | import ( 7 | context "context" 8 | gomock "github.com/golang/mock/gomock" 9 | reflect "reflect" 10 | ) 11 | 12 | // MockStorage is a mock of Storage interface 13 | type MockStorage struct { 14 | ctrl *gomock.Controller 15 | recorder *MockStorageMockRecorder 16 | } 17 | 18 | // MockStorageMockRecorder is the mock recorder for MockStorage 19 | type MockStorageMockRecorder struct { 20 | mock *MockStorage 21 | } 22 | 23 | // NewMockStorage creates a new mock instance 24 | func NewMockStorage(ctrl *gomock.Controller) *MockStorage { 25 | mock := &MockStorage{ctrl: ctrl} 26 | mock.recorder = &MockStorageMockRecorder{mock} 27 | return mock 28 | } 29 | 30 | // EXPECT returns an object that allows the caller to indicate expected use 31 | func (_m *MockStorage) EXPECT() *MockStorageMockRecorder { 32 | return _m.recorder 33 | } 34 | 35 | // UpdateState mocks base method 36 | func (_m *MockStorage) UpdateState(ctx context.Context, owner string, name string, analysisID string, state *State) error { 37 | ret := _m.ctrl.Call(_m, "UpdateState", ctx, owner, name, analysisID, state) 38 | ret0, _ := ret[0].(error) 39 | return ret0 40 | } 41 | 42 | // UpdateState indicates an expected call of UpdateState 43 | func (_mr *MockStorageMockRecorder) UpdateState(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { 44 | return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "UpdateState", reflect.TypeOf((*MockStorage)(nil).UpdateState), arg0, arg1, arg2, arg3, arg4) 45 | } 46 | 47 | // GetState mocks base method 48 | func (_m *MockStorage) GetState(ctx context.Context, owner string, name string, analysisID string) (*State, error) { 49 | ret := _m.ctrl.Call(_m, "GetState", ctx, owner, name, analysisID) 50 | ret0, _ := ret[0].(*State) 51 | ret1, _ := ret[1].(error) 52 | return ret0, ret1 53 | } 54 | 55 | // GetState indicates an expected call of GetState 56 | func (_mr *MockStorageMockRecorder) GetState(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { 57 | return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "GetState", reflect.TypeOf((*MockStorage)(nil).GetState), arg0, arg1, arg2, arg3) 58 | } 59 | -------------------------------------------------------------------------------- /app/analyze/repostate/storage_mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: storage.go 3 | 4 | package repostate 5 | 6 | import ( 7 | context "context" 8 | gomock "github.com/golang/mock/gomock" 9 | reflect "reflect" 10 | ) 11 | 12 | // MockStorage is a mock of Storage interface 13 | type MockStorage struct { 14 | ctrl *gomock.Controller 15 | recorder *MockStorageMockRecorder 16 | } 17 | 18 | // MockStorageMockRecorder is the mock recorder for MockStorage 19 | type MockStorageMockRecorder struct { 20 | mock *MockStorage 21 | } 22 | 23 | // NewMockStorage creates a new mock instance 24 | func NewMockStorage(ctrl *gomock.Controller) *MockStorage { 25 | mock := &MockStorage{ctrl: ctrl} 26 | mock.recorder = &MockStorageMockRecorder{mock} 27 | return mock 28 | } 29 | 30 | // EXPECT returns an object that allows the caller to indicate expected use 31 | func (_m *MockStorage) EXPECT() *MockStorageMockRecorder { 32 | return _m.recorder 33 | } 34 | 35 | // UpdateState mocks base method 36 | func (_m *MockStorage) UpdateState(ctx context.Context, owner string, name string, analysisID string, state *State) error { 37 | ret := _m.ctrl.Call(_m, "UpdateState", ctx, owner, name, analysisID, state) 38 | ret0, _ := ret[0].(error) 39 | return ret0 40 | } 41 | 42 | // UpdateState indicates an expected call of UpdateState 43 | func (_mr *MockStorageMockRecorder) UpdateState(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { 44 | return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "UpdateState", reflect.TypeOf((*MockStorage)(nil).UpdateState), arg0, arg1, arg2, arg3, arg4) 45 | } 46 | 47 | // GetState mocks base method 48 | func (_m *MockStorage) GetState(ctx context.Context, owner string, name string, analysisID string) (*State, error) { 49 | ret := _m.ctrl.Call(_m, "GetState", ctx, owner, name, analysisID) 50 | ret0, _ := ret[0].(*State) 51 | ret1, _ := ret[1].(error) 52 | return ret0, ret1 53 | } 54 | 55 | // GetState indicates an expected call of GetState 56 | func (_mr *MockStorageMockRecorder) GetState(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { 57 | return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "GetState", reflect.TypeOf((*MockStorage)(nil).GetState), arg0, arg1, arg2, arg3) 58 | } 59 | -------------------------------------------------------------------------------- /app/analyze/analyzequeue/consume_test.go: -------------------------------------------------------------------------------- 1 | package analyzequeue 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/golangci/golangci-worker/app/analyze/analyzequeue/consumers" 9 | "github.com/golangci/golangci-worker/app/analyze/analyzequeue/task" 10 | "github.com/golangci/golangci-worker/app/analyze/processors" 11 | "github.com/golangci/golangci-worker/app/lib/github" 12 | "github.com/golangci/golangci-worker/app/lib/queue" 13 | "github.com/golangci/golangci-worker/app/test" 14 | "github.com/stretchr/testify/assert" 15 | ) 16 | 17 | type processorMocker struct { 18 | prevProcessorFactory processors.Factory 19 | } 20 | 21 | func (pm processorMocker) restore() { 22 | consumers.ProcessorFactory = pm.prevProcessorFactory 23 | } 24 | 25 | func mockProcessor(newProcessorFactory processors.Factory) *processorMocker { 26 | ret := &processorMocker{ 27 | prevProcessorFactory: newProcessorFactory, 28 | } 29 | consumers.ProcessorFactory = newProcessorFactory 30 | return ret 31 | } 32 | 33 | type testProcessor struct { 34 | notifyCh chan bool 35 | } 36 | 37 | func (tp testProcessor) Process(ctx context.Context) error { 38 | tp.notifyCh <- true 39 | return nil 40 | } 41 | 42 | type testProcessorFatory struct { 43 | t *testing.T 44 | expTask *task.PRAnalysis 45 | notifyCh chan bool 46 | } 47 | 48 | func (tpf testProcessorFatory) BuildProcessor(ctx context.Context, t *task.PRAnalysis) (processors.Processor, error) { 49 | assert.Equal(tpf.t, tpf.expTask, t) 50 | return testProcessor{ 51 | notifyCh: tpf.notifyCh, 52 | }, nil 53 | } 54 | func TestSendReceiveProcessing(t *testing.T) { 55 | task := &task.PRAnalysis{ 56 | Context: github.FakeContext, 57 | APIRequestID: "req_id", 58 | } 59 | 60 | notifyCh := make(chan bool) 61 | defer mockProcessor(testProcessorFatory{ 62 | t: t, 63 | expTask: task, 64 | notifyCh: notifyCh, 65 | }).restore() 66 | 67 | test.Init() 68 | queue.Init() 69 | RegisterTasks() 70 | go func() { 71 | err := RunWorker() 72 | assert.NoError(t, err) 73 | }() 74 | 75 | assert.NoError(t, SchedulePRAnalysis(task)) 76 | 77 | select { 78 | case <-notifyCh: 79 | return 80 | case <-time.After(time.Second * 1): 81 | t.Fatalf("Timeouted waiting of processing") 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /app/lib/experiments/checker.go: -------------------------------------------------------------------------------- 1 | package experiments 2 | 3 | import ( 4 | "hash/fnv" 5 | "strings" 6 | 7 | "github.com/golangci/golangci-shared/pkg/config" 8 | "github.com/golangci/golangci-shared/pkg/logutil" 9 | "github.com/golangci/golangci-worker/app/lib/github" 10 | ) 11 | 12 | type Checker struct { 13 | cfg config.Config 14 | log logutil.Log 15 | } 16 | 17 | func NewChecker(cfg config.Config, log logutil.Log) *Checker { 18 | return &Checker{cfg: cfg, log: log} 19 | } 20 | 21 | func (c Checker) getConfigKey(name, suffix string) string { 22 | return strings.ToUpper(name + "_" + suffix) 23 | } 24 | 25 | func (c Checker) parseConfigVarToBoolMap(k string) map[string]bool { 26 | elems := c.cfg.GetString(k) 27 | if elems == "" { 28 | return map[string]bool{} 29 | } 30 | 31 | elemList := strings.Split(elems, ",") 32 | ret := map[string]bool{} 33 | for _, e := range elemList { 34 | ret[e] = true 35 | } 36 | 37 | return ret 38 | } 39 | 40 | func (c Checker) IsActiveForAnalysis(name string, repo *github.Repo, forPull bool) bool { 41 | if forPull && !c.cfg.GetBool(c.getConfigKey(name, "for_pulls"), false) { 42 | c.log.Infof("Experiment %s is disabled for pull analyzes", name) 43 | return false 44 | } 45 | 46 | enabledRepos := c.parseConfigVarToBoolMap(c.getConfigKey(name, "repos")) 47 | if enabledRepos[repo.FullName()] { 48 | c.log.Infof("Experiment %s is enabled for repo %s", name, repo.FullName()) 49 | return true 50 | } 51 | 52 | enabledOwners := c.parseConfigVarToBoolMap(c.getConfigKey(name, "owners")) 53 | if enabledOwners[repo.Owner] { 54 | c.log.Infof("Experiment %s is enabled for owner of repo %s", name, repo.FullName()) 55 | return true 56 | } 57 | 58 | percent := c.cfg.GetInt(c.getConfigKey(name, "percent"), 0) 59 | if percent < 0 || percent > 100 { 60 | c.log.Infof("Experiment %s is disabled: invalid percent %d", name, percent) 61 | return false 62 | } 63 | 64 | hash := hash(repo.FullName()) 65 | if uint32(percent) <= (hash % 100) { 66 | c.log.Infof("Experiment %s is disabled by percent for repo %s: %d (percent) <= %d (hash mod 100)", 67 | name, repo.FullName(), percent, hash%100) 68 | return false 69 | } 70 | 71 | c.log.Infof("Experiment %s is enabled by percent for repo %s: %d (percent) > %d (hash mod 100)", 72 | name, repo.FullName(), percent, hash%100) 73 | return true 74 | } 75 | 76 | func hash(s string) uint32 { 77 | h := fnv.New32a() 78 | _, _ = h.Write([]byte(s)) 79 | return h.Sum32() 80 | } 81 | -------------------------------------------------------------------------------- /app/analyze/processors/repo_processor_factory.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | import ( 4 | "github.com/golangci/golangci-shared/pkg/apperrors" 5 | "github.com/golangci/golangci-shared/pkg/config" 6 | "github.com/golangci/golangci-shared/pkg/logutil" 7 | "github.com/golangci/golangci-worker/app/analyze/linters" 8 | "github.com/golangci/golangci-worker/app/analyze/linters/golinters" 9 | "github.com/golangci/golangci-worker/app/analyze/repostate" 10 | "github.com/golangci/golangci-worker/app/lib/experiments" 11 | "github.com/golangci/golangci-worker/app/lib/fetchers" 12 | "github.com/golangci/golangci-worker/app/lib/goutils/workspaces" 13 | "github.com/golangci/golangci-worker/app/lib/httputils" 14 | "github.com/pkg/errors" 15 | ) 16 | 17 | type RepoProcessorFactory struct { 18 | cfg *StaticRepoConfig 19 | noCtxLog logutil.Log 20 | } 21 | 22 | func NewRepoProcessorFactory(cfg *StaticRepoConfig, noCtxLog logutil.Log) *RepoProcessorFactory { 23 | return &RepoProcessorFactory{ 24 | cfg: cfg, 25 | noCtxLog: noCtxLog, 26 | } 27 | } 28 | 29 | func (f RepoProcessorFactory) BuildProcessor(ctx *RepoContext) (*Repo, func(), error) { 30 | cfg := *f.cfg 31 | 32 | if cfg.RepoFetcher == nil { 33 | cfg.RepoFetcher = fetchers.NewGit() 34 | } 35 | 36 | if cfg.Linters == nil { 37 | cfg.Linters = []linters.Linter{ 38 | golinters.GolangciLint{}, 39 | } 40 | } 41 | 42 | if cfg.Runner == nil { 43 | cfg.Runner = linters.SimpleRunner{} 44 | } 45 | 46 | if cfg.State == nil { 47 | cfg.State = repostate.NewAPIStorage(httputils.GrequestsClient{}) 48 | } 49 | 50 | if cfg.Cfg == nil { 51 | envCfg := config.NewEnvConfig(f.noCtxLog) 52 | cfg.Cfg = envCfg 53 | } 54 | 55 | if cfg.Et == nil { 56 | cfg.Et = apperrors.GetTracker(cfg.Cfg, f.noCtxLog, "worker") 57 | } 58 | 59 | lctx := logutil.Context{ 60 | "branch": ctx.Branch, 61 | "analysisGUID": ctx.AnalysisGUID, 62 | "provider": "github", 63 | "repoName": ctx.Repo.FullName(), 64 | "analysisType": "repo", 65 | } 66 | log := logutil.WrapLogWithContext(f.noCtxLog, lctx) 67 | log = apperrors.WrapLogWithTracker(log, lctx, cfg.Et) 68 | 69 | ec := experiments.NewChecker(cfg.Cfg, log) 70 | 71 | exec, err := makeExecutor(ctx.Ctx, ctx.Repo, false, log, ec) 72 | if err != nil { 73 | return nil, nil, errors.Wrap(err, "can't make executor") 74 | } 75 | 76 | cleanup := func() { 77 | exec.Clean() 78 | } 79 | p := NewRepo(&RepoConfig{ 80 | StaticRepoConfig: cfg, 81 | Log: log, 82 | Exec: exec, 83 | Wi: workspaces.NewGo2(exec, log, cfg.RepoFetcher), 84 | Ec: ec, 85 | }) 86 | 87 | return p, cleanup, nil 88 | } 89 | -------------------------------------------------------------------------------- /app/analyze/linters/golinters/golangci_lint.go: -------------------------------------------------------------------------------- 1 | package golinters 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "strings" 8 | 9 | "github.com/golangci/golangci-worker/app/analytics" 10 | "github.com/golangci/golangci-worker/app/analyze/linters/result" 11 | "github.com/golangci/golangci-worker/app/lib/errorutils" 12 | "github.com/golangci/golangci-worker/app/lib/executors" 13 | 14 | "github.com/golangci/golangci-lint/pkg/printers" 15 | ) 16 | 17 | type GolangciLint struct { 18 | PatchPath string 19 | } 20 | 21 | func (g GolangciLint) Name() string { 22 | return "golangci-lint" 23 | } 24 | 25 | func (g GolangciLint) Run(ctx context.Context, exec executors.Executor) (*result.Result, error) { 26 | exec = exec.WithEnv("GOLANGCI_COM_RUN", "1") 27 | 28 | args := []string{ 29 | "run", 30 | "--out-format=json", 31 | "--issues-exit-code=0", 32 | "--print-welcome=false", 33 | "--timeout=5m", 34 | "--new=false", 35 | "--new-from-rev=", 36 | "--new-from-patch=" + g.PatchPath, 37 | } 38 | 39 | out, runErr := exec.Run(ctx, g.Name(), args...) 40 | rawJSON := []byte(out) 41 | 42 | if runErr != nil { 43 | var res printers.JSONResult 44 | if jsonErr := json.Unmarshal(rawJSON, &res); jsonErr == nil && res.Report.Error != "" { 45 | return nil, &errorutils.BadInputError{ 46 | PublicDesc: fmt.Sprintf("can't run golangci-lint: %s", res.Report.Error), 47 | } 48 | } 49 | 50 | const badLoadStr = "failed to load program with go/packages" 51 | if strings.Contains(runErr.Error(), badLoadStr) { 52 | ind := strings.Index(runErr.Error(), badLoadStr) 53 | if ind < len(runErr.Error())-1 { 54 | return nil, &errorutils.BadInputError{ 55 | PublicDesc: runErr.Error()[ind:], 56 | } 57 | } 58 | } 59 | 60 | return nil, &errorutils.InternalError{ 61 | PublicDesc: "can't run golangci-lint", 62 | PrivateDesc: fmt.Sprintf("can't run golangci-lint: %s, %s", runErr, out), 63 | } 64 | } 65 | 66 | var res printers.JSONResult 67 | if jsonErr := json.Unmarshal(rawJSON, &res); jsonErr != nil { 68 | return nil, &errorutils.InternalError{ 69 | PublicDesc: "can't run golangci-lint: invalid output json", 70 | PrivateDesc: fmt.Sprintf("can't run golangci-lint: can't parse json output %s: %s", out, jsonErr), 71 | } 72 | } 73 | 74 | if res.Report != nil && len(res.Report.Warnings) != 0 { 75 | analytics.Log(ctx).Infof("Got golangci-lint warnings: %#v", res.Report.Warnings) 76 | } 77 | 78 | var retIssues []result.Issue 79 | for _, i := range res.Issues { 80 | retIssues = append(retIssues, result.Issue{ 81 | File: i.FilePath(), 82 | LineNumber: i.Line(), 83 | Text: i.Text, 84 | FromLinter: i.FromLinter, 85 | HunkPos: i.HunkPos, 86 | }) 87 | } 88 | return &result.Result{ 89 | Issues: retIssues, 90 | ResultJSON: json.RawMessage(rawJSON), 91 | }, nil 92 | } 93 | -------------------------------------------------------------------------------- /app/analyze/analyzequeue/consumers/analyze_repo.go: -------------------------------------------------------------------------------- 1 | package consumers 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "strings" 8 | "time" 9 | 10 | "github.com/golangci/golangci-worker/app/analytics" 11 | "github.com/golangci/golangci-worker/app/analyze/processors" 12 | "github.com/golangci/golangci-worker/app/lib/experiments" 13 | "github.com/golangci/golangci-worker/app/lib/github" 14 | "github.com/pkg/errors" 15 | ) 16 | 17 | type AnalyzeRepo struct { 18 | baseConsumer 19 | 20 | ec *experiments.Checker 21 | rpf *processors.RepoProcessorFactory 22 | } 23 | 24 | func NewAnalyzeRepo(ec *experiments.Checker, rpf *processors.RepoProcessorFactory) *AnalyzeRepo { 25 | return &AnalyzeRepo{ 26 | baseConsumer: baseConsumer{ 27 | eventName: analytics.EventRepoAnalyzed, 28 | }, 29 | ec: ec, 30 | rpf: rpf, 31 | } 32 | } 33 | 34 | func (c AnalyzeRepo) Consume(ctx context.Context, repoName, analysisGUID, branch string) error { 35 | ctx = c.prepareContext(ctx, map[string]interface{}{ 36 | "repoName": repoName, 37 | "provider": "github", 38 | "analysisGUID": analysisGUID, 39 | "branch": branch, 40 | }) 41 | 42 | if os.Getenv("DISABLE_REPO_ANALYSIS") == "1" { 43 | analytics.Log(ctx).Warnf("Repo analysis is disabled, return error to try it later") 44 | return errors.New("repo analysis is disabled") 45 | } 46 | 47 | return c.wrapConsuming(ctx, func() error { 48 | var cancel context.CancelFunc 49 | // If you change timeout value don't forget to change it 50 | // in golangci-api stale analyzes checker 51 | ctx, cancel = context.WithTimeout(ctx, 10*time.Minute) 52 | defer cancel() 53 | 54 | return c.analyzeRepo(ctx, repoName, analysisGUID, branch) 55 | }) 56 | } 57 | 58 | func (c AnalyzeRepo) analyzeRepo(ctx context.Context, repoName, analysisGUID, branch string) error { 59 | parts := strings.Split(repoName, "/") 60 | repo := &github.Repo{ 61 | Owner: parts[0], 62 | Name: parts[1], 63 | } 64 | if len(parts) != 2 { 65 | return fmt.Errorf("invalid repo name %s", repoName) 66 | } 67 | 68 | if c.ec.IsActiveForAnalysis("use_new_repo_analysis", repo, false) { 69 | repoCtx := &processors.RepoContext{ 70 | Ctx: ctx, 71 | AnalysisGUID: analysisGUID, 72 | Branch: branch, 73 | Repo: repo, 74 | } 75 | p, cleanup, err := c.rpf.BuildProcessor(repoCtx) 76 | if err != nil { 77 | return errors.Wrap(err, "failed to build repo processor") 78 | } 79 | defer cleanup() 80 | 81 | p.Process(repoCtx) 82 | return nil 83 | } 84 | 85 | p, err := processors.NewGithubGoRepo(ctx, processors.GithubGoRepoConfig{}, analysisGUID, repoName, branch) 86 | if err != nil { 87 | return fmt.Errorf("can't make github go repo processor: %s", err) 88 | } 89 | 90 | if err := p.Process(ctx); err != nil { 91 | return fmt.Errorf("can't process repo analysis for %s and branch %s: %s", repoName, branch, err) 92 | } 93 | 94 | return nil 95 | } 96 | -------------------------------------------------------------------------------- /app/lib/goutils/workspaces/go.go: -------------------------------------------------------------------------------- 1 | package workspaces 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "path/filepath" 8 | "strings" 9 | 10 | "github.com/golangci/golangci-api/pkg/goenv/ensuredeps" 11 | "github.com/golangci/golangci-worker/app/analytics" 12 | "github.com/golangci/golangci-worker/app/analyze/repoinfo" 13 | "github.com/golangci/golangci-worker/app/lib/executors" 14 | "github.com/golangci/golangci-worker/app/lib/fetchers" 15 | "github.com/golangci/golangci-worker/app/lib/goutils/environments" 16 | "github.com/pkg/errors" 17 | ) 18 | 19 | type Go struct { 20 | gopath string 21 | exec executors.Executor 22 | infoFetcher repoinfo.Fetcher 23 | } 24 | 25 | func NewGo(exec executors.Executor, infoFetcher repoinfo.Fetcher) *Go { 26 | return &Go{ 27 | exec: exec, 28 | infoFetcher: infoFetcher, 29 | } 30 | } 31 | 32 | func (w *Go) Setup(ctx context.Context, repo *fetchers.Repo, projectPathParts ...string) error { 33 | repoInfo, err := w.infoFetcher.Fetch(ctx, repo, w.exec) 34 | if err != nil { 35 | return errors.Wrap(err, "failed to fetch repo info") 36 | } 37 | 38 | if repoInfo != nil && repoInfo.CanonicalImportPath != "" { 39 | newProjectPathParts := strings.Split(repoInfo.CanonicalImportPath, "/") 40 | analytics.Log(ctx).Infof("change canonical project path: %s -> %s", projectPathParts, newProjectPathParts) 41 | projectPathParts = newProjectPathParts 42 | } 43 | 44 | if _, err := w.exec.Run(ctx, "find", ".", "-delete"); err != nil { 45 | analytics.Log(ctx).Warnf("Failed to cleanup after repo info fetcher: %s", err) 46 | } 47 | 48 | gopath := w.exec.WorkDir() 49 | wdParts := []string{gopath, "src"} 50 | wdParts = append(wdParts, projectPathParts...) 51 | wd := filepath.Join(wdParts...) 52 | if out, err := w.exec.Run(ctx, "mkdir", "-p", wd); err != nil { 53 | return fmt.Errorf("can't create project dir %q: %s, %s", wd, err, out) 54 | } 55 | 56 | goEnv := environments.NewGolang(gopath) 57 | goEnv.Setup(w.exec) 58 | 59 | w.exec = w.exec.WithWorkDir(wd) // XXX: clean gopath, but work in subdir of gopath 60 | 61 | w.gopath = gopath 62 | return nil 63 | } 64 | 65 | func (w Go) Executor() executors.Executor { 66 | return w.exec 67 | } 68 | 69 | func (w Go) Gopath() string { 70 | return w.gopath 71 | } 72 | 73 | func (w Go) FetchDeps(ctx context.Context, fullRepoPath string) (*ensuredeps.Result, error) { 74 | cleanupPath := filepath.Join("/app", "cleanup.sh") 75 | out, err := w.exec.Run(ctx, "bash", cleanupPath) 76 | if err != nil { 77 | return nil, fmt.Errorf("can't call /app/cleanup.sh: %s, %s", err, out) 78 | } 79 | 80 | out, err = w.exec.Run(ctx, "ensuredeps", "--repo", fullRepoPath) 81 | if err != nil { 82 | return nil, fmt.Errorf("can't ensuredeps --repo %s: %s, %s", fullRepoPath, err, out) 83 | } 84 | 85 | var res ensuredeps.Result 86 | if err = json.Unmarshal([]byte(out), &res); err != nil { 87 | return nil, fmt.Errorf("failed to parse res json: %s", err) 88 | } 89 | 90 | return &res, nil 91 | } 92 | 93 | func (w Go) Clean(ctx context.Context) { 94 | out, err := w.exec.Run(ctx, "go", "clean", "-modcache") 95 | if err != nil { 96 | analytics.Log(ctx).Warnf("Can't clean go modcache: %s, %s", err, out) 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![CircleCI](https://circleci.com/gh/golangci/golangci-worker.svg?style=svg&circle-token=94e0eb37b49bb5f87364a50592794eba13f0d95d)](https://circleci.com/gh/golangci/golangci-worker) 2 | [![GolangCI](https://golangci.com/badges/github.com/golangci/golangci-worker.svg)](https://golangci.com) 3 | 4 | # Worker 5 | 6 | This repository contains code of queue worker. Worker runs golangci-lint and reports result to GitHub. 7 | 8 | ## Development 9 | 10 | ### Technologies 11 | 12 | Go (golang), heroku, circleci, docker, redis, postgres. 13 | 14 | ### Preparation 15 | 16 | In [golangci-api](https://github.com/golangci/golangci-api) repo run: 17 | 18 | ```bash 19 | docker-compose up -d 20 | ``` 21 | 22 | It runs postgres and redis needed for both api and worker. 23 | 24 | ### How to run worker 25 | 26 | ```bash 27 | make run_dev 28 | ``` 29 | 30 | ### How to run once on GitHub repo without changing GitHub data: commit status, comments 31 | 32 | ```bash 33 | REPO={OWNER/NAME} PR={PULL_REQUEST_NUMBER} make test_repo_fake_github 34 | ``` 35 | 36 | e.g. `REPO=golangci/golangci-worker PR=39 make test_repo_fake_github` 37 | 38 | ### How to run analysis of pull request locally 39 | 40 | ```bash 41 | # in golangci-api repo 42 | godotenv -f .env go run ./scripts/emulate_webhook/main.go -repo golangci/golangci-lint -pr 292 -sha 7b605d5c6f5a524e6b0a9cc12ad747222375ad54 43 | ``` 44 | 45 | ### Configuration 46 | 47 | Configurate via `.env` file. Dev `.env` may be like this: 48 | 49 | ```bash 50 | REDIS_URL="redis://localhost:6379" 51 | API_URL="https://api.dev.golangci.com" 52 | WEB_ROOT="https://dev.golangci.com" 53 | USE_CONTAINER_EXECUTOR_PERCENT=100 54 | USE_NEW_REPO_ANALYSIS_PERCENT=100 55 | ORCHESTRATOR_ADDR="http://127.0.0.1:8001" 56 | ORCHESTRATOR_TOKEN=secret_token 57 | ``` 58 | 59 | ### Executors 60 | 61 | Executor is an abstration allowing to run arbitrary shell commands. 62 | We support following executor types: 63 | 64 | 1. shell - runs commands on a local machine 65 | 2. remote shell - runs commands on the specified remote host; it's currently the primary executor 66 | 3. container - runs commands by sending them to containers orchestrator; containers orchestrator runs container for executing commands; currently we migrate to this executor type. 67 | 68 | The recommended way to run executors during development: 69 | 70 | ```bash 71 | # in golangci-api repo 72 | TOKEN=secret_token go run ./cmd/containers_orchestrator/main.go 73 | ``` 74 | 75 | ### API 76 | 77 | golangci-api is not needed for running and testing golangci-worker. Not running api can just make log warnings like this: 78 | 79 | ```bash 80 | level=warning msg="Can't get current state: bad status code 404" 81 | ``` 82 | 83 | ### Testing 84 | 85 | To run tests: 86 | 87 | ```bash 88 | make test 89 | ``` 90 | 91 | For more realistic testing than `test_repo_fake_github` use in golangci-api repo GitHub WebHook emulator: 92 | 93 | ```bash 94 | go run app/scripts/emulate_webhook/main.go --repo golangci/golangci-worker --pr 39 --sha fbd0d7bada8a6cfa7adbc58e5901e0d66f7f65b1 95 | ``` 96 | 97 | ## Contributing 98 | 99 | See [CONTRIBUTING](https://github.com/golangci/golangci-worker/blob/master/CONTRIBUTING.md). 100 | -------------------------------------------------------------------------------- /app/lib/executors/remote_shell.go: -------------------------------------------------------------------------------- 1 | package executors 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "os/exec" 8 | "path/filepath" 9 | "strconv" 10 | "strings" 11 | 12 | "github.com/sirupsen/logrus" 13 | ) 14 | 15 | type RemoteShell struct { 16 | envStore 17 | tempWorkDir string 18 | wd string 19 | user, host, keyFilePath string 20 | } 21 | 22 | var _ Executor = &RemoteShell{} 23 | 24 | func NewRemoteShell(user, host, keyFilePath string) *RemoteShell { 25 | return &RemoteShell{ 26 | envStore: envStore{}, 27 | user: user, 28 | host: host, 29 | keyFilePath: keyFilePath, 30 | } 31 | } 32 | 33 | func (s *RemoteShell) SetupTempWorkDir(ctx context.Context) error { 34 | out, err := s.Run(ctx, "mktemp", "-d") 35 | if err != nil { 36 | return err 37 | } 38 | 39 | s.tempWorkDir = strings.TrimSpace(out) 40 | if s.tempWorkDir == "" { 41 | return fmt.Errorf("empty temp dir") 42 | } 43 | 44 | s.wd = s.tempWorkDir 45 | 46 | return nil 47 | } 48 | 49 | func quoteArgs(args []string) []string { 50 | var ret []string 51 | for _, arg := range args { 52 | ret = append(ret, strconv.Quote(arg)) 53 | } 54 | return ret 55 | } 56 | 57 | func sprintArgs(args []string) string { 58 | return strings.Join(quoteArgs(args), " ") 59 | } 60 | 61 | func (s RemoteShell) Run(ctx context.Context, name string, srcArgs ...string) (string, error) { 62 | shellArg := fmt.Sprintf("cd %s; %s %s %s", 63 | s.wd, 64 | strings.Join(s.env, " "), 65 | name, strings.Join(srcArgs, " ")) 66 | args := []string{ 67 | "-i", 68 | s.keyFilePath, 69 | fmt.Sprintf("%s@%s", s.user, s.host), 70 | shellArg, 71 | } 72 | 73 | cmd := exec.CommandContext(ctx, "ssh", args...) 74 | var stderrBuf bytes.Buffer 75 | cmd.Stderr = &stderrBuf 76 | 77 | out, err := cmd.Output() 78 | if err != nil { 79 | return "", fmt.Errorf("can't execute command ssh %s: %s, %s, %s", 80 | sprintArgs(args), err, string(out), stderrBuf.String()) 81 | } 82 | 83 | return string(out), nil 84 | } 85 | 86 | func (s RemoteShell) CopyFile(ctx context.Context, dst, src string) error { 87 | if !filepath.IsAbs(dst) { 88 | dst = filepath.Join(s.WorkDir(), dst) 89 | } 90 | out, err := exec.CommandContext(ctx, "scp", 91 | "-i", s.keyFilePath, 92 | src, 93 | fmt.Sprintf("%s@%s:%s", s.user, s.host, dst), 94 | ).CombinedOutput() 95 | if err != nil { 96 | return fmt.Errorf("can't copy file %s to %s: %s, %s", src, dst, err, out) 97 | } 98 | 99 | return nil 100 | } 101 | 102 | func (s RemoteShell) Clean() { 103 | if s.tempWorkDir == "" { 104 | panic("empty temp work dir") 105 | } 106 | 107 | out, err := s.Run(context.TODO(), "rm", "-r", s.tempWorkDir) 108 | if err != nil { 109 | logrus.Warnf("Can't remove temp work dir in remote shell: %s, %s", err, out) 110 | } 111 | } 112 | 113 | func (s RemoteShell) WithEnv(k, v string) Executor { 114 | eCopy := s 115 | eCopy.SetEnv(k, v) 116 | return &eCopy 117 | } 118 | 119 | func (s RemoteShell) WorkDir() string { 120 | return s.wd 121 | } 122 | 123 | func (s *RemoteShell) SetWorkDir(wd string) { 124 | s.wd = wd 125 | } 126 | 127 | func (s RemoteShell) WithWorkDir(wd string) Executor { 128 | eCopy := s 129 | eCopy.wd = wd 130 | return &eCopy 131 | } 132 | -------------------------------------------------------------------------------- /app/analyze/reporters/github_reviewer.go: -------------------------------------------------------------------------------- 1 | package reporters 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/golangci/golangci-worker/app/analytics" 9 | "github.com/golangci/golangci-worker/app/analyze/linters/result" 10 | "github.com/golangci/golangci-worker/app/lib/github" 11 | gh "github.com/google/go-github/github" 12 | ) 13 | 14 | type GithubReviewer struct { 15 | *github.Context 16 | client github.Client 17 | includeLinterName bool 18 | } 19 | 20 | func NewGithubReviewer(c *github.Context, client github.Client, includeLinterName bool) *GithubReviewer { 21 | accessToken := os.Getenv("GITHUB_REVIEWER_ACCESS_TOKEN") 22 | if accessToken != "" { // review as special user 23 | cCopy := *c 24 | cCopy.GithubAccessToken = accessToken 25 | c = &cCopy 26 | } 27 | ret := &GithubReviewer{ 28 | Context: c, 29 | client: client, 30 | includeLinterName: includeLinterName, 31 | } 32 | return ret 33 | } 34 | 35 | type existingComment struct { 36 | file string 37 | line int 38 | } 39 | 40 | type existingComments []existingComment 41 | 42 | func (ecs existingComments) contains(i *result.Issue) bool { 43 | for _, c := range ecs { 44 | if c.file == i.File && c.line == i.HunkPos { 45 | return true 46 | } 47 | } 48 | 49 | return false 50 | } 51 | 52 | func (gr GithubReviewer) fetchExistingComments(ctx context.Context) (existingComments, error) { 53 | comments, err := gr.client.GetPullRequestComments(ctx, gr.Context) 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | var ret existingComments 59 | for _, c := range comments { 60 | if c.Position == nil { // comment on outdated code, skip it 61 | continue 62 | } 63 | ret = append(ret, existingComment{ 64 | file: c.GetPath(), 65 | line: c.GetPosition(), 66 | }) 67 | } 68 | 69 | return ret, nil 70 | } 71 | 72 | func (gr GithubReviewer) Report(ctx context.Context, ref string, issues []result.Issue) error { 73 | if len(issues) == 0 { 74 | analytics.Log(ctx).Infof("Nothing to report") 75 | return nil 76 | } 77 | 78 | existingComments, err := gr.fetchExistingComments(ctx) 79 | if err != nil { 80 | return err 81 | } 82 | 83 | comments := []*gh.DraftReviewComment{} 84 | for _, i := range issues { 85 | if existingComments.contains(&i) { 86 | continue // don't be annoying: don't comment on the same line twice 87 | } 88 | 89 | text := i.Text 90 | if gr.includeLinterName && i.FromLinter != "" { 91 | text += fmt.Sprintf(" (from `%s`)", i.FromLinter) 92 | } 93 | 94 | comment := &gh.DraftReviewComment{ 95 | Path: gh.String(i.File), 96 | Position: gh.Int(i.HunkPos), 97 | Body: gh.String(text), 98 | } 99 | comments = append(comments, comment) 100 | } 101 | 102 | if len(comments) == 0 { 103 | return nil // all comments are already exist 104 | } 105 | 106 | review := &gh.PullRequestReviewRequest{ 107 | CommitID: gh.String(ref), 108 | Event: gh.String("COMMENT"), 109 | Body: gh.String(""), 110 | Comments: comments, 111 | } 112 | if err := gr.client.CreateReview(ctx, gr.Context, review); err != nil { 113 | return fmt.Errorf("can't create review %+v: %s", review, err) 114 | } 115 | 116 | analytics.Log(ctx).Infof("Submitted review %+v, existing comments: %+v, issues: %+v", 117 | review, existingComments, issues) 118 | return nil 119 | } 120 | -------------------------------------------------------------------------------- /app/lib/executors/shell.go: -------------------------------------------------------------------------------- 1 | package executors 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "io" 8 | "os/exec" 9 | "strconv" 10 | "strings" 11 | "time" 12 | 13 | "github.com/golangci/golangci-worker/app/analytics" 14 | "github.com/shirou/gopsutil/process" 15 | ) 16 | 17 | type shell struct { 18 | envStore 19 | wd string 20 | } 21 | 22 | func newShell(workDir string) *shell { 23 | return &shell{ 24 | wd: workDir, 25 | envStore: *newEnvStore(), 26 | } 27 | } 28 | 29 | func trackMemoryEveryNSeconds(ctx context.Context, name string, pid int) { 30 | rssValues := []uint64{} 31 | ticker := time.NewTicker(100 * time.Millisecond) 32 | for { 33 | p, _ := process.NewProcess(int32(pid)) 34 | mi, err := p.MemoryInfoWithContext(ctx) 35 | if err != nil { 36 | analytics.Log(ctx).Debugf("Can't fetch memory info on subprocess: %s", err) 37 | return 38 | } 39 | 40 | rssValues = append(rssValues, mi.RSS) 41 | 42 | stop := false 43 | select { 44 | case <-ctx.Done(): 45 | stop = true 46 | case <-ticker.C: // track every second 47 | } 48 | 49 | if stop { 50 | break 51 | } 52 | } 53 | 54 | var avg, max uint64 55 | for _, v := range rssValues { 56 | avg += v 57 | if v > max { 58 | max = v 59 | } 60 | } 61 | avg /= uint64(len(rssValues)) 62 | 63 | const MB = 1024 * 1024 64 | maxMB := float64(max) / MB 65 | if maxMB >= 10 { 66 | analytics.Log(ctx).Infof("Subprocess %q memory: got %d rss values, avg is %.1fMB, max is %.1fMB", 67 | name, len(rssValues), float64(avg)/MB, maxMB) 68 | } 69 | } 70 | 71 | func (s shell) wait(ctx context.Context, name string, childPid int, outReader io.Reader) []string { 72 | trackCtx, cancel := context.WithCancel(ctx) 73 | defer cancel() 74 | go trackMemoryEveryNSeconds(trackCtx, name, childPid) 75 | 76 | scanner := bufio.NewScanner(outReader) 77 | lines := []string{} 78 | for scanner.Scan() { 79 | line := scanner.Text() 80 | analytics.Log(ctx).Debugf("%s", line) 81 | lines = append(lines, line) 82 | } 83 | if err := scanner.Err(); err != nil { 84 | analytics.Log(ctx).Warnf("Out lines scanning error: %s", err) 85 | } 86 | 87 | return lines 88 | } 89 | 90 | func (s shell) Run(ctx context.Context, name string, args ...string) (string, error) { 91 | for i := range args { 92 | unquotedArg, err := strconv.Unquote(args[i]) 93 | if err == nil { 94 | args[i] = unquotedArg 95 | } 96 | } 97 | startedAt := time.Now() 98 | pid, outReader, finish, err := s.runAsync(ctx, name, args...) 99 | if err != nil { 100 | return "", err 101 | } 102 | 103 | endCh := make(chan struct{}) 104 | defer close(endCh) 105 | 106 | go func() { 107 | select { 108 | case <-ctx.Done(): 109 | analytics.Log(ctx).Warnf("Closing shell reader on timeout") 110 | if cerr := outReader.Close(); cerr != nil { 111 | analytics.Log(ctx).Warnf("Failed to close shell reader on deadline: %s", cerr) 112 | } 113 | case <-endCh: 114 | } 115 | }() 116 | 117 | lines := s.wait(ctx, name, pid, outReader) 118 | 119 | err = finish() 120 | 121 | logger := analytics.Log(ctx).Debugf 122 | if err != nil { 123 | logger = analytics.Log(ctx).Infof 124 | } 125 | logger("shell[%s]: %s %v executed for %s: %v", s.wd, name, args, time.Since(startedAt), err) 126 | 127 | // XXX: it's important to not change error here, because it holds exit code 128 | return strings.Join(lines, "\n"), err 129 | } 130 | 131 | type finishFunc func() error 132 | 133 | func (s shell) runAsync(ctx context.Context, name string, args ...string) (int, io.ReadCloser, finishFunc, error) { 134 | cmd := exec.CommandContext(ctx, name, args...) 135 | cmd.Env = s.env 136 | cmd.Dir = s.wd 137 | 138 | outReader, err := cmd.StdoutPipe() 139 | if err != nil { 140 | return 0, nil, nil, fmt.Errorf("can't make out pipe: %s", err) 141 | } 142 | 143 | cmd.Stderr = cmd.Stdout // Set the same pipe 144 | if err := cmd.Start(); err != nil { 145 | return 0, nil, nil, err 146 | } 147 | 148 | // XXX: it's important to not change error here, because it holds exit code 149 | return cmd.Process.Pid, outReader, cmd.Wait, nil 150 | } 151 | -------------------------------------------------------------------------------- /app/lib/github/client_mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: client.go 3 | 4 | package github 5 | 6 | import ( 7 | context "context" 8 | gomock "github.com/golang/mock/gomock" 9 | github "github.com/google/go-github/github" 10 | reflect "reflect" 11 | ) 12 | 13 | // MockClient is a mock of Client interface 14 | type MockClient struct { 15 | ctrl *gomock.Controller 16 | recorder *MockClientMockRecorder 17 | } 18 | 19 | // MockClientMockRecorder is the mock recorder for MockClient 20 | type MockClientMockRecorder struct { 21 | mock *MockClient 22 | } 23 | 24 | // NewMockClient creates a new mock instance 25 | func NewMockClient(ctrl *gomock.Controller) *MockClient { 26 | mock := &MockClient{ctrl: ctrl} 27 | mock.recorder = &MockClientMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (_m *MockClient) EXPECT() *MockClientMockRecorder { 33 | return _m.recorder 34 | } 35 | 36 | // GetPullRequest mocks base method 37 | func (_m *MockClient) GetPullRequest(ctx context.Context, c *Context) (*github.PullRequest, error) { 38 | ret := _m.ctrl.Call(_m, "GetPullRequest", ctx, c) 39 | ret0, _ := ret[0].(*github.PullRequest) 40 | ret1, _ := ret[1].(error) 41 | return ret0, ret1 42 | } 43 | 44 | // GetPullRequest indicates an expected call of GetPullRequest 45 | func (_mr *MockClientMockRecorder) GetPullRequest(arg0, arg1 interface{}) *gomock.Call { 46 | return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "GetPullRequest", reflect.TypeOf((*MockClient)(nil).GetPullRequest), arg0, arg1) 47 | } 48 | 49 | // GetPullRequestComments mocks base method 50 | func (_m *MockClient) GetPullRequestComments(ctx context.Context, c *Context) ([]*github.PullRequestComment, error) { 51 | ret := _m.ctrl.Call(_m, "GetPullRequestComments", ctx, c) 52 | ret0, _ := ret[0].([]*github.PullRequestComment) 53 | ret1, _ := ret[1].(error) 54 | return ret0, ret1 55 | } 56 | 57 | // GetPullRequestComments indicates an expected call of GetPullRequestComments 58 | func (_mr *MockClientMockRecorder) GetPullRequestComments(arg0, arg1 interface{}) *gomock.Call { 59 | return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "GetPullRequestComments", reflect.TypeOf((*MockClient)(nil).GetPullRequestComments), arg0, arg1) 60 | } 61 | 62 | // GetPullRequestPatch mocks base method 63 | func (_m *MockClient) GetPullRequestPatch(ctx context.Context, c *Context) (string, error) { 64 | ret := _m.ctrl.Call(_m, "GetPullRequestPatch", ctx, c) 65 | ret0, _ := ret[0].(string) 66 | ret1, _ := ret[1].(error) 67 | return ret0, ret1 68 | } 69 | 70 | // GetPullRequestPatch indicates an expected call of GetPullRequestPatch 71 | func (_mr *MockClientMockRecorder) GetPullRequestPatch(arg0, arg1 interface{}) *gomock.Call { 72 | return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "GetPullRequestPatch", reflect.TypeOf((*MockClient)(nil).GetPullRequestPatch), arg0, arg1) 73 | } 74 | 75 | // CreateReview mocks base method 76 | func (_m *MockClient) CreateReview(ctx context.Context, c *Context, review *github.PullRequestReviewRequest) error { 77 | ret := _m.ctrl.Call(_m, "CreateReview", ctx, c, review) 78 | ret0, _ := ret[0].(error) 79 | return ret0 80 | } 81 | 82 | // CreateReview indicates an expected call of CreateReview 83 | func (_mr *MockClientMockRecorder) CreateReview(arg0, arg1, arg2 interface{}) *gomock.Call { 84 | return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "CreateReview", reflect.TypeOf((*MockClient)(nil).CreateReview), arg0, arg1, arg2) 85 | } 86 | 87 | // SetCommitStatus mocks base method 88 | func (_m *MockClient) SetCommitStatus(ctx context.Context, c *Context, ref string, status Status, desc string, url string) error { 89 | ret := _m.ctrl.Call(_m, "SetCommitStatus", ctx, c, ref, status, desc, url) 90 | ret0, _ := ret[0].(error) 91 | return ret0 92 | } 93 | 94 | // SetCommitStatus indicates an expected call of SetCommitStatus 95 | func (_mr *MockClientMockRecorder) SetCommitStatus(arg0, arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call { 96 | return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "SetCommitStatus", reflect.TypeOf((*MockClient)(nil).SetCommitStatus), arg0, arg1, arg2, arg3, arg4, arg5) 97 | } 98 | -------------------------------------------------------------------------------- /app/lib/executors/executor_mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: executor.go 3 | 4 | package executors 5 | 6 | import ( 7 | context "context" 8 | gomock "github.com/golang/mock/gomock" 9 | reflect "reflect" 10 | ) 11 | 12 | // MockExecutor is a mock of Executor interface 13 | type MockExecutor struct { 14 | ctrl *gomock.Controller 15 | recorder *MockExecutorMockRecorder 16 | } 17 | 18 | // MockExecutorMockRecorder is the mock recorder for MockExecutor 19 | type MockExecutorMockRecorder struct { 20 | mock *MockExecutor 21 | } 22 | 23 | // NewMockExecutor creates a new mock instance 24 | func NewMockExecutor(ctrl *gomock.Controller) *MockExecutor { 25 | mock := &MockExecutor{ctrl: ctrl} 26 | mock.recorder = &MockExecutorMockRecorder{mock} 27 | return mock 28 | } 29 | 30 | // EXPECT returns an object that allows the caller to indicate expected use 31 | func (_m *MockExecutor) EXPECT() *MockExecutorMockRecorder { 32 | return _m.recorder 33 | } 34 | 35 | // Run mocks base method 36 | func (_m *MockExecutor) Run(ctx context.Context, name string, args ...string) (string, error) { 37 | _s := []interface{}{ctx, name} 38 | for _, _x := range args { 39 | _s = append(_s, _x) 40 | } 41 | ret := _m.ctrl.Call(_m, "Run", _s...) 42 | ret0, _ := ret[0].(string) 43 | ret1, _ := ret[1].(error) 44 | return ret0, ret1 45 | } 46 | 47 | // Run indicates an expected call of Run 48 | func (_mr *MockExecutorMockRecorder) Run(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { 49 | _s := append([]interface{}{arg0, arg1}, arg2...) 50 | return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Run", reflect.TypeOf((*MockExecutor)(nil).Run), _s...) 51 | } 52 | 53 | // WithEnv mocks base method 54 | func (_m *MockExecutor) WithEnv(k string, v string) Executor { 55 | ret := _m.ctrl.Call(_m, "WithEnv", k, v) 56 | ret0, _ := ret[0].(Executor) 57 | return ret0 58 | } 59 | 60 | // WithEnv indicates an expected call of WithEnv 61 | func (_mr *MockExecutorMockRecorder) WithEnv(arg0, arg1 interface{}) *gomock.Call { 62 | return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "WithEnv", reflect.TypeOf((*MockExecutor)(nil).WithEnv), arg0, arg1) 63 | } 64 | 65 | // SetEnv mocks base method 66 | func (_m *MockExecutor) SetEnv(k string, v string) { 67 | _m.ctrl.Call(_m, "SetEnv", k, v) 68 | } 69 | 70 | // SetEnv indicates an expected call of SetEnv 71 | func (_mr *MockExecutorMockRecorder) SetEnv(arg0, arg1 interface{}) *gomock.Call { 72 | return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "SetEnv", reflect.TypeOf((*MockExecutor)(nil).SetEnv), arg0, arg1) 73 | } 74 | 75 | // WorkDir mocks base method 76 | func (_m *MockExecutor) WorkDir() string { 77 | ret := _m.ctrl.Call(_m, "WorkDir") 78 | ret0, _ := ret[0].(string) 79 | return ret0 80 | } 81 | 82 | // WorkDir indicates an expected call of WorkDir 83 | func (_mr *MockExecutorMockRecorder) WorkDir() *gomock.Call { 84 | return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "WorkDir", reflect.TypeOf((*MockExecutor)(nil).WorkDir)) 85 | } 86 | 87 | // WithWorkDir mocks base method 88 | func (_m *MockExecutor) WithWorkDir(wd string) Executor { 89 | ret := _m.ctrl.Call(_m, "WithWorkDir", wd) 90 | ret0, _ := ret[0].(Executor) 91 | return ret0 92 | } 93 | 94 | // WithWorkDir indicates an expected call of WithWorkDir 95 | func (_mr *MockExecutorMockRecorder) WithWorkDir(arg0 interface{}) *gomock.Call { 96 | return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "WithWorkDir", reflect.TypeOf((*MockExecutor)(nil).WithWorkDir), arg0) 97 | } 98 | 99 | // CopyFile mocks base method 100 | func (_m *MockExecutor) CopyFile(ctx context.Context, dst string, src string) error { 101 | ret := _m.ctrl.Call(_m, "CopyFile", ctx, dst, src) 102 | ret0, _ := ret[0].(error) 103 | return ret0 104 | } 105 | 106 | // CopyFile indicates an expected call of CopyFile 107 | func (_mr *MockExecutorMockRecorder) CopyFile(arg0, arg1, arg2 interface{}) *gomock.Call { 108 | return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "CopyFile", reflect.TypeOf((*MockExecutor)(nil).CopyFile), arg0, arg1, arg2) 109 | } 110 | 111 | // Clean mocks base method 112 | func (_m *MockExecutor) Clean() { 113 | _m.ctrl.Call(_m, "Clean") 114 | } 115 | 116 | // Clean indicates an expected call of Clean 117 | func (_mr *MockExecutorMockRecorder) Clean() *gomock.Call { 118 | return _mr.mock.ctrl.RecordCallWithMethodType(_mr.mock, "Clean", reflect.TypeOf((*MockExecutor)(nil).Clean)) 119 | } 120 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # CONTRIBUTION LICENSE AGREEMENT 2 | 3 | This Contribution License Agreement (the “CLA”) is between the individual set forth in the signature block (“You”) and Golangci OÜ, (“GolangCI”), effective as of the date of Your signature and sets forth the terms pursuant to which You provides Contributions to GolangCI. 4 | 5 | You accept and agree to the following terms and conditions for Your present and future Contributions submitted to GolangCI. In return, GolangCI will not use Your Contributions in a way that is contrary to GolangCI’s business objectives. Except for the license granted herein to GolangCI and recipients of software distributed by GolangCI, You reserve all right, title, and interest in and to Your Contributions. 6 | 7 | Definitions. “Contribution” means any original work of authorship, including any modifications or additions to an existing work, that You intentionally submit to GolangCI for inclusion in, or documentation of, any of the products owned or managed by GolangCI (the “Work”). “Submit” means any form of electronic, verbal, or written communication sent to GolangCI or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, GolangCI for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by You as “Not a Contribution.” 8 | 9 | Copyright License. Subject to the terms and conditions of this CLA, You hereby grant to GolangCI and to recipients of software distributed by GolangCI a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to use, copy, reproduce, prepare derivative works of, sublicense, distribute and publicly perform and display the Contributions on any licensing terms, including without limitation: (a) open source licenses like the MIT license; and (b) binary, proprietary, or commercial licenses. 10 | 11 | Patent License. Subject to the terms and conditions of this CLA, You hereby grant to GolangCI and to recipients of software distributed by GolangCI a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by You that are necessarily infringed by Your Contribution(s) alone or by combination of Your Contribution(s) with the Work to which such Contribution(s) was submitted. If any entity institutes patent litigation against You or any other entity (including a cross-claim or counterclaim in a lawsuit) alleging that Your Contribution, or the Work to which You have contributed, constitutes direct or contributory patent infringement, then any patent licenses granted to that entity under this CLA for that Contribution or Work will terminate as of the date such litigation is filed. 12 | 13 | Representations and Warranties. You represent and warrant to GolangCI that: 14 | 15 | You are legally entitled to grant the above license, and if Your employer(s) has rights to intellectual property that You create that includes Your Contributions, then You represent and warrant that You have received permission to make Contributions on behalf of that employer, that Your employer has waived such rights for Your Contributions to GolangCI, or that Your employer has executed a separate CLA with GolangCI; 16 | 17 | Each of Your Contributions is Your original creation (see section 6 for submissions on behalf of others); and 18 | 19 | Your Contribution submissions include complete details of any third-party license or other restriction (including, but not limited to, related patents and trademarks) of which You are personally aware and which are associated with any part of Your Contributions. 20 | 21 | Support; Disclaimer. You are not expected to provide support for Your Contributions, except to the extent You desire to do so. You may provide support for free, for a fee, or not at all. Unless required by applicable law or agreed to in writing, You provide Your Contributions on an “AS IS” BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON- INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. 22 | 23 | Third Party Works. If You wish to submit work that is not Your original creation, then You may submit it to GolangCI separately from any Contribution, identifying the complete details of it. -------------------------------------------------------------------------------- /app/lib/fsutils/path_resolver.go: -------------------------------------------------------------------------------- 1 | package fsutils 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "sort" 8 | "strings" 9 | ) 10 | 11 | type PathResolver struct { 12 | excludeDirs map[string]bool 13 | allowedFileExtensions map[string]bool 14 | } 15 | 16 | type pathResolveState struct { 17 | files map[string]bool 18 | dirs map[string]bool 19 | } 20 | 21 | func (s *pathResolveState) addFile(path string) { 22 | s.files[filepath.Clean(path)] = true 23 | } 24 | 25 | func (s *pathResolveState) addDir(path string) { 26 | s.dirs[filepath.Clean(path)] = true 27 | } 28 | 29 | type PathResolveResult struct { 30 | files []string 31 | dirs []string 32 | } 33 | 34 | func (prr PathResolveResult) Files() []string { 35 | return prr.files 36 | } 37 | 38 | func (prr PathResolveResult) Dirs() []string { 39 | return prr.dirs 40 | } 41 | 42 | func (s pathResolveState) toResult() *PathResolveResult { 43 | res := &PathResolveResult{ 44 | files: []string{}, 45 | dirs: []string{}, 46 | } 47 | for f := range s.files { 48 | res.files = append(res.files, f) 49 | } 50 | for d := range s.dirs { 51 | res.dirs = append(res.dirs, d) 52 | } 53 | 54 | sort.Strings(res.files) 55 | sort.Strings(res.dirs) 56 | return res 57 | } 58 | 59 | func NewPathResolver(excludeDirs, allowedFileExtensions []string) *PathResolver { 60 | excludeDirsMap := map[string]bool{} 61 | for _, dir := range excludeDirs { 62 | excludeDirsMap[dir] = true 63 | } 64 | 65 | allowedFileExtensionsMap := map[string]bool{} 66 | for _, fe := range allowedFileExtensions { 67 | allowedFileExtensionsMap[fe] = true 68 | } 69 | 70 | return &PathResolver{ 71 | excludeDirs: excludeDirsMap, 72 | allowedFileExtensions: allowedFileExtensionsMap, 73 | } 74 | } 75 | 76 | func (pr PathResolver) isIgnoredDir(dir string) bool { 77 | dirName := filepath.Base(filepath.Clean(dir)) // ignore dirs on any depth level 78 | 79 | // https://github.com/golang/dep/issues/298 80 | // https://github.com/tools/godep/issues/140 81 | if strings.HasPrefix(dirName, ".") && dirName != "." { 82 | return true 83 | } 84 | if strings.HasPrefix(dirName, "_") { 85 | return true 86 | } 87 | 88 | return pr.excludeDirs[dirName] 89 | } 90 | 91 | func (pr PathResolver) isAllowedFile(path string) bool { 92 | return pr.allowedFileExtensions[filepath.Ext(path)] 93 | } 94 | 95 | func (pr PathResolver) resolveRecursively(root string, state *pathResolveState) error { 96 | walkErr := filepath.Walk(root, func(p string, i os.FileInfo, err error) error { 97 | if err != nil { 98 | return err 99 | } 100 | 101 | if i.IsDir() { 102 | if pr.isIgnoredDir(p) { 103 | return filepath.SkipDir 104 | } 105 | state.addDir(p) 106 | return nil 107 | } 108 | 109 | if pr.isAllowedFile(p) { 110 | state.addFile(p) 111 | } 112 | return nil 113 | }) 114 | 115 | if walkErr != nil { 116 | return fmt.Errorf("can't walk dir %s: %s", root, walkErr) 117 | } 118 | 119 | return nil 120 | } 121 | 122 | func (pr PathResolver) resolveDir(root string, state *pathResolveState) error { 123 | walkErr := filepath.Walk(root, func(p string, i os.FileInfo, err error) error { 124 | if err != nil { 125 | return err 126 | } 127 | 128 | if i.IsDir() { 129 | if filepath.Clean(p) != filepath.Clean(root) { 130 | return filepath.SkipDir 131 | } 132 | state.addDir(p) 133 | return nil 134 | } 135 | 136 | if pr.isAllowedFile(p) { 137 | state.addFile(p) 138 | } 139 | return nil 140 | }) 141 | 142 | if walkErr != nil { 143 | return fmt.Errorf("can't walk dir %s: %s", root, walkErr) 144 | } 145 | 146 | return nil 147 | } 148 | 149 | func (pr PathResolver) Resolve(paths ...string) (*PathResolveResult, error) { 150 | if len(paths) == 0 { 151 | return nil, fmt.Errorf("no paths are set") 152 | } 153 | 154 | state := &pathResolveState{ 155 | files: map[string]bool{}, 156 | dirs: map[string]bool{}, 157 | } 158 | for _, path := range paths { 159 | if strings.HasSuffix(path, "/...") { 160 | if err := pr.resolveRecursively(filepath.Dir(path), state); err != nil { 161 | return nil, fmt.Errorf("can't recursively resolve %s: %s", path, err) 162 | } 163 | continue 164 | } 165 | 166 | fi, err := os.Stat(path) 167 | if err != nil { 168 | return nil, fmt.Errorf("can't find path %s: %s", path, err) 169 | } 170 | 171 | if fi.IsDir() { 172 | if err := pr.resolveDir(path, state); err != nil { 173 | return nil, fmt.Errorf("can't resolve dir %s: %s", path, err) 174 | } 175 | continue 176 | } 177 | 178 | state.addFile(path) 179 | } 180 | 181 | return state.toResult(), nil 182 | } 183 | -------------------------------------------------------------------------------- /app/lib/fsutils/path_resolver_test.go: -------------------------------------------------------------------------------- 1 | package fsutils 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | type fsPreparer struct { 14 | t *testing.T 15 | root string 16 | prevWD string 17 | } 18 | 19 | func (fp fsPreparer) clean() { 20 | err := os.Chdir(fp.prevWD) 21 | assert.NoError(fp.t, err) 22 | 23 | err = os.RemoveAll(fp.root) 24 | assert.NoError(fp.t, err) 25 | } 26 | 27 | func prepareFS(t *testing.T, paths ...string) *fsPreparer { 28 | root, err := ioutil.TempDir("/tmp", "golangci.test.path_resolver") 29 | assert.NoError(t, err) 30 | 31 | prevWD, err := os.Getwd() 32 | assert.NoError(t, err) 33 | 34 | err = os.Chdir(root) 35 | assert.NoError(t, err) 36 | 37 | for _, p := range paths { 38 | err = os.MkdirAll(filepath.Dir(p), os.ModePerm) 39 | assert.NoError(t, err) 40 | 41 | if strings.HasSuffix(p, "/") { 42 | continue 43 | } 44 | 45 | err = ioutil.WriteFile(p, []byte("test"), os.ModePerm) 46 | assert.NoError(t, err) 47 | } 48 | 49 | return &fsPreparer{ 50 | root: root, 51 | t: t, 52 | prevWD: prevWD, 53 | } 54 | } 55 | 56 | func newPR() *PathResolver { 57 | return NewPathResolver([]string{}, []string{}) 58 | } 59 | 60 | func TestPathResolverNoPaths(t *testing.T) { 61 | _, err := newPR().Resolve() 62 | assert.EqualError(t, err, "no paths are set") 63 | } 64 | 65 | func TestPathResolverNotExistingPath(t *testing.T) { 66 | fp := prepareFS(t) 67 | defer fp.clean() 68 | 69 | _, err := newPR().Resolve("a") 70 | assert.EqualError(t, err, "can't find path a: stat a: no such file or directory") 71 | } 72 | 73 | func TestPathResolverCommonCases(t *testing.T) { 74 | type testCase struct { 75 | name string 76 | prepare []string 77 | resolve []string 78 | expFiles []string 79 | expDirs []string 80 | } 81 | 82 | testCases := []testCase{ 83 | { 84 | name: "empty root recursively", 85 | resolve: []string{"./..."}, 86 | expDirs: []string{"."}, 87 | }, 88 | { 89 | name: "empty root", 90 | resolve: []string{"./"}, 91 | expDirs: []string{"."}, 92 | }, 93 | { 94 | name: "vendor is excluded recursively", 95 | prepare: []string{"vendor/a/"}, 96 | resolve: []string{"./..."}, 97 | expDirs: []string{"."}, 98 | }, 99 | { 100 | name: "vendor is excluded", 101 | prepare: []string{"vendor/"}, 102 | resolve: []string{"./..."}, 103 | expDirs: []string{"."}, 104 | }, 105 | { 106 | name: "vendor implicitely resolved", 107 | prepare: []string{"vendor/"}, 108 | resolve: []string{"./vendor"}, 109 | expDirs: []string{"vendor"}, 110 | }, 111 | { 112 | name: "extensions filter recursively", 113 | prepare: []string{"a/b.go", "a/c.txt", "d.go", "e.csv"}, 114 | resolve: []string{"./..."}, 115 | expDirs: []string{".", "a"}, 116 | expFiles: []string{"a/b.go", "d.go"}, 117 | }, 118 | { 119 | name: "extensions filter", 120 | prepare: []string{"a/b.go", "a/c.txt", "d.go"}, 121 | resolve: []string{"a"}, 122 | expDirs: []string{"a"}, 123 | expFiles: []string{"a/b.go"}, 124 | }, 125 | { 126 | name: "one level dirs exclusion", 127 | prepare: []string{"a/b/", "a/c.go"}, 128 | resolve: []string{"./a"}, 129 | expDirs: []string{"a"}, 130 | expFiles: []string{"a/c.go"}, 131 | }, 132 | { 133 | name: "implicitely resolved files", 134 | prepare: []string{"a/b/c.go", "a/d.txt"}, 135 | resolve: []string{"./a/...", "a/d.txt"}, 136 | expDirs: []string{"a", "a/b"}, 137 | expFiles: []string{"a/b/c.go", "a/d.txt"}, 138 | }, 139 | { 140 | name: ".* is always ignored", 141 | prepare: []string{".git/a.go", ".circleci/b.go"}, 142 | resolve: []string{"./..."}, 143 | expDirs: []string{"."}, 144 | }, 145 | { 146 | name: "exclude dirs on any depth level", 147 | prepare: []string{"ok/.git/a.go"}, 148 | resolve: []string{"./..."}, 149 | expDirs: []string{".", "ok"}, 150 | }, 151 | { 152 | name: "ignore _*", 153 | prepare: []string{"_any/a.go"}, 154 | resolve: []string{"./..."}, 155 | expDirs: []string{"."}, 156 | }, 157 | } 158 | 159 | for _, tc := range testCases { 160 | t.Run(tc.name, func(t *testing.T) { 161 | fp := prepareFS(t, tc.prepare...) 162 | defer fp.clean() 163 | 164 | pr := NewPathResolver([]string{"vendor"}, []string{".go"}) 165 | res, err := pr.Resolve(tc.resolve...) 166 | assert.NoError(t, err) 167 | 168 | if tc.expFiles == nil { 169 | assert.Empty(t, res.files) 170 | } else { 171 | assert.Equal(t, tc.expFiles, res.files) 172 | } 173 | 174 | if tc.expDirs == nil { 175 | assert.Empty(t, res.dirs) 176 | } else { 177 | assert.Equal(t, tc.expDirs, res.dirs) 178 | } 179 | }) 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /app/lib/github/client.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "net/http" 8 | "time" 9 | 10 | "github.com/cenkalti/backoff" 11 | gh "github.com/google/go-github/github" 12 | "github.com/sirupsen/logrus" 13 | ) 14 | 15 | //go:generate mockgen -package github -source client.go -destination client_mock.go 16 | 17 | type Status string 18 | 19 | var ErrPRNotFound = errors.New("no such pull request") 20 | var ErrUnauthorized = errors.New("invalid authorization") 21 | 22 | func IsRecoverableError(err error) bool { 23 | return err != ErrPRNotFound && err != ErrUnauthorized 24 | } 25 | 26 | const ( 27 | StatusPending Status = "pending" 28 | StatusFailure Status = "failure" 29 | StatusError Status = "error" 30 | StatusSuccess Status = "success" 31 | ) 32 | 33 | type Client interface { 34 | GetPullRequest(ctx context.Context, c *Context) (*gh.PullRequest, error) 35 | GetPullRequestComments(ctx context.Context, c *Context) ([]*gh.PullRequestComment, error) 36 | GetPullRequestPatch(ctx context.Context, c *Context) (string, error) 37 | CreateReview(ctx context.Context, c *Context, review *gh.PullRequestReviewRequest) error 38 | SetCommitStatus(ctx context.Context, c *Context, ref string, status Status, desc, url string) error 39 | } 40 | 41 | type MyClient struct{} 42 | 43 | var _ Client = &MyClient{} 44 | 45 | func NewMyClient() *MyClient { 46 | return &MyClient{} 47 | } 48 | 49 | func transformGithubError(err error) error { 50 | if er, ok := err.(*gh.ErrorResponse); ok { 51 | if er.Response.StatusCode == http.StatusNotFound { 52 | logrus.Warnf("Got 404 from github: %+v", er) 53 | return ErrPRNotFound 54 | } 55 | if er.Response.StatusCode == http.StatusUnauthorized { 56 | logrus.Warnf("Got 401 from github: %+v", er) 57 | return ErrUnauthorized 58 | } 59 | } 60 | 61 | return nil 62 | } 63 | 64 | func retryGet(f func() error) error { 65 | b := backoff.NewExponentialBackOff() 66 | b.MaxElapsedTime = 2 * time.Minute 67 | 68 | bmr := backoff.WithMaxRetries(b, 5) 69 | 70 | if err := backoff.Retry(f, bmr); err != nil { 71 | logrus.Warnf("Github operation failed to retry with %v and took %s: %s", b, b.GetElapsedTime(), err) 72 | return err 73 | } 74 | 75 | return nil 76 | } 77 | 78 | func (gc *MyClient) GetPullRequest(ctx context.Context, c *Context) (*gh.PullRequest, error) { 79 | var retPR *gh.PullRequest 80 | 81 | f := func() error { 82 | pr, _, err := c.GetClient(ctx).PullRequests.Get(ctx, c.Repo.Owner, c.Repo.Name, c.PullRequestNumber) 83 | if err != nil { 84 | return err 85 | } 86 | 87 | retPR = pr 88 | return nil 89 | } 90 | 91 | if err := retryGet(f); err != nil { 92 | if terr := transformGithubError(err); terr != nil { 93 | return nil, terr 94 | } 95 | 96 | return nil, fmt.Errorf("can't get pull request %d from github: %s", c.PullRequestNumber, err) 97 | } 98 | 99 | return retPR, nil 100 | } 101 | 102 | func (gc *MyClient) CreateReview(ctx context.Context, c *Context, review *gh.PullRequestReviewRequest) error { 103 | _, _, err := c.GetClient(ctx).PullRequests.CreateReview(ctx, c.Repo.Owner, c.Repo.Name, c.PullRequestNumber, review) 104 | if err != nil { 105 | if terr := transformGithubError(err); terr != nil { 106 | return terr 107 | } 108 | 109 | return fmt.Errorf("can't create github review: %s", err) 110 | } 111 | 112 | return nil 113 | } 114 | 115 | func (gc *MyClient) GetPullRequestPatch(ctx context.Context, c *Context) (string, error) { 116 | var ret string 117 | 118 | f := func() error { 119 | opts := gh.RawOptions{Type: gh.Diff} 120 | raw, _, err := c.GetClient(ctx).PullRequests.GetRaw(ctx, c.Repo.Owner, c.Repo.Name, 121 | c.PullRequestNumber, opts) 122 | if err != nil { 123 | return err 124 | } 125 | 126 | ret = raw 127 | return nil 128 | } 129 | 130 | if err := retryGet(f); err != nil { 131 | if terr := transformGithubError(err); terr != nil { 132 | return "", terr 133 | } 134 | 135 | return "", fmt.Errorf("can't get patch for pull request: %s", err) 136 | } 137 | 138 | return ret, nil 139 | } 140 | 141 | func (gc *MyClient) SetCommitStatus(ctx context.Context, c *Context, ref string, status Status, desc, url string) error { 142 | rs := &gh.RepoStatus{ 143 | Description: gh.String(desc), 144 | State: gh.String(string(status)), 145 | Context: gh.String("GolangCI"), 146 | } 147 | if url != "" { 148 | rs.TargetURL = gh.String(url) 149 | } 150 | _, _, err := c.GetClient(ctx).Repositories.CreateStatus(ctx, c.Repo.Owner, c.Repo.Name, ref, rs) 151 | if err != nil { 152 | if terr := transformGithubError(err); terr != nil { 153 | return terr 154 | } 155 | 156 | return fmt.Errorf("can't set commit %s status %s: %s", ref, status, err) 157 | } 158 | 159 | return nil 160 | } 161 | 162 | func (gc *MyClient) GetPullRequestComments(ctx context.Context, c *Context) ([]*gh.PullRequestComment, error) { 163 | var ret []*gh.PullRequestComment 164 | 165 | f := func() error { 166 | opt := &gh.PullRequestListCommentsOptions{ 167 | ListOptions: gh.ListOptions{ 168 | PerPage: 100, // max allowed value, TODO: fetch all comments if >100 169 | }, 170 | } 171 | comments, _, err := c.GetClient(ctx).PullRequests.ListComments(ctx, c.Repo.Owner, c.Repo.Name, c.PullRequestNumber, opt) 172 | if err != nil { 173 | return err 174 | } 175 | 176 | ret = comments 177 | return nil 178 | } 179 | 180 | if err := retryGet(f); err != nil { 181 | if terr := transformGithubError(err); terr != nil { 182 | return nil, terr 183 | } 184 | 185 | return nil, fmt.Errorf("can't get pull request %d comments from github: %s", c.PullRequestNumber, err) 186 | } 187 | 188 | return ret, nil 189 | } 190 | -------------------------------------------------------------------------------- /app/lib/executors/container.go: -------------------------------------------------------------------------------- 1 | package executors 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "strings" 9 | "time" 10 | 11 | "github.com/golangci/golangci-api/pkg/app/buildagent/build" 12 | "github.com/golangci/golangci-api/pkg/app/buildagent/containers" 13 | "github.com/golangci/golangci-shared/pkg/logutil" 14 | 15 | "github.com/levigross/grequests" 16 | "github.com/pkg/errors" 17 | ) 18 | 19 | type Container struct { 20 | envStore 21 | wd string 22 | 23 | orchestratorAddr string 24 | token string 25 | 26 | containerID containers.ContainerID 27 | log logutil.Log 28 | } 29 | 30 | var _ Executor = &Container{} 31 | 32 | func NewContainer(log logutil.Log) (*Container, error) { 33 | orchestratorAddr := os.Getenv("ORCHESTRATOR_ADDR") 34 | if orchestratorAddr == "" { 35 | return nil, errors.New("no ORCHESTRATOR_ADDR env var") 36 | } 37 | if strings.HasSuffix(orchestratorAddr, "/") { 38 | return nil, errors.New("ORCHESTRATOR_ADDR shouldn't end with /") 39 | } 40 | 41 | token := os.Getenv("ORCHESTRATOR_TOKEN") 42 | if token == "" { 43 | return nil, errors.New("no ORCHESTRATOR_TOKEN env var") 44 | } 45 | 46 | return &Container{ 47 | envStore: envStore{}, 48 | orchestratorAddr: orchestratorAddr, 49 | token: token, 50 | log: log, 51 | }, nil 52 | } 53 | 54 | func (c *Container) Setup(ctx context.Context) error { 55 | resp, err := grequests.Post(fmt.Sprintf("%s/setup", c.orchestratorAddr), &grequests.RequestOptions{ 56 | Context: ctx, 57 | JSON: containers.SetupContainerRequest{ 58 | TimeoutMs: 30 * 1000, // 30s 59 | }, 60 | Headers: map[string]string{ 61 | containers.TokenHeaderName: c.token, 62 | }, 63 | }) 64 | if err != nil { 65 | return errors.Wrap(err, "failed to make request to orchestrator") 66 | } 67 | 68 | var setupResp containers.SetupContainerResponse 69 | if err = resp.JSON(&setupResp); err != nil { 70 | return errors.Wrap(err, "failed to parse json of setup response") 71 | } 72 | 73 | if setupResp.Error != "" { 74 | return fmt.Errorf("failed to setup container: %s", setupResp.Error) 75 | } 76 | 77 | c.log.Infof("Setup of container: id is %#v", setupResp.ContainerID) 78 | c.containerID = setupResp.ContainerID 79 | return nil 80 | } 81 | 82 | func (c Container) Run(ctx context.Context, name string, args ...string) (string, error) { 83 | deadline, ok := ctx.Deadline() 84 | if !ok { 85 | return "", errors.New("no deadline was set for context") 86 | } 87 | now := time.Now() 88 | if deadline.Before(now) { 89 | return "", errors.New("deadline exceeded: it's before now") 90 | } 91 | 92 | req := containers.BuildCommandRequest{ 93 | ContainerID: c.containerID, 94 | Request: build.Request{ 95 | TimeoutMs: uint(deadline.Sub(now) / time.Millisecond), 96 | WorkDir: c.wd, 97 | Env: c.env, 98 | Kind: build.RequestKindRun, 99 | Args: append([]string{name}, args...), 100 | }, 101 | } 102 | 103 | return c.runBuildCommand(ctx, &req) 104 | } 105 | 106 | func (c Container) runBuildCommand(ctx context.Context, req *containers.BuildCommandRequest) (string, error) { 107 | resp, err := grequests.Post(fmt.Sprintf("%s/buildcommand", c.orchestratorAddr), &grequests.RequestOptions{ 108 | Context: ctx, 109 | JSON: req, 110 | Headers: map[string]string{ 111 | containers.TokenHeaderName: c.token, 112 | }, 113 | }) 114 | if err != nil { 115 | return "", errors.Wrapf(err, "failed to make request to orchestrator with req %#v", req) 116 | } 117 | 118 | var containerResp containers.BuildCommandResponse 119 | if err = resp.JSON(&containerResp); err != nil { 120 | return "", errors.Wrap(err, "failed to parse json of container response") 121 | } 122 | 123 | if containerResp.Error != "" { 124 | return "", fmt.Errorf("failed to run container build command with req %#v: %s", 125 | req, containerResp.Error) 126 | } 127 | 128 | buildResp := containerResp.BuildResponse 129 | if buildResp.Error != "" { 130 | return "", fmt.Errorf("failed to run build command with req %#v: %s", req, buildResp.Error) 131 | } 132 | 133 | if buildResp.CommandError != "" { 134 | return buildResp.StdOut, fmt.Errorf("build command for req %#v complete with error: %s", 135 | req, buildResp.CommandError) 136 | } 137 | 138 | return buildResp.StdOut, nil 139 | } 140 | 141 | func (c Container) CopyFile(ctx context.Context, dst, src string) error { 142 | srcContent, err := ioutil.ReadFile(src) 143 | if err != nil { 144 | return errors.Wrapf(err, "failed to read file %s", src) 145 | } 146 | 147 | req := containers.BuildCommandRequest{ 148 | ContainerID: c.containerID, 149 | Request: build.Request{ 150 | WorkDir: c.wd, 151 | Env: c.env, 152 | Kind: build.RequestKindCopy, 153 | Args: []string{dst, string(srcContent)}, 154 | }, 155 | } 156 | 157 | _, err = c.runBuildCommand(ctx, &req) 158 | return err 159 | } 160 | 161 | func (c Container) Clean() { 162 | err := func() error { 163 | ctx, finish := context.WithTimeout(context.TODO(), time.Second*30) 164 | defer finish() 165 | 166 | resp, err := grequests.Post(fmt.Sprintf("%s/shutdown", c.orchestratorAddr), &grequests.RequestOptions{ 167 | Context: ctx, 168 | JSON: containers.ShutdownContainerRequest{ 169 | TimeoutMs: 30 * 1000, // 30s 170 | ContainerID: c.containerID, 171 | }, 172 | Headers: map[string]string{ 173 | containers.TokenHeaderName: c.token, 174 | }, 175 | }) 176 | if err != nil { 177 | return errors.Wrap(err, "failed to make request to orchestrator") 178 | } 179 | 180 | var shutdownResp containers.ShutdownContainerResponse 181 | if err = resp.JSON(&shutdownResp); err != nil { 182 | return errors.Wrap(err, "failed to parse json of shutdown response") 183 | } 184 | 185 | if shutdownResp.Error != "" { 186 | return fmt.Errorf("failed to shutdown container: %s", shutdownResp.Error) 187 | } 188 | 189 | c.log.Infof("Shutdowned container with id %#v", c.containerID) 190 | return nil 191 | }() 192 | if err != nil { 193 | c.log.Warnf("Failed to shutdown container: %s", err) 194 | } 195 | } 196 | 197 | func (c Container) WithEnv(k, v string) Executor { 198 | eCopy := c 199 | eCopy.SetEnv(k, v) 200 | return &eCopy 201 | } 202 | 203 | func (c Container) WorkDir() string { 204 | return c.wd 205 | } 206 | 207 | func (c *Container) SetWorkDir(wd string) { 208 | c.wd = wd 209 | } 210 | 211 | func (c Container) WithWorkDir(wd string) Executor { 212 | eCopy := c 213 | eCopy.wd = wd 214 | return &eCopy 215 | } 216 | -------------------------------------------------------------------------------- /app/analyze/processors/repo_processor.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "runtime/debug" 8 | "strings" 9 | "time" 10 | 11 | "github.com/golangci/golangci-api/pkg/goenv/result" 12 | "github.com/golangci/golangci-shared/pkg/apperrors" 13 | "github.com/golangci/golangci-shared/pkg/config" 14 | "github.com/golangci/golangci-shared/pkg/logutil" 15 | "github.com/golangci/golangci-worker/app/analyze/linters" 16 | lintersResult "github.com/golangci/golangci-worker/app/analyze/linters/result" 17 | "github.com/golangci/golangci-worker/app/analyze/repostate" 18 | "github.com/golangci/golangci-worker/app/lib/errorutils" 19 | "github.com/golangci/golangci-worker/app/lib/executors" 20 | "github.com/golangci/golangci-worker/app/lib/experiments" 21 | "github.com/golangci/golangci-worker/app/lib/fetchers" 22 | "github.com/golangci/golangci-worker/app/lib/github" 23 | "github.com/golangci/golangci-worker/app/lib/goutils/workspaces" 24 | 25 | "github.com/pkg/errors" 26 | ) 27 | 28 | type StaticRepoConfig struct { 29 | RepoFetcher fetchers.Fetcher 30 | Linters []linters.Linter 31 | Runner linters.Runner 32 | State repostate.Storage 33 | Cfg config.Config 34 | Et apperrors.Tracker 35 | } 36 | 37 | type RepoConfig struct { 38 | StaticRepoConfig 39 | 40 | Log logutil.Log 41 | Exec executors.Executor 42 | Wi workspaces.Installer 43 | Ec *experiments.Checker 44 | } 45 | 46 | type Repo struct { 47 | RepoConfig 48 | } 49 | 50 | type RepoContext struct { 51 | Ctx context.Context 52 | 53 | AnalysisGUID string 54 | Branch string 55 | Repo *github.Repo // TODO: abstract from repo provider 56 | } 57 | 58 | type repoResult struct { 59 | resultCollector 60 | prepareLog *result.Log 61 | lintRes *lintersResult.Result 62 | } 63 | 64 | func NewRepo(cfg *RepoConfig) *Repo { 65 | return &Repo{ 66 | RepoConfig: *cfg, 67 | } 68 | } 69 | 70 | func (r Repo) Process(ctx *RepoContext) { 71 | res, err := r.processPanicSafe(ctx) 72 | if res == nil { 73 | res = &repoResult{} 74 | } 75 | 76 | r.submitResult(ctx, res, err) 77 | } 78 | 79 | func (r Repo) processPanicSafe(ctx *RepoContext) (retRes *repoResult, err error) { 80 | defer func() { 81 | if rerr := recover(); rerr != nil { 82 | retRes = nil 83 | err = &errorutils.InternalError{ 84 | PublicDesc: "internal error", 85 | PrivateDesc: fmt.Sprintf("panic occured: %s, %s", rerr, debug.Stack()), 86 | } 87 | } 88 | }() 89 | 90 | var res repoResult 91 | r.updateStatusToInQueue(ctx, &res) 92 | 93 | if err := r.prepare(ctx, &res); err != nil { 94 | return nil, errors.Wrap(err, "failed to prepare repo") 95 | } 96 | 97 | if err := r.analyze(ctx, &res); err != nil { 98 | return nil, errors.Wrap(err, "failed to analyze repo") 99 | } 100 | 101 | return &res, nil 102 | } 103 | 104 | func (r Repo) updateStatusToInQueue(ctx *RepoContext, res *repoResult) { 105 | curState, err := r.State.GetState(ctx.Ctx, ctx.Repo.Owner, ctx.Repo.Name, ctx.AnalysisGUID) 106 | if err != nil { 107 | r.Log.Warnf("Can't get current state: %s", err) 108 | } else if curState.Status == statusSentToQueue { 109 | res.addTimingFrom("In Queue", fromDBTime(curState.CreatedAt)) 110 | curState.Status = statusProcessing 111 | if err = r.State.UpdateState(ctx.Ctx, ctx.Repo.Owner, ctx.Repo.Name, ctx.AnalysisGUID, curState); err != nil { 112 | r.Log.Warnf("Can't update repo analysis %s state with setting status to 'processing': %s", ctx.AnalysisGUID, err) 113 | } 114 | } 115 | } 116 | 117 | func (r *Repo) prepare(ctx *RepoContext, res *repoResult) error { 118 | defer res.addTimingFrom("Prepare", time.Now()) 119 | 120 | fr := buildFetchersRepo(ctx) 121 | exec, resLog, err := r.Wi.Setup(ctx.Ctx, fr, "github.com", ctx.Repo.Owner, ctx.Repo.Name) 122 | if err != nil { 123 | return errors.Wrap(err, "failed to setup workspace") 124 | } 125 | 126 | r.Exec = exec 127 | res.prepareLog = resLog 128 | return nil 129 | } 130 | 131 | func (r Repo) analyze(ctx *RepoContext, res *repoResult) error { 132 | defer res.addTimingFrom("Analysis", time.Now()) 133 | 134 | lintRes, err := r.Runner.Run(ctx.Ctx, r.Linters, r.Exec) 135 | if err != nil { 136 | return errors.Wrap(err, "failed running linters") 137 | } 138 | 139 | res.lintRes = lintRes 140 | return nil 141 | } 142 | 143 | func buildFetchersRepo(ctx *RepoContext) *fetchers.Repo { 144 | repo := ctx.Repo 145 | return &fetchers.Repo{ 146 | CloneURL: fmt.Sprintf("https://github.com/%s/%s.git", repo.Owner, repo.Name), 147 | Ref: ctx.Branch, 148 | FullPath: fmt.Sprintf("github.com/%s/%s", repo.Owner, repo.Name), 149 | } 150 | } 151 | 152 | // TODO: migrate to golangci-lint linter runner when pr processor will have the same code 153 | func (r Repo) transformError(err error) error { 154 | if err == nil { 155 | return nil 156 | } 157 | 158 | causeErr := errors.Cause(err) 159 | if causeErr == fetchers.ErrNoBranchOrRepo { 160 | return causeErr 161 | } 162 | 163 | if ierr, ok := causeErr.(*errorutils.InternalError); ok { 164 | if strings.Contains(ierr.PrivateDesc, noGoFilesToAnalyzeErr) { 165 | return errNothingToAnalyze 166 | } 167 | 168 | return ierr 169 | } 170 | 171 | return err 172 | } 173 | 174 | func (r Repo) errorToStatus(err error) string { 175 | if err == nil { 176 | return statusProcessed 177 | } 178 | 179 | if err == errNothingToAnalyze { 180 | return statusProcessed 181 | } 182 | 183 | if _, ok := err.(*errorutils.BadInputError); ok { 184 | return statusProcessed 185 | } 186 | 187 | if _, ok := err.(*errorutils.InternalError); ok { 188 | return string(github.StatusError) 189 | } 190 | 191 | if err == fetchers.ErrNoBranchOrRepo { 192 | return statusNotFound 193 | } 194 | 195 | return string(github.StatusError) 196 | } 197 | 198 | func (r Repo) buildPublicError(err error) string { 199 | if err == nil { 200 | return "" 201 | } 202 | 203 | if err == errNothingToAnalyze { 204 | return err.Error() 205 | } 206 | 207 | if ierr, ok := err.(*errorutils.InternalError); ok { 208 | r.Log.Warnf("Internal error: %s", ierr.PrivateDesc) 209 | return ierr.PublicDesc 210 | } 211 | 212 | if berr, ok := err.(*errorutils.BadInputError); ok { 213 | return escapeErrorText(berr.PublicDesc, buildSecrets()) 214 | } 215 | 216 | return internalError 217 | } 218 | 219 | func (r Repo) submitResult(ctx *RepoContext, res *repoResult, err error) { 220 | err = r.transformError(err) 221 | status := r.errorToStatus(err) 222 | publicErrorText := r.buildPublicError(err) 223 | 224 | if err == nil { 225 | r.Log.Infof("Succeeded repo analysis, timings: %v", res.timings) 226 | } else { 227 | r.Log.Errorf("Failed repo analysis: %s, timings: %v", err, res.timings) 228 | } 229 | 230 | if res.prepareLog != nil { 231 | for _, sg := range res.prepareLog.Groups { 232 | for _, s := range sg.Steps { 233 | if s.Error != "" { 234 | text := fmt.Sprintf("%s error: %s", s.Description, s.Error) 235 | text = escapeErrorText(text, buildSecrets()) 236 | res.publicWarn(sg.Name, text) 237 | } 238 | } 239 | } 240 | } 241 | 242 | resJSON := &resultJSON{ 243 | Version: 1, 244 | WorkerRes: workerRes{ 245 | Timings: res.timings, 246 | Warnings: res.warnings, 247 | Error: publicErrorText, 248 | }, 249 | } 250 | 251 | if res.lintRes != nil { 252 | resJSON.GolangciLintRes = res.lintRes.ResultJSON 253 | } 254 | s := &repostate.State{ 255 | Status: status, 256 | ResultJSON: resJSON, 257 | } 258 | 259 | jsonBytes, err := json.Marshal(*resJSON) 260 | if err != nil { 261 | r.Log.Warnf("Failed to marshal json: %s", err) 262 | } else { 263 | r.Log.Infof("Save repo analysis status: status=%s, result_json=%s", status, string(jsonBytes)) 264 | } 265 | 266 | updateCtx := context.Background() // no timeout for state and status saving: it must be durable 267 | if err = r.State.UpdateState(updateCtx, ctx.Repo.Owner, ctx.Repo.Name, ctx.AnalysisGUID, s); err != nil { 268 | r.Log.Warnf("Can't set analysis %s status to '%v': %s", ctx.AnalysisGUID, s, err) 269 | } 270 | } 271 | -------------------------------------------------------------------------------- /app/analyze/processors/github_go_repo.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "os" 8 | "runtime/debug" 9 | "strings" 10 | 11 | "github.com/golangci/golangci-api/pkg/goenv/ensuredeps" 12 | "github.com/golangci/golangci-worker/app/analytics" 13 | "github.com/golangci/golangci-worker/app/analyze/linters" 14 | "github.com/golangci/golangci-worker/app/analyze/linters/golinters" 15 | "github.com/golangci/golangci-worker/app/analyze/linters/result" 16 | "github.com/golangci/golangci-worker/app/analyze/repoinfo" 17 | "github.com/golangci/golangci-worker/app/analyze/repostate" 18 | "github.com/golangci/golangci-worker/app/lib/errorutils" 19 | "github.com/golangci/golangci-worker/app/lib/executors" 20 | "github.com/golangci/golangci-worker/app/lib/fetchers" 21 | "github.com/golangci/golangci-worker/app/lib/github" 22 | "github.com/golangci/golangci-worker/app/lib/goutils/workspaces" 23 | "github.com/golangci/golangci-worker/app/lib/httputils" 24 | "github.com/pkg/errors" 25 | ) 26 | 27 | type GithubGoRepoConfig struct { 28 | repoFetcher fetchers.Fetcher 29 | infoFetcher repoinfo.Fetcher 30 | linters []linters.Linter 31 | runner linters.Runner 32 | exec executors.Executor 33 | state repostate.Storage 34 | } 35 | 36 | type GithubGoRepo struct { 37 | analysisGUID string 38 | branch string 39 | gw *workspaces.Go 40 | repo *github.Repo 41 | 42 | GithubGoRepoConfig 43 | resultCollector 44 | } 45 | 46 | func NewGithubGoRepo(ctx context.Context, cfg GithubGoRepoConfig, analysisGUID, repoName, branch string) (*GithubGoRepo, error) { 47 | parts := strings.Split(repoName, "/") 48 | repo := &github.Repo{ 49 | Owner: parts[0], 50 | Name: parts[1], 51 | } 52 | if len(parts) != 2 { 53 | return nil, fmt.Errorf("invalid repo name %s", repoName) 54 | } 55 | 56 | if cfg.exec == nil { 57 | var err error 58 | cfg.exec, err = makeExecutor(ctx, repo, true, nil, nil) 59 | if err != nil { 60 | return nil, fmt.Errorf("can't make executor: %s", err) 61 | } 62 | } 63 | 64 | if cfg.repoFetcher == nil { 65 | cfg.repoFetcher = fetchers.NewGit() 66 | } 67 | 68 | if cfg.infoFetcher == nil { 69 | cfg.infoFetcher = repoinfo.NewCloningFetcher(cfg.repoFetcher) 70 | } 71 | 72 | if cfg.linters == nil { 73 | cfg.linters = []linters.Linter{ 74 | golinters.GolangciLint{}, 75 | } 76 | } 77 | 78 | if cfg.runner == nil { 79 | cfg.runner = linters.SimpleRunner{} 80 | } 81 | 82 | if cfg.state == nil { 83 | cfg.state = repostate.NewAPIStorage(httputils.GrequestsClient{}) 84 | } 85 | 86 | return &GithubGoRepo{ 87 | GithubGoRepoConfig: cfg, 88 | analysisGUID: analysisGUID, 89 | branch: branch, 90 | repo: repo, 91 | }, nil 92 | } 93 | 94 | func (g *GithubGoRepo) getRepo() *fetchers.Repo { 95 | return &fetchers.Repo{ 96 | CloneURL: fmt.Sprintf("https://github.com/%s/%s.git", g.repo.Owner, g.repo.Name), 97 | Ref: g.branch, 98 | FullPath: fmt.Sprintf("github.com/%s/%s", g.repo.Owner, g.repo.Name), 99 | } 100 | } 101 | 102 | func (g *GithubGoRepo) prepareRepo(ctx context.Context) error { 103 | repo := g.getRepo() 104 | var err error 105 | g.trackTiming("Clone", func() { 106 | err = g.repoFetcher.Fetch(ctx, repo, g.exec) 107 | }) 108 | if err != nil { 109 | return &errorutils.InternalError{ 110 | PublicDesc: "can't clone git repo", 111 | PrivateDesc: fmt.Sprintf("can't clone git repo: %s", err), 112 | } 113 | } 114 | 115 | var depsRes *ensuredeps.Result 116 | g.trackTiming("Deps", func() { 117 | depsRes, err = g.gw.FetchDeps(ctx, repo.FullPath) 118 | }) 119 | if err != nil { 120 | // don't public warn: it's an internal error 121 | analytics.Log(ctx).Warnf("Internal error fetching deps: %s", err) 122 | } else { 123 | analytics.Log(ctx).Infof("Got deps result: %#v", depsRes) 124 | 125 | for _, w := range depsRes.Warnings { 126 | warnText := fmt.Sprintf("Fetch deps: %s: %s", w.Kind, w.Text) 127 | warnText = escapeErrorText(warnText, g.buildSecrets()) 128 | g.publicWarn("prepare repo", warnText) 129 | 130 | analytics.Log(ctx).Infof("Fetch deps warning: [%s]: %s", w.Kind, w.Text) 131 | } 132 | } 133 | 134 | return nil 135 | } 136 | 137 | func (g GithubGoRepo) updateAnalysisState(ctx context.Context, res *result.Result, status, publicError string) { 138 | resJSON := &resultJSON{ 139 | Version: 1, 140 | WorkerRes: workerRes{ 141 | Timings: g.timings, 142 | Warnings: g.warnings, 143 | Error: publicError, 144 | }, 145 | } 146 | 147 | if res != nil { 148 | resJSON.GolangciLintRes = res.ResultJSON 149 | } 150 | s := &repostate.State{ 151 | Status: status, 152 | ResultJSON: resJSON, 153 | } 154 | 155 | jsonBytes, err := json.Marshal(*resJSON) 156 | if err == nil { 157 | analytics.Log(ctx).Infof("Save repo analysis status: status=%s, result_json=%s", status, string(jsonBytes)) 158 | } 159 | 160 | if err := g.state.UpdateState(ctx, g.repo.Owner, g.repo.Name, g.analysisGUID, s); err != nil { 161 | analytics.Log(ctx).Warnf("Can't set analysis %s status to '%v': %s", g.analysisGUID, s, err) 162 | } 163 | } 164 | 165 | func (g *GithubGoRepo) processWithGuaranteedGithubStatus(ctx context.Context) error { 166 | res, err := g.work(ctx) 167 | analytics.Log(ctx).Infof("timings: %s", g.timings) 168 | 169 | ctx = context.Background() // no timeout for state and status saving: it must be durable 170 | 171 | var status string 172 | var publicError string 173 | if err != nil { 174 | if ierr, ok := err.(*errorutils.InternalError); ok { 175 | if strings.Contains(ierr.PrivateDesc, noGoFilesToAnalyzeErr) { 176 | publicError = noGoFilesToAnalyzeMessage 177 | status = statusProcessed 178 | err = nil 179 | } else { 180 | status = string(github.StatusError) 181 | publicError = ierr.PublicDesc 182 | } 183 | } else if berr, ok := err.(*errorutils.BadInputError); ok { 184 | berr.PublicDesc = escapeErrorText(berr.PublicDesc, g.buildSecrets()) 185 | publicError = berr.PublicDesc 186 | status = statusProcessed 187 | err = nil 188 | analytics.Log(ctx).Warnf("Repo analysis bad input error: %s", berr) 189 | } else { 190 | status = string(github.StatusError) 191 | publicError = internalError 192 | } 193 | } else { 194 | status = statusProcessed 195 | } 196 | 197 | g.updateAnalysisState(ctx, res, status, publicError) 198 | return err 199 | } 200 | 201 | func (g GithubGoRepo) buildSecrets() map[string]string { 202 | const hidden = "{hidden}" 203 | ret := map[string]string{ 204 | g.gw.Gopath(): "$GOPATH", 205 | } 206 | 207 | for _, kv := range os.Environ() { 208 | parts := strings.Split(kv, "=") 209 | if len(parts) != 2 { 210 | continue 211 | } 212 | 213 | v := parts[1] 214 | if len(v) >= 6 { 215 | ret[v] = hidden 216 | } 217 | } 218 | 219 | return ret 220 | } 221 | 222 | func (g *GithubGoRepo) work(ctx context.Context) (res *result.Result, err error) { 223 | defer func() { 224 | if rerr := recover(); rerr != nil { 225 | err = &errorutils.InternalError{ 226 | PublicDesc: "golangci-worker panic-ed", 227 | PrivateDesc: fmt.Sprintf("panic occured: %s, %s", rerr, debug.Stack()), 228 | } 229 | } 230 | }() 231 | 232 | if err = g.prepareRepo(ctx); err != nil { 233 | return nil, err // don't wrap error, need to save it's type 234 | } 235 | 236 | g.trackTiming("Analysis", func() { 237 | res, err = g.runner.Run(ctx, g.linters, g.exec) 238 | }) 239 | if err != nil { 240 | return nil, err // don't wrap error, need to save it's type 241 | } 242 | 243 | return res, nil 244 | } 245 | 246 | func (g GithubGoRepo) Process(ctx context.Context) error { 247 | defer g.exec.Clean() 248 | 249 | curState, err := g.state.GetState(ctx, g.repo.Owner, g.repo.Name, g.analysisGUID) 250 | if err != nil { 251 | return fmt.Errorf("can't get current state: %s", err) 252 | } 253 | 254 | g.gw = workspaces.NewGo(g.exec, g.infoFetcher) 255 | defer g.gw.Clean(ctx) 256 | if err = g.gw.Setup(ctx, g.getRepo(), "github.com", g.repo.Owner, g.repo.Name); err != nil { 257 | if errors.Cause(err) == fetchers.ErrNoBranchOrRepo { 258 | curState.Status = statusNotFound 259 | if updateErr := g.state.UpdateState(ctx, g.repo.Owner, g.repo.Name, g.analysisGUID, curState); updateErr != nil { 260 | analytics.Log(ctx).Warnf("Can't update repo analysis %s state with setting status to 'not_found': %s", 261 | g.analysisGUID, updateErr) 262 | } 263 | analytics.Log(ctx).Warnf("Branch or repo doesn't exist, set status not_found") 264 | return nil 265 | } 266 | return fmt.Errorf("can't setup go workspace: %s", err) 267 | } 268 | g.exec = g.gw.Executor() 269 | 270 | if curState.Status == statusSentToQueue { 271 | g.addTimingFrom("In Queue", fromDBTime(curState.CreatedAt)) 272 | curState.Status = statusProcessing 273 | if err = g.state.UpdateState(ctx, g.repo.Owner, g.repo.Name, g.analysisGUID, curState); err != nil { 274 | analytics.Log(ctx).Warnf("Can't update repo analysis %s state with setting status to 'processing': %s", g.analysisGUID, err) 275 | } 276 | } 277 | 278 | return g.processWithGuaranteedGithubStatus(ctx) 279 | } 280 | -------------------------------------------------------------------------------- /app/analyze/processors/github_go_pr_test.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "strconv" 9 | "strings" 10 | "testing" 11 | "time" 12 | 13 | "github.com/golang/mock/gomock" 14 | "github.com/golangci/golangci-worker/app/analytics" 15 | "github.com/golangci/golangci-worker/app/analyze/linters" 16 | "github.com/golangci/golangci-worker/app/analyze/linters/result" 17 | "github.com/golangci/golangci-worker/app/analyze/prstate" 18 | "github.com/golangci/golangci-worker/app/analyze/repoinfo" 19 | "github.com/golangci/golangci-worker/app/analyze/reporters" 20 | "github.com/golangci/golangci-worker/app/lib/executors" 21 | "github.com/golangci/golangci-worker/app/lib/fetchers" 22 | "github.com/golangci/golangci-worker/app/lib/github" 23 | "github.com/golangci/golangci-worker/app/test" 24 | gh "github.com/google/go-github/github" 25 | "github.com/stretchr/testify/assert" 26 | ) 27 | 28 | var testCtxMatcher = gomock.Any() 29 | var testCtx = analytics.ContextWithEventPropsCollector(context.Background(), analytics.EventPRChecked) 30 | 31 | var any = gomock.Any() 32 | var fakeChangedIssue = result.NewIssue("linter2", "F1 issue", "main.go", 10, 11) 33 | var fakeChangedIssues = []result.Issue{ 34 | result.NewIssue("linter2", "F1 issue", "main.go", 9, 10), 35 | result.NewIssue("linter3", "F1 issue", "main.go", 10, 11), 36 | } 37 | 38 | var testSHA = "testSHA" 39 | var testBranch = "testBranch" 40 | var testPR = &gh.PullRequest{ 41 | Head: &gh.PullRequestBranch{ 42 | Ref: gh.String(testBranch), 43 | SHA: gh.String(testSHA), 44 | }, 45 | Base: &gh.PullRequestBranch{ 46 | Repo: &gh.Repository{ 47 | Private: gh.Bool(false), 48 | }, 49 | }, 50 | Number: gh.Int(7), 51 | } 52 | var testAnalysisGUID = "test-guid" 53 | 54 | func getFakeLinters(ctrl *gomock.Controller, issues ...result.Issue) []linters.Linter { 55 | a := linters.NewMockLinter(ctrl) 56 | a.EXPECT(). 57 | Run(testCtxMatcher, any). 58 | Return(&result.Result{ 59 | Issues: issues, 60 | }, nil) 61 | return []linters.Linter{a} 62 | } 63 | 64 | func getNopFetcher(ctrl *gomock.Controller) fetchers.Fetcher { 65 | f := fetchers.NewMockFetcher(ctrl) 66 | f.EXPECT().Fetch(testCtxMatcher, any, any).Return(nil) 67 | return f 68 | } 69 | 70 | func getNopReporter(ctrl *gomock.Controller) reporters.Reporter { 71 | r := reporters.NewMockReporter(ctrl) 72 | r.EXPECT().Report(testCtxMatcher, any, any).AnyTimes().Return(nil) 73 | return r 74 | } 75 | 76 | func getNopInfoFetcher(ctrl *gomock.Controller) repoinfo.Fetcher { 77 | r := repoinfo.NewMockFetcher(ctrl) 78 | r.EXPECT().Fetch(testCtxMatcher, any, any).AnyTimes().Return(&repoinfo.Info{}, nil) 79 | return r 80 | } 81 | 82 | func getErroredReporter(ctrl *gomock.Controller) reporters.Reporter { 83 | r := reporters.NewMockReporter(ctrl) 84 | r.EXPECT().Report(testCtxMatcher, any, any).Return(fmt.Errorf("can't report")) 85 | return r 86 | } 87 | 88 | func getNopState(ctrl *gomock.Controller) prstate.Storage { 89 | r := prstate.NewMockStorage(ctrl) 90 | r.EXPECT().UpdateState(any, any, any, any, any).AnyTimes().Return(nil) 91 | r.EXPECT().GetState(any, any, any, any).AnyTimes().Return(&prstate.State{ 92 | Status: statusSentToQueue, 93 | }, nil) 94 | return r 95 | } 96 | 97 | func getNopExecutor(ctrl *gomock.Controller) executors.Executor { 98 | e := executors.NewMockExecutor(ctrl) 99 | e.EXPECT().WorkDir().Return("").AnyTimes() 100 | e.EXPECT().WithWorkDir(any).Return(e).AnyTimes() 101 | e.EXPECT().Run(testCtxMatcher, any, any).Return("", nil).AnyTimes() 102 | e.EXPECT().Run(testCtxMatcher, any, any, any).Return("", nil).AnyTimes() 103 | e.EXPECT().Clean().AnyTimes() 104 | e.EXPECT().SetEnv(any, any).AnyTimes() 105 | e.EXPECT().CopyFile(any, any, any).Return(nil) 106 | return e 107 | } 108 | 109 | func getFakePatch(t *testing.T) string { 110 | patch, err := ioutil.ReadFile(fmt.Sprintf("test/%d.patch", github.FakeContext.PullRequestNumber)) 111 | assert.Nil(t, err) 112 | return string(patch) 113 | } 114 | 115 | func getFakeStatusGithubClient(t *testing.T, ctrl *gomock.Controller, status github.Status, statusDesc string) github.Client { 116 | c := &github.FakeContext 117 | gc := github.NewMockClient(ctrl) 118 | gc.EXPECT().GetPullRequest(testCtxMatcher, c).Return(testPR, nil) 119 | 120 | scsPending := gc.EXPECT().SetCommitStatus(testCtxMatcher, c, testSHA, 121 | github.StatusPending, "GolangCI is reviewing your Pull Request...", ""). 122 | Return(nil) 123 | 124 | gc.EXPECT().GetPullRequestPatch(any, any).AnyTimes().Return(getFakePatch(t), nil) 125 | 126 | test.Init() 127 | url := fmt.Sprintf("%s/r/github.com/%s/%s/pulls/%d", os.Getenv("WEB_ROOT"), c.Repo.Owner, c.Repo.Name, testPR.GetNumber()) 128 | gc.EXPECT().SetCommitStatus(testCtxMatcher, c, testSHA, status, statusDesc, url).After(scsPending) 129 | 130 | return gc 131 | } 132 | 133 | func getNopGithubClient(t *testing.T, ctrl *gomock.Controller) github.Client { 134 | c := &github.FakeContext 135 | 136 | gc := github.NewMockClient(ctrl) 137 | gc.EXPECT().CreateReview(any, any, any).AnyTimes() 138 | gc.EXPECT().GetPullRequest(testCtxMatcher, c).AnyTimes().Return(testPR, nil) 139 | gc.EXPECT().GetPullRequestPatch(any, any).AnyTimes().Return(getFakePatch(t)) 140 | gc.EXPECT().SetCommitStatus(any, any, testSHA, any, any, any).AnyTimes() 141 | return gc 142 | } 143 | 144 | func fillWithNops(t *testing.T, ctrl *gomock.Controller, cfg *githubGoPRConfig) { 145 | if cfg.client == nil { 146 | cfg.client = getNopGithubClient(t, ctrl) 147 | } 148 | if cfg.exec == nil { 149 | cfg.exec = getNopExecutor(ctrl) 150 | } 151 | if cfg.linters == nil { 152 | cfg.linters = getFakeLinters(ctrl) 153 | } 154 | if cfg.repoFetcher == nil { 155 | cfg.repoFetcher = getNopFetcher(ctrl) 156 | } 157 | if cfg.infoFetcher == nil { 158 | cfg.infoFetcher = getNopInfoFetcher(ctrl) 159 | } 160 | if cfg.reporter == nil { 161 | cfg.reporter = getNopReporter(ctrl) 162 | } 163 | if cfg.state == nil { 164 | cfg.state = getNopState(ctrl) 165 | } 166 | } 167 | 168 | func getNopedProcessor(t *testing.T, ctrl *gomock.Controller, cfg githubGoPRConfig) *githubGoPR { 169 | fillWithNops(t, ctrl, &cfg) 170 | 171 | p, err := newGithubGoPR(testCtx, &github.FakeContext, cfg, testAnalysisGUID) 172 | assert.NoError(t, err) 173 | 174 | return p 175 | } 176 | 177 | func testProcessor(t *testing.T, ctrl *gomock.Controller, cfg githubGoPRConfig) { 178 | p := getNopedProcessor(t, ctrl, cfg) 179 | 180 | err := p.Process(testCtx) 181 | assert.NoError(t, err) 182 | } 183 | 184 | func TestSetCommitStatusSuccess(t *testing.T) { 185 | ctrl := gomock.NewController(t) 186 | defer ctrl.Finish() 187 | 188 | testProcessor(t, ctrl, githubGoPRConfig{ 189 | linters: getFakeLinters(ctrl), 190 | client: getFakeStatusGithubClient(t, ctrl, github.StatusSuccess, "No issues found!"), 191 | }) 192 | } 193 | 194 | func TestSetCommitStatusFailureOneIssue(t *testing.T) { 195 | ctrl := gomock.NewController(t) 196 | defer ctrl.Finish() 197 | 198 | testProcessor(t, ctrl, githubGoPRConfig{ 199 | linters: getFakeLinters(ctrl, fakeChangedIssue), 200 | client: getFakeStatusGithubClient(t, ctrl, github.StatusFailure, "1 issue found"), 201 | }) 202 | } 203 | 204 | func TestSetCommitStatusFailureTwoIssues(t *testing.T) { 205 | ctrl := gomock.NewController(t) 206 | defer ctrl.Finish() 207 | 208 | testProcessor(t, ctrl, githubGoPRConfig{ 209 | linters: getFakeLinters(ctrl, fakeChangedIssues...), 210 | client: getFakeStatusGithubClient(t, ctrl, github.StatusFailure, "2 issues found"), 211 | }) 212 | } 213 | 214 | func TestSetCommitStatusOnReportingError(t *testing.T) { 215 | ctrl := gomock.NewController(t) 216 | defer ctrl.Finish() 217 | 218 | p := getNopedProcessor(t, ctrl, githubGoPRConfig{ 219 | linters: getFakeLinters(ctrl, fakeChangedIssue), 220 | reporter: getErroredReporter(ctrl), 221 | client: getFakeStatusGithubClient(t, ctrl, 222 | github.StatusError, "can't send pull request comments to github"), 223 | }) 224 | assert.Error(t, p.Process(testCtx)) 225 | } 226 | 227 | func getRealisticTestProcessor(ctx context.Context, t *testing.T, ctrl *gomock.Controller) *githubGoPR { 228 | c := getTestingRepo(t) 229 | cloneURL := fmt.Sprintf("git@github.com:%s/%s.git", c.Repo.Owner, c.Repo.Name) 230 | pr := &gh.PullRequest{ 231 | Head: &gh.PullRequestBranch{ 232 | Ref: gh.String("master"), 233 | Repo: &gh.Repository{ 234 | SSHURL: gh.String(cloneURL), 235 | }, 236 | }, 237 | } 238 | gc := github.NewMockClient(ctrl) 239 | gc.EXPECT().GetPullRequest(testCtxMatcher, c).Return(pr, nil).AnyTimes() 240 | gc.EXPECT().GetPullRequestPatch(any, any).AnyTimes().Return(getFakePatch(t), nil) 241 | gc.EXPECT().SetCommitStatus(any, any, any, any, any, any).AnyTimes() 242 | 243 | exec, err := executors.NewTempDirShell("gopath") 244 | assert.NoError(t, err) 245 | 246 | cfg := githubGoPRConfig{ 247 | exec: exec, 248 | runner: linters.SimpleRunner{}, 249 | reporter: getNopReporter(ctrl), 250 | client: gc, 251 | } 252 | 253 | p, err := newGithubGoPR(ctx, c, cfg, testAnalysisGUID) 254 | assert.NoError(t, err) 255 | 256 | return p 257 | } 258 | 259 | func TestProcessorTimeout(t *testing.T) { 260 | test.Init() 261 | 262 | startedAt := time.Now() 263 | ctrl := gomock.NewController(t) 264 | defer ctrl.Finish() 265 | 266 | ctx, cancel := context.WithTimeout(testCtx, 100*time.Millisecond) 267 | defer cancel() 268 | p := getRealisticTestProcessor(ctx, t, ctrl) 269 | 270 | assert.Error(t, p.Process(ctx)) 271 | assert.True(t, time.Since(startedAt) < 300*time.Millisecond) 272 | } 273 | 274 | func getTestingRepo(t *testing.T) *github.Context { 275 | repo := os.Getenv("REPO") 276 | if repo == "" { 277 | repo = "golangci/golangci-worker" 278 | } 279 | 280 | repoParts := strings.Split(repo, "/") 281 | assert.Len(t, repoParts, 2) 282 | 283 | pr := os.Getenv("PR") 284 | if pr == "" { 285 | pr = "1" 286 | } 287 | prNumber, err := strconv.Atoi(pr) 288 | assert.NoError(t, err) 289 | 290 | c := &github.Context{ 291 | Repo: github.Repo{ 292 | Owner: repoParts[0], 293 | Name: repoParts[1], 294 | }, 295 | PullRequestNumber: prNumber, 296 | GithubAccessToken: os.Getenv("TEST_GITHUB_TOKEN"), 297 | } 298 | 299 | return c 300 | } 301 | 302 | func getTestProcessorWithFakeGithub(ctx context.Context, t *testing.T, ctrl *gomock.Controller) *githubGoPR { 303 | c := getTestingRepo(t) 304 | 305 | realGc := github.NewMyClient() 306 | patch, err := realGc.GetPullRequestPatch(ctx, c) 307 | assert.NoError(t, err) 308 | pr, err := realGc.GetPullRequest(ctx, c) 309 | assert.NoError(t, err) 310 | 311 | gc := github.NewMockClient(ctrl) 312 | gc.EXPECT().GetPullRequestPatch(any, any).AnyTimes().Return(patch, nil) 313 | gc.EXPECT().GetPullRequest(testCtxMatcher, c).Return(pr, nil) 314 | gc.EXPECT().SetCommitStatus(any, any, any, any, any, any).AnyTimes() 315 | 316 | cfg := githubGoPRConfig{ 317 | reporter: getNopReporter(ctrl), 318 | client: gc, 319 | } 320 | 321 | p, err := newGithubGoPR(ctx, c, cfg, testAnalysisGUID) 322 | assert.NoError(t, err) 323 | 324 | return p 325 | } 326 | 327 | func TestProcessRepoWithFakeGithub(t *testing.T) { 328 | test.Init() 329 | test.MarkAsSlow(t) 330 | 331 | ctrl := gomock.NewController(t) 332 | defer ctrl.Finish() 333 | 334 | p := getTestProcessorWithFakeGithub(testCtx, t, ctrl) 335 | err := p.Process(testCtx) 336 | assert.NoError(t, err) 337 | } 338 | -------------------------------------------------------------------------------- /app/analyze/processors/github_go_pr.go: -------------------------------------------------------------------------------- 1 | package processors 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "runtime/debug" 9 | "strings" 10 | "time" 11 | 12 | "github.com/golangci/golangci-api/pkg/goenv/ensuredeps" 13 | goenvresult "github.com/golangci/golangci-api/pkg/goenv/result" 14 | "github.com/golangci/golangci-worker/app/analytics" 15 | "github.com/golangci/golangci-worker/app/analyze/linters" 16 | "github.com/golangci/golangci-worker/app/analyze/linters/golinters" 17 | "github.com/golangci/golangci-worker/app/analyze/linters/result" 18 | "github.com/golangci/golangci-worker/app/analyze/prstate" 19 | "github.com/golangci/golangci-worker/app/analyze/repoinfo" 20 | "github.com/golangci/golangci-worker/app/analyze/reporters" 21 | "github.com/golangci/golangci-worker/app/lib/errorutils" 22 | "github.com/golangci/golangci-worker/app/lib/executors" 23 | "github.com/golangci/golangci-worker/app/lib/fetchers" 24 | "github.com/golangci/golangci-worker/app/lib/github" 25 | "github.com/golangci/golangci-worker/app/lib/goutils/workspaces" 26 | "github.com/golangci/golangci-worker/app/lib/httputils" 27 | gh "github.com/google/go-github/github" 28 | 29 | "github.com/golangci/golangci-shared/pkg/config" 30 | "github.com/golangci/golangci-shared/pkg/logutil" 31 | "github.com/golangci/golangci-worker/app/lib/experiments" 32 | ) 33 | 34 | const ( 35 | patchPath = "../changes.patch" 36 | ) 37 | 38 | type githubGoPRConfig struct { 39 | repoFetcher fetchers.Fetcher 40 | infoFetcher repoinfo.Fetcher 41 | linters []linters.Linter 42 | runner linters.Runner 43 | reporter reporters.Reporter 44 | exec executors.Executor 45 | client github.Client 46 | state prstate.Storage 47 | } 48 | 49 | type githubGoPR struct { 50 | pr *gh.PullRequest 51 | analysisGUID string 52 | 53 | context *github.Context 54 | gw *workspaces.Go 55 | 56 | resLog *goenvresult.Log 57 | 58 | githubGoPRConfig 59 | resultCollector 60 | 61 | newWorkspaceInstaller workspaces.Installer 62 | ec *experiments.Checker 63 | } 64 | 65 | //nolint:gocyclo 66 | func newGithubGoPR(ctx context.Context, c *github.Context, cfg githubGoPRConfig, analysisGUID string) (*githubGoPR, error) { 67 | if cfg.client == nil { 68 | cfg.client = github.NewMyClient() 69 | } 70 | 71 | if cfg.exec == nil { 72 | var err error 73 | cfg.exec, err = makeExecutor(ctx, &c.Repo, true, nil, nil) 74 | if err != nil { 75 | return nil, fmt.Errorf("can't make executor: %s", err) 76 | } 77 | } 78 | 79 | if cfg.repoFetcher == nil { 80 | cfg.repoFetcher = fetchers.NewGit() 81 | } 82 | 83 | if cfg.infoFetcher == nil { 84 | cfg.infoFetcher = repoinfo.NewCloningFetcher(cfg.repoFetcher) 85 | } 86 | 87 | if cfg.linters == nil { 88 | cfg.linters = []linters.Linter{ 89 | golinters.GolangciLint{ 90 | PatchPath: patchPath, 91 | }, 92 | } 93 | } 94 | 95 | log := logutil.NewStderrLog("executor") 96 | log.SetLevel(logutil.LogLevelInfo) 97 | envCfg := config.NewEnvConfig(log) 98 | ec := experiments.NewChecker(envCfg, log) 99 | 100 | if cfg.reporter == nil { 101 | includeLinterName := ec.IsActiveForAnalysis("include_linter_name_in_comment", &c.Repo, true) 102 | cfg.reporter = reporters.NewGithubReviewer(c, cfg.client, includeLinterName) 103 | } 104 | 105 | if cfg.runner == nil { 106 | cfg.runner = linters.SimpleRunner{} 107 | } 108 | 109 | if cfg.state == nil { 110 | cfg.state = prstate.NewAPIStorage(httputils.GrequestsClient{}) 111 | } 112 | 113 | var wi workspaces.Installer 114 | 115 | if ec.IsActiveForAnalysis("new_pr_prepare", &c.Repo, true) { 116 | wi = workspaces.NewGo2(cfg.exec, log, cfg.repoFetcher) 117 | } 118 | 119 | return &githubGoPR{ 120 | context: c, 121 | githubGoPRConfig: cfg, 122 | analysisGUID: analysisGUID, 123 | newWorkspaceInstaller: wi, 124 | ec: ec, 125 | }, nil 126 | } 127 | 128 | func storePatch(ctx context.Context, patch string, exec executors.Executor) error { 129 | f, err := ioutil.TempFile("/tmp", "golangci.diff") 130 | defer os.Remove(f.Name()) 131 | 132 | if err != nil { 133 | return fmt.Errorf("can't create temp file for patch: %s", err) 134 | } 135 | if err = ioutil.WriteFile(f.Name(), []byte(patch), os.ModePerm); err != nil { 136 | return fmt.Errorf("can't write patch to temp file %s: %s", f.Name(), err) 137 | } 138 | 139 | if err = exec.CopyFile(ctx, patchPath, f.Name()); err != nil { 140 | return fmt.Errorf("can't copy patch file: %s", err) 141 | } 142 | 143 | return nil 144 | } 145 | 146 | func (g githubGoPR) getRepo() *fetchers.Repo { 147 | return &fetchers.Repo{ 148 | CloneURL: g.context.GetCloneURL(g.pr.GetHead().GetRepo()), 149 | Ref: g.pr.GetHead().GetRef(), 150 | FullPath: fmt.Sprintf("github.com/%s/%s", g.context.Repo.Owner, g.context.Repo.Name), 151 | } 152 | } 153 | 154 | func (g *githubGoPR) prepareRepo(ctx context.Context) error { 155 | if g.newWorkspaceInstaller != nil { 156 | if g.resLog != nil { 157 | for _, sg := range g.resLog.Groups { 158 | for _, s := range sg.Steps { 159 | if s.Error != "" { 160 | text := fmt.Sprintf("%s error: %s", s.Description, s.Error) 161 | text = escapeErrorText(text, g.buildSecrets()) 162 | g.publicWarn(sg.Name, text) 163 | } 164 | } 165 | } 166 | } 167 | 168 | return nil 169 | } 170 | 171 | repo := g.getRepo() 172 | var err error 173 | g.trackTiming("Clone", func() { 174 | err = g.repoFetcher.Fetch(ctx, repo, g.exec) 175 | }) 176 | if err != nil { 177 | return &errorutils.InternalError{ 178 | PublicDesc: "can't clone git repo", 179 | PrivateDesc: fmt.Sprintf("can't clone git repo: %s", err), 180 | } 181 | } 182 | 183 | var depsRes *ensuredeps.Result 184 | g.trackTiming("Deps", func() { 185 | depsRes, err = g.gw.FetchDeps(ctx, repo.FullPath) 186 | }) 187 | if err != nil { 188 | // don't public warn: it's an internal error 189 | analytics.Log(ctx).Warnf("Internal error fetching deps: %s", err) 190 | } else { 191 | analytics.Log(ctx).Infof("Got deps result: %#v", depsRes) 192 | 193 | for _, w := range depsRes.Warnings { 194 | warnText := fmt.Sprintf("Fetch deps: %s: %s", w.Kind, w.Text) 195 | warnText = escapeErrorText(warnText, g.buildSecrets()) 196 | g.publicWarn("prepare repo", warnText) 197 | 198 | analytics.Log(ctx).Infof("Fetch deps warning: [%s]: %s", w.Kind, w.Text) 199 | } 200 | } 201 | 202 | return nil 203 | } 204 | 205 | func (g githubGoPR) updateAnalysisState(ctx context.Context, res *result.Result, status github.Status, publicError string) { 206 | resJSON := &resultJSON{ 207 | Version: 1, 208 | WorkerRes: workerRes{ 209 | Timings: g.timings, 210 | Warnings: g.warnings, 211 | Error: publicError, 212 | }, 213 | } 214 | 215 | issuesCount := 0 216 | if res != nil { 217 | resJSON.GolangciLintRes = res.ResultJSON 218 | issuesCount = len(res.Issues) 219 | } 220 | s := &prstate.State{ 221 | Status: "processed/" + string(status), 222 | ReportedIssuesCount: issuesCount, 223 | ResultJSON: resJSON, 224 | } 225 | 226 | if err := g.state.UpdateState(ctx, g.context.Repo.Owner, g.context.Repo.Name, g.analysisGUID, s); err != nil { 227 | analytics.Log(ctx).Warnf("Can't set analysis %s status to '%v': %s", g.analysisGUID, s, err) 228 | } 229 | } 230 | 231 | func getGithubStatusForIssues(issues []result.Issue) (github.Status, string) { 232 | switch len(issues) { 233 | case 0: 234 | return github.StatusSuccess, "No issues found!" 235 | case 1: 236 | return github.StatusFailure, "1 issue found" 237 | default: 238 | return github.StatusFailure, fmt.Sprintf("%d issues found", len(issues)) 239 | } 240 | } 241 | 242 | func (g githubGoPR) buildSecrets() map[string]string { 243 | const hidden = "{hidden}" 244 | ret := map[string]string{ 245 | g.context.GithubAccessToken: hidden, 246 | g.analysisGUID: hidden, 247 | } 248 | if g.newWorkspaceInstaller == nil { 249 | ret[g.gw.Gopath()] = "$GOPATH" 250 | } 251 | 252 | for _, kv := range os.Environ() { 253 | parts := strings.Split(kv, "=") 254 | if len(parts) != 2 { 255 | continue 256 | } 257 | 258 | v := parts[1] 259 | if len(v) >= 6 { 260 | ret[v] = hidden 261 | } 262 | } 263 | 264 | return ret 265 | } 266 | 267 | func (g *githubGoPR) processWithGuaranteedGithubStatus(ctx context.Context) error { 268 | res, err := g.work(ctx) 269 | analytics.Log(ctx).Infof("timings: %s", g.timings) 270 | 271 | ctx = context.Background() // no timeout for state and status saving: it must be durable 272 | 273 | var status github.Status 274 | var statusDesc, publicError string 275 | if err != nil { 276 | if serr, ok := err.(*IgnoredError); ok { 277 | status, statusDesc = serr.Status, serr.StatusDesc 278 | if !serr.IsRecoverable { 279 | err = nil 280 | } 281 | // already must have warning, don't set publicError 282 | } else if ierr, ok := err.(*errorutils.InternalError); ok { 283 | if strings.Contains(ierr.PrivateDesc, noGoFilesToAnalyzeErr) { 284 | status, statusDesc = github.StatusSuccess, noGoFilesToAnalyzeMessage 285 | err = nil 286 | } else { 287 | status, statusDesc = github.StatusError, ierr.PublicDesc 288 | } 289 | publicError = statusDesc 290 | } else if berr, ok := err.(*errorutils.BadInputError); ok { 291 | berr.PublicDesc = escapeErrorText(berr.PublicDesc, g.buildSecrets()) 292 | status, statusDesc = github.StatusError, "can't analyze" 293 | publicError = berr.PublicDesc 294 | err = nil 295 | analytics.Log(ctx).Warnf("PR analysis bad input error: %s", berr) 296 | } else { 297 | status, statusDesc = github.StatusError, internalError 298 | publicError = statusDesc 299 | } 300 | } else { 301 | status, statusDesc = getGithubStatusForIssues(res.Issues) 302 | } 303 | 304 | // update of state must be before commit status update: user can open details link before: race condition 305 | g.updateAnalysisState(ctx, res, status, publicError) 306 | g.setCommitStatus(ctx, status, statusDesc) 307 | 308 | return err 309 | } 310 | 311 | func (g *githubGoPR) work(ctx context.Context) (res *result.Result, err error) { 312 | defer func() { 313 | if rerr := recover(); rerr != nil { 314 | err = &errorutils.InternalError{ 315 | PublicDesc: "golangci-worker panic-ed", 316 | PrivateDesc: fmt.Sprintf("panic occured: %s, %s", rerr, debug.Stack()), 317 | } 318 | } 319 | }() 320 | 321 | prState := strings.ToUpper(g.pr.GetState()) 322 | if prState == "MERGED" || prState == "CLOSED" { 323 | // branch can be deleted: will be an error; no need to analyze 324 | g.publicWarn("process", fmt.Sprintf("Pull Request is already %s, skip analysis", prState)) 325 | analytics.Log(ctx).Warnf("Pull Request is already %s, skip analysis", prState) 326 | return nil, &IgnoredError{ 327 | Status: github.StatusSuccess, 328 | StatusDesc: fmt.Sprintf("Pull Request is already %s", strings.ToLower(prState)), 329 | IsRecoverable: false, 330 | } 331 | } 332 | 333 | if err = g.prepareRepo(ctx); err != nil { 334 | return nil, err // don't wrap error, need to save it's type 335 | } 336 | 337 | g.trackTiming("Analysis", func() { 338 | res, err = g.runner.Run(ctx, g.linters, g.exec) 339 | }) 340 | if err != nil { 341 | return nil, err // don't wrap error, need to save it's type 342 | } 343 | 344 | issues := res.Issues 345 | analytics.SaveEventProp(ctx, analytics.EventPRChecked, "reportedIssues", len(issues)) 346 | 347 | if len(issues) == 0 { 348 | analytics.Log(ctx).Infof("Linters found no issues") 349 | } else { 350 | analytics.Log(ctx).Infof("Linters found %d issues: %+v", len(issues), issues) 351 | } 352 | 353 | if err = g.reporter.Report(ctx, g.pr.GetHead().GetSHA(), issues); err != nil { 354 | return nil, &errorutils.InternalError{ 355 | PublicDesc: "can't send pull request comments to github", 356 | PrivateDesc: fmt.Sprintf("can't send pull request comments to github: %s", err), 357 | } 358 | } 359 | 360 | return res, nil 361 | } 362 | 363 | func (g githubGoPR) setCommitStatus(ctx context.Context, status github.Status, desc string) { 364 | var url string 365 | if status == github.StatusFailure || status == github.StatusSuccess || status == github.StatusError { 366 | c := g.context 367 | url = fmt.Sprintf("%s/r/github.com/%s/%s/pulls/%d", 368 | os.Getenv("WEB_ROOT"), c.Repo.Owner, c.Repo.Name, g.pr.GetNumber()) 369 | } 370 | err := g.client.SetCommitStatus(ctx, g.context, g.pr.GetHead().GetSHA(), status, desc, url) 371 | if err != nil { 372 | g.publicWarn("github", "Can't set github commit status") 373 | analytics.Log(ctx).Warnf("Can't set github commit status: %s", err) 374 | } 375 | } 376 | 377 | //nolint:gocyclo 378 | func (g githubGoPR) Process(ctx context.Context) error { 379 | defer g.exec.Clean() 380 | 381 | var err error 382 | g.pr, err = g.client.GetPullRequest(ctx, g.context) 383 | if err != nil { 384 | if !github.IsRecoverableError(err) { 385 | return err // preserve error 386 | } 387 | return fmt.Errorf("can't get pull request: %s", err) 388 | } 389 | 390 | g.setCommitStatus(ctx, github.StatusPending, "GolangCI is reviewing your Pull Request...") 391 | 392 | if g.newWorkspaceInstaller == nil { 393 | g.gw = workspaces.NewGo(g.exec, g.infoFetcher) 394 | if err = g.gw.Setup(ctx, g.getRepo(), "github.com", g.context.Repo.Owner, g.context.Repo.Name); err != nil { 395 | publicError := fmt.Sprintf("failed to setup workspace: %s", err) 396 | publicError = escapeErrorText(publicError, g.buildSecrets()) 397 | g.updateAnalysisState(ctx, nil, github.StatusError, publicError) 398 | g.setCommitStatus(ctx, github.StatusError, "failed to setup") 399 | 400 | return fmt.Errorf("can't setup go workspace: %s", err) 401 | } 402 | defer g.gw.Clean(ctx) 403 | g.exec = g.gw.Executor() 404 | } else { 405 | startedAt := time.Now() 406 | exec, resLog, err := g.newWorkspaceInstaller.Setup(ctx, g.getRepo(), "github.com", g.context.Repo.Owner, g.context.Repo.Name) //nolint:govet 407 | if err != nil { 408 | publicError := fmt.Sprintf("failed to setup workspace: %s", err) 409 | publicError = escapeErrorText(publicError, g.buildSecrets()) 410 | g.updateAnalysisState(ctx, nil, github.StatusError, publicError) 411 | g.setCommitStatus(ctx, github.StatusError, "failed to setup") 412 | 413 | return nil 414 | } 415 | g.exec = exec 416 | g.resLog = resLog 417 | g.addTimingFrom("Prepare", startedAt) 418 | } 419 | 420 | patch, err := g.client.GetPullRequestPatch(ctx, g.context) 421 | if err != nil { 422 | if !github.IsRecoverableError(err) { 423 | return err // preserve error 424 | } 425 | return fmt.Errorf("can't get patch: %s", err) 426 | } 427 | 428 | if err = storePatch(ctx, patch, g.exec); err != nil { 429 | return fmt.Errorf("can't store patch: %s", err) 430 | } 431 | 432 | curState, err := g.state.GetState(ctx, g.context.Repo.Owner, g.context.Repo.Name, g.analysisGUID) 433 | if err != nil { 434 | analytics.Log(ctx).Warnf("Can't get current state: %s", err) 435 | } else if curState.Status == statusSentToQueue { 436 | g.addTimingFrom("In Queue", fromDBTime(curState.CreatedAt)) 437 | inQueue := time.Since(fromDBTime(curState.CreatedAt)) 438 | analytics.SaveEventProp(ctx, analytics.EventPRChecked, "inQueueSeconds", int(inQueue/time.Second)) 439 | curState.Status = statusProcessing 440 | if err = g.state.UpdateState(ctx, g.context.Repo.Owner, g.context.Repo.Name, g.analysisGUID, curState); err != nil { 441 | analytics.Log(ctx).Warnf("Can't update analysis %s state with setting status to 'processing': %s", g.analysisGUID, err) 442 | } 443 | } 444 | 445 | return g.processWithGuaranteedGithubStatus(ctx) 446 | } 447 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.33.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= 4 | github.com/OpenPeeDeeP/depguard v0.0.0-20180806142446-a69c782687b2/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o= 5 | github.com/RichardKnop/logging v0.0.0-20171219150333-66aaaba18258 h1:HDg14QRScUCSAFBFHM62YHJTj9JXapEWHsAmWCynYOE= 6 | github.com/RichardKnop/logging v0.0.0-20171219150333-66aaaba18258/go.mod h1:GN1ovZ77t2jiz0kTaWhgtQe271GODCgheqxlxGt7wIo= 7 | github.com/RichardKnop/machinery v0.0.0-20180221144734-c5e057032f00 h1:EnaaOull/1mlC4vEpHyI+x6DUIRVPGwYMuJHZgMhsnA= 8 | github.com/RichardKnop/machinery v0.0.0-20180221144734-c5e057032f00/go.mod h1:EGygLd9H7AmtBgSQiHkRjtPiOVsyofRSBQftWEGLZXs= 9 | github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= 10 | github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= 11 | github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= 12 | github.com/aws/aws-lambda-go v1.6.0/go.mod h1:zUsUQhAUjYzR8AuduJPCfhBuKWUaDbQiPOG+ouzmE1A= 13 | github.com/aws/aws-sdk-go v0.0.0-20180126231901-00cca3f093a8 h1:xqzpCUtYUsmKRnKjKOmGh/kqa+zdhaCYuYAEwXzHCSY= 14 | github.com/aws/aws-sdk-go v0.0.0-20180126231901-00cca3f093a8/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= 15 | github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d h1:7IjN4QP3c38xhg6wz8R3YjoU+6S9e7xBc0DAVLLIpHE= 16 | github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= 17 | github.com/cenkalti/backoff v2.0.0+incompatible h1:5IIPUHhlnUZbcHQsQou5k1Tn58nJkeJL9U+ig5CHJbY= 18 | github.com/cenkalti/backoff v2.0.0+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= 19 | github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 h1:6/yVvBsKeAw05IUj4AzvrxaCnDjN4nUqKjW9+w5wixg= 20 | github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= 21 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 22 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 23 | github.com/denisenkom/go-mssqldb v0.0.0-20181014144952-4e0d7dc8888f/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc= 24 | github.com/docker/distribution v2.6.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= 25 | github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= 26 | github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= 27 | github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= 28 | github.com/dukex/mixpanel v0.0.0-20170510165255-53bfdf679eec h1:G2HAXmBBLzV75BtSKugJSFDGs6C/64ew/A3cUEL4+30= 29 | github.com/dukex/mixpanel v0.0.0-20170510165255-53bfdf679eec/go.mod h1:AgMMmOoSoKDavirJHvIHNcaPq2S9QvZKnuN0We/Hwyo= 30 | github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= 31 | github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= 32 | github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= 33 | github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= 34 | github.com/fatih/structs v0.0.0-20180123065059-ebf56d35bba7/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= 35 | github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= 36 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 37 | github.com/garyburd/redigo v1.5.0 h1:OcZhiwwjKtBe7TO4TlXpj/1E3I2RVg1uLxwMT4VFF5w= 38 | github.com/garyburd/redigo v1.5.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= 39 | github.com/gavv/httpexpect v0.0.0-20170820080527-c44a6d7bb636/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= 40 | github.com/gavv/monotime v0.0.0-20171021193802-6f8212e8d10d/go.mod h1:vmp8DIyckQMXOPl0AQVHt+7n5h7Gb7hS6CUydiV8QeA= 41 | github.com/getsentry/raven-go v0.0.0-20180801005657-7535a8fa2ace h1:M5ZUuRO+XFqhTa9PlaqyWgfzMNWKSraCWm7z4PzM1GA= 42 | github.com/getsentry/raven-go v0.0.0-20180801005657-7535a8fa2ace/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= 43 | github.com/go-ini/ini v1.32.0 h1:/MArBHSS0TFR28yPPDK1vPIjt4wUnPBfb81i6iiyKvA= 44 | github.com/go-ini/ini v1.32.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= 45 | github.com/go-kit/kit v0.7.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 46 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 47 | github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= 48 | github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= 49 | github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= 50 | github.com/go-stack/stack v1.7.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 51 | github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= 52 | github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 53 | github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8= 54 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 55 | github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= 56 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 57 | github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= 58 | github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk= 59 | github.com/golangci/errcheck v0.0.0-20181003203344-1765131d5be5/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0= 60 | github.com/golangci/getrepoinfo v0.0.0-20180818083854-2a0c71df2c85 h1:kh6Zg84MFnU1Bci0Y+2BDuD1ex5HA6oKE3XJueUcSik= 61 | github.com/golangci/getrepoinfo v0.0.0-20180818083854-2a0c71df2c85/go.mod h1:UZDiy3xYhBUFnSteK1X+o24ICTNBbuF/798thHx9A/0= 62 | github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8= 63 | github.com/golangci/go-tools v0.0.0-20180902103155-93eecd106a0b/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM= 64 | github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o= 65 | github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= 66 | github.com/golangci/gofmt v0.0.0-20180506063654-2076e05ced53/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= 67 | github.com/golangci/golangci-api v0.0.0-20181118193359-820cf3a69851 h1:9R9sIUe+D6TjyKVvuf/CLgkq/x9bWzlJZ5490eRlcrs= 68 | github.com/golangci/golangci-api v0.0.0-20181118193359-820cf3a69851/go.mod h1:4yQy6xyRI1QHiWp3lxX7lzJ6RaPJgoEirx/AwY3Auhg= 69 | github.com/golangci/golangci-lint v0.0.0-20181114200623-a84578d603c7 h1:R+aQZIXiC8tOSz7RXdn1hr2HwH6n/pEmjNzv6iqCdPc= 70 | github.com/golangci/golangci-lint v0.0.0-20181114200623-a84578d603c7/go.mod h1:+H6w0IjZlyIPNH1F4I9T0i3KokCnnSbwxr1pKbEhbyQ= 71 | github.com/golangci/golangci-shared v0.0.0-20181003182622-9200811537b3 h1:fAka2SfnHfj6mNEMC/0pjJVCVuIlQ4z/T2TIwWxCP9U= 72 | github.com/golangci/golangci-shared v0.0.0-20181003182622-9200811537b3/go.mod h1:FZeWixWAzURoZ2rRsvR5G+aNPQ1E/MjrtzAXXbXyzTQ= 73 | github.com/golangci/golangci-worker v0.0.0-20180812155933-97fc92d30cca/go.mod h1:tKZadiV6tdkKEacPo9Rp4HgMRpD5IuvyBud1MPd+eyc= 74 | github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb/go.mod h1:ON/c2UR0VAAv6ZEAFKhjCLplESSmRFfZcDLASbI1GWo= 75 | github.com/golangci/govet v0.0.0-20180818181408-44ddbe260190/go.mod h1:pPwb+AK755h3/r73avHz5bEN6sa51/2HEZlLaV53hCo= 76 | github.com/golangci/ineffassign v0.0.0-20180808204949-2ee8f2867dde/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU= 77 | github.com/golangci/interfacer v0.0.0-20180902080945-01958817a6ec/go.mod h1:yBorupihJ5OYDFE7/EZwrslyNyZaaidqqVptYTcNxnk= 78 | github.com/golangci/lint v0.0.0-20170908181259-c2187e7932b5/go.mod h1:zs8jPuoOp76KrjiydDqO3CGeS4v9gq77HNNiYcxxTGw= 79 | github.com/golangci/lint v0.0.0-20180902080404-c2187e7932b5/go.mod h1:zs8jPuoOp76KrjiydDqO3CGeS4v9gq77HNNiYcxxTGw= 80 | github.com/golangci/lint-1 v0.0.0-20180610141402-4bf9709227d1/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= 81 | github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o= 82 | github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= 83 | github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI= 84 | github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4= 85 | github.com/golangci/tools v0.0.0-20180902102414-98e75f53b4b9/go.mod h1:zgj6NOYXOC1cexsdtDceI4/mj3aXK4JOVg9AV3C5LWI= 86 | github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= 87 | github.com/golangci/unparam v0.0.0-20180902112548-7ad9dbcccc16/go.mod h1:KW2L33j82vo0S0U6RP6uUQSuat+0Q457Yf+1mXC98/M= 88 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 89 | github.com/google/go-github v0.0.0-20180123235826-b1f138353a62 h1:+aYgS2LQXaXBpM2cOkq8kEML87Xj+pMCPMu4ZhggYBM= 90 | github.com/google/go-github v0.0.0-20180123235826-b1f138353a62/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= 91 | github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 h1:zLTLjkaOFEFIOxY5BWLFLwh+cL8vOBW4XJ2aqLE/Tf0= 92 | github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= 93 | github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg= 94 | github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 95 | github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= 96 | github.com/gorilla/mux v0.0.0-20180120075819-c0091a029979/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= 97 | github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= 98 | github.com/gorilla/sessions v0.0.0-20180115173807-fe21b6a095cd/go.mod h1:+WVp8kdw6VhyKExm03PAMRn2ZxnPtm58pV0dBVPdhHE= 99 | github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= 100 | github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= 101 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 102 | github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= 103 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 104 | github.com/jinzhu/gorm v1.9.1/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= 105 | github.com/jinzhu/inflection v0.0.0-20170102125226-1c35d901db3d/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= 106 | github.com/jinzhu/now v0.0.0-20180511015916-ed742868f2ae/go.mod h1:oHTiXerJ20+SfYcrdlBO7rzZRJWGwSTQ0iUY2jI6Gfc= 107 | github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE= 108 | github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= 109 | github.com/joho/godotenv v0.0.0-20180115024921-6bb08516677f h1:lEn+aojk7YXen0nSV6Xsl6KpmIedcArHRyn+PTtPI6I= 110 | github.com/joho/godotenv v0.0.0-20180115024921-6bb08516677f/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= 111 | github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= 112 | github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 113 | github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= 114 | github.com/kelseyhightower/envconfig v0.0.0-20170918161510-462fda1f11d8 h1:OwrhvQ6R+Wr02rrdU8fr9toIvbPLAYxXiNp8xLacqaM= 115 | github.com/kelseyhightower/envconfig v0.0.0-20170918161510-462fda1f11d8/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= 116 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 117 | github.com/klauspost/compress v0.0.0-20180110203047-b88785bfd699/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= 118 | github.com/klauspost/cpuid v0.0.0-20180102081000-ae832f27941a/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= 119 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 120 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 121 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 122 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 123 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 124 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 125 | github.com/levigross/grequests v0.0.0-20180717012718-3f841d606c5a h1:6x67pbxt5Axddz4DTYyzwOlkQF9jtvYRxaVrJlDCWhI= 126 | github.com/levigross/grequests v0.0.0-20180717012718-3f841d606c5a/go.mod h1:uCZIhROSrVmuF/BPYFPwDeiiQ6juSLp0kikFoEcNcEs= 127 | github.com/lib/pq v0.0.0-20180201184707-88edab080323/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 128 | github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 129 | github.com/markbates/goth v0.0.0-20180113214406-24f8ac10e57e/go.mod h1:ERjpUjiHOcJUNTBjgUhpKzkay5qNGcMdjRHYOIpF5Uk= 130 | github.com/mattes/migrate v0.0.0-20171208214826-d23f71b03c4a/go.mod h1:LJcqgpj1jQoxv3m2VXd3drv0suK5CbN/RCX7MXwgnVI= 131 | github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= 132 | github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= 133 | github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI= 134 | github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 135 | github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= 136 | github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= 137 | github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 138 | github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= 139 | github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= 140 | github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= 141 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 142 | github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I= 143 | github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 144 | github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= 145 | github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= 146 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 147 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 148 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 149 | github.com/rs/cors v0.0.0-20170801073201-eabcc6af4bbe/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= 150 | github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= 151 | github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= 152 | github.com/savaki/amplitude-go v0.0.0-20160610055645-f62e3b57c0e4 h1:97Sfylvx7jxNwMDa6m3wRYQ0sCwP8R7ZGneF57jTSP0= 153 | github.com/savaki/amplitude-go v0.0.0-20160610055645-f62e3b57c0e4/go.mod h1:+dADDNKPvI1c6TuMfh+UakKF37l2i4ok/+wpEVT2KII= 154 | github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= 155 | github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= 156 | github.com/shirou/gopsutil v0.0.0-20180801053943-8048a2e9c577 h1:fgCv3khdlkkaSfAehroQ2qpqJaM4eBFl6MhCWOWQNpY= 157 | github.com/shirou/gopsutil v0.0.0-20180801053943-8048a2e9c577/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= 158 | github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= 159 | github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= 160 | github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= 161 | github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= 162 | github.com/sirupsen/logrus v1.0.5 h1:8c8b5uO0zS4X6RPl/sd1ENwSkIc0/H2PaHxE3udaE8I= 163 | github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= 164 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= 165 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= 166 | github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c h1:Ho+uVpkel/udgjbwB5Lktg9BtvJSh2DT0Hi6LPSyI2w= 167 | github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= 168 | github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= 169 | github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= 170 | github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= 171 | github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= 172 | github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 173 | github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= 174 | github.com/stevvooe/resumable v0.0.0-20180830230917-22b14a53ba50/go.mod h1:1pdIZTAHUz+HDKDVZ++5xg/duPlhKAIzw9qy42CWYp4= 175 | github.com/streadway/amqp v0.0.0-20180131094250-fc7fda2371f5 h1:3ljEj+C1EXsFlQO+bPBZ2DxJ+e+x5Wwnk1zFd/ErKuM= 176 | github.com/streadway/amqp v0.0.0-20180131094250-fc7fda2371f5/go.mod h1:1WNBiOZtZQLpVAyu0iTduoJL9hEsMloAK5XWrtW0xdY= 177 | github.com/stretchr/testify v1.2.1 h1:52QO5WkIUcHGIR7EnGagH88x1bUzqGXTC5/1bDTUQ7U= 178 | github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 179 | github.com/stvp/rollbar v0.5.1 h1:qvyWbd0RNL5V27MBumqCXlcU7ohmHeEtKX+Czc8oeuw= 180 | github.com/stvp/rollbar v0.5.1/go.mod h1:/fyFC854GgkbHRz/rSsiYc6h84o0G5hxBezoQqRK7Ho= 181 | github.com/stvp/tempredis v0.0.0-20160122230306-83f7aae7ea49 h1:aBiMfOQ47GCfoTbNcnVyph/dnUuTWN0rSqEKFq3T58Y= 182 | github.com/stvp/tempredis v0.0.0-20160122230306-83f7aae7ea49/go.mod h1:oqN97ltKNihBbwlX8dLpwxCl3+HnXKV/R0e+sRLd9C8= 183 | github.com/urfave/negroni v0.0.0-20180105164225-ff85fb036d90/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= 184 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 185 | github.com/valyala/fasthttp v0.0.0-20171207120941-e5f51c11919d/go.mod h1:+g/po7GqyG5E+1CNgquiIxJnsXEi5vwFn5weFujbO78= 186 | github.com/xeipuuv/gojsonpointer v0.0.0-20170225233418-6fe8760cad35/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= 187 | github.com/xeipuuv/gojsonreference v0.0.0-20150808065054-e02fc20de94c/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= 188 | github.com/xeipuuv/gojsonschema v0.0.0-20171230112544-511d08a359d1/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= 189 | github.com/yalp/jsonpath v0.0.0-20150812003900-31a79c7593bb/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= 190 | github.com/yudai/gojsondiff v0.0.0-20171126075747-e21612694bdd/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= 191 | github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= 192 | github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= 193 | golang.org/x/crypto v0.0.0-20180505025534-4ec37c66abab h1:w4c/LoOA2vE8SYwh8wEEQVRUwpph7TtcjH7AtZvOjy0= 194 | golang.org/x/crypto v0.0.0-20180505025534-4ec37c66abab/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 195 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA= 196 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 197 | golang.org/x/oauth2 v0.0.0-20180118004544-b28fcf2b08a1 h1:gRThnsUxGd2h5EB2AOiqLcAxfMF3Y2LQCeru8REL+p0= 198 | golang.org/x/oauth2 v0.0.0-20180118004544-b28fcf2b08a1/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 199 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 200 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= 201 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 202 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs= 203 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 204 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 205 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 206 | golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 207 | golang.org/x/tools v0.0.0-20180826000951-f6ba57429505/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 208 | golang.org/x/tools v0.0.0-20180831211245-5d4988d199e2/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 209 | google.golang.org/appengine v1.0.0 h1:dN4LljjBKVChsv0XCSI+zbyzdqrkEwX5LQFUMRSGqOc= 210 | google.golang.org/appengine v1.0.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 211 | gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo= 212 | gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= 213 | gopkg.in/boj/redistore.v1 v1.0.0-20160128113310-fc113767cd6b/go.mod h1:fgfIZMlsafAHpspcks2Bul+MWUNw/2dyQmjC2faKjtg= 214 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 215 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 216 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 217 | gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= 218 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 219 | gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0= 220 | gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= 221 | gopkg.in/ini.v1 v1.39.0 h1:Jf2sFGT+sAd7i+4ftUN1Jz90uw8XNH8NXbbOY16taA8= 222 | gopkg.in/ini.v1 v1.39.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 223 | gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528 h1:/saqWwm73dLmuzbNhe92F0QsZ/KiFND+esHco2v1hiY= 224 | gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= 225 | gopkg.in/redsync.v1 v1.0.1 h1:5pQPAP8QgEnCbX09zhG204v9Y4AKXdqvovdUdfhXtCY= 226 | gopkg.in/redsync.v1 v1.0.1/go.mod h1:vJHDHbiLriSzwa/ydqeuTZiOl6CdMPZNbPlsXi9yv4I= 227 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 228 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 229 | gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= 230 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 231 | sourcegraph.com/sourcegraph/go-diff v0.0.0-20171119081133-3f415a150aec/go.mod h1:R09mWeb9JcPbO+A3cYDc11xjz0wp6r9+KnqdqROAoRU= 232 | sourcegraph.com/sqs/pbtypes v0.0.0-20160107090929-4d1b9dc7ffc3/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= 233 | --------------------------------------------------------------------------------