├── .dockerignore ├── docs ├── .gitignore ├── sources │ ├── robots.txt │ ├── images │ │ ├── srclib_symbol.svg │ │ └── srclib_symbol_white.svg │ └── install.md ├── theme │ ├── images │ │ ├── favicon.png │ │ ├── editors │ │ │ ├── atom.png │ │ │ └── sublime.png │ │ ├── sourcegraph-emacs.png │ │ ├── layers.svg │ │ ├── whiteasterisk.svg │ │ ├── asterisk.svg │ │ ├── banner.svg │ │ ├── stack.svg │ │ ├── features.svg │ │ ├── integrations.svg │ │ └── dots.svg │ ├── newsletter.html │ ├── toc.html │ ├── content.html │ ├── newsletter2.html │ ├── base.html │ ├── community.html │ ├── navbar.html │ ├── docsnav.html │ └── layout.html ├── build.sh ├── test.sh ├── mkdocs.yml ├── deploy.sh ├── buildsite.py └── README.md ├── .gitignore ├── TODOs.no-docker.txt ├── ann ├── doc.go ├── gen.go ├── annotation_test.go ├── annotation.go └── ann.proto ├── store ├── misc.go ├── test_util.go ├── pb │ ├── gen.go │ ├── srcstore.proto │ ├── mock │ │ └── srcstore.pb_mock.go │ └── helpers.go ├── repo_store_mock.go ├── tree_store_mock.go ├── log.go ├── unit_store_mock.go ├── counter.go ├── primary_index_test.go ├── indexes_test.go ├── memory_test.go ├── misc_test.go ├── fs_test.go ├── fs_store_mock.go ├── fs_store_test.go ├── repo_paths_test.go ├── indexed_test.go ├── store.go ├── phtable │ ├── slicereader_safe.go │ └── slicereader_fast.go ├── multi_repo_store_mock.go ├── multi_repo_store.go ├── pbio │ ├── io.go │ └── io_test.go ├── units_index.go ├── tree_stores.go ├── repo_paths.go ├── primary_index.go ├── index_cache_test.go ├── index_test.go ├── ref_file_index.go └── index_cache.go ├── unit ├── doc.go ├── gen.go └── source_unit_test.go ├── util ├── makefile_other.go ├── makefile_windows.go ├── makefile.go ├── ancestordirs.go └── homedir.go ├── contrib └── completion │ ├── _srclib │ └── srclib-completion.bash ├── NOTICE ├── graph ├── gen.go ├── errors.go ├── proto_test.go ├── output.proto ├── doc.go ├── tree_path_test.go ├── doc.proto ├── format.go ├── formatter_test.go ├── ref.proto ├── repo_test.go ├── ref.go ├── repo.go └── stats.go ├── TODO.md ├── cli ├── cmdutil.go ├── testdata │ └── strip-code │ │ ├── expected-source.txt │ │ └── input-source.txt ├── build_data.go ├── version_cmd.go ├── repo_cmds.go ├── makefile_cmd.go ├── repo_opt.go ├── doall_cmd.go ├── gen_data_cmd.go ├── info_cmds.go ├── cli.go ├── coverage_test.go ├── misc.go ├── tool_cmd.go ├── util_test.go ├── repo_config.go └── util.go ├── .goxc.json ├── plan ├── data.go ├── util.go ├── makefile_test.go └── makefile.go ├── toolchain ├── bundle_test.go ├── choose_test.go ├── tool.go ├── choose.go ├── find_test.go ├── toolchain.go └── config.go ├── gendata └── common.go ├── config ├── validate.go ├── validate_test.go ├── external.go ├── cached.go └── config.go ├── cmd └── srclib │ └── srclib.go ├── toolref.go ├── grapher ├── registry.go ├── validate_test.go └── grapher.go ├── .travis.yml ├── dep ├── dep.go ├── rule.go ├── resolve.go └── doc.go ├── env.go ├── cvg └── coverage.go ├── DCO ├── Makefile ├── flagutil ├── marshal.go └── marshal_test.go ├── CONTRIBUTING.md └── scan └── scan.go /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | site/ 2 | *.css 3 | *.scssc 4 | -------------------------------------------------------------------------------- /docs/sources/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Allow: / 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /release/ 2 | /.tmp-selfupdate/ 3 | *.test 4 | vendor/*/ -------------------------------------------------------------------------------- /TODOs.no-docker.txt: -------------------------------------------------------------------------------- 1 | * remove dependency on `git`, don't require knowing the commit ID 2 | -------------------------------------------------------------------------------- /ann/doc.go: -------------------------------------------------------------------------------- 1 | // Package ann defines types that represent source code annotations. 2 | package ann 3 | -------------------------------------------------------------------------------- /docs/theme/images/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sourcegraph/srclib/HEAD/docs/theme/images/favicon.png -------------------------------------------------------------------------------- /docs/theme/images/editors/atom.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sourcegraph/srclib/HEAD/docs/theme/images/editors/atom.png -------------------------------------------------------------------------------- /store/misc.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | func min(a, b int) int { 4 | if a < b { 5 | return a 6 | } 7 | return b 8 | } 9 | -------------------------------------------------------------------------------- /docs/theme/images/editors/sublime.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sourcegraph/srclib/HEAD/docs/theme/images/editors/sublime.png -------------------------------------------------------------------------------- /docs/theme/images/sourcegraph-emacs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sourcegraph/srclib/HEAD/docs/theme/images/sourcegraph-emacs.png -------------------------------------------------------------------------------- /unit/doc.go: -------------------------------------------------------------------------------- 1 | // Package unit provides a source unit abstraction over distribution packages in 2 | // various languages. 3 | package unit 4 | -------------------------------------------------------------------------------- /util/makefile_other.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | package util 4 | 5 | func safeCommandName(command string) string { 6 | return command 7 | } 8 | -------------------------------------------------------------------------------- /ann/gen.go: -------------------------------------------------------------------------------- 1 | package ann 2 | 3 | //go:generate gopathexec protoc -I$GOPATH/src -I$GOPATH/src/github.com/gogo/protobuf/protobuf -I. --gogo_out=. ann.proto 4 | -------------------------------------------------------------------------------- /contrib/completion/_srclib: -------------------------------------------------------------------------------- 1 | #compdef srclib 2 | 3 | _srclib() { 4 | compadd "$@" $(GO_FLAGS_COMPLETION=1 $words[0] "$words[@]") 5 | } 6 | 7 | _srclib 8 | -------------------------------------------------------------------------------- /unit/gen.go: -------------------------------------------------------------------------------- 1 | package unit 2 | 3 | //go:generate gopathexec protoc -I$GOPATH/src -I$GOPATH/src/github.com/gogo/protobuf/protobuf -I. --gogo_out=. unit.proto 4 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | srclib Project 2 | Copyright 2014 Sourcegraph, Inc 3 | 4 | This product includes software developed at Sourcegraph, Inc. 5 | (https://sourcegraph.com/). 6 | -------------------------------------------------------------------------------- /graph/gen.go: -------------------------------------------------------------------------------- 1 | package graph 2 | 3 | //go:generate gopathexec protoc -I$GOPATH/src -I$GOPATH/src/github.com/gogo/protobuf/protobuf -I. --gogo_out=. def.proto doc.proto output.proto ref.proto 4 | -------------------------------------------------------------------------------- /graph/errors.go: -------------------------------------------------------------------------------- 1 | package graph 2 | 3 | import "errors" 4 | 5 | var ErrDefNotExist = errors.New("def does not exist") 6 | 7 | func IsNotExist(err error) bool { 8 | return err == ErrDefNotExist 9 | } 10 | -------------------------------------------------------------------------------- /store/test_util.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import "encoding/json" 4 | 5 | func deepEqual(u, v interface{}) bool { 6 | u_, _ := json.Marshal(u) 7 | v_, _ := json.Marshal(v) 8 | return string(u_) == string(v_) 9 | } 10 | -------------------------------------------------------------------------------- /docs/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Create site dir if it does not exist 4 | mkdir -p site 5 | 6 | # Compile the css file 7 | sass theme/styles.scss:theme/styles.css 8 | 9 | # Build the other parts of the site 10 | python buildsite.py 11 | -------------------------------------------------------------------------------- /util/makefile_windows.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | 3 | package util 4 | 5 | import ( 6 | "path/filepath" 7 | "strings" 8 | ) 9 | 10 | func safeCommandName(command string) string { 11 | return strings.Replace(filepath.ToSlash(command), ":", "\\:", 1) 12 | } 13 | -------------------------------------------------------------------------------- /contrib/completion/srclib-completion.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | _src() { 4 | args=("${COMP_WORDS[@]:1:$COMP_CWORD}") 5 | 6 | local IFS=$'\n' 7 | COMPREPLY=($(GO_FLAGS_COMPLETION=1 ${COMP_WORDS[0]} "${args[@]}")) 8 | return 1 9 | } 10 | 11 | complete -F _srclib srclib 12 | -------------------------------------------------------------------------------- /store/pb/gen.go: -------------------------------------------------------------------------------- 1 | package pb 2 | 3 | //go:generate gopathexec protoc -I$GOPATH/src -I$GOPATH/src/github.com/gogo/protobuf/protobuf -I../../graph -I. --gogo_out=plugins=grpc:. srcstore.proto 4 | //go:generate gen-mocks -w -i=.+(Server|Client|Service)$ -o mock -outpkg mock -name_prefix= -no_pass_args=opts 5 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | # TODOs 2 | 3 | ## Simplify build data 4 | 5 | Build data is currently only ever stored on local filesystems, and 6 | it's unlikely that will change (because any virtual/remote access 7 | would be to the store, not to the build data tree). That means it's 8 | probably unnecessary to use a VFS interface to access it. 9 | -------------------------------------------------------------------------------- /store/repo_store_mock.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | type MockRepoStore struct { 4 | Versions_ func(...VersionFilter) ([]*Version, error) 5 | MockTreeStore 6 | } 7 | 8 | func (m MockRepoStore) Versions(f ...VersionFilter) ([]*Version, error) { 9 | return m.Versions_(f...) 10 | } 11 | 12 | var _ RepoStore = MockRepoStore{} 13 | -------------------------------------------------------------------------------- /docs/theme/newsletter.html: -------------------------------------------------------------------------------- 1 |
2 | SIGN UP FOR OUR DEVELOPER NEWSLETTER

3 |
4 | 5 |
6 |
7 | -------------------------------------------------------------------------------- /store/tree_store_mock.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import "sourcegraph.com/sourcegraph/srclib/unit" 4 | 5 | type MockTreeStore struct { 6 | Units_ func(...UnitFilter) ([]*unit.SourceUnit, error) 7 | MockUnitStore 8 | } 9 | 10 | func (m MockTreeStore) Units(f ...UnitFilter) ([]*unit.SourceUnit, error) { 11 | return m.Units_(f...) 12 | } 13 | 14 | var _ TreeStore = MockTreeStore{} 15 | -------------------------------------------------------------------------------- /util/makefile.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | // Makes command name safe for shell script (and for makefiles). 4 | // For example, Cygwin does not like when you trying to execute C:/foo/bar.exe arguments 5 | // so we replacing first : with \: there. On Unix/Darwin there is no need to perform replacements 6 | func SafeCommandName(command string) string { 7 | return safeCommandName(command) 8 | } 9 | -------------------------------------------------------------------------------- /cli/cmdutil.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "log" 5 | 6 | "sourcegraph.com/sourcegraph/go-flags" 7 | ) 8 | 9 | func SetOptionDefaultValue(g *flags.Group, longName string, defaultVal ...string) { 10 | for _, opt := range g.Options() { 11 | if opt.LongName == longName { 12 | opt.Default = defaultVal 13 | return 14 | } 15 | } 16 | log.Fatalf("Failed to set default value %v for option %q (not found).", defaultVal, longName) 17 | } 18 | -------------------------------------------------------------------------------- /store/log.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "io" 5 | "io/ioutil" 6 | "log" 7 | "os" 8 | "strconv" 9 | ) 10 | 11 | func vlogWriter() io.Writer { 12 | if v, _ := strconv.ParseBool(os.Getenv("V")); v { 13 | return os.Stderr 14 | } 15 | return ioutil.Discard 16 | } 17 | 18 | func cyan(s string) string { 19 | return "\x1b[36m" + s + "\x1b[0m" 20 | } 21 | 22 | var vlog = log.New(vlogWriter(), cyan("▶ "), log.Lmicroseconds|log.Lshortfile) 23 | -------------------------------------------------------------------------------- /docs/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | # Create site dir if it does not exist 6 | mkdir -p site 7 | 8 | # Python server 9 | cd site 10 | python -m SimpleHTTPServer & 11 | cd .. 12 | 13 | # Kill python server on exit 14 | trap "exit" INT TERM 15 | trap "kill 0" EXIT 16 | 17 | while true; do 18 | echo "Building site..." 19 | sassc theme/styles.scss theme/styles.css 20 | #mkdocs build 21 | python buildsite.py 22 | 23 | sleep 1 24 | done 25 | -------------------------------------------------------------------------------- /.goxc.json: -------------------------------------------------------------------------------- 1 | { 2 | "AppName": "srclib", 3 | "ArtifactsDest": "release", 4 | "OutPath": "{{.Dest}}{{.PS}}{{.Version}}{{.PS}}{{.Os}}-{{.Arch}}", 5 | "Tasks": [ 6 | "xc" 7 | ], 8 | "Arch": "amd64", 9 | "Os": "linux darwin", 10 | "ConfigVersion": "0.9", 11 | "BuildSettings": { 12 | "LdFlagsXVars": { 13 | "Version": "sourcegraph.com/sourcegraph/srclib/cli.Version" 14 | }, 15 | "ExtraArgs": ["-a"] 16 | }, 17 | "Env": [ 18 | "CGO_ENABLED=0" 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /cli/testdata/strip-code/expected-source.txt: -------------------------------------------------------------------------------- 1 | 2 | package retrofit2http 3 | 4 | import javalangannotationDocumented 5 | import javalangannotationRetention 6 | import javalangannotationTarget 7 | import okhttp3HttpUrl 8 | 9 | import static javalangannotationElementTypeMETHOD 10 | import static javalangannotationRetentionPolicyRUNTIME 11 | 12 | 13 | Documented 14 | TargetMETHOD 15 | RetentionRUNTIME 16 | public interface DELETE 17 | 18 | String value default 19 | 20 | -------------------------------------------------------------------------------- /docs/mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: srclib 2 | site_description: srclib is a hackable, polyglot code analysis library. 3 | 4 | docs_dir: sources 5 | 6 | repo_url: https://sourcegraph.com/sourcegraph/srclib/ 7 | 8 | dev_addr: '0.0.0.0:8000' 9 | 10 | theme_dir: ./theme/ 11 | 12 | google_analytics: ['UA-40540747-5', 'srclib.org'] 13 | 14 | toolchains: 15 | - lang: Go 16 | repo: srclib/srclib-go 17 | - lang: Java 18 | repo: srclib/srclib-java 19 | 20 | pages: 21 | - ['install.md', 'Installation'] 22 | -------------------------------------------------------------------------------- /docs/theme/toc.html: -------------------------------------------------------------------------------- 1 |
2 | 12 | -------------------------------------------------------------------------------- /docs/theme/content.html: -------------------------------------------------------------------------------- 1 |
2 | 3 | Edit 4 | 5 |
6 | 7 | {% if meta.source %} 8 | 13 | {% endif %} 14 | 15 | {{ content }} 16 | 17 | -------------------------------------------------------------------------------- /store/unit_store_mock.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import "sourcegraph.com/sourcegraph/srclib/graph" 4 | 5 | type MockUnitStore struct { 6 | Defs_ func(...DefFilter) ([]*graph.Def, error) 7 | Refs_ func(...RefFilter) ([]*graph.Ref, error) 8 | } 9 | 10 | func (m MockUnitStore) Defs(f ...DefFilter) ([]*graph.Def, error) { 11 | return m.Defs_(f...) 12 | } 13 | 14 | func (m MockUnitStore) Refs(f ...RefFilter) ([]*graph.Ref, error) { 15 | return m.Refs_(f...) 16 | } 17 | 18 | var _ UnitStore = MockUnitStore{} 19 | -------------------------------------------------------------------------------- /cli/build_data.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "sourcegraph.com/sourcegraph/rwvfs" 5 | "sourcegraph.com/sourcegraph/srclib/buildstore" 6 | ) 7 | 8 | func GetBuildDataFS(commitID string) (rwvfs.FileSystem, error) { 9 | lrepo, err := OpenLocalRepo() 10 | if lrepo == nil || lrepo.RootDir == "" || commitID == "" { 11 | return nil, err 12 | } 13 | localStore, err := buildstore.LocalRepo(lrepo.RootDir) 14 | if err != nil { 15 | return nil, err 16 | } 17 | return localStore.Commit(commitID), nil 18 | } 19 | -------------------------------------------------------------------------------- /docs/theme/newsletter2.html: -------------------------------------------------------------------------------- 1 |
2 |
3 | 4 |
5 | 6 | 9 |
10 | -------------------------------------------------------------------------------- /store/counter.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import "sync/atomic" 4 | 5 | // counter is a simple thread-safe integer. 6 | type counter struct { 7 | count *int64 8 | } 9 | 10 | // increment increments the counter by one. 11 | func (c counter) increment() { 12 | atomic.AddInt64(c.count, 1) 13 | } 14 | 15 | // get returns the counter's current value. 16 | func (c *counter) get() int { 17 | return int(atomic.LoadInt64(c.count)) 18 | } 19 | 20 | // set sets the counter value. 21 | func (c *counter) set(i int) { 22 | atomic.StoreInt64(c.count, int64(i)) 23 | } 24 | -------------------------------------------------------------------------------- /store/primary_index_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import "testing" 4 | 5 | func TestDefPathIndex_Covers(t *testing.T) { 6 | x := &defPathIndex{} 7 | c := x.Covers([]DefFilter{ByDefPath("p")}) 8 | if want := 1; c != want { 9 | t.Errorf("got coverage %d, want %d", c, want) 10 | } 11 | 12 | c = x.Covers([]DefFilter{}) 13 | if want := 0; c != want { 14 | t.Errorf("got coverage %d, want %d", c, want) 15 | } 16 | 17 | c = x.Covers([]DefFilter{ByRepos("r")}) 18 | if want := 0; c != want { 19 | t.Errorf("got coverage %d, want %d", c, want) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /plan/data.go: -------------------------------------------------------------------------------- 1 | package plan 2 | 3 | import ( 4 | "path/filepath" 5 | 6 | "sourcegraph.com/sourcegraph/srclib/buildstore" 7 | "sourcegraph.com/sourcegraph/srclib/unit" 8 | ) 9 | 10 | func RepositoryCommitDataFilename(emptyData interface{}) string { 11 | return buildstore.DataTypeSuffix(emptyData) 12 | } 13 | 14 | func SourceUnitDataFilename(emptyData interface{}, u *unit.SourceUnit) string { 15 | if u.Name == "" { 16 | return u.Type + "." + buildstore.DataTypeSuffix(emptyData) 17 | } 18 | return filepath.Join(u.Name, u.Type+"."+buildstore.DataTypeSuffix(emptyData)) 19 | } 20 | -------------------------------------------------------------------------------- /util/ancestordirs.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "path/filepath" 5 | ) 6 | 7 | // AncestorDirs returns a list of p's ancestor 8 | // directories (optionally including itself) excluding the root ("." or "/")). 9 | func AncestorDirs(p string, self bool) []string { 10 | if p == "" { 11 | return nil 12 | } 13 | var dirs []string 14 | dir := filepath.Dir(p) 15 | for dir != "." && dir[len(dir)-1:] != string(filepath.Separator) { 16 | dirs = append([]string{dir}, dirs...) 17 | dir = filepath.Dir(dir) 18 | } 19 | if self { 20 | dirs = append(dirs, p) 21 | } 22 | return dirs 23 | } 24 | -------------------------------------------------------------------------------- /plan/util.go: -------------------------------------------------------------------------------- 1 | package plan 2 | 3 | import "sourcegraph.com/sourcegraph/makex" 4 | 5 | // ruleSort sorts rules by target name, alphabetically. It is used to 6 | // enforce stable ordering so that Makefiles are consistently 7 | // reproducible given the same set of inputs to CreateMakefile. 8 | type ruleSort struct { 9 | Rules []makex.Rule 10 | } 11 | 12 | func (s ruleSort) Len() int { return len(s.Rules) } 13 | func (s ruleSort) Less(i, j int) bool { 14 | return s.Rules[i].Target() < s.Rules[j].Target() 15 | } 16 | func (s ruleSort) Swap(i, j int) { 17 | s.Rules[i], s.Rules[j] = s.Rules[j], s.Rules[i] 18 | } 19 | -------------------------------------------------------------------------------- /store/indexes_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | func TestIndexes(t *testing.T) { 9 | t.Skip("TODO(sqs): add test cases") 10 | 11 | tests := []struct { 12 | store interface{} 13 | want []IndexStatus 14 | }{} 15 | for _, test := range tests { 16 | xs, err := Indexes(test.store, IndexCriteria{}, nil) 17 | if err != nil { 18 | t.Errorf("%s: Indexes: %s", test.store, err) 19 | continue 20 | } 21 | if !reflect.DeepEqual(xs, test.want) { 22 | t.Errorf("%s: got index statuses %v, want %v", test.store, xs, test.want) 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /unit/source_unit_test.go: -------------------------------------------------------------------------------- 1 | package unit 2 | 3 | import "testing" 4 | 5 | func TestParseID(t *testing.T) { 6 | tests := []struct { 7 | input string 8 | wantName, wantType string 9 | }{ 10 | {"a@b", "a", "b"}, 11 | {"a%2Fb@b", "a/b", "b"}, 12 | } 13 | for _, test := range tests { 14 | name, typ, err := ParseID(test.input) 15 | if err != nil { 16 | t.Fatal(err) 17 | } 18 | if name != test.wantName { 19 | t.Errorf("got name %q, want %q", name, test.wantName) 20 | } 21 | if typ != test.wantType { 22 | t.Errorf("got type %q, want %q", typ, test.wantType) 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /docs/theme/base.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | 3 | {% block title %}{{ page_title }}{% endblock %} 4 | 5 | {% block content %} 6 | {% include "docsnav.html" %} 7 | 8 |
9 |
10 | {% if meta.no_toc %} 11 |
{% include "content.html" %}
12 | {% else %} 13 |
{% include "toc.html" %}
14 |
{% include "content.html" %}
15 | {% endif %} 16 |
17 |
{% include "newsletter.html" %} 18 |
19 | 20 | 21 | 22 | 23 | 24 | {% endblock %} 25 | -------------------------------------------------------------------------------- /docs/sources/images/srclib_symbol.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 8 | 9 | -------------------------------------------------------------------------------- /docs/sources/images/srclib_symbol_white.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 8 | 9 | -------------------------------------------------------------------------------- /store/memory_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import "testing" 4 | 5 | func TestMemoryUnitStore(t *testing.T) { 6 | testUnitStore(t, func() UnitStoreImporter { 7 | return &memoryUnitStore{} 8 | }) 9 | } 10 | 11 | func TestMemoryTreeStore(t *testing.T) { 12 | testTreeStore(t, func() TreeStoreImporter { 13 | return newMemoryTreeStore() 14 | }) 15 | } 16 | 17 | func TestMemoryRepoStore(t *testing.T) { 18 | testRepoStore(t, func() RepoStoreImporter { 19 | return newMemoryRepoStore() 20 | }) 21 | } 22 | 23 | func TestMemoryMultiRepoStore(t *testing.T) { 24 | testMultiRepoStore(t, func() MultiRepoStoreImporter { 25 | return newMemoryMultiRepoStore() 26 | }) 27 | } 28 | -------------------------------------------------------------------------------- /util/homedir.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "os" 5 | "os/user" 6 | "runtime" 7 | ) 8 | 9 | // CurrentUserHomeDir tries to get the current user's home directory in a 10 | // cross-platform manner. 11 | func CurrentUserHomeDir() string { 12 | user, err := user.Current() 13 | if err == nil && user.HomeDir != "" { 14 | return user.HomeDir 15 | } 16 | 17 | // from http://stackoverflow.com/questions/7922270/obtain-users-home-directory 18 | if runtime.GOOS == "windows" { 19 | home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH") 20 | if home == "" { 21 | home = os.Getenv("USERPROFILE") 22 | } 23 | return home 24 | } 25 | return os.Getenv("HOME") 26 | } 27 | -------------------------------------------------------------------------------- /store/misc_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "path/filepath" 5 | "reflect" 6 | "testing" 7 | 8 | "sourcegraph.com/sourcegraph/srclib/util" 9 | ) 10 | 11 | func TestAncestorDirsExceptRoot(t *testing.T) { 12 | tests := map[string][]string{ 13 | ".": nil, 14 | "/": nil, 15 | "": nil, 16 | "a": nil, 17 | "a/": []string{"a"}, // maybe we don't want this behavior 18 | "a/b": []string{"a"}, 19 | "a/b/c": []string{"a", filepath.FromSlash("a/b")}, 20 | } 21 | for p, want := range tests { 22 | dirs := util.AncestorDirs(p, false) 23 | if !reflect.DeepEqual(dirs, want) { 24 | t.Errorf("%v: got %v, want %v", p, dirs, want) 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /graph/proto_test.go: -------------------------------------------------------------------------------- 1 | package graph 2 | 3 | import ( 4 | "log" 5 | "reflect" 6 | "testing" 7 | 8 | "sourcegraph.com/sourcegraph/srclib/ann" 9 | 10 | "github.com/gogo/protobuf/proto" 11 | ) 12 | 13 | func TestProtobufMarshal(t *testing.T) { 14 | o := Output{ 15 | Defs: []*Def{{File: "f1"}}, 16 | Refs: []*Ref{{File: "f2"}}, 17 | Docs: []*Doc{{File: "f3"}}, 18 | Anns: []*ann.Ann{{Unit: "foo"}}, 19 | } 20 | 21 | b, err := proto.Marshal(&o) 22 | if err != nil { 23 | t.Fatal(err) 24 | } 25 | 26 | var o2 Output 27 | if err := proto.Unmarshal(b, &o2); err != nil { 28 | log.Fatal(err) 29 | } 30 | 31 | if !reflect.DeepEqual(o2, o) { 32 | t.Errorf("got %#v, want %#v", o2, o) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /docs/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | # Create site dir if it does not exist 6 | mkdir -p site 7 | 8 | # Compile the css file 9 | sassc theme/styles.scss theme/styles.css 10 | 11 | # Build the other parts of the site 12 | python buildsite.py 13 | 14 | # Sync site with S3 bucket 15 | aws s3 sync site/ s3://srclib.org/ 16 | 17 | echo < 8 |
9 | get invited to the srclib Slack team 10 |
11 | srclib on GitHub 12 |
13 | srclib-dev mailing list 14 | 15 |
16 | Local meetups in San Francisco 17 | 18 | 20 |
21 | 22 | 23 | {% endblock %} 24 | -------------------------------------------------------------------------------- /cli/version_cmd.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "log" 5 | 6 | "github.com/alexsaveliev/go-colorable-wrapper" 7 | "sourcegraph.com/sourcegraph/go-flags" 8 | ) 9 | 10 | // Version of srclib. 11 | // 12 | // For releases, this is set using the -X flag to `go tool ld`. See 13 | // http://stackoverflow.com/a/11355611. 14 | var Version = "dev" 15 | 16 | func init() { 17 | cliInit = append(cliInit, func(cli *flags.Command) { 18 | _, err := cli.AddCommand("version", 19 | "show version", 20 | "The version subcommand displays the current version of this srclib program.", 21 | &versionCmd{}, 22 | ) 23 | if err != nil { 24 | log.Fatal(err) 25 | } 26 | }) 27 | } 28 | 29 | type versionCmd struct{} 30 | 31 | func (v *versionCmd) Execute(_ []string) error { 32 | colorable.Println(Version) 33 | return nil 34 | } 35 | -------------------------------------------------------------------------------- /cli/repo_cmds.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "log" 5 | 6 | "github.com/alexsaveliev/go-colorable-wrapper" 7 | "sourcegraph.com/sourcegraph/go-flags" 8 | ) 9 | 10 | func init() { 11 | cliInit = append(cliInit, func(cli *flags.Command) { 12 | _, err := cli.AddCommand("repo", 13 | "display current repo info", 14 | "The repo subcommand displays autodetected info about the current repo.", 15 | &repoCmd{}, 16 | ) 17 | if err != nil { 18 | log.Fatal(err) 19 | } 20 | }) 21 | } 22 | 23 | type repoCmd struct{} 24 | 25 | func (c *repoCmd) Execute(args []string) error { 26 | repo, err := OpenRepo(".") 27 | if err != nil { 28 | return err 29 | } 30 | colorable.Println("# Current repository:") 31 | colorable.Println("VCS:", repo.VCSType) 32 | colorable.Println("Root dir:", repo.RootDir) 33 | colorable.Println("Commit ID:", repo.CommitID) 34 | return nil 35 | } 36 | -------------------------------------------------------------------------------- /docs/buildsite.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import jinja2 4 | import os 5 | 6 | import mkdocs.build 7 | from mkdocs.build import build 8 | from mkdocs.config import load_config 9 | 10 | if __name__ == "__main__": 11 | # Build documentation 12 | config = load_config(options=None) 13 | build(config) 14 | 15 | # Load templates 16 | template_env = jinja2.Environment(loader = jinja2.FileSystemLoader(os.path.join(os.path.dirname(__file__), 'theme'))) 17 | index_template = template_env.get_template('home.html') 18 | community_template = template_env.get_template('community.html') 19 | 20 | # Home page 21 | with open('site/index.html', 'w') as f: 22 | f.write(index_template.render( 23 | page="home" 24 | )) 25 | 26 | # Community page 27 | with open('site/community.html', 'w') as f: 28 | f.write(community_template.render( 29 | page="community" 30 | )) 31 | -------------------------------------------------------------------------------- /cmd/srclib/srclib.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "runtime/pprof" 8 | "strings" 9 | 10 | "sourcegraph.com/sourcegraph/go-flags" 11 | 12 | "sourcegraph.com/sourcegraph/srclib/cli" 13 | _ "sourcegraph.com/sourcegraph/srclib/dep" 14 | _ "sourcegraph.com/sourcegraph/srclib/scan" 15 | ) 16 | 17 | func main() { 18 | if cpuprof := os.Getenv("CPUPROF"); cpuprof != "" { 19 | f, err := os.Create(cpuprof) 20 | if err != nil { 21 | log.Fatal("CPUPROF:", err) 22 | } 23 | if err := pprof.StartCPUProfile(f); err != nil { 24 | log.Fatal("StartCPUProfile:", err) 25 | } 26 | defer func() { 27 | pprof.StopCPUProfile() 28 | f.Close() 29 | }() 30 | } 31 | 32 | if err := cli.Main(); err != nil { 33 | if _, ok := err.(*flags.Error); !ok { 34 | fmt.Fprintf(os.Stderr, "FAILED: %s (%s)\n", strings.Join(os.Args, " "), err) 35 | } 36 | os.Exit(1) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /toolchain/choose_test.go: -------------------------------------------------------------------------------- 1 | package toolchain 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "sourcegraph.com/sourcegraph/srclib" 8 | ) 9 | 10 | func TestChooseTool(t *testing.T) { 11 | // TODO(sqs): implement this test 12 | tests := []struct { 13 | toolchains []*Info 14 | op, unitType string 15 | want *srclib.ToolRef 16 | wantErr error 17 | }{} 18 | for _, test := range tests { 19 | tool, err := chooseTool(test.op, test.unitType, test.toolchains) 20 | if err != nil { 21 | if test.wantErr == nil { 22 | t.Errorf("got error %q, want no error", err) 23 | continue 24 | } 25 | if test.wantErr != nil && err.Error() != test.wantErr.Error() { 26 | t.Errorf("got error %q, want %q", err, test.wantErr) 27 | continue 28 | } 29 | } 30 | 31 | if !reflect.DeepEqual(tool, test.want) { 32 | t.Errorf("got tool %+v, want %+v", tool, test.want) 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /cli/makefile_cmd.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "log" 5 | "os" 6 | 7 | "sourcegraph.com/sourcegraph/go-flags" 8 | 9 | "sourcegraph.com/sourcegraph/makex" 10 | ) 11 | 12 | func init() { 13 | cliInit = append(cliInit, func(cli *flags.Command) { 14 | _, err := cli.AddCommand("makefile", 15 | "prints the Makefile that the `make` subcommand executes", 16 | "The makefile command prints the Makefile that the `make` subcommand will execute.", 17 | &makefileCmd, 18 | ) 19 | if err != nil { 20 | log.Fatal(err) 21 | } 22 | }) 23 | } 24 | 25 | type MakefileCmd struct{} 26 | 27 | var makefileCmd MakefileCmd 28 | 29 | func (c *MakefileCmd) Execute(args []string) error { 30 | mf, err := CreateMakefile() 31 | if err != nil { 32 | return err 33 | } 34 | 35 | mfData, err := makex.Marshal(mf) 36 | if err != nil { 37 | return err 38 | } 39 | _, err = os.Stdout.Write(mfData) 40 | return err 41 | } 42 | -------------------------------------------------------------------------------- /toolref.go: -------------------------------------------------------------------------------- 1 | package srclib 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strings" 7 | ) 8 | 9 | // A ToolRef identifies a tool inside a specific toolchain. It can be used to 10 | // look up the tool. 11 | type ToolRef struct { 12 | // Toolchain is the toolchain path of the toolchain that contains this tool. 13 | Toolchain string 14 | 15 | // Subcmd is the name of the toolchain subcommand that runs this tool. 16 | Subcmd string 17 | } 18 | 19 | func (t ToolRef) String() string { return fmt.Sprintf("%s %s", t.Toolchain, t.Subcmd) } 20 | 21 | func (t *ToolRef) UnmarshalFlag(value string) error { 22 | parts := strings.Split(value, ":") 23 | if len(parts) != 2 { 24 | return errors.New("expected format 'TOOLCHAIN:TOOL' (separated by 1 colon)") 25 | } 26 | t.Toolchain = parts[0] 27 | t.Subcmd = parts[1] 28 | return nil 29 | } 30 | 31 | func (t ToolRef) MarshalFlag() (string, error) { 32 | return t.Toolchain + ":" + t.Subcmd, nil 33 | } 34 | -------------------------------------------------------------------------------- /grapher/registry.go: -------------------------------------------------------------------------------- 1 | // TODO(sqs): remove this file 2 | package grapher 3 | 4 | import ( 5 | "reflect" 6 | 7 | "sourcegraph.com/sourcegraph/srclib/unit" 8 | ) 9 | 10 | // Graphers holds all registered graphers. 11 | var Graphers = make(map[reflect.Type]Grapher) 12 | 13 | // Register sets the grapher to be used for source units of the given type. 14 | // If Register is called twice with the same name or if grapher is nil, it 15 | // panics 16 | func Register(emptySourceUnit unit.SourceUnit, grapher Grapher) { 17 | typ := ptrTo(emptySourceUnit) 18 | if _, dup := Graphers[typ]; dup { 19 | panic("grapher: Register called twice for source unit type " + typ.String()) 20 | } 21 | if grapher == nil { 22 | panic("grapher: Register grapher is nil") 23 | } 24 | Graphers[typ] = grapher 25 | } 26 | 27 | func ptrTo(v interface{}) reflect.Type { 28 | typ := reflect.TypeOf(v) 29 | if typ.Kind() != reflect.Ptr { 30 | typ = reflect.PtrTo(typ) 31 | } 32 | return typ 33 | } 34 | -------------------------------------------------------------------------------- /grapher/validate_test.go: -------------------------------------------------------------------------------- 1 | package grapher 2 | 3 | import ( 4 | "testing" 5 | 6 | "sourcegraph.com/sourcegraph/srclib/graph" 7 | ) 8 | 9 | func TestValidateDocs_ok(t *testing.T) { 10 | docs := []*graph.Doc{ 11 | { 12 | DefKey: graph.DefKey{Path: "p"}, 13 | Format: "f", 14 | Data: "d", 15 | }, 16 | { 17 | DefKey: graph.DefKey{Path: "p"}, 18 | Format: "f2", 19 | Data: "d", 20 | }, 21 | { 22 | DefKey: graph.DefKey{Path: "p2"}, 23 | Format: "f", 24 | Data: "d", 25 | }, 26 | } 27 | if err := ValidateDocs(docs); err != nil { 28 | t.Fatal(err) 29 | } 30 | } 31 | 32 | func TestValidateDocs_dup(t *testing.T) { 33 | docs := []*graph.Doc{ 34 | { 35 | DefKey: graph.DefKey{Path: "p"}, 36 | Format: "f", 37 | Data: "d", 38 | }, 39 | { 40 | DefKey: graph.DefKey{Path: "p"}, 41 | Format: "f", 42 | Data: "d", 43 | }, 44 | } 45 | if err := ValidateDocs(docs); err == nil { 46 | t.Fatalf("got nil err, want validation error") 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /graph/doc.go: -------------------------------------------------------------------------------- 1 | package graph 2 | 3 | import "encoding/json" 4 | 5 | // Key returns the unique key for the doc. 6 | func (d *Doc) Key() DocKey { 7 | return DocKey{DefKey: d.DefKey, Format: d.Format, Start: d.Start, File: d.File} 8 | } 9 | 10 | // DocKey is the unique key for a doc. Each doc within a source unit 11 | // must have a unique DocKey. 12 | // 13 | // Freestanding comments will not have an associated DefKey, but they 14 | // *must* provide 'Start' and 'File'. 15 | type DocKey struct { 16 | DefKey 17 | Format string 18 | File string 19 | Start uint32 20 | } 21 | 22 | func (d DocKey) String() string { 23 | b, _ := json.Marshal(d) 24 | return string(b) 25 | } 26 | 27 | func (d *Doc) sortKey() string { return d.Key().String() } 28 | 29 | // Sorting 30 | 31 | type Docs []*Doc 32 | 33 | func (vs Docs) Len() int { return len(vs) } 34 | func (vs Docs) Swap(i, j int) { vs[i], vs[j] = vs[j], vs[i] } 35 | func (vs Docs) Less(i, j int) bool { return vs[i].sortKey() < vs[j].sortKey() } 36 | -------------------------------------------------------------------------------- /config/validate_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "path/filepath" 5 | "runtime" 6 | "testing" 7 | 8 | "sourcegraph.com/sourcegraph/srclib/unit" 9 | ) 10 | 11 | func TestTree_validate(t *testing.T) { 12 | var absPath string 13 | if runtime.GOOS == "windows" { 14 | absPath = "C:\\foo" 15 | } else { 16 | absPath = "/foo" 17 | } 18 | 19 | tests := map[string]*Tree{ 20 | "absolute path": &Tree{SourceUnits: []*unit.SourceUnit{{Info: unit.Info{Files: []string{absPath}}}}}, 21 | "relative path above root": &Tree{SourceUnits: []*unit.SourceUnit{{Info: unit.Info{Files: []string{filepath.Join("..", "foo")}}}}}, 22 | "bad path after being cleaned": &Tree{SourceUnits: []*unit.SourceUnit{{Info: unit.Info{Files: []string{filepath.Join("foo", "bar", "..", "..", "..", "..", "baz")}}}}}, 23 | } 24 | 25 | for label, tree := range tests { 26 | if err := tree.validate(); err != ErrInvalidFilePath { 27 | t.Errorf("%s: got err %v, want ErrInvalidFilePath", label, err) 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /docs/theme/navbar.html: -------------------------------------------------------------------------------- 1 | 22 | -------------------------------------------------------------------------------- /docs/theme/docsnav.html: -------------------------------------------------------------------------------- 1 | 25 | -------------------------------------------------------------------------------- /store/fs_store_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import "testing" 4 | 5 | func TestFSUnitStore(t *testing.T) { 6 | useIndexedStore = false 7 | testUnitStore(t, func() UnitStoreImporter { 8 | return &fsUnitStore{fs: newTestFS()} 9 | }) 10 | } 11 | 12 | func TestFSTreeStore(t *testing.T) { 13 | useIndexedStore = false 14 | testTreeStore(t, func() TreeStoreImporter { 15 | return newFSTreeStore(newTestFS()) 16 | }) 17 | } 18 | 19 | func TestFSRepoStore(t *testing.T) { 20 | useIndexedStore = false 21 | testRepoStore(t, func() RepoStoreImporter { 22 | return NewFSRepoStore(newTestFS()) 23 | }) 24 | } 25 | 26 | func TestFSMultiRepoStore(t *testing.T) { 27 | useIndexedStore = false 28 | testMultiRepoStore(t, func() MultiRepoStoreImporter { 29 | return NewFSMultiRepoStore(newTestFS(), nil) 30 | }) 31 | } 32 | 33 | func TestFSMultiRepoStore_customRepoPaths(t *testing.T) { 34 | useIndexedStore = false 35 | testMultiRepoStore(t, func() MultiRepoStoreImporter { 36 | return NewFSMultiRepoStore(newTestFS(), &FSMultiRepoStoreConf{RepoPaths: &customRepoPaths{}}) 37 | }) 38 | } 39 | -------------------------------------------------------------------------------- /store/repo_paths_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "encoding/hex" 5 | "fmt" 6 | 7 | "sourcegraph.com/sourcegraph/rwvfs" 8 | ) 9 | 10 | type customRepoPaths struct{} 11 | 12 | // RepoToPath implements RepoPaths. 13 | func (e *customRepoPaths) RepoToPath(repo string) []string { 14 | h := make([]byte, hex.EncodedLen(len(repo))) 15 | hex.Encode(h, []byte(repo)) 16 | return []string{string(h)} 17 | } 18 | 19 | // PathToRepo implements RepoPaths. 20 | func (e *customRepoPaths) PathToRepo(path []string) string { 21 | c, err := hex.DecodeString(path[0]) 22 | if err != nil { 23 | panic(fmt.Sprintf("hex-decoding path %q: %s", path, err)) 24 | } 25 | return string(c) 26 | } 27 | 28 | // ListRepoPaths implements RepoPaths. 29 | func (e *customRepoPaths) ListRepoPaths(vfs rwvfs.WalkableFileSystem, after string, max int) ([][]string, error) { 30 | entries, err := vfs.ReadDir(".") 31 | if err != nil { 32 | return nil, err 33 | } 34 | paths := make([][]string, len(entries)) 35 | for i, e := range entries { 36 | paths[i] = []string{e.Name()} 37 | } 38 | return paths, nil 39 | } 40 | -------------------------------------------------------------------------------- /cli/repo_opt.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import "sourcegraph.com/sourcegraph/go-flags" 4 | 5 | var ( 6 | localRepo *Repo 7 | localRepoErr error 8 | 9 | // CacheLocalRepo controls whether OpenLocalRepo caches the 10 | // results of OpenRepo the first time it runs and returns the same 11 | // repo for all subsequent calls (even if you call os.Chdir, for 12 | // example). 13 | CacheLocalRepo = true 14 | ) 15 | 16 | // OpenLocalRepo opens the VCS repository in or above the current 17 | // directory. 18 | func OpenLocalRepo() (*Repo, error) { 19 | if !CacheLocalRepo { 20 | return OpenRepo(".") 21 | } 22 | 23 | // Only try to open the current-dir repo once (we'd get the same result each 24 | // time, since we never modify it). 25 | if localRepo == nil && localRepoErr == nil { 26 | localRepo, localRepoErr = OpenRepo(".") 27 | } 28 | return localRepo, localRepoErr 29 | } 30 | 31 | func SetDefaultCommitIDOpt(c *flags.Command) { 32 | OpenLocalRepo() 33 | if localRepo != nil { 34 | if localRepo.CommitID != "" { 35 | SetOptionDefaultValue(c.Group, "commit", localRepo.CommitID) 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /cli/doall_cmd.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "log" 5 | "os" 6 | 7 | "sourcegraph.com/sourcegraph/go-flags" 8 | ) 9 | 10 | func init() { 11 | cliInit = append(cliInit, func(cli *flags.Command) { 12 | // TODO(sqs): "do-all" is a stupid name 13 | _, err := cli.AddCommand("do-all", 14 | "fully process (config, plan, execute, and import)", 15 | `Fully processes a tree: configures it, plans the execution, executes all analysis steps, and imports the data.`, 16 | &doAllCmd, 17 | ) 18 | if err != nil { 19 | log.Fatal(err) 20 | } 21 | }) 22 | } 23 | 24 | type DoAllCmd struct { 25 | Dir Directory `short:"C" long:"directory" description:"change to DIR before doing anything" value-name:"DIR"` 26 | } 27 | 28 | var doAllCmd DoAllCmd 29 | 30 | func (c *DoAllCmd) Execute(args []string) error { 31 | if c.Dir != "" { 32 | if err := os.Chdir(c.Dir.String()); err != nil { 33 | return err 34 | } 35 | } 36 | 37 | // config 38 | configCmd := &ConfigCmd{} 39 | if err := configCmd.Execute(nil); err != nil { 40 | return err 41 | } 42 | 43 | // make 44 | makeCmd := &MakeCmd{} 45 | if err := makeCmd.Execute(nil); err != nil { 46 | return err 47 | } 48 | 49 | return nil 50 | } 51 | -------------------------------------------------------------------------------- /graph/tree_path_test.go: -------------------------------------------------------------------------------- 1 | package graph 2 | 3 | import "testing" 4 | 5 | func Test_TreePath(t *testing.T) { 6 | treePaths := []string{ 7 | "foo", 8 | "foo/bar", 9 | "foo/-/bar", 10 | 11 | "commonjs/lib/async.js", 12 | "commonjs/lib/async.js/-/all", 13 | "commonjs/test/test-async.js/-/all alias", 14 | "commonjs/test/test-async.js/-/queue", 15 | "commonjs/test/test-async.js/-/queue too many callbacks", 16 | "file/lib/async.js/-/@231/@local/@231/@local/@1011/@local/arr/", 17 | "file/lib/async.js/-/@231/@local/@231/@local/@22540/@local/_insert", 18 | "global/-/console.", 19 | "global/-/Function.prototype.bind", 20 | "global/-/process.nextTick", 21 | 22 | "flask", 23 | "flask/app", 24 | "flask/app/Flask", 25 | "flask/app/Flask/add_template_filter", 26 | 27 | ".", 28 | "./foo/bar", 29 | } 30 | notTreePaths := []string{"", "/", "///", "foo//bar", "/foo/bar"} 31 | 32 | for _, tp := range treePaths { 33 | if !IsValidTreePath(tp) { 34 | t.Errorf("%s should be valid, but was invalid", tp) 35 | } 36 | } 37 | 38 | for _, ntp := range notTreePaths { 39 | if IsValidTreePath(ntp) { 40 | t.Errorf("%s should be invalid, but was valid", ntp) 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 1.6 3 | sudo: required 4 | 5 | before_install: 6 | - mkdir -p $HOME/gopath/src/sourcegraph.com/sourcegraph 7 | - mv $TRAVIS_BUILD_DIR $HOME/gopath/src/sourcegraph.com/sourcegraph/srclib 8 | - export TRAVIS_BUILD_DIR=$HOME/gopath/src/sourcegraph.com/sourcegraph/srclib 9 | - export PATH=$PATH:$HOME/gopath/bin 10 | - export GOBIN=$HOME/gopath/bin 11 | # Ruby 12 | - curl -L https://get.rvm.io | bash -s stable --ruby 13 | - rvm reload 14 | - rvm install 2.2.2 15 | - rvm use 2.2.2 16 | - gem update bundler 17 | # JavaScript and TypeScript 18 | - curl -sL https://deb.nodesource.com/setup_6.x | sudo -E bash - 19 | # Java and Basic 20 | - sudo add-apt-repository -y ppa:openjdk-r/ppa 21 | - sudo add-apt-repository -y ppa:cwchien/gradle 22 | # Python 23 | - sudo add-apt-repository -y ppa:fkrull/deadsnakes 24 | - sudo apt-get update 25 | # Java, Basic, JavaScript, TypeScript, and Python 26 | - sudo apt-get install -y openjdk-8-jdk gradle-2.13 nodejs python3.5 27 | - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ 28 | # TypeScript 29 | - npm install -g typescript 30 | 31 | install: 32 | - make 33 | 34 | script: 35 | - make test 36 | # TODO: add csharp 37 | - srclib toolchain install go java javascript typescript basic python ruby css 38 | -------------------------------------------------------------------------------- /cli/gen_data_cmd.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "log" 5 | 6 | "sourcegraph.com/sourcegraph/go-flags" 7 | 8 | "sourcegraph.com/sourcegraph/srclib/gendata" 9 | ) 10 | 11 | func init() { 12 | cliInit = append(cliInit, func(cli *flags.Command) { 13 | c, err := cli.AddCommand("gen-data", 14 | "generates fake data", 15 | `generates fake data for testing and benchmarking purposes. Run this command inside an empty or expendable directory.`, 16 | &gendata.GenDataCmd{}, 17 | ) 18 | if err != nil { 19 | log.Fatal(err) 20 | } 21 | c.Aliases = []string{"c"} 22 | 23 | _, err = c.AddCommand("simple", 24 | "generates a simple repository", 25 | "generates a simple repository with the specified source unit and file structure, with the given number of defs and refs to those defs in each file", 26 | &gendata.SimpleRepoCmd{}, 27 | ) 28 | if err != nil { 29 | log.Fatal(err) 30 | } 31 | 32 | _, err = c.AddCommand("urefs", 33 | "generates a repository with cross-unit references", 34 | "generates a repository with cross-unit references with the given unit and file structure, the number of defs in each file, and refs from each source unit to each def", 35 | &gendata.URefsRepoCmd{}, 36 | ) 37 | if err != nil { 38 | log.Fatal(err) 39 | } 40 | }) 41 | } 42 | -------------------------------------------------------------------------------- /cli/info_cmds.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "log" 5 | 6 | "sourcegraph.com/sourcegraph/go-flags" 7 | 8 | "sourcegraph.com/sourcegraph/srclib" 9 | "sourcegraph.com/sourcegraph/srclib/buildstore" 10 | "sourcegraph.com/sourcegraph/srclib/plan" 11 | ) 12 | 13 | func init() { 14 | cliInit = append(cliInit, func(cli *flags.Command) { 15 | _, err := cli.AddCommand("info", 16 | "show info about enabled capabilities", 17 | "Shows information about enabled capabilities in this tool as well as system information.", 18 | &infoCmd, 19 | ) 20 | if err != nil { 21 | log.Fatal(err) 22 | } 23 | }) 24 | } 25 | 26 | type InfoCmd struct{} 27 | 28 | var infoCmd InfoCmd 29 | 30 | func (c *InfoCmd) Execute(args []string) error { 31 | log.Printf("srclib v%s\n", Version) 32 | log.Println("https://sourcegraph.com/sourcegraph/srclib") 33 | log.Println() 34 | 35 | log.Printf("SRCLIBPATH=%q", srclib.Path) 36 | 37 | log.Println() 38 | log.Printf("Build data types (%d)", len(buildstore.DataTypes)) 39 | for name, _ := range buildstore.DataTypes { 40 | log.Printf(" - %s", name) 41 | } 42 | log.Println() 43 | 44 | log.Printf("Build rule makers (%d)", len(plan.RuleMakers)) 45 | for name, _ := range plan.RuleMakers { 46 | log.Printf(" - %s", name) 47 | } 48 | 49 | return nil 50 | } 51 | -------------------------------------------------------------------------------- /dep/dep.go: -------------------------------------------------------------------------------- 1 | package dep 2 | 3 | type ResolvedDep struct { 4 | // FromRepo is the repository from which this dependency originates. 5 | FromRepo string `json:",omitempty"` 6 | 7 | // FromCommitID is the VCS commit in the repository that this dep was found 8 | // in. 9 | FromCommitID string `json:",omitempty"` 10 | 11 | // FromUnit is the source unit name from which this dependency originates. 12 | FromUnit string 13 | 14 | // FromUnitType is the source unit type from which this dependency originates. 15 | FromUnitType string 16 | 17 | // ToRepo is the repository containing the source unit that is depended on. 18 | // 19 | // TODO(sqs): include repo clone URLs as well, so we can add new 20 | // repositories from seen deps. 21 | ToRepo string 22 | 23 | // ToUnit is the name of the source unit that is depended on. 24 | ToUnit string 25 | 26 | // ToUnitType is the type of the source unit that is depended on. 27 | ToUnitType string 28 | 29 | // ToVersion is the version of the dependent repository (if known), 30 | // according to whatever version string specifier is used by FromRepo's 31 | // dependency management system. 32 | ToVersionString string 33 | 34 | // ToRevSpec specifies the desired VCS revision of the dependent repository 35 | // (if known). 36 | ToRevSpec string 37 | } 38 | -------------------------------------------------------------------------------- /cli/cli.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "log" 5 | 6 | "github.com/alexsaveliev/go-colorable-wrapper" 7 | 8 | "sourcegraph.com/sourcegraph/go-flags" 9 | "sourcegraph.com/sourcegraph/srclib" 10 | ) 11 | 12 | var cliInit []func(*flags.Command) 13 | 14 | // AddCommands adds all of the "srclib ..." subcommands (toolchain, 15 | // tool, make, etc.) as subcommands on c. 16 | // 17 | // It is used to create the "srclib" command, but it can also be used 18 | // to mount the srclib CLI underneath any other CLI tool's command, 19 | // such as "mytool srclib ...". 20 | func AddCommands(c *flags.Command) { 21 | for _, f := range cliInit { 22 | f(c) 23 | } 24 | } 25 | 26 | // GlobalOpt contains global options. 27 | var GlobalOpt struct { 28 | Verbose bool `short:"v" description:"show verbose output"` 29 | } 30 | 31 | func Main() error { 32 | log.SetFlags(0) 33 | log.SetPrefix("") 34 | log.SetOutput(colorable.Stderr) 35 | 36 | cli := flags.NewNamedParser(srclib.CommandName, flags.Default ^ flags.PrintErrors) 37 | cli.LongDescription = "srclib builds projects, analyzes source code, and queries Sourcegraph." 38 | cli.AddGroup("Global options", "", &GlobalOpt) 39 | AddCommands(cli.Command) 40 | 41 | _, err := cli.Parse() 42 | if err != nil { 43 | colorable.Println(err) 44 | } 45 | return err 46 | } 47 | -------------------------------------------------------------------------------- /store/indexed_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import "testing" 4 | 5 | func TestIndexedUnitStore(t *testing.T) { 6 | useIndexedStore = true 7 | testUnitStore(t, func() UnitStoreImporter { 8 | return newIndexedUnitStore(newTestFS(), "") 9 | }) 10 | } 11 | 12 | func TestIndexedTreeStore(t *testing.T) { 13 | useIndexedStore = true 14 | testTreeStore(t, func() TreeStoreImporter { 15 | return newIndexedTreeStore(newTestFS(), "test") 16 | }) 17 | } 18 | 19 | func TestIndexedFSTreeStore(t *testing.T) { 20 | useIndexedStore = true 21 | testTreeStore(t, func() TreeStoreImporter { 22 | return newFSTreeStore(newTestFS()) 23 | }) 24 | } 25 | 26 | func TestIndexedFSRepoStore(t *testing.T) { 27 | useIndexedStore = true 28 | testRepoStore(t, func() RepoStoreImporter { 29 | return NewFSRepoStore(newTestFS()) 30 | }) 31 | } 32 | 33 | func TestIndexedFSMultiRepoStore(t *testing.T) { 34 | useIndexedStore = true 35 | testMultiRepoStore(t, func() MultiRepoStoreImporter { 36 | return NewFSMultiRepoStore(newTestFS(), nil) 37 | }) 38 | } 39 | 40 | func TestIndexedFSMultiRepoStore_customRepoPaths(t *testing.T) { 41 | useIndexedStore = true 42 | testMultiRepoStore(t, func() MultiRepoStoreImporter { 43 | return NewFSMultiRepoStore(newTestFS(), &FSMultiRepoStoreConf{RepoPaths: &customRepoPaths{}}) 44 | }) 45 | } 46 | -------------------------------------------------------------------------------- /store/store.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "os" 5 | "strings" 6 | ) 7 | 8 | // isStoreNotExist returns a boolean indicating whether err is known 9 | // to report that a store does not exist. It can be used to determine 10 | // whether a "not exists" error should be returned in combined stores 11 | // (repoStores, and treeStores, and unitStores types) that open 12 | // lower-level stores during lookups. 13 | func isStoreNotExist(err error) bool { 14 | if err == nil { 15 | return false 16 | } 17 | if _, ok := err.(*errIndexNotExist); ok { 18 | return true 19 | } 20 | return isOSOrVFSNotExist(err) || err == errRepoNoInit || err == errTreeNoInit || err == errMultiRepoStoreNoInit || err == errUnitNoInit 21 | } 22 | 23 | // isOSOrVFSNotExist returns a boolean indicating whether err is known 24 | // to be an OS- or VFS-level error reporting that a file or dir does 25 | // not exist. It is like os.IsNotExist but also handles common errors 26 | // produced by VFSes. 27 | func isOSOrVFSNotExist(err error) bool { 28 | if perr, ok := err.(*os.PathError); ok { 29 | if strings.HasPrefix(perr.Err.Error(), "unwanted http status 404") { 30 | return true 31 | } 32 | } 33 | return os.IsNotExist(err) 34 | } 35 | 36 | // storeFetchPar is the max number of parallel fetches to child stores 37 | // in xyzStores calls. 38 | const storeFetchPar = 15 39 | -------------------------------------------------------------------------------- /cli/coverage_test.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "io/ioutil" 5 | "testing" 6 | ) 7 | 8 | func TestStripCode(t *testing.T) { 9 | 10 | source, _ := ioutil.ReadFile("testdata/strip-code/input-source.txt") 11 | target, _ := ioutil.ReadFile("testdata/strip-code/expected-source.txt") 12 | expected := string(target) 13 | 14 | actual := string(stripCode(source)) 15 | if actual != expected { 16 | t.Errorf("got\n%v\n, want\n%v\n", actual, expected) 17 | } 18 | 19 | source = []byte("abc\n//def\nfgh") 20 | expected = "abc\n\nfgh" 21 | 22 | actual = string(stripCode(source)) 23 | if actual != expected { 24 | t.Errorf("got\n%v\n, want\n%v\n", actual, expected) 25 | } 26 | 27 | } 28 | 29 | func TestNumLines(t *testing.T) { 30 | tests := []struct { 31 | data string 32 | expected int 33 | }{ 34 | { 35 | "do\ncats\neat\nbats", 36 | 4, 37 | }, 38 | { 39 | "do\n\n\n\ncats\neat\nbats", 40 | 4, 41 | }, 42 | { 43 | "do\r\n\r\n\r\ncats\neat\nbats", 44 | 4, 45 | }, 46 | { 47 | "", 48 | 0, 49 | }, 50 | { 51 | "abc\n//def\nfgh", 52 | 2, 53 | }, 54 | { 55 | "", 56 | 0, 57 | }, 58 | } 59 | 60 | for _, test := range tests { 61 | actual := numLines([]byte(test.data)) 62 | if actual != test.expected { 63 | t.Errorf("%s, got %v, want %v", test.data, actual, test.expected) 64 | } 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /env.go: -------------------------------------------------------------------------------- 1 | package srclib 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "path/filepath" 7 | 8 | "sourcegraph.com/sourcegraph/srclib/util" 9 | ) 10 | 11 | var ( 12 | // Path is SRCLIBPATH, a colon-separated list of directories that lists 13 | // places to look for srclib toolchains and cache build data. It is 14 | // initialized from the SRCLIBPATH environment variable; if empty, it 15 | // defaults to $HOME/.srclib. 16 | Path = os.Getenv("SRCLIBPATH") 17 | 18 | // CacheDir stores cached build results. It is initialized from the 19 | // SRCLIBCACHE environment variable; if empty, it defaults to DIR/.cache, 20 | // where DIR is the first entry in Path (SRCLIBPATH). 21 | CacheDir = os.Getenv("SRCLIBCACHE") 22 | 23 | // CommandName holds the commands that will be used to call self when generating 24 | // Makefiles and updating toolchains. 25 | CommandName = "srclib" 26 | ) 27 | 28 | func init() { 29 | if Path == "" { 30 | homeDir := util.CurrentUserHomeDir() 31 | if homeDir == "" { 32 | log.Fatalf("Fatal: No SRCLIBPATH and current user has no home directory.") 33 | } 34 | Path = filepath.Join(homeDir, ".srclib") 35 | if err := os.Setenv("SRCLIBPATH", Path); err != nil { 36 | log.Fatalf("Fatal: Could not set SRCLIBPATH environment variable to %q.", Path) 37 | } 38 | } 39 | 40 | if CacheDir == "" { 41 | dirs := filepath.SplitList(Path) 42 | CacheDir = filepath.Join(dirs[0], ".cache") 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /graph/doc.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package graph; 3 | 4 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 5 | import "def.proto"; 6 | 7 | option (gogoproto.goproto_getters_all) = false; 8 | option (gogoproto.unmarshaler_all) = true; 9 | option (gogoproto.marshaler_all) = true; 10 | option (gogoproto.sizer_all) = true; 11 | 12 | // Doc is documentation on a Def. 13 | message Doc { 14 | // DefKey points to the Def that this documentation pertains to. 15 | DefKey Key = 1 [(gogoproto.nullable) = false, (gogoproto.embed) = true, (gogoproto.jsontag) = ""]; 16 | 17 | // Format is the the MIME-type that the documentation is stored 18 | // in. Valid formats include 'text/html', 'text/plain', 19 | // 'text/x-markdown', text/x-rst'. 20 | string Format = 2 [(gogoproto.jsontag) = "Format"]; 21 | 22 | // Data is the actual documentation text. 23 | string Data = 3 [(gogoproto.jsontag) = "Data"]; 24 | 25 | // File is the filename where this Doc exists. 26 | string File = 4 [(gogoproto.jsontag) = "File,omitempty"]; 27 | 28 | // Start is the byte offset of this Doc's first byte in File. 29 | uint32 Start = 5 [(gogoproto.jsontag) = "Start,omitempty"]; 30 | 31 | // End is the byte offset of this Doc's last byte in File. 32 | uint32 End = 6 [(gogoproto.jsontag) = "End,omitempty"]; 33 | 34 | // DocUnit is the source unit containing this Doc. 35 | string DocUnit = 7 [(gogoproto.jsontag) = "DocUnit,omitempty"]; 36 | }; 37 | -------------------------------------------------------------------------------- /cvg/coverage.go: -------------------------------------------------------------------------------- 1 | package cvg 2 | 3 | type Coverage struct { 4 | FileScore float64 // % files successfully processed 5 | RefScore float64 // % internal refs that resolve to a def 6 | TokDensity float64 // average number of refs/defs per LoC 7 | UncoveredFiles []string `json:",omitempty"` // files for which srclib data was not successfully generated (best-effort guess) 8 | UndiscoveredFiles []string `json:",omitempty"` // files weren't detected by toolchain(s) (best-effort guess) 9 | } 10 | 11 | func (c *Coverage) FileScorePass() bool { return c.FileScore > 0.8 } 12 | func (c *Coverage) RefScorePass() bool { return c.RefScore > 0.95 } 13 | func (c *Coverage) TokDensityPass() bool { return c.TokDensity > 1.0 } 14 | 15 | // HasRegressed determines if coverage has regressed from one indexing 16 | // to the next. 17 | func HasRegressed(prev map[string]*Coverage, next map[string]*Coverage) bool { 18 | for lang, prevCov := range prev { 19 | cov := next[lang] 20 | if cov == nil { 21 | cov = &Coverage{} 22 | } 23 | 24 | if prevCov.FileScorePass() && (!cov.FileScorePass() || cov.FileScore-prevCov.FileScore < -0.1) { 25 | return true 26 | } 27 | if prevCov.RefScorePass() && (!cov.RefScorePass() || cov.RefScore-prevCov.RefScore < -0.1) { 28 | return true 29 | } 30 | if prevCov.TokDensityPass() && (!cov.TokDensityPass() || cov.TokDensity-prevCov.TokDensity < -0.5) { 31 | return true 32 | } 33 | } 34 | 35 | return false 36 | } 37 | -------------------------------------------------------------------------------- /store/pb/srcstore.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package pb; 3 | 4 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 5 | import "sourcegraph.com/sourcegraph/srclib/unit/unit.proto"; 6 | import "sourcegraph.com/sourcegraph/srclib/graph/output.proto"; 7 | import "sourcegraph.com/sqs/pbtypes/void.proto"; 8 | 9 | option (gogoproto.goproto_getters_all) = false; 10 | option (gogoproto.unmarshaler_all) = true; 11 | option (gogoproto.marshaler_all) = true; 12 | option (gogoproto.sizer_all) = true; 13 | 14 | // A MultiRepoImporter imports srclib build data for a repository's 15 | // source unit at a specific version into a RepoStore. It allows 16 | // performing import operations over gRPC. 17 | service MultiRepoImporter { 18 | // Import imports srclib build data for a source unit at a 19 | // specific version into the store. 20 | rpc Import(ImportOp) returns (pbtypes.Void); 21 | 22 | // CreateVersion creates the version entry for the given commit. All other data (including 23 | // indexes) needs to exist before this gets called. 24 | rpc CreateVersion(CreateVersionOp) returns (pbtypes.Void); 25 | 26 | // Index builds indexes for a specific repo at a specific version. 27 | rpc Index(IndexOp) returns (pbtypes.Void); 28 | } 29 | 30 | message ImportOp { 31 | string Repo = 1; 32 | string CommitID = 2; 33 | unit.SourceUnit Unit = 3; 34 | graph.Output Data = 4; 35 | } 36 | 37 | message CreateVersionOp { 38 | string Repo = 1; 39 | string CommitID = 2; 40 | } 41 | 42 | message IndexOp { 43 | string Repo = 1; 44 | string CommitID = 2; 45 | } 46 | -------------------------------------------------------------------------------- /DCO: -------------------------------------------------------------------------------- 1 | Developer Certificate of Origin 2 | Version 1.1 3 | 4 | Copyright (C) 2004, 2006 The Linux Foundation and its contributors. 5 | 660 York Street, Suite 102, 6 | San Francisco, CA 94110 USA 7 | 8 | Everyone is permitted to copy and distribute verbatim copies of this 9 | license document, but changing it is not allowed. 10 | 11 | 12 | Developer's Certificate of Origin 1.1 13 | 14 | By making a contribution to this project, I certify that: 15 | 16 | (a) The contribution was created in whole or in part by me and I 17 | have the right to submit it under the open source license 18 | indicated in the file; or 19 | 20 | (b) The contribution is based upon previous work that, to the best 21 | of my knowledge, is covered under an appropriate open source 22 | license and I have the right under that license to submit that 23 | work with modifications, whether created in whole or in part 24 | by me, under the same open source license (unless I am 25 | permitted to submit under a different license), as indicated 26 | in the file; or 27 | 28 | (c) The contribution was provided directly to me by some other 29 | person who certified (a), (b) or (c) and I have not modified 30 | it. 31 | 32 | (d) I understand and agree that this project and the contribution 33 | are public and that a record of the contribution (including all 34 | personal information I submit with it, including my sign-off) is 35 | maintained indefinitely and may be redistributed consistent with 36 | this project or the open source license(s) involved. 37 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ifndef GOBIN 2 | ifeq ($(OS),Windows_NT) 3 | GOBIN := $(shell cmd /C "echo %GOPATH%| cut -d';' -f1") 4 | GOBIN := $(subst \,/,$(GOBIN))/bin 5 | else 6 | GOBIN := $(shell echo $$GOPATH | cut -d':' -f1 )/bin 7 | endif 8 | endif 9 | 10 | ifeq ($(OS),Windows_NT) 11 | EXE := srclib.exe 12 | else 13 | EXE := srclib 14 | endif 15 | 16 | MAKEFLAGS+=--no-print-directory 17 | 18 | .PHONY: default install srclib release upload-release check-release test 19 | 20 | default: govendor install 21 | 22 | install: srclib 23 | 24 | srclib: ${GOBIN}/${EXE} 25 | 26 | ${GOBIN}/${EXE}: $(shell /usr/bin/find . -type f -and -name '*.go') 27 | go install ./cmd/srclib 28 | 29 | govendor: 30 | go get github.com/kardianos/govendor 31 | govendor sync 32 | 33 | release: upload-release check-release 34 | 35 | upload-release: 36 | @bash -c 'if [[ "$(V)" == "" ]]; then echo Must specify version: make release V=x.y.z; exit 1; fi' 37 | go get github.com/laher/goxc 38 | goxc -q -pv="$(V)" 39 | git tag v$(V) 40 | git push --tags 41 | 42 | check-release: 43 | @bash -c 'if [[ "$(V)" == "" ]]; then echo Must specify version: make release V=x.y.z; exit 1; fi' 44 | @rm -rf /tmp/srclib-$(V).gz 45 | curl -Lo /tmp/srclib-$(V).gz "https://srclib-release.s3.amazonaws.com/srclib/$(V)/$(shell go env GOOS)-$(shell go env GOARCH)/srclib.gz" 46 | cd /tmp && gunzip -f srclib-$(V).gz && chmod +x srclib-$(V) 47 | echo; echo 48 | /tmp/srclib-$(V) version 49 | echo; echo 50 | @echo Released srclib $(V) 51 | 52 | test: 53 | go test -race -v $(go list ./... | grep -v /vendor/) 54 | -------------------------------------------------------------------------------- /cli/testdata/strip-code/input-source.txt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2013 Square, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package retrofit2.http; 17 | 18 | import java.lang.annotation.Documented; 19 | import java.lang.annotation.Retention; 20 | import java.lang.annotation.Target; 21 | import okhttp3.HttpUrl; 22 | 23 | import static java.lang.annotation.ElementType.METHOD; 24 | import static java.lang.annotation.RetentionPolicy.RUNTIME; 25 | 26 | /** Make a DELETE request. */ 27 | @Documented 28 | @Target(METHOD) 29 | @Retention(RUNTIME) 30 | public @interface DELETE { 31 | /** 32 | * A relative or absolute path, or full URL of the endpoint. This value is optional if the first 33 | * parameter of the method is annotated with {@link Url @Url}. 34 | *

35 | * See {@linkplain retrofit2.Retrofit.Builder#baseUrl(HttpUrl) base URL} for details of how 36 | * this is resolved against a base URL to create the full endpoint URL. 37 | */ 38 | String value() default ""; 39 | } 40 | -------------------------------------------------------------------------------- /store/phtable/slicereader_safe.go: -------------------------------------------------------------------------------- 1 | // +build !386,!amd64,!arm 2 | 3 | package phtable 4 | 5 | import ( 6 | "encoding/binary" 7 | "fmt" 8 | ) 9 | 10 | // Read values and typed vectors from a byte slice without copying where possible. 11 | type sliceReader struct { 12 | b []byte 13 | start, end uint64 14 | } 15 | 16 | func (b *sliceReader) Read(size uint64) []byte { 17 | b.start, b.end = b.end, b.end+size 18 | if b.start == b.end { 19 | return nil 20 | } 21 | return b.b[b.start:b.end] 22 | } 23 | 24 | func (b *sliceReader) ReadUint64Array(n uint64) []uint64 { 25 | b.start, b.end = b.end, b.end+n*8 26 | out := make([]uint64, n) 27 | buf := b.b[b.start:b.end] 28 | for i := 0; i < len(buf); i += 8 { 29 | out[i>>3] = binary.LittleEndian.Uint64(buf[i : i+8]) 30 | } 31 | return out 32 | } 33 | 34 | func (b *sliceReader) ReadUint16Array(n uint64) []uint16 { 35 | b.start, b.end = b.end, b.end+n*2 36 | out := make([]uint16, n) 37 | buf := b.b[b.start:b.end] 38 | for i := 0; i < len(buf); i += 2 { 39 | out[i>>1] = binary.LittleEndian.Uint16(buf[i : i+2]) 40 | } 41 | return out 42 | } 43 | 44 | func (b *sliceReader) ReadInt() uint64 { 45 | return uint64(binary.LittleEndian.Uint32(b.Read(4))) 46 | } 47 | 48 | func (b *sliceReader) ReadInt32() uint32 { 49 | return binary.LittleEndian.Uint32(b.Read(4)) 50 | } 51 | 52 | func (b *sliceReader) ReadUvarint() uint64 { 53 | v, n := binary.Uvarint(b.b[b.end:]) 54 | if n <= 0 { 55 | panic(fmt.Sprintf("Uvarint error: n == %d (buf size %d)", n, b.end-b.start)) 56 | } 57 | b.start, b.end = b.end, b.end+uint64(n) 58 | return v 59 | } 60 | -------------------------------------------------------------------------------- /cli/misc.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "path/filepath" 7 | 8 | "sourcegraph.com/sourcegraph/go-flags" 9 | ) 10 | 11 | // Directory is flags.Completer that provides directory name 12 | // completion. Do not convert Directory to a string type manually, 13 | // always use Directory.String(). Only use Directory for 14 | // go-flags.Command fields, not for internal functions. 15 | // 16 | // TODO(sqs): this is annoying. it only completes the dir name and doesn't let 17 | // you keep typing the arg. 18 | type Directory string 19 | 20 | // Complete implements flags.Completer and returns a list of existing 21 | // directories with the given prefix. 22 | func (d Directory) Complete(match string) []flags.Completion { 23 | names, err := filepath.Glob(match + "*") 24 | if err != nil { 25 | log.Println(err) 26 | return nil 27 | } 28 | 29 | var dirs []flags.Completion 30 | for _, name := range names { 31 | if fi, err := os.Stat(name); err == nil && fi.Mode().IsDir() { 32 | dirs = append(dirs, flags.Completion{Item: name + string(filepath.Separator)}) 33 | } 34 | } 35 | return dirs 36 | } 37 | 38 | // String returns the uncleaned string representation of d. If d is 39 | // empty, "." is returned. Never convert Directories to strings 40 | // manually, always call String. 41 | func (d Directory) String() string { 42 | if d == "" { 43 | return "." 44 | } 45 | dir, file := filepath.Split(string(d)) 46 | if file == "" { 47 | return dir 48 | } 49 | if file == "." || file == ".." { 50 | return dir + file 51 | } 52 | return dir + file + string(filepath.Separator) 53 | } 54 | -------------------------------------------------------------------------------- /toolchain/tool.go: -------------------------------------------------------------------------------- 1 | package toolchain 2 | 3 | import "sourcegraph.com/sourcegraph/srclib" 4 | 5 | // ToolInfo describes a tool in a toolchain. 6 | type ToolInfo struct { 7 | // Subcmd is the subcommand name of this tool. 8 | // 9 | // By convention, this is the same as Op in toolchains that only have one 10 | // tool that performs this operation (e.g., a toolchain's "graph" subcommand 11 | // performs the "graph" operation). 12 | Subcmd string 13 | 14 | // Op is the operation that this tool performs (e.g., "scan", "graph", 15 | // "deplist", etc.). 16 | Op string 17 | 18 | // SourceUnitTypes is a list of source unit types (e.g., "GoPackage") that 19 | // this tool can operate on. 20 | // 21 | // If this tool doesn't operate on source units (for example, it operates on 22 | // directories or repositories, such as the "blame" tools), then this will 23 | // be empty. 24 | // 25 | // TODO(sqs): determine how repository- or directory-level tools will be 26 | // defined. 27 | SourceUnitTypes []string `json:",omitempty"` 28 | } 29 | 30 | // ListTools lists all tools in all available toolchains (returned by List). If 31 | // op is non-empty, only tools that perform that operation are returned. 32 | func ListTools(op string) ([]*srclib.ToolRef, error) { 33 | tcs, err := List() 34 | if err != nil { 35 | return nil, err 36 | } 37 | 38 | var tools []*srclib.ToolRef 39 | for _, tc := range tcs { 40 | c, err := tc.ReadConfig() 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | for _, tool := range c.Tools { 46 | if op == "" || tool.Op == op { 47 | tools = append(tools, &srclib.ToolRef{Toolchain: tc.Path, Subcmd: tool.Subcmd}) 48 | } 49 | } 50 | } 51 | return tools, nil 52 | } 53 | -------------------------------------------------------------------------------- /store/multi_repo_store_mock.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "sourcegraph.com/sourcegraph/srclib/graph" 5 | "sourcegraph.com/sourcegraph/srclib/unit" 6 | ) 7 | 8 | type MockMultiRepoStore struct { 9 | Repos_ func(...RepoFilter) ([]string, error) 10 | Versions_ func(...VersionFilter) ([]*Version, error) 11 | Units_ func(...UnitFilter) ([]*unit.SourceUnit, error) 12 | Defs_ func(...DefFilter) ([]*graph.Def, error) 13 | Refs_ func(...RefFilter) ([]*graph.Ref, error) 14 | 15 | Import_ func(repo, commitID string, unit *unit.SourceUnit, data graph.Output) error 16 | Index_ func(repo, commitID string) error 17 | CreateVersion_ func(repo, commit string) error 18 | } 19 | 20 | func (m MockMultiRepoStore) Repos(f ...RepoFilter) ([]string, error) { 21 | return m.Repos_(f...) 22 | } 23 | 24 | func (m MockMultiRepoStore) Versions(f ...VersionFilter) ([]*Version, error) { 25 | return m.Versions_(f...) 26 | } 27 | 28 | func (m MockMultiRepoStore) Units(f ...UnitFilter) ([]*unit.SourceUnit, error) { 29 | return m.Units_(f...) 30 | } 31 | 32 | func (m MockMultiRepoStore) Defs(f ...DefFilter) ([]*graph.Def, error) { 33 | return m.Defs_(f...) 34 | } 35 | 36 | func (m MockMultiRepoStore) Refs(f ...RefFilter) ([]*graph.Ref, error) { 37 | return m.Refs_(f...) 38 | } 39 | 40 | func (m MockMultiRepoStore) Import(repo, commitID string, unit *unit.SourceUnit, data graph.Output) error { 41 | return m.Import_(repo, commitID, unit, data) 42 | } 43 | 44 | func (m MockMultiRepoStore) Index(repo, commitID string) error { 45 | return m.Index_(repo, commitID) 46 | } 47 | 48 | func (m MockMultiRepoStore) CreateVersion(repo, commitID string) error { 49 | return m.CreateVersion_(repo, commitID) 50 | } 51 | 52 | var _ MultiRepoStoreImporterIndexer = MockMultiRepoStore{} 53 | -------------------------------------------------------------------------------- /toolchain/choose.go: -------------------------------------------------------------------------------- 1 | package toolchain 2 | 3 | import ( 4 | "fmt" 5 | 6 | "sourcegraph.com/sourcegraph/srclib" 7 | ) 8 | 9 | // ChooseTool determines which toolchain and tool to use to run op (graph, 10 | // depresolve, etc.) on a source unit of the given type. If no tools fit the 11 | // criteria, an error is returned. 12 | // 13 | // The selection algorithm is currently very simplistic: if exactly one tool is 14 | // found that can perform op on the source unit type, it is returned. If zero or 15 | // more than 1 are found, then an error is returned. TODO(sqs): extend this to 16 | // choose the "best" tool when multiple tools would suffice. 17 | var ChooseTool = func(op, unitType string) (*srclib.ToolRef, error) { 18 | tcs, err := List() 19 | if err != nil { 20 | return nil, err 21 | } 22 | return chooseTool(op, unitType, tcs) 23 | } 24 | 25 | // chooseTool is like ChooseTool but the list of tools is provided as an 26 | // argument instead of being obtained by calling List. 27 | func chooseTool(op, unitType string, tcs []*Info) (*srclib.ToolRef, error) { 28 | var satisfying []*srclib.ToolRef 29 | for _, tc := range tcs { 30 | cfg, err := tc.ReadConfig() 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | for _, tool := range cfg.Tools { 36 | if tool.Op == op { 37 | for _, u := range tool.SourceUnitTypes { 38 | if u == unitType { 39 | satisfying = append(satisfying, &srclib.ToolRef{Toolchain: tc.Path, Subcmd: tool.Subcmd}) 40 | } 41 | } 42 | } 43 | } 44 | } 45 | 46 | if n := len(satisfying); n > 1 { 47 | return nil, fmt.Errorf("%d tools satisfy op %q for source unit type %q (refusing to choose between multiple possibilities)", n, op, unitType) 48 | } else if n == 0 { 49 | return nil, nil 50 | } 51 | return satisfying[0], nil 52 | } 53 | -------------------------------------------------------------------------------- /store/phtable/slicereader_fast.go: -------------------------------------------------------------------------------- 1 | // +build 386 amd64 arm 2 | 3 | package phtable 4 | 5 | import ( 6 | "encoding/binary" 7 | "fmt" 8 | 9 | "github.com/alecthomas/unsafeslice" 10 | ) 11 | 12 | // Read values and typed vectors from a byte slice without copying where 13 | // possible. This implementation directly references the underlying byte slice 14 | // for array operations, making them essentially zero copy. As the data is 15 | // written in little endian form, this of course means that this will only 16 | // work on little-endian architectures. 17 | type sliceReader struct { 18 | b []byte 19 | start, end uint64 20 | } 21 | 22 | func (b *sliceReader) Read(size uint64) []byte { 23 | b.start, b.end = b.end, b.end+size 24 | if b.start == b.end { 25 | return nil 26 | } 27 | return b.b[b.start:b.end] 28 | } 29 | 30 | func (b *sliceReader) ReadUint64Array(n uint64) []uint64 { 31 | b.start, b.end = b.end, b.end+n*8 32 | return unsafeslice.Uint64SliceFromByteSlice(b.b[b.start:b.end]) 33 | } 34 | 35 | func (b *sliceReader) ReadUint16Array(n uint64) []uint16 { 36 | b.start, b.end = b.end, b.end+n*2 37 | return unsafeslice.Uint16SliceFromByteSlice(b.b[b.start:b.end]) 38 | } 39 | 40 | // Despite returning a uint64, this actually reads a uint32. All table indices 41 | // and lengths are stored as uint32 values. 42 | func (b *sliceReader) ReadInt() uint64 { 43 | return uint64(binary.LittleEndian.Uint32(b.Read(4))) 44 | } 45 | 46 | func (b *sliceReader) ReadInt32() uint32 { 47 | return binary.LittleEndian.Uint32(b.Read(4)) 48 | } 49 | 50 | func (b *sliceReader) ReadUvarint() uint64 { 51 | v, n := binary.Uvarint(b.b[b.end:]) 52 | if n <= 0 { 53 | panic(fmt.Sprintf("Uvarint error: n == %d (buf size %d)", n, b.end-b.start)) 54 | } 55 | b.start, b.end = b.end, b.end+uint64(n) 56 | return v 57 | } 58 | -------------------------------------------------------------------------------- /dep/rule.go: -------------------------------------------------------------------------------- 1 | package dep 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | 7 | "sourcegraph.com/sourcegraph/makex" 8 | "sourcegraph.com/sourcegraph/srclib" 9 | "sourcegraph.com/sourcegraph/srclib/buildstore" 10 | "sourcegraph.com/sourcegraph/srclib/config" 11 | "sourcegraph.com/sourcegraph/srclib/plan" 12 | "sourcegraph.com/sourcegraph/srclib/toolchain" 13 | "sourcegraph.com/sourcegraph/srclib/unit" 14 | "sourcegraph.com/sourcegraph/srclib/util" 15 | ) 16 | 17 | const depresolveOp = "depresolve" 18 | 19 | func init() { 20 | plan.RegisterRuleMaker(depresolveOp, makeDepRules) 21 | buildstore.RegisterDataType("depresolve", []*ResolvedDep{}) 22 | } 23 | 24 | func makeDepRules(c *config.Tree, dataDir string, existing []makex.Rule) ([]makex.Rule, error) { 25 | const op = depresolveOp 26 | var rules []makex.Rule 27 | for _, u := range c.SourceUnits { 28 | toolRef, err := toolchain.ChooseTool(depresolveOp, u.Type) 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | rules = append(rules, &ResolveDepsRule{dataDir, u, toolRef}) 34 | } 35 | return rules, nil 36 | } 37 | 38 | type ResolveDepsRule struct { 39 | dataDir string 40 | Unit *unit.SourceUnit 41 | Tool *srclib.ToolRef 42 | } 43 | 44 | func (r *ResolveDepsRule) Target() string { 45 | return filepath.ToSlash(filepath.Join(r.dataDir, plan.SourceUnitDataFilename([]*ResolvedDep{}, r.Unit))) 46 | } 47 | 48 | func (r *ResolveDepsRule) Prereqs() []string { 49 | return []string{filepath.ToSlash(filepath.Join(r.dataDir, plan.SourceUnitDataFilename(unit.SourceUnit{}, r.Unit)))} 50 | } 51 | 52 | func (r *ResolveDepsRule) Recipes() []string { 53 | if r.Tool == nil { 54 | return nil 55 | } 56 | return []string{ 57 | fmt.Sprintf("%s tool %q %q < $^ 1> $@", util.SafeCommandName(srclib.CommandName), r.Tool.Toolchain, r.Tool.Subcmd), 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /config/external.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "log" 7 | "os" 8 | "path/filepath" 9 | 10 | "sourcegraph.com/sourcegraph/srclib" 11 | "sourcegraph.com/sourcegraph/srclib/toolchain" 12 | ) 13 | 14 | // An External configuration file, represented by this struct, can set system- 15 | // and user-level settings for srclib. 16 | type External struct { 17 | // Scanners is the default set of scanners to use. If not specified, all 18 | // scanners in the SRCLIBPATH will be used. 19 | Scanners []*srclib.ToolRef 20 | } 21 | 22 | const srclibconfigFile = ".srclibconfig" 23 | 24 | // SrclibPathConfig gets the srclib path configuration (which lists 25 | // all available scanners). It reads it from SRCLIBPATH/.srclibconfig 26 | // if that file exists, and otherwise it walks SRCLIBPATH for 27 | // available scanners. 28 | func SrclibPathConfig() (*External, error) { 29 | var x External 30 | 31 | // Try reading from .srclibconfig. 32 | dir := filepath.SplitList(srclib.Path)[0] 33 | configFile := filepath.Join(dir, srclibconfigFile) 34 | f, err := os.Open(configFile) 35 | if os.IsNotExist(err) { 36 | // do nothing 37 | } else if err != nil { 38 | log.Printf("Warning: unable to open config file at %s: %s. Continuing without this config.", configFile, err) 39 | } else { 40 | defer f.Close() 41 | if err := json.NewDecoder(f).Decode(&x); err != nil { 42 | log.Printf("Warning: unable to decode config file at %s: %s. Continuing without this config.", configFile, err) 43 | } 44 | } 45 | 46 | // Default to using all available scanners. 47 | if len(x.Scanners) == 0 { 48 | var err error 49 | x.Scanners, err = toolchain.ListTools("scan") 50 | if err != nil && !os.IsNotExist(err) { 51 | return nil, fmt.Errorf("failed to find scanners in SRCLIBPATH: %s", err) 52 | } 53 | } 54 | 55 | return &x, nil 56 | } 57 | -------------------------------------------------------------------------------- /graph/format.go: -------------------------------------------------------------------------------- 1 | package graph 2 | 3 | // RepositoryListingDef holds rendered display text to show on the "package" 4 | // listing page of a repository. 5 | type RepositoryListingDef struct { 6 | // Name is the full name shown on the page. 7 | Name string 8 | 9 | // NameLabel is a label displayed next to the Name, such as "(main package)" 10 | // to denote that a package is a Go main package. 11 | NameLabel string 12 | 13 | // Language is the source language of the def, with any additional 14 | // specifiers, such as "JavaScript (node.js)". 15 | Language string 16 | 17 | // SortKey is the key used to lexicographically sort all of the defs on 18 | // the page. 19 | SortKey string 20 | } 21 | 22 | // // FormatAndSortDefsForRepositoryListing uses DefFormatters registered by 23 | // // the various toolchains to format and sort defs for display on the 24 | // // "package" listing page of a repository. The provided defs slice is sorted 25 | // // in-place. 26 | // func FormatAndSortDefsForRepositoryListing(defs []*Def) map[*Def]RepositoryListingDef { 27 | // m := make(map[*Def]RepositoryListingDef, len(defs)) 28 | // for _, s := range defs { 29 | // sf, present := DefFormatters[s.UnitType] 30 | // if !present { 31 | // panic("no DefFormatter for def with UnitType " + s.UnitType) 32 | // } 33 | 34 | // m[s] = sf.RepositoryListing(s) 35 | // } 36 | 37 | // // sort 38 | // ss := &repositoryListingDefs{m, defs} 39 | // sort.Sort(ss) 40 | // return m 41 | // } 42 | 43 | // type repositoryListingDefs struct { 44 | // info map[*Def]RepositoryListingDef 45 | // defs []*Def 46 | // } 47 | 48 | // func (s *repositoryListingDefs) Len() int { return len(s.defs) } 49 | // func (s *repositoryListingDefs) Swap(i, j int) { 50 | // s.defs[i], s.defs[j] = s.defs[j], s.defs[i] 51 | // } 52 | // func (s *repositoryListingDefs) Less(i, j int) bool { 53 | // return s.info[s.defs[i]].SortKey < s.info[s.defs[j]].SortKey 54 | // } 55 | -------------------------------------------------------------------------------- /store/multi_repo_store.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "sourcegraph.com/sourcegraph/srclib/graph" 5 | "sourcegraph.com/sourcegraph/srclib/unit" 6 | ) 7 | 8 | // MultiRepoStore provides access to RepoStores for multiple 9 | // repositories. 10 | // 11 | // Using this interface instead of directly accessing a single 12 | // RepoStore allows aliasing repository URIs and supporting both ID 13 | // and URI lookups. 14 | type MultiRepoStore interface { 15 | // Repos returns all repositories that match the RepoFilter. 16 | Repos(...RepoFilter) ([]string, error) 17 | 18 | // RepoStore's methods call the corresponding methods on the 19 | // RepoStore of each repository contained within this multi-repo 20 | // store. The combined results are returned (in undefined order). 21 | RepoStore 22 | } 23 | 24 | // A MultiRepoImporter imports srclib build data for a repository's 25 | // source unit at a specific version into a RepoStore. 26 | type MultiRepoImporter interface { 27 | // Import imports srclib build data for a source unit at a 28 | // specific version into the store. 29 | Import(repo, commitID string, unit *unit.SourceUnit, data graph.Output) error 30 | 31 | // CreateVersion creates the version entry for the given commit. All other data (including 32 | // indexes) needs to exist before this gets called. 33 | CreateVersion(repo, commitID string) error 34 | } 35 | 36 | type MultiRepoIndexer interface { 37 | // Index builds indexes for the store. 38 | Index(repo, commitID string) error 39 | } 40 | 41 | // A MultiRepoStoreImporter implements both MultiRepoStore and 42 | // MultiRepoImporter. 43 | type MultiRepoStoreImporter interface { 44 | MultiRepoStore 45 | MultiRepoImporter 46 | } 47 | 48 | // A MultiRepoStoreImporterIndexer implements all 3 interfaces. 49 | type MultiRepoStoreImporterIndexer interface { 50 | MultiRepoStore 51 | MultiRepoImporter 52 | MultiRepoIndexer 53 | } 54 | 55 | // TODO(sqs): What should the Repo type be? Right now it is just string. 56 | -------------------------------------------------------------------------------- /store/pbio/io.go: -------------------------------------------------------------------------------- 1 | // Extensions for Protocol Buffers to create more go like structures. 2 | // 3 | // Copyright (c) 2013, Vastech SA (PTY) LTD. All rights reserved. 4 | // http://github.com/gogo/protobuf/gogoproto 5 | // 6 | // Redistribution and use in source and binary forms, with or without 7 | // modification, are permitted provided that the following conditions are 8 | // met: 9 | // 10 | // * Redistributions of source code must retain the above copyright 11 | // notice, this list of conditions and the following disclaimer. 12 | // * Redistributions in binary form must reproduce the above 13 | // copyright notice, this list of conditions and the following disclaimer 14 | // in the documentation and/or other materials provided with the 15 | // distribution. 16 | // 17 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | package pbio 30 | 31 | import "github.com/gogo/protobuf/proto" 32 | 33 | type Writer interface { 34 | WriteMsg(proto.Message) (uint64, error) 35 | } 36 | 37 | type Reader interface { 38 | ReadMsg(msg proto.Message) (uint64, error) 39 | } 40 | 41 | type marshaler interface { 42 | MarshalTo(data []byte) (n int, err error) 43 | Sizer 44 | } 45 | 46 | type Sizer interface { 47 | Size() (n int) 48 | } 49 | -------------------------------------------------------------------------------- /store/pb/mock/srcstore.pb_mock.go: -------------------------------------------------------------------------------- 1 | // generated by gen-mocks; DO NOT EDIT 2 | 3 | package mock 4 | 5 | import ( 6 | "golang.org/x/net/context" 7 | "google.golang.org/grpc" 8 | "sourcegraph.com/sourcegraph/srclib/store/pb" 9 | "sourcegraph.com/sqs/pbtypes" 10 | ) 11 | 12 | type MultiRepoImporterClient struct { 13 | Import_ func(ctx context.Context, in *pb.ImportOp) (*pbtypes.Void, error) 14 | CreateVersion_ func(ctx context.Context, in *pb.CreateVersionOp) (*pbtypes.Void, error) 15 | Index_ func(ctx context.Context, in *pb.IndexOp) (*pbtypes.Void, error) 16 | } 17 | 18 | func (s *MultiRepoImporterClient) Import(ctx context.Context, in *pb.ImportOp, opts ...grpc.CallOption) (*pbtypes.Void, error) { 19 | return s.Import_(ctx, in) 20 | } 21 | 22 | func (s *MultiRepoImporterClient) CreateVersion(ctx context.Context, in *pb.CreateVersionOp, opts ...grpc.CallOption) (*pbtypes.Void, error) { 23 | return s.CreateVersion_(ctx, in) 24 | } 25 | 26 | func (s *MultiRepoImporterClient) Index(ctx context.Context, in *pb.IndexOp, opts ...grpc.CallOption) (*pbtypes.Void, error) { 27 | return s.Index_(ctx, in) 28 | } 29 | 30 | var _ pb.MultiRepoImporterClient = (*MultiRepoImporterClient)(nil) 31 | 32 | type MultiRepoImporterServer struct { 33 | Import_ func(v0 context.Context, v1 *pb.ImportOp) (*pbtypes.Void, error) 34 | CreateVersion_ func(v0 context.Context, v1 *pb.CreateVersionOp) (*pbtypes.Void, error) 35 | Index_ func(v0 context.Context, v1 *pb.IndexOp) (*pbtypes.Void, error) 36 | } 37 | 38 | func (s *MultiRepoImporterServer) Import(v0 context.Context, v1 *pb.ImportOp) (*pbtypes.Void, error) { 39 | return s.Import_(v0, v1) 40 | } 41 | 42 | func (s *MultiRepoImporterServer) CreateVersion(v0 context.Context, v1 *pb.CreateVersionOp) (*pbtypes.Void, error) { 43 | return s.CreateVersion_(v0, v1) 44 | } 45 | 46 | func (s *MultiRepoImporterServer) Index(v0 context.Context, v1 *pb.IndexOp) (*pbtypes.Void, error) { 47 | return s.Index_(v0, v1) 48 | } 49 | 50 | var _ pb.MultiRepoImporterServer = (*MultiRepoImporterServer)(nil) 51 | -------------------------------------------------------------------------------- /store/units_index.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | 8 | "sourcegraph.com/sourcegraph/srclib/unit" 9 | ) 10 | 11 | // unitsIndex makes it fast list all source units without needing to 12 | // traverse directories. 13 | type unitsIndex struct { 14 | units []*unit.SourceUnit 15 | ready bool 16 | } 17 | 18 | var _ interface { 19 | Index 20 | persistedIndex 21 | unitIndexBuilder 22 | unitFullIndex 23 | } = (*unitsIndex)(nil) 24 | 25 | var c_unitsIndex_listUnits = &counter{count: new(int64)} 26 | 27 | func (x *unitsIndex) String() string { return fmt.Sprintf("unitsIndex(ready=%v)", x.ready) } 28 | 29 | // Covers returns -1 because it should never be selected over a more 30 | // specific index. When the indexed stores use the unitsIndex, they do 31 | // so through a different code path from the one that selects other 32 | // indexes. 33 | func (x *unitsIndex) Covers(filters interface{}) int { return -1 } 34 | 35 | // Units implements unitFullIndex. 36 | func (x *unitsIndex) Units(fs ...UnitFilter) ([]*unit.SourceUnit, error) { 37 | if x.units == nil { 38 | panic("units not built/read") 39 | } 40 | 41 | c_unitsIndex_listUnits.increment() 42 | 43 | var units []*unit.SourceUnit 44 | for _, u := range x.units { 45 | if unitFilters(fs).SelectUnit(u) { 46 | units = append(units, u) 47 | } 48 | } 49 | 50 | return units, nil 51 | } 52 | 53 | // Build implements unitIndexBuilder. 54 | func (x *unitsIndex) Build(units []*unit.SourceUnit) error { 55 | x.units = units 56 | x.ready = true 57 | return nil 58 | } 59 | 60 | // Write implements persistedIndex. 61 | func (x *unitsIndex) Write(w io.Writer) error { 62 | if x.units == nil { 63 | panic("no units to write") 64 | } 65 | return json.NewEncoder(w).Encode(x.units) 66 | } 67 | 68 | // Read implements persistedIndex. 69 | func (x *unitsIndex) Read(r io.Reader) error { 70 | err := json.NewDecoder(r).Decode(&x.units) 71 | x.ready = (err == nil) 72 | return err 73 | } 74 | 75 | // Ready implements persistedIndex. 76 | func (x *unitsIndex) Ready() bool { return x.ready } 77 | -------------------------------------------------------------------------------- /dep/resolve.go: -------------------------------------------------------------------------------- 1 | package dep 2 | 3 | import ( 4 | "encoding/json" 5 | ) 6 | 7 | // START ResolvedTarget OMIT 8 | // ResolvedTarget represents a resolved dependency target. 9 | type ResolvedTarget struct { 10 | // ToRepoCloneURL is the clone URL of the repository that is depended on. 11 | // 12 | // When graphers emit ResolvedDependencies, they should fill in this field, 13 | // not ToRepo, so that the dependent repository can be added if it doesn't 14 | // exist. The ToRepo URI alone does not specify enough information to add 15 | // the repository (because it doesn't specify the VCS type, scheme, etc.). 16 | ToRepoCloneURL string 17 | 18 | // ToUnit is the name of the source unit that is depended on. 19 | ToUnit string 20 | 21 | // ToUnitType is the type of the source unit that is depended on. 22 | ToUnitType string 23 | 24 | // ToVersion is the version of the dependent repository (if known), 25 | // according to whatever version string specifier is used by FromRepo's 26 | // dependency management system. 27 | ToVersionString string 28 | 29 | // ToRevSpec specifies the desired VCS revision of the dependent repository 30 | // (if known). 31 | ToRevSpec string 32 | } 33 | 34 | // END ResolvedTarget OMIT 35 | 36 | // START Resolution OMIT 37 | // Resolution is the result of dependency resolution: either a successfully 38 | // resolved target or an error. 39 | type Resolution struct { 40 | // Raw is the original raw dep that this was resolution was attempted on. 41 | Raw interface{} 42 | 43 | // Target is the resolved dependency, if resolution succeeds. 44 | Target *ResolvedTarget `json:",omitempty"` 45 | 46 | // Error is the resolution error, if any. 47 | Error string `json:",omitempty"` 48 | } 49 | 50 | // END Resolution OMIT 51 | 52 | func (r *Resolution) KeyId() string { 53 | return r.Target.ToRepoCloneURL + r.Target.ToUnit + r.Target.ToUnitType + r.Target.ToVersionString + r.Target.ToRevSpec 54 | } 55 | 56 | func (r *Resolution) RawKeyId() (string, error) { 57 | b, err := json.Marshal(r.Raw) 58 | if err != nil { 59 | return "", err 60 | } 61 | return string(b), nil 62 | } 63 | -------------------------------------------------------------------------------- /plan/makefile_test.go: -------------------------------------------------------------------------------- 1 | package plan_test 2 | 3 | import ( 4 | "bytes" 5 | "strings" 6 | "testing" 7 | 8 | "sourcegraph.com/sourcegraph/makex" 9 | "sourcegraph.com/sourcegraph/srclib" 10 | "sourcegraph.com/sourcegraph/srclib/config" 11 | _ "sourcegraph.com/sourcegraph/srclib/config" 12 | _ "sourcegraph.com/sourcegraph/srclib/dep" 13 | _ "sourcegraph.com/sourcegraph/srclib/grapher" 14 | "sourcegraph.com/sourcegraph/srclib/plan" 15 | "sourcegraph.com/sourcegraph/srclib/toolchain" 16 | "sourcegraph.com/sourcegraph/srclib/unit" 17 | ) 18 | 19 | func TestCreateMakefile(t *testing.T) { 20 | oldChooseTool := toolchain.ChooseTool 21 | defer func() { toolchain.ChooseTool = oldChooseTool }() 22 | 23 | toolchain.ChooseTool = func(op, unitType string) (*srclib.ToolRef, error) { 24 | return &srclib.ToolRef{ 25 | Toolchain: "tc", 26 | Subcmd: "t", 27 | }, nil 28 | } 29 | buildDataDir := "testdata" 30 | c := &config.Tree{ 31 | SourceUnits: []*unit.SourceUnit{ 32 | { 33 | Key: unit.Key{ 34 | Name: "n", 35 | Type: "t", 36 | }, 37 | Info: unit.Info{ 38 | Files: []string{"f"}, 39 | Ops: map[string][]byte{ 40 | "graph": nil, 41 | "depresolve": nil, 42 | }, 43 | }, 44 | }, 45 | }, 46 | } 47 | 48 | mf, err := plan.CreateMakefile(buildDataDir, nil, "", c) 49 | if err != nil { 50 | t.Fatal(err) 51 | } 52 | 53 | want := ` 54 | .PHONY: all 55 | 56 | all: testdata/n/t.depresolve.json testdata/n/t.graph.json 57 | 58 | testdata/n/t.depresolve.json: testdata/n/t.unit.json 59 | srclib tool "tc" "t" < $^ 1> $@ 60 | 61 | testdata/n/t.graph.json: testdata/n/t.unit.json 62 | srclib tool "tc" "t" < $< | srclib internal normalize-graph-data --unit-type "t" --dir . 1> $@ 63 | 64 | .DELETE_ON_ERROR: 65 | ` 66 | 67 | gotBytes, err := makex.Marshal(mf) 68 | if err != nil { 69 | t.Fatal(err) 70 | } 71 | 72 | want = strings.TrimSpace(want) 73 | got := string(bytes.TrimSpace(gotBytes)) 74 | 75 | if got != want { 76 | t.Errorf("got makefile:\n==========\n%s\n==========\n\nwant makefile:\n==========\n%s\n==========", got, want) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /cli/tool_cmd.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "os/exec" 7 | "strings" 8 | 9 | "sourcegraph.com/sourcegraph/go-flags" 10 | 11 | "sourcegraph.com/sourcegraph/srclib/toolchain" 12 | ) 13 | 14 | func init() { 15 | cliInit = append(cliInit, func(cli *flags.Command) { 16 | c, err := cli.AddCommand("tool", 17 | "run a tool", 18 | "Run a srclib tool with the specified arguments.", 19 | &toolCmd, 20 | ) 21 | if err != nil { 22 | log.Fatal(err) 23 | } 24 | c.ArgsRequired = true 25 | }) 26 | } 27 | 28 | type ToolCmd struct { 29 | Args struct { 30 | Toolchain ToolchainPath `name:"TOOLCHAIN" description:"toolchain path of the toolchain to run"` 31 | Tool ToolName `name:"TOOL" description:"tool subcommand name to run (in TOOLCHAIN)"` 32 | ToolArgs []string `name:"ARGS" description:"args to pass to TOOL"` 33 | } `positional-args:"yes" required:"yes"` 34 | } 35 | 36 | var toolCmd ToolCmd 37 | 38 | func (c *ToolCmd) Execute(args []string) error { 39 | cmdName, err := toolchain.Command(string(c.Args.Toolchain)) 40 | if err != nil { 41 | log.Fatal(err) 42 | } 43 | 44 | cmd := exec.Command(cmdName) 45 | if c.Args.Tool != "" { 46 | cmd.Args = append(cmd.Args, string(c.Args.Tool)) 47 | cmd.Args = append(cmd.Args, c.Args.ToolArgs...) 48 | } 49 | cmd.Stdin = os.Stdin 50 | cmd.Stdout = os.Stdout 51 | cmd.Stderr = os.Stderr 52 | if GlobalOpt.Verbose { 53 | log.Printf("Running tool: %v", cmd.Args) 54 | } 55 | return cmd.Run() 56 | } 57 | 58 | type ToolName string 59 | 60 | func (t ToolName) Complete(match string) []flags.Completion { 61 | // Assume toolchain is the last arg. 62 | toolchainPath := os.Args[len(os.Args)-2] 63 | tc, err := toolchain.Lookup(toolchainPath) 64 | if err != nil { 65 | log.Println(err) 66 | return nil 67 | } 68 | c, err := tc.ReadConfig() 69 | if err != nil { 70 | log.Println(err) 71 | return nil 72 | } 73 | var comps []flags.Completion 74 | for _, tt := range c.Tools { 75 | if strings.HasPrefix(tt.Subcmd, match) { 76 | comps = append(comps, flags.Completion{Item: tt.Subcmd}) 77 | } 78 | } 79 | return comps 80 | } 81 | -------------------------------------------------------------------------------- /toolchain/find_test.go: -------------------------------------------------------------------------------- 1 | package toolchain 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path" 7 | "path/filepath" 8 | "reflect" 9 | "runtime" 10 | "testing" 11 | 12 | "sourcegraph.com/sourcegraph/srclib" 13 | ) 14 | 15 | func TestList_program(t *testing.T) { 16 | tmpdir, err := ioutil.TempDir("", "srclib-toolchain-test") 17 | if err != nil { 18 | t.Fatal(err) 19 | } 20 | defer os.RemoveAll(tmpdir) 21 | 22 | defer func(orig string) { 23 | srclib.Path = orig 24 | }(srclib.Path) 25 | srclib.Path = tmpdir 26 | 27 | var extension string 28 | if runtime.GOOS == "windows" { 29 | extension = ".exe" 30 | } else { 31 | extension = "" 32 | } 33 | files := map[string]os.FileMode{ 34 | // ok 35 | filepath.Join("a", "a", ".bin", "a"+extension): 0700, 36 | filepath.Join("a", "a", "Srclibtoolchain"): 0700, 37 | 38 | // not executable 39 | filepath.Join("b", "b", ".bin", "z"+extension): 0600, 40 | filepath.Join("b", "b", "Srclibtoolchain"): 0600, 41 | 42 | // not in .bin 43 | filepath.Join("c", "c", "c"+extension): 0700, 44 | filepath.Join("c", "c", "Srclibtoolchain"): 0700, 45 | } 46 | for f, mode := range files { 47 | f = filepath.Join(tmpdir, f) 48 | if err := os.MkdirAll(filepath.Dir(f), 0700); err != nil { 49 | t.Fatal(err) 50 | } 51 | if err := ioutil.WriteFile(f, nil, mode); err != nil { 52 | t.Fatal(err) 53 | } 54 | } 55 | 56 | // Put a file symlink in srclib DIR path. 57 | oldp := filepath.Join(tmpdir, "a", "a", ".bin", "a"+extension) 58 | newp := filepath.Join(tmpdir, "link") 59 | if err := os.Symlink(oldp, newp); err != nil { 60 | t.Fatal(err) 61 | } 62 | 63 | toolchains, err := List() 64 | if err != nil { 65 | t.Fatal(err) 66 | } 67 | 68 | got := toolchainPathsWithProgram(toolchains) 69 | want := []string{path.Join("a", "a")} 70 | if !reflect.DeepEqual(got, want) { 71 | t.Errorf("got toolchains %v, want %v", got, want) 72 | } 73 | } 74 | 75 | func toolchainPathsWithProgram(toolchains []*Info) []string { 76 | paths := make([]string, 0, len(toolchains)) 77 | for _, toolchain := range toolchains { 78 | if toolchain.Program != "" { 79 | paths = append(paths, toolchain.Path) 80 | } 81 | } 82 | return paths 83 | } 84 | -------------------------------------------------------------------------------- /flagutil/marshal.go: -------------------------------------------------------------------------------- 1 | package flagutil 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "strings" 7 | 8 | "sourcegraph.com/sourcegraph/go-flags" 9 | ) 10 | 11 | // MarshalArgs takes a struct with go-flags field tags and turns it into an 12 | // equivalent []string for use as command-line args. 13 | func MarshalArgs(v interface{}) ([]string, error) { 14 | parser := flags.NewParser(nil, flags.None) 15 | group, err := parser.AddGroup("", "", v) 16 | if err != nil { 17 | return nil, err 18 | } 19 | return marshalArgsInGroup(group, "") 20 | } 21 | 22 | func marshalArgsInGroup(group *flags.Group, prefix string) ([]string, error) { 23 | var args []string 24 | for _, opt := range group.Options() { 25 | flagStr := opt.String() 26 | 27 | // handle flags with both short and long (just get the long) 28 | if i := strings.Index(flagStr, ", --"); i != -1 { 29 | flagStr = flagStr[i+2:] 30 | } 31 | 32 | switch v := opt.Value().(type) { 33 | case flags.Marshaler: 34 | s, err := v.MarshalFlag() 35 | if err != nil { 36 | return nil, err 37 | } 38 | args = append(args, flagStr, s) 39 | case []string: 40 | for _, s := range v { 41 | args = append(args, flagStr, s) 42 | } 43 | case bool: 44 | if v { 45 | args = append(args, flagStr) 46 | } 47 | default: 48 | if !isDefaultValue(opt, fmt.Sprintf("%v", v)) { 49 | args = append(args, flagStr, fmt.Sprintf("%v", v)) 50 | } 51 | } 52 | } 53 | for _, g := range group.Groups() { 54 | // TODO(sqs): assumes that the NamespaceDelimiter is "." 55 | const namespaceDelimiter = "." 56 | groupArgs, err := marshalArgsInGroup(g, g.Namespace+namespaceDelimiter) 57 | if err != nil { 58 | return nil, err 59 | } 60 | args = append(args, groupArgs...) 61 | } 62 | 63 | return args, nil 64 | } 65 | 66 | func isDefaultValue(opt *flags.Option, val string) bool { 67 | var defaultVal string 68 | 69 | switch len(opt.Default) { 70 | case 0: 71 | if k := reflect.TypeOf(opt.Value()).Kind(); k >= reflect.Int && k <= reflect.Uint64 { 72 | defaultVal = "0" 73 | } 74 | case 1: 75 | defaultVal = fmt.Sprintf("%v", opt.Default[0]) 76 | default: 77 | return false 78 | } 79 | 80 | return defaultVal == fmt.Sprintf("%v", val) 81 | } 82 | -------------------------------------------------------------------------------- /ann/annotation.go: -------------------------------------------------------------------------------- 1 | package ann 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/url" 7 | "strconv" 8 | "strings" 9 | ) 10 | 11 | const ( 12 | // Link is a type of annotation that refers to an arbitrary URL 13 | // (typically pointing to an external web page). 14 | Link = "link" 15 | ) 16 | 17 | // LinkURL parses and returns a's link URL, if a's type is Link and if 18 | // its Data contains a valid URL (encoded as a JSON string). 19 | func (a *Ann) LinkURL() (*url.URL, error) { 20 | if a.Type != Link { 21 | return nil, &ErrType{Expected: Link, Actual: a.Type, Op: "LinkURL"} 22 | } 23 | var urlStr string 24 | if err := json.Unmarshal(a.Data, &urlStr); err != nil { 25 | return nil, fmt.Errorf("could not unmarshal annotation link URL: %s. JSON was %v", err, a.Data) 26 | } 27 | return url.Parse(urlStr) 28 | } 29 | 30 | // SetLinkURL sets a's Type to Link and Data to the JSON 31 | // representation of the URL string. If the URL is invalid, an error 32 | // is returned. 33 | func (a *Ann) SetLinkURL(urlStr string) error { 34 | u, err := url.Parse(urlStr) 35 | if err != nil { 36 | return err 37 | } 38 | b, err := json.Marshal(u.String()) 39 | if err != nil { 40 | return err 41 | } 42 | a.Type = Link 43 | a.Data = b 44 | return nil 45 | } 46 | 47 | // ErrType indicates that an operation performed on an annotation 48 | // expected the annotation to be a different type (e.g., calling 49 | // LinkURL on a non-link annotation). 50 | type ErrType struct { 51 | Expected, Actual string // Expected and actual types 52 | Op string // The name of the operation or method that was called 53 | } 54 | 55 | func (e *ErrType) Error() string { 56 | return fmt.Sprintf("%s called on annotation type %q, expected type %q", e.Op, e.Actual, e.Expected) 57 | } 58 | 59 | func (a *Ann) sortKey() string { 60 | return strings.Join([]string{a.Repo, a.CommitID, a.UnitType, a.Unit, a.Type, a.File, strconv.Itoa(int(a.StartLine)), strconv.Itoa(int(a.EndLine))}, ":") 61 | } 62 | 63 | // Sorting 64 | 65 | type Anns []*Ann 66 | 67 | func (vs Anns) Len() int { return len(vs) } 68 | func (vs Anns) Swap(i, j int) { vs[i], vs[j] = vs[j], vs[i] } 69 | func (vs Anns) Less(i, j int) bool { return vs[i].sortKey() < vs[j].sortKey() } 70 | -------------------------------------------------------------------------------- /docs/theme/images/layers.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 18 | 20 | 42 | 44 | 45 | 47 | image/svg+xml 48 | 50 | 51 | 52 | 53 | 54 | 59 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /config/cached.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | "runtime" 8 | "sort" 9 | "strings" 10 | 11 | "github.com/neelance/parallel" 12 | 13 | "github.com/kr/fs" 14 | "golang.org/x/tools/godoc/vfs" 15 | "sourcegraph.com/sourcegraph/rwvfs" 16 | "sourcegraph.com/sourcegraph/srclib/buildstore" 17 | "sourcegraph.com/sourcegraph/srclib/unit" 18 | ) 19 | 20 | // ReadCached reads a Tree's configuration from all of its source unit 21 | // definition files (which may either be in a local VFS rooted at a 22 | // .srclib-cache/ dir, or a remote VFS). It does not read 23 | // the Srcfile; the Srcfile's directives are already accounted for in 24 | // the cached source unit definition files. 25 | // 26 | // bdfs should be a VFS obtained from a call to 27 | // (buildstore.RepoBuildStore).Commit. 28 | func ReadCached(bdfs vfs.FileSystem) (*Tree, error) { 29 | if _, err := bdfs.Lstat("."); os.IsNotExist(err) { 30 | return nil, fmt.Errorf("build cache dir does not exist (did you run `srclib config` to create it)?") 31 | } else if err != nil { 32 | return nil, err 33 | } 34 | 35 | // Collect all **/*.unit.json files. 36 | var unitFiles []string 37 | unitSuffix := buildstore.DataTypeSuffix(unit.SourceUnit{}) 38 | w := fs.WalkFS(".", rwvfs.Walkable(rwvfs.ReadOnly(bdfs))) 39 | for w.Step() { 40 | if err := w.Err(); err != nil { 41 | return nil, err 42 | } 43 | if path := w.Path(); strings.HasSuffix(path, unitSuffix) { 44 | unitFiles = append(unitFiles, path) 45 | } 46 | } 47 | 48 | // Parse units 49 | sort.Strings(unitFiles) 50 | units := make([]*unit.SourceUnit, len(unitFiles)) 51 | par := parallel.NewRun(runtime.GOMAXPROCS(0)) 52 | for i_, unitFile_ := range unitFiles { 53 | i, unitFile := i_, unitFile_ 54 | par.Acquire() 55 | go func() { 56 | defer par.Release() 57 | f, err := bdfs.Open(unitFile) 58 | if err != nil { 59 | par.Error(err) 60 | return 61 | } 62 | if err := json.NewDecoder(f).Decode(&units[i]); err != nil { 63 | f.Close() 64 | par.Error(err) 65 | return 66 | } 67 | if err := f.Close(); err != nil { 68 | par.Error(err) 69 | return 70 | } 71 | }() 72 | } 73 | if err := par.Wait(); err != nil { 74 | return nil, err 75 | } 76 | return &Tree{SourceUnits: units}, nil 77 | } 78 | -------------------------------------------------------------------------------- /docs/theme/images/whiteasterisk.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 18 | 20 | 42 | 44 | 45 | 47 | image/svg+xml 48 | 50 | 51 | 52 | 53 | 54 | 59 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /docs/theme/images/asterisk.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 18 | 20 | 42 | 44 | 45 | 47 | image/svg+xml 48 | 50 | 51 | 52 | 53 | 54 | 59 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /toolchain/toolchain.go: -------------------------------------------------------------------------------- 1 | package toolchain 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | "path/filepath" 7 | 8 | "sourcegraph.com/sourcegraph/srclib" 9 | ) 10 | 11 | // Dir returns the directory where the named toolchain lives (under 12 | // the SRCLIBPATH). If the toolchain already exists in any of the 13 | // entries of SRCLIBPATH, that directory is returned. Otherwise a 14 | // nonexistent directory in the first SRCLIBPATH entry is returned. 15 | func Dir(toolchainPath string) (string, error) { 16 | toolchainPath = filepath.Clean(toolchainPath) 17 | 18 | dir, err := lookupToolchain(toolchainPath) 19 | if os.IsNotExist(err) { 20 | return filepath.Join(filepath.SplitList(srclib.Path)[0], toolchainPath), nil 21 | } 22 | if err != nil { 23 | err = &os.PathError{Op: "toolchain.Dir", Path: toolchainPath, Err: err} 24 | } 25 | return dir, err 26 | } 27 | 28 | // Info describes a toolchain. 29 | type Info struct { 30 | // Path is the toolchain's path (not a directory path) underneath the 31 | // SRCLIBPATH. It consists of the URI of this repository's toolchain plus 32 | // its subdirectory path within the repository. E.g., "github.com/foo/bar" 33 | // for a toolchain defined in the root directory of that repository. 34 | Path string 35 | 36 | // Dir is the filesystem directory that defines this toolchain. 37 | Dir string 38 | 39 | // ConfigFile is the path to the Srclibtoolchain file, relative to Dir. 40 | ConfigFile string 41 | 42 | // Program is the path to the executable program (relative to Dir) to run to 43 | // invoke this toolchain. 44 | Program string `json:",omitempty"` 45 | } 46 | 47 | // ReadConfig reads and parses the Srclibtoolchain config file for the 48 | // toolchain. 49 | func (t *Info) ReadConfig() (*Config, error) { 50 | f, err := os.Open(filepath.Join(t.Dir, t.ConfigFile)) 51 | if err != nil { 52 | return nil, err 53 | } 54 | defer f.Close() 55 | 56 | var c *Config 57 | if err := json.NewDecoder(f).Decode(&c); err != nil { 58 | return nil, err 59 | } 60 | return c, nil 61 | } 62 | 63 | // Command returns the path to the executable program for the 64 | // toolchain with the given path. 65 | func Command(path string) (string, error) { 66 | tc, err := Lookup(path) 67 | if err != nil { 68 | return "", err 69 | } 70 | return filepath.Join(tc.Dir, tc.Program), nil 71 | } 72 | -------------------------------------------------------------------------------- /graph/formatter_test.go: -------------------------------------------------------------------------------- 1 | package graph 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | type testFormatter struct{} 9 | 10 | func (_ testFormatter) Name(qual Qualification) string { 11 | switch qual { 12 | case Unqualified: 13 | return "name" 14 | case ScopeQualified: 15 | return "scope.name" 16 | case DepQualified: 17 | return "imp.scope.name" 18 | case RepositoryWideQualified: 19 | return "dir/lib.scope.name" 20 | case LanguageWideQualified: 21 | return "lib.scope.name" 22 | } 23 | panic("Name: unrecognized Qualification: " + fmt.Sprint(qual)) 24 | } 25 | 26 | func (_ testFormatter) Type(qual Qualification) string { 27 | switch qual { 28 | case Unqualified: 29 | return "typeName" 30 | case ScopeQualified: 31 | return "scope.typeName" 32 | case DepQualified: 33 | return "imp.scope.typeName" 34 | case RepositoryWideQualified: 35 | return "dir/lib.scope.typeName" 36 | case LanguageWideQualified: 37 | return "lib.scope.typeName" 38 | } 39 | panic("Type: unrecognized Qualification: " + fmt.Sprint(qual)) 40 | } 41 | 42 | func (_ testFormatter) Language() string { return "lang" } 43 | func (_ testFormatter) DefKeyword() string { return "defkw" } 44 | func (_ testFormatter) NameAndTypeSeparator() string { return "_" } 45 | func (_ testFormatter) Kind() string { return "kind" } 46 | 47 | func TestPrintFormatter(t *testing.T) { 48 | const unitType = "TestFormatter" 49 | RegisterMakeDefFormatter("TestFormatter", func(*Def) DefFormatter { return testFormatter{} }) 50 | def := &Def{DefKey: DefKey{UnitType: unitType}} 51 | tests := []struct { 52 | format string 53 | want string 54 | }{ 55 | {"%n", "name"}, 56 | {"%.0n", "name"}, 57 | {"%.1n", "scope.name"}, 58 | {"%.2n", "imp.scope.name"}, 59 | {"%.3n", "dir/lib.scope.name"}, 60 | {"%.4n", "lib.scope.name"}, 61 | {"%t", "typeName"}, 62 | {"%.0t", "typeName"}, 63 | {"%.1t", "scope.typeName"}, 64 | {"%.2t", "imp.scope.typeName"}, 65 | {"%.3t", "dir/lib.scope.typeName"}, 66 | {"%.4t", "lib.scope.typeName"}, 67 | {"% t", "_typeName"}, 68 | {"%w", "defkw"}, 69 | {"%k", "kind"}, 70 | } 71 | for _, test := range tests { 72 | str := fmt.Sprintf(test.format, PrintFormatter(def)) 73 | if str != test.want { 74 | t.Errorf("Sprintf(%q, def): got %q, want %q", test.format, str, test.want) 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # srclib Documentation Site 2 | 3 | The srclib documentation files are written in Markdown and converted to a 4 | browsable HTML static site, hosted at [srclib.org](http://srclib.org/). 5 | **[MkDocs](http://www.mkdocs.org/)**, installable through pip, is used to 6 | generate the HTML. 7 | 8 | Run `pip install mkdocs==0.9.0` to install MkDocs, version 0.9.0. 9 | 10 | You will also need SASS, in order to compile the style sheets. This can be 11 | obtained with `gem install sass`. In order to use the testing script `test.sh`, 12 | you will need to run `sudo apt-get install inotify-tools`. 13 | 14 | ## WARNING 15 | 16 | **Doc tags (like "START SomeTag") are NOT preserved when protobuf 17 | files are generated. You will need to add them by hand.** 18 | 19 | 20 | ## Testing Documentation 21 | 22 | To start testing, run `./test.sh`. This will watch for changes, building into 23 | `site/` whenever a file is modified. Additionally, `python -m SimpleHTTPServer` 24 | is used to start a local server on port 8000. 25 | 26 | ## Editing 27 | 28 | Markdown files are located in the `sources` directory, with subdirectories 29 | intended to correspond with the drop-down menus on the site. The actual 30 | categories, however, are specified through configuration in `mkdocs.yml`. 31 | 32 | ### Embedding Code Segments 33 | In the docs, you can embed code taken directly from the repository source. 34 | The format is as follows: 35 | ``` 36 | [[.code "grapher/grapher.go" 23 30]] 37 | ``` 38 | Note that the file name is specified as relative to the repository root. 39 | You can also specify two strings instead of two integers. 40 | ``` 41 | [[.code "grapher/grapher.go" "START REGION" "END REGION"]] 42 | ``` 43 | This will look for the first lines that contain those strings (case insensitive). 44 | 45 | Like in the go-present tool, lines ending with the word `OMIT` are omitted from the 46 | embedded code. If you have a segment as follows: 47 | ```go 48 | // START RegionName OMIT 49 | Go code here... 50 | // END RegionName OMIT 51 | ``` 52 | You can use the following short-hand: 53 | ``` 54 | [[.code "grapher/grapher.go" "RegionName"]] 55 | ``` 56 | 57 | ## Deploying 58 | 59 | The site can be deployed to srclib.org simply by running `./deploy.sh`. This 60 | builds the SASS stylesheet, builds the docs, and builds the other static pages 61 | of the site. It then uses `aws s3 sync` to store these in the correct bucket. 62 | -------------------------------------------------------------------------------- /cli/util_test.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | "os" 7 | "path/filepath" 8 | "testing" 9 | 10 | "golang.org/x/tools/godoc/vfs" 11 | ) 12 | 13 | func TestReadJSONFileEmpty(t *testing.T) { 14 | f, err := ioutil.TempFile("", "read-json") 15 | if err != nil { 16 | t.Fatal(err) 17 | } 18 | defer f.Close() 19 | defer os.Remove(f.Name()) 20 | err = readJSONFile(f.Name(), struct{}{}) 21 | if err != errEmptyJSONFile { 22 | t.Errorf("Expected empty JSON file error, got %v", err) 23 | } 24 | fs := vfs.OS(filepath.Dir(f.Name())) 25 | err = readJSONFileFS(fs, filepath.Base(f.Name()), struct{}{}) 26 | if err != errEmptyJSONFile { 27 | t.Errorf("Expected empty JSON file error (VFS), got %v", err) 28 | } 29 | } 30 | 31 | func TestReadJSONFileNotFound(t *testing.T) { 32 | err := readJSONFile("doesnotexist", struct{}{}) 33 | if !os.IsNotExist(err) { 34 | t.Errorf("Expected does not exist error, got %v", err) 35 | } 36 | wd, err := os.Getwd() 37 | if err != nil { 38 | t.Fatal(err) 39 | } 40 | err = readJSONFileFS(vfs.OS(wd), "doesnotexist", struct{}{}) 41 | if !os.IsNotExist(err) { 42 | t.Errorf("Expected does not exist error (VFS), got %v", err) 43 | } 44 | } 45 | 46 | func TestReadJSONFile(t *testing.T) { 47 | f, err := ioutil.TempFile("", "read-json") 48 | if err != nil { 49 | t.Fatal(err) 50 | } 51 | defer f.Close() 52 | defer os.Remove(f.Name()) 53 | 54 | // Test valid decode. 55 | type msg struct{ A, B int } 56 | in := msg{5, 3} 57 | err = json.NewEncoder(f).Encode(in) 58 | if err != nil { 59 | t.Fatal(err) 60 | } 61 | out := new(msg) 62 | err = readJSONFile(f.Name(), out) 63 | if err != nil { 64 | t.Fatal(err) 65 | } 66 | if in != *out { 67 | t.Errorf("Read JSON file %+v, expected %+v", out, in) 68 | } 69 | fs := vfs.OS(filepath.Dir(f.Name())) 70 | outFS := new(msg) 71 | err = readJSONFileFS(fs, filepath.Base(f.Name()), outFS) 72 | if err != nil { 73 | t.Fatal(err) 74 | } 75 | if in != *outFS { 76 | t.Errorf("Read JSON file %+v (VFS), expected %+v", outFS, in) 77 | } 78 | 79 | // Test invalid decode. 80 | err = readJSONFile(f.Name(), &struct{ A, B bool }{}) 81 | if err == nil { 82 | t.Error("Expected error because of invalid decode") 83 | } 84 | err = readJSONFileFS(fs, filepath.Base(f.Name()), &struct{ A, B bool }{}) 85 | if err == nil { 86 | t.Error("Expected erorr because of invalid decode (VFS)") 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /dep/doc.go: -------------------------------------------------------------------------------- 1 | // Package dep defines an interface for listing raw dependencies and resolving 2 | // them, and registering handlers to perform these tasks. 3 | // 4 | // Toolchains should register two handlers to list and resolve dependencies: 5 | // 6 | // *Lister:* this performs a quick pass over the declared dependencies of a 7 | // source unit and emits whatever information can be determined without 8 | // an expensive dependency resolution process. For example, the Go lister 9 | // simply lists the packages imported by each file in the source unit. 10 | // 11 | // *Resolver:* this resolves a RawDependency (emitted from a Lister), 12 | // determining the defining repository, source unit, version, etc. For example, 13 | // the Ruby resolver queries rubygems.org (and consults a hard-coded list) to 14 | // determine this information. Note that the only necessary field in 15 | // ResolvedTarget is ToRepoCloneURL; the other fields are for future use. 16 | // 17 | // In code, this looks like: 18 | // 19 | // package my_toolchain 20 | // 21 | // func init() { 22 | // dep.RegisterLister(PythonPackage{}, pip) 23 | // dep.RegisterResolver("pip-package", pip) 24 | // } 25 | // 26 | // type pip struct {} 27 | // 28 | // func (p *pip) List(dir string, unit unit.SourceUnit, c *config.Repository) ([]*dep.RawDependency, error) { 29 | // // return each line in requirements.txt, perhaps 30 | // // return something like: 31 | // return []*dep.RawDependency{{TargetType: "pip-package", Target: "Django==1.6"}}, nil 32 | // } 33 | // 34 | // func (p *pip) Resolve(rawDep *dep.RawDependency, c *config.Repository) (*dep.ResolvedTarget, error) { 35 | // // query pypi.python.org or use cheerio, and then return something like: 36 | // return &dep.ResolvedTarget{ 37 | // ToRepoCloneURL: "git://github.com/django/django.git", 38 | // }, nil 39 | // } 40 | // 41 | // Separating the list and resolution steps allows us to more easily pare down 42 | // the total number of unique resolutions we must perform. If multiple source 43 | // units all depend on the same external library, then those duplicate 44 | // RawDependencies only yield a single call to Resolve. This saves even more 45 | // work when finding dependencies for all of a repository's history (because 46 | // most commits likely won't add or update dependency declarations). 47 | // 48 | // TODO(sqs): update these docs after we removed deplist 49 | package dep 50 | -------------------------------------------------------------------------------- /toolchain/config.go: -------------------------------------------------------------------------------- 1 | package toolchain 2 | 3 | // ConfigFilename is the filename of the toolchain configuration file. The 4 | // presence of this file in a directory signifies that a srclib toolchain is 5 | // defined in that directory. 6 | const ConfigFilename = "Srclibtoolchain" 7 | 8 | // Config represents a Srclibtoolchain file, which defines a srclib toolchain. 9 | type Config struct { 10 | // Tools is the list of this toolchain's tools and their definitions. 11 | Tools []*ToolInfo 12 | 13 | // Bundle configures the way that this toolchain is built and 14 | // archived. If Bundle is not set, it means that the toolchain 15 | // can't be bundled. 16 | Bundle *struct { 17 | // Paths is a list of path globs whose matches are included in 18 | // the toolchain bundle archive file (created, e.g., using 19 | // "srclib toolchain bundle "). It should contain all of 20 | // the files necessary to execute the toolchain. For 21 | // toolchains whose entrypoint is a static binary or JAR, this 22 | // is typically the entrypoint plus any shell scripts or 23 | // support files necessary. 24 | // 25 | // Go's filepath.Glob is used for globbing. In addition, if a 26 | // dir is specified, all of its entries are included 27 | // recursively. 28 | // 29 | // "Srclibtoolchain" and ".bin/{basename}" (where {basename} 30 | // is this toolchain's path's final path component) MUST 31 | // always be included in this list, or else when unarchived 32 | // the bundle won't be a valid toolchain. 33 | Paths []string `json:",omitempty"` 34 | 35 | // Commands is the list of commands to run in order to build 36 | // the files to archive. (E.g., "go build ...".) Sometimes 37 | // these commands just consist of a "make install" or similar 38 | // invocation. 39 | // 40 | // All commands are passed to `sh -c`. 41 | Commands []string `json:",omitempty"` 42 | 43 | // Variants is the set of possible bundles. Typically these 44 | // define products such as "linux-amd64", "linux-386", 45 | // "darwin-amd64", etc., for binary outputs. 46 | // 47 | // The key-value pairs specified in each variant are available 48 | // to the commands (in the Commands list). Each command's 49 | // variable references ($foo or ${foo}) are expanded using the 50 | // values in the Variant, and they are run with the Variant's 51 | // properties set as environment variables. 52 | Variants []Variant `json:",omitempty"` 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /store/tree_stores.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | // scopeTrees returns a list of commit IDs that are matched by the 4 | // filters. If potentially all commits could match, or if enough 5 | // commits could potentially match that it would probably be cheaper 6 | // to iterate through all of them, then a nil slice is returned. If 7 | // none match, an empty slice is returned. 8 | // 9 | // scopeTrees is used to select which TreeStores to query. 10 | // 11 | // TODO(sqs): return an error if the filters are mutually exclusive? 12 | func scopeTrees(filters []interface{}) ([]string, error) { 13 | commitIDs := map[string]struct{}{} 14 | everHadAny := false // whether unitIDs ever contained any commitIDs 15 | 16 | for _, f := range filters { 17 | switch f := f.(type) { 18 | case ByCommitIDsFilter: 19 | if len(commitIDs) == 0 && !everHadAny { 20 | everHadAny = true 21 | for _, c := range f.ByCommitIDs() { 22 | commitIDs[c] = struct{}{} 23 | } 24 | } else { 25 | // Intersect. 26 | newCommitIDs := make(map[string]struct{}, (len(commitIDs)+len(f.ByCommitIDs()))/2) 27 | for _, c := range f.ByCommitIDs() { 28 | if _, present := commitIDs[c]; present { 29 | newCommitIDs[c] = struct{}{} 30 | } 31 | } 32 | commitIDs = newCommitIDs 33 | } 34 | } 35 | } 36 | 37 | if len(commitIDs) == 0 && !everHadAny { 38 | // No unit scoping filters were present, so scope includes 39 | // potentially all commitIDs. 40 | return nil, nil 41 | } 42 | 43 | ids := make([]string, 0, len(commitIDs)) 44 | for commitID := range commitIDs { 45 | ids = append(ids, commitID) 46 | } 47 | return ids, nil 48 | } 49 | 50 | // A treeStoreOpener opens the TreeStore for the specified tree. 51 | type treeStoreOpener interface { 52 | openTreeStore(commitID string) TreeStore 53 | openAllTreeStores() (map[string]TreeStore, error) 54 | } 55 | 56 | // openCommitstores is a helper func that calls o.openTreeStore for 57 | // each tree returned by scopeTrees(filters...). 58 | func openTreeStores(o treeStoreOpener, filters interface{}) (map[string]TreeStore, error) { 59 | commitIDs, err := scopeTrees(storeFilters(filters)) 60 | if err != nil { 61 | return nil, err 62 | } 63 | 64 | if commitIDs == nil { 65 | return o.openAllTreeStores() 66 | } 67 | 68 | tss := make(map[string]TreeStore, len(commitIDs)) 69 | for _, commitID := range commitIDs { 70 | tss[commitID] = o.openTreeStore(commitID) 71 | } 72 | return tss, nil 73 | } 74 | -------------------------------------------------------------------------------- /cli/repo_config.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "path/filepath" 7 | 8 | "os" 9 | "os/exec" 10 | "strings" 11 | 12 | "sourcegraph.com/sourcegraph/srclib/util" 13 | ) 14 | 15 | type Repo struct { 16 | RootDir string // Root directory containing repository being analyzed 17 | VCSType string // VCS type (git or hg) 18 | CommitID string // CommitID of current working directory 19 | } 20 | 21 | func OpenRepo(dir string) (*Repo, error) { 22 | if fi, err := os.Stat(dir); err != nil || !fi.Mode().IsDir() { 23 | return nil, fmt.Errorf("not a directory: %q", dir) 24 | } 25 | 26 | rc := new(Repo) 27 | 28 | // VCS and root directory 29 | var err error 30 | rc.RootDir, rc.VCSType, err = getRootDir(dir) 31 | if err != nil { 32 | return nil, fmt.Errorf("detecting git/hg repository in %s: %s", dir, err) 33 | } 34 | if rc.RootDir == "" { 35 | return nil, fmt.Errorf("no git/hg repository found in or above %s", dir) 36 | } 37 | 38 | rc.CommitID, err = resolveWorkingTreeRevision(rc.VCSType, rc.RootDir) 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | return rc, nil 44 | } 45 | 46 | func resolveWorkingTreeRevision(vcsType string, dir string) (string, error) { 47 | var cmd *exec.Cmd 48 | switch vcsType { 49 | case "git": 50 | cmd = exec.Command("git", "rev-parse", "HEAD") 51 | case "hg": 52 | cmd = exec.Command("hg", "--config", "trusted.users=root", "identify", "--debug", "-i") 53 | default: 54 | return "", fmt.Errorf("unknown vcs type: %q", vcsType) 55 | } 56 | cmd.Dir = dir 57 | 58 | out, err := cmd.CombinedOutput() 59 | if err != nil { 60 | return "", fmt.Errorf("exec %v failed: %s. Output was:\n\n%s", cmd.Args, err, out) 61 | } 62 | // hg adds a "+" if the wd is dirty 63 | return strings.TrimSuffix(string(bytes.TrimSpace(out)), "+"), nil 64 | } 65 | 66 | func getRootDir(dir string) (rootDir string, vcsType string, err error) { 67 | dir, err = filepath.Abs(dir) 68 | if err != nil { 69 | return "", "", err 70 | } 71 | ancestors := util.AncestorDirs(dir, true) 72 | 73 | vcsTypes := []string{"git", "hg"} 74 | for i := len(ancestors) - 1; i >= 0; i-- { 75 | ancDir := ancestors[i] 76 | for _, vt := range vcsTypes { 77 | // Don't check that the FileInfo is a dir because git 78 | // submodules have a .git file. 79 | if _, err := os.Stat(filepath.Join(ancDir, "."+vt)); err == nil { 80 | return ancDir, vt, nil 81 | } 82 | } 83 | } 84 | return "", "", nil 85 | } 86 | -------------------------------------------------------------------------------- /docs/theme/layout.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | {% block title %}srclib{% endblock %} 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | {% include "navbar.html" %} 32 | 33 | {% block content %}{% endblock %} 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 51 | 52 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /ann/ann.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package ann; 3 | 4 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 5 | 6 | option (gogoproto.goproto_getters_all) = false; 7 | option (gogoproto.unmarshaler_all) = true; 8 | option (gogoproto.marshaler_all) = true; 9 | option (gogoproto.sizer_all) = true; 10 | 11 | // An Ann is a source code annotation. 12 | // 13 | // Annotations are unique on (Repo, CommitID, UnitType, Unit, File, 14 | // StartLine, EndLine, Type). 15 | message Ann { 16 | // Repo is the VCS repository in which this ann exists. 17 | string Repo = 1 [(gogoproto.jsontag) = "Repo,omitempty"]; 18 | 19 | // CommitID is the ID of the VCS commit that this ann exists 20 | // in. The CommitID is always a full commit ID (40 hexadecimal 21 | // characters for git and hg), never a branch or tag name. 22 | string CommitID = 2 [(gogoproto.jsontag) = "CommitID,omitempty"]; 23 | 24 | // UnitType is the source unit type that the annotation exists 25 | // on. It is either the source unit type during whose processing 26 | // the annotation was detected/created. Multiple annotations may 27 | // exist on the same file from different source unit types if a 28 | // file is contained in multiple source units. 29 | string UnitType = 3 [(gogoproto.jsontag) = "UnitType,omitempty"]; 30 | 31 | // Unit is the name of the source unit that this ann exists in. 32 | string Unit = 4 [(gogoproto.jsontag) = "Unit,omitempty"]; 33 | 34 | // File is the filename in which this Ann exists. 35 | string File = 5 [(gogoproto.jsontag) = "File,omitempty"]; 36 | 37 | // TODO(sqs): Add byte offset fields. Add back byte offset fixups to grapher/grapher.go when done. 38 | 39 | // StartLine is the line number (inclusive, 1-indexed) of the 40 | // beginning of the annotation. 41 | uint32 StartLine = 6 [(gogoproto.jsontag) = "StartLine"]; 42 | 43 | // EndLine is the line number (inclusive, 1-indexed) of the end of 44 | // the annotation. 45 | uint32 EndLine = 7 [(gogoproto.jsontag) = "EndLine"]; 46 | 47 | // Type is the type of the annotation. See this package's type 48 | // constants for a list of possible types. 49 | string Type = 8 [(gogoproto.jsontag) = "Type"]; 50 | 51 | // Data contains arbitrary JSON data that is specific to this 52 | // annotation type (e.g., the link URL for Link annotations). 53 | bytes Data = 9 [(gogoproto.casttype) = "sourcegraph.com/sqs/pbtypes.RawMessage", (gogoproto.jsontag) = "Data,omitempty"]; 54 | }; 55 | -------------------------------------------------------------------------------- /docs/theme/images/banner.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 18 | 20 | 42 | 44 | 45 | 47 | image/svg+xml 48 | 50 | 51 | 52 | 53 | 54 | 59 | 66 | 73 | 80 | 81 | 82 | -------------------------------------------------------------------------------- /docs/theme/images/stack.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 18 | 20 | 42 | 44 | 45 | 47 | image/svg+xml 48 | 50 | 51 | 52 | 53 | 54 | 59 | 62 | 65 | 68 | 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /store/pb/helpers.go: -------------------------------------------------------------------------------- 1 | package pb 2 | 3 | import ( 4 | "golang.org/x/net/context" 5 | "sourcegraph.com/sourcegraph/srclib/graph" 6 | "sourcegraph.com/sourcegraph/srclib/store" 7 | "sourcegraph.com/sourcegraph/srclib/unit" 8 | "sourcegraph.com/sqs/pbtypes" 9 | ) 10 | 11 | // A MultiRepoImporterIndexer implements both store.MultiRepoImporter 12 | // and store.MultiRepoIndexer. 13 | type MultiRepoImporterIndexer interface { 14 | store.MultiRepoImporter 15 | store.MultiRepoIndexer 16 | } 17 | 18 | // Client wraps a gRPC MultiRepoImporterClient and makes it implement 19 | // store.MultiRepoImporter. Clients should not be long-lived (because 20 | // they must hold the ctx). 21 | func Client(ctx context.Context, c MultiRepoImporterClient) MultiRepoImporterIndexer { 22 | return &client{ctx, c} 23 | } 24 | 25 | type client struct { 26 | ctx context.Context 27 | u MultiRepoImporterClient 28 | } 29 | 30 | func (c *client) Import(repo, commitID string, u *unit.SourceUnit, data graph.Output) error { 31 | _, err := c.u.Import(c.ctx, &ImportOp{ 32 | Repo: repo, 33 | CommitID: commitID, 34 | Unit: u, 35 | Data: &data, 36 | }) 37 | return err 38 | } 39 | 40 | func (c *client) CreateVersion(repo, commitID string) error { 41 | _, err := c.u.CreateVersion(c.ctx, &CreateVersionOp{ 42 | Repo: repo, 43 | CommitID: commitID, 44 | }) 45 | return err 46 | } 47 | 48 | func (c *client) Index(repo, commitID string) error { 49 | _, err := c.u.Index(c.ctx, &IndexOp{Repo: repo, CommitID: commitID}) 50 | return err 51 | } 52 | 53 | // Server wraps a store.MultiRepoImporter and makes it implement 54 | // MultiRepoImporterServer. 55 | func Server(s MultiRepoImporterIndexer) MultiRepoImporterServer { return &server{s} } 56 | 57 | type server struct{ u MultiRepoImporterIndexer } 58 | 59 | func (s *server) Import(ctx context.Context, op *ImportOp) (*pbtypes.Void, error) { 60 | if op.Data == nil { 61 | op.Data = &graph.Output{} 62 | } 63 | if err := s.u.Import(op.Repo, op.CommitID, op.Unit, *op.Data); err != nil { 64 | return nil, err 65 | } 66 | return &pbtypes.Void{}, nil 67 | } 68 | 69 | func (s *server) CreateVersion(ctx context.Context, op *CreateVersionOp) (*pbtypes.Void, error) { 70 | if err := s.u.CreateVersion(op.Repo, op.CommitID); err != nil { 71 | return nil, err 72 | } 73 | return &pbtypes.Void{}, nil 74 | } 75 | 76 | func (s *server) Index(ctx context.Context, op *IndexOp) (*pbtypes.Void, error) { 77 | if err := s.u.Index(op.Repo, op.CommitID); err != nil { 78 | return nil, err 79 | } 80 | return &pbtypes.Void{}, nil 81 | } 82 | -------------------------------------------------------------------------------- /plan/makefile.go: -------------------------------------------------------------------------------- 1 | package plan 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | 7 | "sourcegraph.com/sourcegraph/makex" 8 | "sourcegraph.com/sourcegraph/srclib/buildstore" 9 | "sourcegraph.com/sourcegraph/srclib/config" 10 | ) 11 | 12 | type RuleMaker func(c *config.Tree, dataDir string, existing []makex.Rule) ([]makex.Rule, error) 13 | 14 | var ( 15 | RuleMakers = make(map[string]RuleMaker) 16 | ruleMakerNames []string 17 | orderedRuleMakers []RuleMaker 18 | ) 19 | 20 | // RegisterRuleMaker adds a function that creates a list of build rules for a 21 | // repository. If RegisterRuleMaker is called twice with the same target or 22 | // target name, if name is empty, or if r is nil, it panics. 23 | func RegisterRuleMaker(name string, r RuleMaker) { 24 | if _, dup := RuleMakers[name]; dup { 25 | panic("build: Register called twice for target lister " + name) 26 | } 27 | if r == nil { 28 | panic("build: Register target is nil") 29 | } 30 | RuleMakers[name] = r 31 | ruleMakerNames = append(ruleMakerNames, name) 32 | orderedRuleMakers = append(orderedRuleMakers, r) 33 | } 34 | 35 | // CreateMakefile creates the makefiles for the source units in c. 36 | func CreateMakefile(buildDataDir string, buildStore buildstore.RepoBuildStore, vcsType string, c *config.Tree) (*makex.Makefile, error) { 37 | var allRules []makex.Rule 38 | for i, r := range orderedRuleMakers { 39 | name := ruleMakerNames[i] 40 | rules, err := r(c, buildDataDir, allRules) 41 | if err != nil { 42 | return nil, fmt.Errorf("rule maker %s: %s", name, err) 43 | } 44 | sort.Sort(ruleSort{rules}) 45 | allRules = append(allRules, rules...) 46 | } 47 | 48 | // Add a ".PHONY" and "all" rules at the very beginning. 49 | allTargets := make([]string, len(allRules)) 50 | for i, rule := range allRules { 51 | allTargets[i] = rule.Target() 52 | } 53 | 54 | allRule := &makex.BasicRule{TargetFile: "all", PrereqFiles: allTargets} 55 | allRules = append([]makex.Rule{allRule}, allRules...) 56 | 57 | phonyRule := &makex.BasicRule{TargetFile: ".PHONY", PrereqFiles: []string{"all"}} 58 | allRules = append([]makex.Rule{phonyRule}, allRules...) 59 | 60 | // DELETE_ON_ERROR makes it so that the targets for failed recipes are 61 | // deleted. This lets us do "1> $@" to write to the target file without 62 | // erroneously satisfying the target if the recipe fails. makex has this 63 | // behavior by default and does not heed .DELETE_ON_ERROR. 64 | allRules = append(allRules, &makex.BasicRule{TargetFile: ".DELETE_ON_ERROR"}) 65 | 66 | mf := &makex.Makefile{Rules: allRules} 67 | 68 | return mf, nil 69 | } 70 | -------------------------------------------------------------------------------- /graph/ref.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package graph; 3 | 4 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 5 | 6 | option (gogoproto.goproto_getters_all) = false; 7 | option (gogoproto.unmarshaler_all) = true; 8 | option (gogoproto.marshaler_all) = true; 9 | option (gogoproto.sizer_all) = true; 10 | 11 | // Ref represents a reference from source code to a Def. 12 | message Ref { 13 | // DefRepo is the repository URI of the Def that this Ref refers 14 | // to. 15 | string DefRepo = 1 [(gogoproto.jsontag) = "DefRepo,omitempty"]; 16 | 17 | // DefUnitType is the source unit type of the Def that this Ref refers to. 18 | string DefUnitType = 3 [(gogoproto.jsontag) = "DefUnitType,omitempty"]; 19 | 20 | // DefUnit is the name of the source unit that this ref exists in. 21 | string DefUnit = 4 [(gogoproto.jsontag) = "DefUnit,omitempty"]; 22 | 23 | // Path is the path of the Def that this ref refers to. 24 | string DefPath = 5 [(gogoproto.jsontag) = "DefPath"]; 25 | 26 | 27 | // Repo is the VCS repository in which this ref exists. 28 | string Repo = 6 [(gogoproto.jsontag) = "Repo,omitempty"]; 29 | 30 | // CommitID is the ID of the VCS commit that this ref exists 31 | // in. The CommitID is always a full commit ID (40 hexadecimal 32 | // characters for git and hg), never a branch or tag name. 33 | string CommitID = 7 [(gogoproto.jsontag) = "CommitID,omitempty"]; 34 | 35 | // UnitType is the type name of the source unit that this ref 36 | // exists in. 37 | string UnitType = 8 [(gogoproto.jsontag) = "UnitType,omitempty"]; 38 | 39 | // Unit is the name of the source unit that this ref exists in. 40 | string Unit = 9 [(gogoproto.jsontag) = "Unit,omitempty"]; 41 | 42 | // Def is true if this Ref spans the name of the Def it points to. 43 | bool Def = 17 [(gogoproto.jsontag) = "Def,omitempty"]; 44 | 45 | // File is the filename in which this Ref exists. 46 | string File = 10 [(gogoproto.jsontag) = "File,omitempty"]; 47 | 48 | // Start is the byte offset of this ref's first byte in File. 49 | uint32 Start = 11 [(gogoproto.jsontag) = "Start"]; 50 | 51 | // End is the byte offset of this ref's last byte in File. 52 | uint32 End = 12 [(gogoproto.jsontag) = "End"]; 53 | }; 54 | 55 | message RefDefKey { 56 | string DefRepo = 1 [(gogoproto.jsontag) = "DefRepo,omitempty"]; 57 | string DefUnitType = 3 [(gogoproto.jsontag) = "DefUnitType,omitempty"]; 58 | string DefUnit = 4 [(gogoproto.jsontag) = "DefUnit,omitempty"]; 59 | string DefPath = 5 [(gogoproto.jsontag) = "DefPath"]; 60 | }; 61 | -------------------------------------------------------------------------------- /docs/sources/install.md: -------------------------------------------------------------------------------- 1 | # Getting started 2 | 3 | 4 | ## Install a prebuilt binary 5 | 6 | To install the **srclib** program, download one of the prebuilt binaries or build 7 | it from source (see next section). 8 | 9 | srclib binary downloads: 10 | 11 | * [Linux amd64](https://api.equinox.io/1/Applications/ap_BQxVz1iWMxmjQnbVGd85V58qz6/Updates/Asset/srclib.zip?os=linux&arch=amd64&channel=stable) 12 | * [Linux i386](https://api.equinox.io/1/Applications/ap_BQxVz1iWMxmjQnbVGd85V58qz6/Updates/Asset/srclib.zip?os=linux&arch=386&channel=stable) 13 | * [Mac OS X](https://api.equinox.io/1/Applications/ap_BQxVz1iWMxmjQnbVGd85V58qz6/Updates/Asset/srclib.zip?os=darwin&arch=amd64&channel=stable) 14 | 15 | After downloading the file, unzip it and place the `srclib` program in your 16 | `$PATH`. Run `srclib --help` to verify that it's installed. 17 | 18 | 19 | ## Building from source 20 | 21 | First, install [install Go](http://golang.org/doc/install) (version 1.5 or newer). 22 | 23 | Then: 24 | 25 | ``` 26 | go get -u -v sourcegraph.com/sourcegraph/srclib/cmd/srclib 27 | ``` 28 | 29 |
30 | 31 | ##Language Toolchains 32 | 33 | To install the language analysis toolchains for 34 | ([Go](toolchains/go.md), [Ruby](toolchains/ruby.md), 35 | [JavaScript](toolchains/javascript.md), and [Python](toolchains/python.md)), run: 36 | 37 | ``` 38 | srclib toolchain install go ruby javascript python 39 | ``` 40 | 41 | If this command fails, please 42 | [file an issue](https://github.com/sourcegraph/srclib/issues) or skip 43 | one of the languages if you don't need it. 44 | 45 | `srclib toolchain list` helps to verify the currently installed language toolchains like the following example 46 | 47 | ``` 48 | $ srclib toolchain list 49 | sourcegraph.com/sourcegraph/srclib-python 50 | sourcegraph.com/sourcegraph/srclib-go 51 | ``` 52 | 53 |
54 | 55 | ##Testing srclib 56 | 57 | In order to test srclib we can use it to analyze the already fetched source code for the Go toolchain `srclib-go`. 58 | 59 | First, you need to initialize the git submodules in the root directory of srclib-go 60 | ``` 61 | cd src/sourcegraph.com/sourcegraph/srclib-go 62 | git submodule update --init 63 | ``` 64 | 65 | Now you can test the srclib-go toolchain with: 66 | 67 | ``` 68 | $ srclib do-all 69 | $ srclib store import 70 | ``` 71 | 72 | You should have a .srclib-cache directory inside srclib-go that has all of the 73 | build data for the repository. You should also have a .srclib-store directory 74 | corresponding to the analysis information. 75 | 76 | ``` 77 | srclib store defs 78 | ``` 79 | 80 | should show you the definitions. 81 | 82 |
83 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | Sourcegraph projects are [MIT licensed](LICENSE) and accept contributions 4 | via GitHub pull requests. This document outlines some of the conventions on 5 | development workflow, commit message formatting, contact points and other 6 | resources to make it easier to get your contribution accepted. 7 | 8 | # Certificate of Origin 9 | 10 | By contributing to this project you agree to the Developer Certificate of Origin 11 | (DCO). This document was created by the Linux Kernel community and is a simple 12 | statement that you, as a contributor, have the legal right to make the 13 | contribution. See the [DCO](DCO) file for details. 14 | 15 | # Email and Chat 16 | 17 | The project currently uses the 18 | [srclib-dev email list](https://groups.google.com/forum/#!forum/srclib-dev). 19 | 20 | Please avoid emailing maintainers found in the MAINTAINERS file directly. They 21 | are very busy and read the mailing lists. 22 | 23 | ## Getting Started 24 | 25 | - Fork the repository on GitHub 26 | - Read the [README](README.md) for build and test instructions 27 | - Play with the project, submit bugs, submit patches! 28 | 29 | ## Contribution Flow 30 | 31 | This is a rough outline of what a contributor's workflow looks like: 32 | 33 | - Create a topic branch from where you want to base your work (usually master). 34 | - Make commits of logical units. 35 | - Make sure your commit messages are in the proper format (see below). 36 | - Push your changes to a topic branch in your fork of the repository. 37 | - Make sure the tests pass, and add any new tests as appropriate. 38 | - Submit a pull request to the original repository. 39 | 40 | Thanks for your contributions! 41 | 42 | ### Format of the Commit Message 43 | 44 | We follow a rough convention for commit messages that is designed to answer two 45 | questions: what changed and why. The subject line should feature the what and 46 | the body of the commit should describe the why. 47 | 48 | ``` 49 | scripts: add the test-cluster command 50 | 51 | this uses tmux to setup a test cluster that you can easily kill and 52 | start for debugging. 53 | 54 | Fixes #38 55 | ``` 56 | 57 | The format can be described more formally as follows: 58 | 59 | ``` 60 | : 61 | 62 | 63 | 64 |