├── .dockerignore ├── docs ├── .gitignore ├── sources │ ├── robots.txt │ ├── images │ │ ├── srclib_symbol.svg │ │ └── srclib_symbol_white.svg │ └── install.md ├── theme │ ├── images │ │ ├── favicon.png │ │ ├── editors │ │ │ ├── atom.png │ │ │ └── sublime.png │ │ ├── sourcegraph-emacs.png │ │ ├── layers.svg │ │ ├── whiteasterisk.svg │ │ ├── asterisk.svg │ │ ├── banner.svg │ │ ├── stack.svg │ │ ├── features.svg │ │ ├── integrations.svg │ │ └── dots.svg │ ├── newsletter.html │ ├── toc.html │ ├── content.html │ ├── newsletter2.html │ ├── base.html │ ├── community.html │ ├── navbar.html │ ├── docsnav.html │ └── layout.html ├── build.sh ├── test.sh ├── mkdocs.yml ├── deploy.sh ├── buildsite.py └── README.md ├── .gitignore ├── TODOs.no-docker.txt ├── ann ├── doc.go ├── gen.go ├── annotation_test.go ├── annotation.go └── ann.proto ├── store ├── misc.go ├── test_util.go ├── pb │ ├── gen.go │ ├── srcstore.proto │ ├── mock │ │ └── srcstore.pb_mock.go │ └── helpers.go ├── repo_store_mock.go ├── tree_store_mock.go ├── log.go ├── unit_store_mock.go ├── counter.go ├── primary_index_test.go ├── indexes_test.go ├── memory_test.go ├── misc_test.go ├── fs_test.go ├── fs_store_mock.go ├── fs_store_test.go ├── repo_paths_test.go ├── indexed_test.go ├── store.go ├── phtable │ ├── slicereader_safe.go │ └── slicereader_fast.go ├── multi_repo_store_mock.go ├── multi_repo_store.go ├── pbio │ ├── io.go │ └── io_test.go ├── units_index.go ├── tree_stores.go ├── repo_paths.go ├── primary_index.go ├── index_cache_test.go ├── index_test.go ├── ref_file_index.go └── index_cache.go ├── unit ├── doc.go ├── gen.go └── source_unit_test.go ├── util ├── makefile_other.go ├── makefile_windows.go ├── makefile.go ├── ancestordirs.go └── homedir.go ├── contrib └── completion │ ├── _srclib │ └── srclib-completion.bash ├── NOTICE ├── graph ├── gen.go ├── errors.go ├── proto_test.go ├── output.proto ├── doc.go ├── tree_path_test.go ├── doc.proto ├── format.go ├── formatter_test.go ├── ref.proto ├── repo_test.go ├── ref.go ├── repo.go └── stats.go ├── TODO.md ├── cli ├── cmdutil.go ├── testdata │ └── strip-code │ │ ├── expected-source.txt │ │ └── input-source.txt ├── build_data.go ├── version_cmd.go ├── repo_cmds.go ├── makefile_cmd.go ├── repo_opt.go ├── doall_cmd.go ├── gen_data_cmd.go ├── info_cmds.go ├── cli.go ├── coverage_test.go ├── misc.go ├── tool_cmd.go ├── util_test.go ├── repo_config.go └── util.go ├── .goxc.json ├── plan ├── data.go ├── util.go ├── makefile_test.go └── makefile.go ├── toolchain ├── bundle_test.go ├── choose_test.go ├── tool.go ├── choose.go ├── find_test.go ├── toolchain.go └── config.go ├── gendata └── common.go ├── config ├── validate.go ├── validate_test.go ├── external.go ├── cached.go └── config.go ├── cmd └── srclib │ └── srclib.go ├── toolref.go ├── grapher ├── registry.go ├── validate_test.go └── grapher.go ├── .travis.yml ├── dep ├── dep.go ├── rule.go ├── resolve.go └── doc.go ├── env.go ├── cvg └── coverage.go ├── DCO ├── Makefile ├── flagutil ├── marshal.go └── marshal_test.go ├── CONTRIBUTING.md └── scan └── scan.go /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | site/ 2 | *.css 3 | *.scssc 4 | -------------------------------------------------------------------------------- /docs/sources/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Allow: / 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /release/ 2 | /.tmp-selfupdate/ 3 | *.test 4 | vendor/*/ -------------------------------------------------------------------------------- /TODOs.no-docker.txt: -------------------------------------------------------------------------------- 1 | * remove dependency on `git`, don't require knowing the commit ID 2 | -------------------------------------------------------------------------------- /ann/doc.go: -------------------------------------------------------------------------------- 1 | // Package ann defines types that represent source code annotations. 2 | package ann 3 | -------------------------------------------------------------------------------- /docs/theme/images/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sourcegraph/srclib/HEAD/docs/theme/images/favicon.png -------------------------------------------------------------------------------- /docs/theme/images/editors/atom.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sourcegraph/srclib/HEAD/docs/theme/images/editors/atom.png -------------------------------------------------------------------------------- /store/misc.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | func min(a, b int) int { 4 | if a < b { 5 | return a 6 | } 7 | return b 8 | } 9 | -------------------------------------------------------------------------------- /docs/theme/images/editors/sublime.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sourcegraph/srclib/HEAD/docs/theme/images/editors/sublime.png -------------------------------------------------------------------------------- /docs/theme/images/sourcegraph-emacs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sourcegraph/srclib/HEAD/docs/theme/images/sourcegraph-emacs.png -------------------------------------------------------------------------------- /unit/doc.go: -------------------------------------------------------------------------------- 1 | // Package unit provides a source unit abstraction over distribution packages in 2 | // various languages. 3 | package unit 4 | -------------------------------------------------------------------------------- /util/makefile_other.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | package util 4 | 5 | func safeCommandName(command string) string { 6 | return command 7 | } 8 | -------------------------------------------------------------------------------- /ann/gen.go: -------------------------------------------------------------------------------- 1 | package ann 2 | 3 | //go:generate gopathexec protoc -I$GOPATH/src -I$GOPATH/src/github.com/gogo/protobuf/protobuf -I. --gogo_out=. ann.proto 4 | -------------------------------------------------------------------------------- /contrib/completion/_srclib: -------------------------------------------------------------------------------- 1 | #compdef srclib 2 | 3 | _srclib() { 4 | compadd "$@" $(GO_FLAGS_COMPLETION=1 $words[0] "$words[@]") 5 | } 6 | 7 | _srclib 8 | -------------------------------------------------------------------------------- /unit/gen.go: -------------------------------------------------------------------------------- 1 | package unit 2 | 3 | //go:generate gopathexec protoc -I$GOPATH/src -I$GOPATH/src/github.com/gogo/protobuf/protobuf -I. --gogo_out=. unit.proto 4 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | srclib Project 2 | Copyright 2014 Sourcegraph, Inc 3 | 4 | This product includes software developed at Sourcegraph, Inc. 5 | (https://sourcegraph.com/). 6 | -------------------------------------------------------------------------------- /graph/gen.go: -------------------------------------------------------------------------------- 1 | package graph 2 | 3 | //go:generate gopathexec protoc -I$GOPATH/src -I$GOPATH/src/github.com/gogo/protobuf/protobuf -I. --gogo_out=. def.proto doc.proto output.proto ref.proto 4 | -------------------------------------------------------------------------------- /graph/errors.go: -------------------------------------------------------------------------------- 1 | package graph 2 | 3 | import "errors" 4 | 5 | var ErrDefNotExist = errors.New("def does not exist") 6 | 7 | func IsNotExist(err error) bool { 8 | return err == ErrDefNotExist 9 | } 10 | -------------------------------------------------------------------------------- /store/test_util.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import "encoding/json" 4 | 5 | func deepEqual(u, v interface{}) bool { 6 | u_, _ := json.Marshal(u) 7 | v_, _ := json.Marshal(v) 8 | return string(u_) == string(v_) 9 | } 10 | -------------------------------------------------------------------------------- /docs/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Create site dir if it does not exist 4 | mkdir -p site 5 | 6 | # Compile the css file 7 | sass theme/styles.scss:theme/styles.css 8 | 9 | # Build the other parts of the site 10 | python buildsite.py 11 | -------------------------------------------------------------------------------- /util/makefile_windows.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | 3 | package util 4 | 5 | import ( 6 | "path/filepath" 7 | "strings" 8 | ) 9 | 10 | func safeCommandName(command string) string { 11 | return strings.Replace(filepath.ToSlash(command), ":", "\\:", 1) 12 | } 13 | -------------------------------------------------------------------------------- /contrib/completion/srclib-completion.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | _src() { 4 | args=("${COMP_WORDS[@]:1:$COMP_CWORD}") 5 | 6 | local IFS=$'\n' 7 | COMPREPLY=($(GO_FLAGS_COMPLETION=1 ${COMP_WORDS[0]} "${args[@]}")) 8 | return 1 9 | } 10 | 11 | complete -F _srclib srclib 12 | -------------------------------------------------------------------------------- /store/pb/gen.go: -------------------------------------------------------------------------------- 1 | package pb 2 | 3 | //go:generate gopathexec protoc -I$GOPATH/src -I$GOPATH/src/github.com/gogo/protobuf/protobuf -I../../graph -I. --gogo_out=plugins=grpc:. srcstore.proto 4 | //go:generate gen-mocks -w -i=.+(Server|Client|Service)$ -o mock -outpkg mock -name_prefix= -no_pass_args=opts 5 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | # TODOs 2 | 3 | ## Simplify build data 4 | 5 | Build data is currently only ever stored on local filesystems, and 6 | it's unlikely that will change (because any virtual/remote access 7 | would be to the store, not to the build data tree). That means it's 8 | probably unnecessary to use a VFS interface to access it. 9 | -------------------------------------------------------------------------------- /store/repo_store_mock.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | type MockRepoStore struct { 4 | Versions_ func(...VersionFilter) ([]*Version, error) 5 | MockTreeStore 6 | } 7 | 8 | func (m MockRepoStore) Versions(f ...VersionFilter) ([]*Version, error) { 9 | return m.Versions_(f...) 10 | } 11 | 12 | var _ RepoStore = MockRepoStore{} 13 | -------------------------------------------------------------------------------- /docs/theme/newsletter.html: -------------------------------------------------------------------------------- 1 |
7 | -------------------------------------------------------------------------------- /store/tree_store_mock.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import "sourcegraph.com/sourcegraph/srclib/unit" 4 | 5 | type MockTreeStore struct { 6 | Units_ func(...UnitFilter) ([]*unit.SourceUnit, error) 7 | MockUnitStore 8 | } 9 | 10 | func (m MockTreeStore) Units(f ...UnitFilter) ([]*unit.SourceUnit, error) { 11 | return m.Units_(f...) 12 | } 13 | 14 | var _ TreeStore = MockTreeStore{} 15 | -------------------------------------------------------------------------------- /util/makefile.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | // Makes command name safe for shell script (and for makefiles). 4 | // For example, Cygwin does not like when you trying to execute C:/foo/bar.exe arguments 5 | // so we replacing first : with \: there. On Unix/Darwin there is no need to perform replacements 6 | func SafeCommandName(command string) string { 7 | return safeCommandName(command) 8 | } 9 | -------------------------------------------------------------------------------- /cli/cmdutil.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "log" 5 | 6 | "sourcegraph.com/sourcegraph/go-flags" 7 | ) 8 | 9 | func SetOptionDefaultValue(g *flags.Group, longName string, defaultVal ...string) { 10 | for _, opt := range g.Options() { 11 | if opt.LongName == longName { 12 | opt.Default = defaultVal 13 | return 14 | } 15 | } 16 | log.Fatalf("Failed to set default value %v for option %q (not found).", defaultVal, longName) 17 | } 18 | -------------------------------------------------------------------------------- /store/log.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "io" 5 | "io/ioutil" 6 | "log" 7 | "os" 8 | "strconv" 9 | ) 10 | 11 | func vlogWriter() io.Writer { 12 | if v, _ := strconv.ParseBool(os.Getenv("V")); v { 13 | return os.Stderr 14 | } 15 | return ioutil.Discard 16 | } 17 | 18 | func cyan(s string) string { 19 | return "\x1b[36m" + s + "\x1b[0m" 20 | } 21 | 22 | var vlog = log.New(vlogWriter(), cyan("▶ "), log.Lmicroseconds|log.Lshortfile) 23 | -------------------------------------------------------------------------------- /docs/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | # Create site dir if it does not exist 6 | mkdir -p site 7 | 8 | # Python server 9 | cd site 10 | python -m SimpleHTTPServer & 11 | cd .. 12 | 13 | # Kill python server on exit 14 | trap "exit" INT TERM 15 | trap "kill 0" EXIT 16 | 17 | while true; do 18 | echo "Building site..." 19 | sassc theme/styles.scss theme/styles.css 20 | #mkdocs build 21 | python buildsite.py 22 | 23 | sleep 1 24 | done 25 | -------------------------------------------------------------------------------- /.goxc.json: -------------------------------------------------------------------------------- 1 | { 2 | "AppName": "srclib", 3 | "ArtifactsDest": "release", 4 | "OutPath": "{{.Dest}}{{.PS}}{{.Version}}{{.PS}}{{.Os}}-{{.Arch}}", 5 | "Tasks": [ 6 | "xc" 7 | ], 8 | "Arch": "amd64", 9 | "Os": "linux darwin", 10 | "ConfigVersion": "0.9", 11 | "BuildSettings": { 12 | "LdFlagsXVars": { 13 | "Version": "sourcegraph.com/sourcegraph/srclib/cli.Version" 14 | }, 15 | "ExtraArgs": ["-a"] 16 | }, 17 | "Env": [ 18 | "CGO_ENABLED=0" 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /cli/testdata/strip-code/expected-source.txt: -------------------------------------------------------------------------------- 1 | 2 | package retrofit2http 3 | 4 | import javalangannotationDocumented 5 | import javalangannotationRetention 6 | import javalangannotationTarget 7 | import okhttp3HttpUrl 8 | 9 | import static javalangannotationElementTypeMETHOD 10 | import static javalangannotationRetentionPolicyRUNTIME 11 | 12 | 13 | Documented 14 | TargetMETHOD 15 | RetentionRUNTIME 16 | public interface DELETE 17 | 18 | String value default 19 | 20 | -------------------------------------------------------------------------------- /docs/mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: srclib 2 | site_description: srclib is a hackable, polyglot code analysis library. 3 | 4 | docs_dir: sources 5 | 6 | repo_url: https://sourcegraph.com/sourcegraph/srclib/ 7 | 8 | dev_addr: '0.0.0.0:8000' 9 | 10 | theme_dir: ./theme/ 11 | 12 | google_analytics: ['UA-40540747-5', 'srclib.org'] 13 | 14 | toolchains: 15 | - lang: Go 16 | repo: srclib/srclib-go 17 | - lang: Java 18 | repo: srclib/srclib-java 19 | 20 | pages: 21 | - ['install.md', 'Installation'] 22 | -------------------------------------------------------------------------------- /docs/theme/toc.html: -------------------------------------------------------------------------------- 1 |
35 | * See {@linkplain retrofit2.Retrofit.Builder#baseUrl(HttpUrl) base URL} for details of how
36 | * this is resolved against a base URL to create the full endpoint URL.
37 | */
38 | String value() default "";
39 | }
40 |
--------------------------------------------------------------------------------
/store/phtable/slicereader_safe.go:
--------------------------------------------------------------------------------
1 | // +build !386,!amd64,!arm
2 |
3 | package phtable
4 |
5 | import (
6 | "encoding/binary"
7 | "fmt"
8 | )
9 |
10 | // Read values and typed vectors from a byte slice without copying where possible.
11 | type sliceReader struct {
12 | b []byte
13 | start, end uint64
14 | }
15 |
16 | func (b *sliceReader) Read(size uint64) []byte {
17 | b.start, b.end = b.end, b.end+size
18 | if b.start == b.end {
19 | return nil
20 | }
21 | return b.b[b.start:b.end]
22 | }
23 |
24 | func (b *sliceReader) ReadUint64Array(n uint64) []uint64 {
25 | b.start, b.end = b.end, b.end+n*8
26 | out := make([]uint64, n)
27 | buf := b.b[b.start:b.end]
28 | for i := 0; i < len(buf); i += 8 {
29 | out[i>>3] = binary.LittleEndian.Uint64(buf[i : i+8])
30 | }
31 | return out
32 | }
33 |
34 | func (b *sliceReader) ReadUint16Array(n uint64) []uint16 {
35 | b.start, b.end = b.end, b.end+n*2
36 | out := make([]uint16, n)
37 | buf := b.b[b.start:b.end]
38 | for i := 0; i < len(buf); i += 2 {
39 | out[i>>1] = binary.LittleEndian.Uint16(buf[i : i+2])
40 | }
41 | return out
42 | }
43 |
44 | func (b *sliceReader) ReadInt() uint64 {
45 | return uint64(binary.LittleEndian.Uint32(b.Read(4)))
46 | }
47 |
48 | func (b *sliceReader) ReadInt32() uint32 {
49 | return binary.LittleEndian.Uint32(b.Read(4))
50 | }
51 |
52 | func (b *sliceReader) ReadUvarint() uint64 {
53 | v, n := binary.Uvarint(b.b[b.end:])
54 | if n <= 0 {
55 | panic(fmt.Sprintf("Uvarint error: n == %d (buf size %d)", n, b.end-b.start))
56 | }
57 | b.start, b.end = b.end, b.end+uint64(n)
58 | return v
59 | }
60 |
--------------------------------------------------------------------------------
/cli/misc.go:
--------------------------------------------------------------------------------
1 | package cli
2 |
3 | import (
4 | "log"
5 | "os"
6 | "path/filepath"
7 |
8 | "sourcegraph.com/sourcegraph/go-flags"
9 | )
10 |
11 | // Directory is flags.Completer that provides directory name
12 | // completion. Do not convert Directory to a string type manually,
13 | // always use Directory.String(). Only use Directory for
14 | // go-flags.Command fields, not for internal functions.
15 | //
16 | // TODO(sqs): this is annoying. it only completes the dir name and doesn't let
17 | // you keep typing the arg.
18 | type Directory string
19 |
20 | // Complete implements flags.Completer and returns a list of existing
21 | // directories with the given prefix.
22 | func (d Directory) Complete(match string) []flags.Completion {
23 | names, err := filepath.Glob(match + "*")
24 | if err != nil {
25 | log.Println(err)
26 | return nil
27 | }
28 |
29 | var dirs []flags.Completion
30 | for _, name := range names {
31 | if fi, err := os.Stat(name); err == nil && fi.Mode().IsDir() {
32 | dirs = append(dirs, flags.Completion{Item: name + string(filepath.Separator)})
33 | }
34 | }
35 | return dirs
36 | }
37 |
38 | // String returns the uncleaned string representation of d. If d is
39 | // empty, "." is returned. Never convert Directories to strings
40 | // manually, always call String.
41 | func (d Directory) String() string {
42 | if d == "" {
43 | return "."
44 | }
45 | dir, file := filepath.Split(string(d))
46 | if file == "" {
47 | return dir
48 | }
49 | if file == "." || file == ".." {
50 | return dir + file
51 | }
52 | return dir + file + string(filepath.Separator)
53 | }
54 |
--------------------------------------------------------------------------------
/toolchain/tool.go:
--------------------------------------------------------------------------------
1 | package toolchain
2 |
3 | import "sourcegraph.com/sourcegraph/srclib"
4 |
5 | // ToolInfo describes a tool in a toolchain.
6 | type ToolInfo struct {
7 | // Subcmd is the subcommand name of this tool.
8 | //
9 | // By convention, this is the same as Op in toolchains that only have one
10 | // tool that performs this operation (e.g., a toolchain's "graph" subcommand
11 | // performs the "graph" operation).
12 | Subcmd string
13 |
14 | // Op is the operation that this tool performs (e.g., "scan", "graph",
15 | // "deplist", etc.).
16 | Op string
17 |
18 | // SourceUnitTypes is a list of source unit types (e.g., "GoPackage") that
19 | // this tool can operate on.
20 | //
21 | // If this tool doesn't operate on source units (for example, it operates on
22 | // directories or repositories, such as the "blame" tools), then this will
23 | // be empty.
24 | //
25 | // TODO(sqs): determine how repository- or directory-level tools will be
26 | // defined.
27 | SourceUnitTypes []string `json:",omitempty"`
28 | }
29 |
30 | // ListTools lists all tools in all available toolchains (returned by List). If
31 | // op is non-empty, only tools that perform that operation are returned.
32 | func ListTools(op string) ([]*srclib.ToolRef, error) {
33 | tcs, err := List()
34 | if err != nil {
35 | return nil, err
36 | }
37 |
38 | var tools []*srclib.ToolRef
39 | for _, tc := range tcs {
40 | c, err := tc.ReadConfig()
41 | if err != nil {
42 | return nil, err
43 | }
44 |
45 | for _, tool := range c.Tools {
46 | if op == "" || tool.Op == op {
47 | tools = append(tools, &srclib.ToolRef{Toolchain: tc.Path, Subcmd: tool.Subcmd})
48 | }
49 | }
50 | }
51 | return tools, nil
52 | }
53 |
--------------------------------------------------------------------------------
/store/multi_repo_store_mock.go:
--------------------------------------------------------------------------------
1 | package store
2 |
3 | import (
4 | "sourcegraph.com/sourcegraph/srclib/graph"
5 | "sourcegraph.com/sourcegraph/srclib/unit"
6 | )
7 |
8 | type MockMultiRepoStore struct {
9 | Repos_ func(...RepoFilter) ([]string, error)
10 | Versions_ func(...VersionFilter) ([]*Version, error)
11 | Units_ func(...UnitFilter) ([]*unit.SourceUnit, error)
12 | Defs_ func(...DefFilter) ([]*graph.Def, error)
13 | Refs_ func(...RefFilter) ([]*graph.Ref, error)
14 |
15 | Import_ func(repo, commitID string, unit *unit.SourceUnit, data graph.Output) error
16 | Index_ func(repo, commitID string) error
17 | CreateVersion_ func(repo, commit string) error
18 | }
19 |
20 | func (m MockMultiRepoStore) Repos(f ...RepoFilter) ([]string, error) {
21 | return m.Repos_(f...)
22 | }
23 |
24 | func (m MockMultiRepoStore) Versions(f ...VersionFilter) ([]*Version, error) {
25 | return m.Versions_(f...)
26 | }
27 |
28 | func (m MockMultiRepoStore) Units(f ...UnitFilter) ([]*unit.SourceUnit, error) {
29 | return m.Units_(f...)
30 | }
31 |
32 | func (m MockMultiRepoStore) Defs(f ...DefFilter) ([]*graph.Def, error) {
33 | return m.Defs_(f...)
34 | }
35 |
36 | func (m MockMultiRepoStore) Refs(f ...RefFilter) ([]*graph.Ref, error) {
37 | return m.Refs_(f...)
38 | }
39 |
40 | func (m MockMultiRepoStore) Import(repo, commitID string, unit *unit.SourceUnit, data graph.Output) error {
41 | return m.Import_(repo, commitID, unit, data)
42 | }
43 |
44 | func (m MockMultiRepoStore) Index(repo, commitID string) error {
45 | return m.Index_(repo, commitID)
46 | }
47 |
48 | func (m MockMultiRepoStore) CreateVersion(repo, commitID string) error {
49 | return m.CreateVersion_(repo, commitID)
50 | }
51 |
52 | var _ MultiRepoStoreImporterIndexer = MockMultiRepoStore{}
53 |
--------------------------------------------------------------------------------
/toolchain/choose.go:
--------------------------------------------------------------------------------
1 | package toolchain
2 |
3 | import (
4 | "fmt"
5 |
6 | "sourcegraph.com/sourcegraph/srclib"
7 | )
8 |
9 | // ChooseTool determines which toolchain and tool to use to run op (graph,
10 | // depresolve, etc.) on a source unit of the given type. If no tools fit the
11 | // criteria, an error is returned.
12 | //
13 | // The selection algorithm is currently very simplistic: if exactly one tool is
14 | // found that can perform op on the source unit type, it is returned. If zero or
15 | // more than 1 are found, then an error is returned. TODO(sqs): extend this to
16 | // choose the "best" tool when multiple tools would suffice.
17 | var ChooseTool = func(op, unitType string) (*srclib.ToolRef, error) {
18 | tcs, err := List()
19 | if err != nil {
20 | return nil, err
21 | }
22 | return chooseTool(op, unitType, tcs)
23 | }
24 |
25 | // chooseTool is like ChooseTool but the list of tools is provided as an
26 | // argument instead of being obtained by calling List.
27 | func chooseTool(op, unitType string, tcs []*Info) (*srclib.ToolRef, error) {
28 | var satisfying []*srclib.ToolRef
29 | for _, tc := range tcs {
30 | cfg, err := tc.ReadConfig()
31 | if err != nil {
32 | return nil, err
33 | }
34 |
35 | for _, tool := range cfg.Tools {
36 | if tool.Op == op {
37 | for _, u := range tool.SourceUnitTypes {
38 | if u == unitType {
39 | satisfying = append(satisfying, &srclib.ToolRef{Toolchain: tc.Path, Subcmd: tool.Subcmd})
40 | }
41 | }
42 | }
43 | }
44 | }
45 |
46 | if n := len(satisfying); n > 1 {
47 | return nil, fmt.Errorf("%d tools satisfy op %q for source unit type %q (refusing to choose between multiple possibilities)", n, op, unitType)
48 | } else if n == 0 {
49 | return nil, nil
50 | }
51 | return satisfying[0], nil
52 | }
53 |
--------------------------------------------------------------------------------
/store/phtable/slicereader_fast.go:
--------------------------------------------------------------------------------
1 | // +build 386 amd64 arm
2 |
3 | package phtable
4 |
5 | import (
6 | "encoding/binary"
7 | "fmt"
8 |
9 | "github.com/alecthomas/unsafeslice"
10 | )
11 |
12 | // Read values and typed vectors from a byte slice without copying where
13 | // possible. This implementation directly references the underlying byte slice
14 | // for array operations, making them essentially zero copy. As the data is
15 | // written in little endian form, this of course means that this will only
16 | // work on little-endian architectures.
17 | type sliceReader struct {
18 | b []byte
19 | start, end uint64
20 | }
21 |
22 | func (b *sliceReader) Read(size uint64) []byte {
23 | b.start, b.end = b.end, b.end+size
24 | if b.start == b.end {
25 | return nil
26 | }
27 | return b.b[b.start:b.end]
28 | }
29 |
30 | func (b *sliceReader) ReadUint64Array(n uint64) []uint64 {
31 | b.start, b.end = b.end, b.end+n*8
32 | return unsafeslice.Uint64SliceFromByteSlice(b.b[b.start:b.end])
33 | }
34 |
35 | func (b *sliceReader) ReadUint16Array(n uint64) []uint16 {
36 | b.start, b.end = b.end, b.end+n*2
37 | return unsafeslice.Uint16SliceFromByteSlice(b.b[b.start:b.end])
38 | }
39 |
40 | // Despite returning a uint64, this actually reads a uint32. All table indices
41 | // and lengths are stored as uint32 values.
42 | func (b *sliceReader) ReadInt() uint64 {
43 | return uint64(binary.LittleEndian.Uint32(b.Read(4)))
44 | }
45 |
46 | func (b *sliceReader) ReadInt32() uint32 {
47 | return binary.LittleEndian.Uint32(b.Read(4))
48 | }
49 |
50 | func (b *sliceReader) ReadUvarint() uint64 {
51 | v, n := binary.Uvarint(b.b[b.end:])
52 | if n <= 0 {
53 | panic(fmt.Sprintf("Uvarint error: n == %d (buf size %d)", n, b.end-b.start))
54 | }
55 | b.start, b.end = b.end, b.end+uint64(n)
56 | return v
57 | }
58 |
--------------------------------------------------------------------------------
/dep/rule.go:
--------------------------------------------------------------------------------
1 | package dep
2 |
3 | import (
4 | "fmt"
5 | "path/filepath"
6 |
7 | "sourcegraph.com/sourcegraph/makex"
8 | "sourcegraph.com/sourcegraph/srclib"
9 | "sourcegraph.com/sourcegraph/srclib/buildstore"
10 | "sourcegraph.com/sourcegraph/srclib/config"
11 | "sourcegraph.com/sourcegraph/srclib/plan"
12 | "sourcegraph.com/sourcegraph/srclib/toolchain"
13 | "sourcegraph.com/sourcegraph/srclib/unit"
14 | "sourcegraph.com/sourcegraph/srclib/util"
15 | )
16 |
17 | const depresolveOp = "depresolve"
18 |
19 | func init() {
20 | plan.RegisterRuleMaker(depresolveOp, makeDepRules)
21 | buildstore.RegisterDataType("depresolve", []*ResolvedDep{})
22 | }
23 |
24 | func makeDepRules(c *config.Tree, dataDir string, existing []makex.Rule) ([]makex.Rule, error) {
25 | const op = depresolveOp
26 | var rules []makex.Rule
27 | for _, u := range c.SourceUnits {
28 | toolRef, err := toolchain.ChooseTool(depresolveOp, u.Type)
29 | if err != nil {
30 | return nil, err
31 | }
32 |
33 | rules = append(rules, &ResolveDepsRule{dataDir, u, toolRef})
34 | }
35 | return rules, nil
36 | }
37 |
38 | type ResolveDepsRule struct {
39 | dataDir string
40 | Unit *unit.SourceUnit
41 | Tool *srclib.ToolRef
42 | }
43 |
44 | func (r *ResolveDepsRule) Target() string {
45 | return filepath.ToSlash(filepath.Join(r.dataDir, plan.SourceUnitDataFilename([]*ResolvedDep{}, r.Unit)))
46 | }
47 |
48 | func (r *ResolveDepsRule) Prereqs() []string {
49 | return []string{filepath.ToSlash(filepath.Join(r.dataDir, plan.SourceUnitDataFilename(unit.SourceUnit{}, r.Unit)))}
50 | }
51 |
52 | func (r *ResolveDepsRule) Recipes() []string {
53 | if r.Tool == nil {
54 | return nil
55 | }
56 | return []string{
57 | fmt.Sprintf("%s tool %q %q < $^ 1> $@", util.SafeCommandName(srclib.CommandName), r.Tool.Toolchain, r.Tool.Subcmd),
58 | }
59 | }
60 |
--------------------------------------------------------------------------------
/config/external.go:
--------------------------------------------------------------------------------
1 | package config
2 |
3 | import (
4 | "encoding/json"
5 | "fmt"
6 | "log"
7 | "os"
8 | "path/filepath"
9 |
10 | "sourcegraph.com/sourcegraph/srclib"
11 | "sourcegraph.com/sourcegraph/srclib/toolchain"
12 | )
13 |
14 | // An External configuration file, represented by this struct, can set system-
15 | // and user-level settings for srclib.
16 | type External struct {
17 | // Scanners is the default set of scanners to use. If not specified, all
18 | // scanners in the SRCLIBPATH will be used.
19 | Scanners []*srclib.ToolRef
20 | }
21 |
22 | const srclibconfigFile = ".srclibconfig"
23 |
24 | // SrclibPathConfig gets the srclib path configuration (which lists
25 | // all available scanners). It reads it from SRCLIBPATH/.srclibconfig
26 | // if that file exists, and otherwise it walks SRCLIBPATH for
27 | // available scanners.
28 | func SrclibPathConfig() (*External, error) {
29 | var x External
30 |
31 | // Try reading from .srclibconfig.
32 | dir := filepath.SplitList(srclib.Path)[0]
33 | configFile := filepath.Join(dir, srclibconfigFile)
34 | f, err := os.Open(configFile)
35 | if os.IsNotExist(err) {
36 | // do nothing
37 | } else if err != nil {
38 | log.Printf("Warning: unable to open config file at %s: %s. Continuing without this config.", configFile, err)
39 | } else {
40 | defer f.Close()
41 | if err := json.NewDecoder(f).Decode(&x); err != nil {
42 | log.Printf("Warning: unable to decode config file at %s: %s. Continuing without this config.", configFile, err)
43 | }
44 | }
45 |
46 | // Default to using all available scanners.
47 | if len(x.Scanners) == 0 {
48 | var err error
49 | x.Scanners, err = toolchain.ListTools("scan")
50 | if err != nil && !os.IsNotExist(err) {
51 | return nil, fmt.Errorf("failed to find scanners in SRCLIBPATH: %s", err)
52 | }
53 | }
54 |
55 | return &x, nil
56 | }
57 |
--------------------------------------------------------------------------------
/graph/format.go:
--------------------------------------------------------------------------------
1 | package graph
2 |
3 | // RepositoryListingDef holds rendered display text to show on the "package"
4 | // listing page of a repository.
5 | type RepositoryListingDef struct {
6 | // Name is the full name shown on the page.
7 | Name string
8 |
9 | // NameLabel is a label displayed next to the Name, such as "(main package)"
10 | // to denote that a package is a Go main package.
11 | NameLabel string
12 |
13 | // Language is the source language of the def, with any additional
14 | // specifiers, such as "JavaScript (node.js)".
15 | Language string
16 |
17 | // SortKey is the key used to lexicographically sort all of the defs on
18 | // the page.
19 | SortKey string
20 | }
21 |
22 | // // FormatAndSortDefsForRepositoryListing uses DefFormatters registered by
23 | // // the various toolchains to format and sort defs for display on the
24 | // // "package" listing page of a repository. The provided defs slice is sorted
25 | // // in-place.
26 | // func FormatAndSortDefsForRepositoryListing(defs []*Def) map[*Def]RepositoryListingDef {
27 | // m := make(map[*Def]RepositoryListingDef, len(defs))
28 | // for _, s := range defs {
29 | // sf, present := DefFormatters[s.UnitType]
30 | // if !present {
31 | // panic("no DefFormatter for def with UnitType " + s.UnitType)
32 | // }
33 |
34 | // m[s] = sf.RepositoryListing(s)
35 | // }
36 |
37 | // // sort
38 | // ss := &repositoryListingDefs{m, defs}
39 | // sort.Sort(ss)
40 | // return m
41 | // }
42 |
43 | // type repositoryListingDefs struct {
44 | // info map[*Def]RepositoryListingDef
45 | // defs []*Def
46 | // }
47 |
48 | // func (s *repositoryListingDefs) Len() int { return len(s.defs) }
49 | // func (s *repositoryListingDefs) Swap(i, j int) {
50 | // s.defs[i], s.defs[j] = s.defs[j], s.defs[i]
51 | // }
52 | // func (s *repositoryListingDefs) Less(i, j int) bool {
53 | // return s.info[s.defs[i]].SortKey < s.info[s.defs[j]].SortKey
54 | // }
55 |
--------------------------------------------------------------------------------
/store/multi_repo_store.go:
--------------------------------------------------------------------------------
1 | package store
2 |
3 | import (
4 | "sourcegraph.com/sourcegraph/srclib/graph"
5 | "sourcegraph.com/sourcegraph/srclib/unit"
6 | )
7 |
8 | // MultiRepoStore provides access to RepoStores for multiple
9 | // repositories.
10 | //
11 | // Using this interface instead of directly accessing a single
12 | // RepoStore allows aliasing repository URIs and supporting both ID
13 | // and URI lookups.
14 | type MultiRepoStore interface {
15 | // Repos returns all repositories that match the RepoFilter.
16 | Repos(...RepoFilter) ([]string, error)
17 |
18 | // RepoStore's methods call the corresponding methods on the
19 | // RepoStore of each repository contained within this multi-repo
20 | // store. The combined results are returned (in undefined order).
21 | RepoStore
22 | }
23 |
24 | // A MultiRepoImporter imports srclib build data for a repository's
25 | // source unit at a specific version into a RepoStore.
26 | type MultiRepoImporter interface {
27 | // Import imports srclib build data for a source unit at a
28 | // specific version into the store.
29 | Import(repo, commitID string, unit *unit.SourceUnit, data graph.Output) error
30 |
31 | // CreateVersion creates the version entry for the given commit. All other data (including
32 | // indexes) needs to exist before this gets called.
33 | CreateVersion(repo, commitID string) error
34 | }
35 |
36 | type MultiRepoIndexer interface {
37 | // Index builds indexes for the store.
38 | Index(repo, commitID string) error
39 | }
40 |
41 | // A MultiRepoStoreImporter implements both MultiRepoStore and
42 | // MultiRepoImporter.
43 | type MultiRepoStoreImporter interface {
44 | MultiRepoStore
45 | MultiRepoImporter
46 | }
47 |
48 | // A MultiRepoStoreImporterIndexer implements all 3 interfaces.
49 | type MultiRepoStoreImporterIndexer interface {
50 | MultiRepoStore
51 | MultiRepoImporter
52 | MultiRepoIndexer
53 | }
54 |
55 | // TODO(sqs): What should the Repo type be? Right now it is just string.
56 |
--------------------------------------------------------------------------------
/store/pbio/io.go:
--------------------------------------------------------------------------------
1 | // Extensions for Protocol Buffers to create more go like structures.
2 | //
3 | // Copyright (c) 2013, Vastech SA (PTY) LTD. All rights reserved.
4 | // http://github.com/gogo/protobuf/gogoproto
5 | //
6 | // Redistribution and use in source and binary forms, with or without
7 | // modification, are permitted provided that the following conditions are
8 | // met:
9 | //
10 | // * Redistributions of source code must retain the above copyright
11 | // notice, this list of conditions and the following disclaimer.
12 | // * Redistributions in binary form must reproduce the above
13 | // copyright notice, this list of conditions and the following disclaimer
14 | // in the documentation and/or other materials provided with the
15 | // distribution.
16 | //
17 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 |
29 | package pbio
30 |
31 | import "github.com/gogo/protobuf/proto"
32 |
33 | type Writer interface {
34 | WriteMsg(proto.Message) (uint64, error)
35 | }
36 |
37 | type Reader interface {
38 | ReadMsg(msg proto.Message) (uint64, error)
39 | }
40 |
41 | type marshaler interface {
42 | MarshalTo(data []byte) (n int, err error)
43 | Sizer
44 | }
45 |
46 | type Sizer interface {
47 | Size() (n int)
48 | }
49 |
--------------------------------------------------------------------------------
/store/pb/mock/srcstore.pb_mock.go:
--------------------------------------------------------------------------------
1 | // generated by gen-mocks; DO NOT EDIT
2 |
3 | package mock
4 |
5 | import (
6 | "golang.org/x/net/context"
7 | "google.golang.org/grpc"
8 | "sourcegraph.com/sourcegraph/srclib/store/pb"
9 | "sourcegraph.com/sqs/pbtypes"
10 | )
11 |
12 | type MultiRepoImporterClient struct {
13 | Import_ func(ctx context.Context, in *pb.ImportOp) (*pbtypes.Void, error)
14 | CreateVersion_ func(ctx context.Context, in *pb.CreateVersionOp) (*pbtypes.Void, error)
15 | Index_ func(ctx context.Context, in *pb.IndexOp) (*pbtypes.Void, error)
16 | }
17 |
18 | func (s *MultiRepoImporterClient) Import(ctx context.Context, in *pb.ImportOp, opts ...grpc.CallOption) (*pbtypes.Void, error) {
19 | return s.Import_(ctx, in)
20 | }
21 |
22 | func (s *MultiRepoImporterClient) CreateVersion(ctx context.Context, in *pb.CreateVersionOp, opts ...grpc.CallOption) (*pbtypes.Void, error) {
23 | return s.CreateVersion_(ctx, in)
24 | }
25 |
26 | func (s *MultiRepoImporterClient) Index(ctx context.Context, in *pb.IndexOp, opts ...grpc.CallOption) (*pbtypes.Void, error) {
27 | return s.Index_(ctx, in)
28 | }
29 |
30 | var _ pb.MultiRepoImporterClient = (*MultiRepoImporterClient)(nil)
31 |
32 | type MultiRepoImporterServer struct {
33 | Import_ func(v0 context.Context, v1 *pb.ImportOp) (*pbtypes.Void, error)
34 | CreateVersion_ func(v0 context.Context, v1 *pb.CreateVersionOp) (*pbtypes.Void, error)
35 | Index_ func(v0 context.Context, v1 *pb.IndexOp) (*pbtypes.Void, error)
36 | }
37 |
38 | func (s *MultiRepoImporterServer) Import(v0 context.Context, v1 *pb.ImportOp) (*pbtypes.Void, error) {
39 | return s.Import_(v0, v1)
40 | }
41 |
42 | func (s *MultiRepoImporterServer) CreateVersion(v0 context.Context, v1 *pb.CreateVersionOp) (*pbtypes.Void, error) {
43 | return s.CreateVersion_(v0, v1)
44 | }
45 |
46 | func (s *MultiRepoImporterServer) Index(v0 context.Context, v1 *pb.IndexOp) (*pbtypes.Void, error) {
47 | return s.Index_(v0, v1)
48 | }
49 |
50 | var _ pb.MultiRepoImporterServer = (*MultiRepoImporterServer)(nil)
51 |
--------------------------------------------------------------------------------
/store/units_index.go:
--------------------------------------------------------------------------------
1 | package store
2 |
3 | import (
4 | "encoding/json"
5 | "fmt"
6 | "io"
7 |
8 | "sourcegraph.com/sourcegraph/srclib/unit"
9 | )
10 |
11 | // unitsIndex makes it fast list all source units without needing to
12 | // traverse directories.
13 | type unitsIndex struct {
14 | units []*unit.SourceUnit
15 | ready bool
16 | }
17 |
18 | var _ interface {
19 | Index
20 | persistedIndex
21 | unitIndexBuilder
22 | unitFullIndex
23 | } = (*unitsIndex)(nil)
24 |
25 | var c_unitsIndex_listUnits = &counter{count: new(int64)}
26 |
27 | func (x *unitsIndex) String() string { return fmt.Sprintf("unitsIndex(ready=%v)", x.ready) }
28 |
29 | // Covers returns -1 because it should never be selected over a more
30 | // specific index. When the indexed stores use the unitsIndex, they do
31 | // so through a different code path from the one that selects other
32 | // indexes.
33 | func (x *unitsIndex) Covers(filters interface{}) int { return -1 }
34 |
35 | // Units implements unitFullIndex.
36 | func (x *unitsIndex) Units(fs ...UnitFilter) ([]*unit.SourceUnit, error) {
37 | if x.units == nil {
38 | panic("units not built/read")
39 | }
40 |
41 | c_unitsIndex_listUnits.increment()
42 |
43 | var units []*unit.SourceUnit
44 | for _, u := range x.units {
45 | if unitFilters(fs).SelectUnit(u) {
46 | units = append(units, u)
47 | }
48 | }
49 |
50 | return units, nil
51 | }
52 |
53 | // Build implements unitIndexBuilder.
54 | func (x *unitsIndex) Build(units []*unit.SourceUnit) error {
55 | x.units = units
56 | x.ready = true
57 | return nil
58 | }
59 |
60 | // Write implements persistedIndex.
61 | func (x *unitsIndex) Write(w io.Writer) error {
62 | if x.units == nil {
63 | panic("no units to write")
64 | }
65 | return json.NewEncoder(w).Encode(x.units)
66 | }
67 |
68 | // Read implements persistedIndex.
69 | func (x *unitsIndex) Read(r io.Reader) error {
70 | err := json.NewDecoder(r).Decode(&x.units)
71 | x.ready = (err == nil)
72 | return err
73 | }
74 |
75 | // Ready implements persistedIndex.
76 | func (x *unitsIndex) Ready() bool { return x.ready }
77 |
--------------------------------------------------------------------------------
/dep/resolve.go:
--------------------------------------------------------------------------------
1 | package dep
2 |
3 | import (
4 | "encoding/json"
5 | )
6 |
7 | // START ResolvedTarget OMIT
8 | // ResolvedTarget represents a resolved dependency target.
9 | type ResolvedTarget struct {
10 | // ToRepoCloneURL is the clone URL of the repository that is depended on.
11 | //
12 | // When graphers emit ResolvedDependencies, they should fill in this field,
13 | // not ToRepo, so that the dependent repository can be added if it doesn't
14 | // exist. The ToRepo URI alone does not specify enough information to add
15 | // the repository (because it doesn't specify the VCS type, scheme, etc.).
16 | ToRepoCloneURL string
17 |
18 | // ToUnit is the name of the source unit that is depended on.
19 | ToUnit string
20 |
21 | // ToUnitType is the type of the source unit that is depended on.
22 | ToUnitType string
23 |
24 | // ToVersion is the version of the dependent repository (if known),
25 | // according to whatever version string specifier is used by FromRepo's
26 | // dependency management system.
27 | ToVersionString string
28 |
29 | // ToRevSpec specifies the desired VCS revision of the dependent repository
30 | // (if known).
31 | ToRevSpec string
32 | }
33 |
34 | // END ResolvedTarget OMIT
35 |
36 | // START Resolution OMIT
37 | // Resolution is the result of dependency resolution: either a successfully
38 | // resolved target or an error.
39 | type Resolution struct {
40 | // Raw is the original raw dep that this was resolution was attempted on.
41 | Raw interface{}
42 |
43 | // Target is the resolved dependency, if resolution succeeds.
44 | Target *ResolvedTarget `json:",omitempty"`
45 |
46 | // Error is the resolution error, if any.
47 | Error string `json:",omitempty"`
48 | }
49 |
50 | // END Resolution OMIT
51 |
52 | func (r *Resolution) KeyId() string {
53 | return r.Target.ToRepoCloneURL + r.Target.ToUnit + r.Target.ToUnitType + r.Target.ToVersionString + r.Target.ToRevSpec
54 | }
55 |
56 | func (r *Resolution) RawKeyId() (string, error) {
57 | b, err := json.Marshal(r.Raw)
58 | if err != nil {
59 | return "", err
60 | }
61 | return string(b), nil
62 | }
63 |
--------------------------------------------------------------------------------
/plan/makefile_test.go:
--------------------------------------------------------------------------------
1 | package plan_test
2 |
3 | import (
4 | "bytes"
5 | "strings"
6 | "testing"
7 |
8 | "sourcegraph.com/sourcegraph/makex"
9 | "sourcegraph.com/sourcegraph/srclib"
10 | "sourcegraph.com/sourcegraph/srclib/config"
11 | _ "sourcegraph.com/sourcegraph/srclib/config"
12 | _ "sourcegraph.com/sourcegraph/srclib/dep"
13 | _ "sourcegraph.com/sourcegraph/srclib/grapher"
14 | "sourcegraph.com/sourcegraph/srclib/plan"
15 | "sourcegraph.com/sourcegraph/srclib/toolchain"
16 | "sourcegraph.com/sourcegraph/srclib/unit"
17 | )
18 |
19 | func TestCreateMakefile(t *testing.T) {
20 | oldChooseTool := toolchain.ChooseTool
21 | defer func() { toolchain.ChooseTool = oldChooseTool }()
22 |
23 | toolchain.ChooseTool = func(op, unitType string) (*srclib.ToolRef, error) {
24 | return &srclib.ToolRef{
25 | Toolchain: "tc",
26 | Subcmd: "t",
27 | }, nil
28 | }
29 | buildDataDir := "testdata"
30 | c := &config.Tree{
31 | SourceUnits: []*unit.SourceUnit{
32 | {
33 | Key: unit.Key{
34 | Name: "n",
35 | Type: "t",
36 | },
37 | Info: unit.Info{
38 | Files: []string{"f"},
39 | Ops: map[string][]byte{
40 | "graph": nil,
41 | "depresolve": nil,
42 | },
43 | },
44 | },
45 | },
46 | }
47 |
48 | mf, err := plan.CreateMakefile(buildDataDir, nil, "", c)
49 | if err != nil {
50 | t.Fatal(err)
51 | }
52 |
53 | want := `
54 | .PHONY: all
55 |
56 | all: testdata/n/t.depresolve.json testdata/n/t.graph.json
57 |
58 | testdata/n/t.depresolve.json: testdata/n/t.unit.json
59 | srclib tool "tc" "t" < $^ 1> $@
60 |
61 | testdata/n/t.graph.json: testdata/n/t.unit.json
62 | srclib tool "tc" "t" < $< | srclib internal normalize-graph-data --unit-type "t" --dir . 1> $@
63 |
64 | .DELETE_ON_ERROR:
65 | `
66 |
67 | gotBytes, err := makex.Marshal(mf)
68 | if err != nil {
69 | t.Fatal(err)
70 | }
71 |
72 | want = strings.TrimSpace(want)
73 | got := string(bytes.TrimSpace(gotBytes))
74 |
75 | if got != want {
76 | t.Errorf("got makefile:\n==========\n%s\n==========\n\nwant makefile:\n==========\n%s\n==========", got, want)
77 | }
78 | }
79 |
--------------------------------------------------------------------------------
/cli/tool_cmd.go:
--------------------------------------------------------------------------------
1 | package cli
2 |
3 | import (
4 | "log"
5 | "os"
6 | "os/exec"
7 | "strings"
8 |
9 | "sourcegraph.com/sourcegraph/go-flags"
10 |
11 | "sourcegraph.com/sourcegraph/srclib/toolchain"
12 | )
13 |
14 | func init() {
15 | cliInit = append(cliInit, func(cli *flags.Command) {
16 | c, err := cli.AddCommand("tool",
17 | "run a tool",
18 | "Run a srclib tool with the specified arguments.",
19 | &toolCmd,
20 | )
21 | if err != nil {
22 | log.Fatal(err)
23 | }
24 | c.ArgsRequired = true
25 | })
26 | }
27 |
28 | type ToolCmd struct {
29 | Args struct {
30 | Toolchain ToolchainPath `name:"TOOLCHAIN" description:"toolchain path of the toolchain to run"`
31 | Tool ToolName `name:"TOOL" description:"tool subcommand name to run (in TOOLCHAIN)"`
32 | ToolArgs []string `name:"ARGS" description:"args to pass to TOOL"`
33 | } `positional-args:"yes" required:"yes"`
34 | }
35 |
36 | var toolCmd ToolCmd
37 |
38 | func (c *ToolCmd) Execute(args []string) error {
39 | cmdName, err := toolchain.Command(string(c.Args.Toolchain))
40 | if err != nil {
41 | log.Fatal(err)
42 | }
43 |
44 | cmd := exec.Command(cmdName)
45 | if c.Args.Tool != "" {
46 | cmd.Args = append(cmd.Args, string(c.Args.Tool))
47 | cmd.Args = append(cmd.Args, c.Args.ToolArgs...)
48 | }
49 | cmd.Stdin = os.Stdin
50 | cmd.Stdout = os.Stdout
51 | cmd.Stderr = os.Stderr
52 | if GlobalOpt.Verbose {
53 | log.Printf("Running tool: %v", cmd.Args)
54 | }
55 | return cmd.Run()
56 | }
57 |
58 | type ToolName string
59 |
60 | func (t ToolName) Complete(match string) []flags.Completion {
61 | // Assume toolchain is the last arg.
62 | toolchainPath := os.Args[len(os.Args)-2]
63 | tc, err := toolchain.Lookup(toolchainPath)
64 | if err != nil {
65 | log.Println(err)
66 | return nil
67 | }
68 | c, err := tc.ReadConfig()
69 | if err != nil {
70 | log.Println(err)
71 | return nil
72 | }
73 | var comps []flags.Completion
74 | for _, tt := range c.Tools {
75 | if strings.HasPrefix(tt.Subcmd, match) {
76 | comps = append(comps, flags.Completion{Item: tt.Subcmd})
77 | }
78 | }
79 | return comps
80 | }
81 |
--------------------------------------------------------------------------------
/toolchain/find_test.go:
--------------------------------------------------------------------------------
1 | package toolchain
2 |
3 | import (
4 | "io/ioutil"
5 | "os"
6 | "path"
7 | "path/filepath"
8 | "reflect"
9 | "runtime"
10 | "testing"
11 |
12 | "sourcegraph.com/sourcegraph/srclib"
13 | )
14 |
15 | func TestList_program(t *testing.T) {
16 | tmpdir, err := ioutil.TempDir("", "srclib-toolchain-test")
17 | if err != nil {
18 | t.Fatal(err)
19 | }
20 | defer os.RemoveAll(tmpdir)
21 |
22 | defer func(orig string) {
23 | srclib.Path = orig
24 | }(srclib.Path)
25 | srclib.Path = tmpdir
26 |
27 | var extension string
28 | if runtime.GOOS == "windows" {
29 | extension = ".exe"
30 | } else {
31 | extension = ""
32 | }
33 | files := map[string]os.FileMode{
34 | // ok
35 | filepath.Join("a", "a", ".bin", "a"+extension): 0700,
36 | filepath.Join("a", "a", "Srclibtoolchain"): 0700,
37 |
38 | // not executable
39 | filepath.Join("b", "b", ".bin", "z"+extension): 0600,
40 | filepath.Join("b", "b", "Srclibtoolchain"): 0600,
41 |
42 | // not in .bin
43 | filepath.Join("c", "c", "c"+extension): 0700,
44 | filepath.Join("c", "c", "Srclibtoolchain"): 0700,
45 | }
46 | for f, mode := range files {
47 | f = filepath.Join(tmpdir, f)
48 | if err := os.MkdirAll(filepath.Dir(f), 0700); err != nil {
49 | t.Fatal(err)
50 | }
51 | if err := ioutil.WriteFile(f, nil, mode); err != nil {
52 | t.Fatal(err)
53 | }
54 | }
55 |
56 | // Put a file symlink in srclib DIR path.
57 | oldp := filepath.Join(tmpdir, "a", "a", ".bin", "a"+extension)
58 | newp := filepath.Join(tmpdir, "link")
59 | if err := os.Symlink(oldp, newp); err != nil {
60 | t.Fatal(err)
61 | }
62 |
63 | toolchains, err := List()
64 | if err != nil {
65 | t.Fatal(err)
66 | }
67 |
68 | got := toolchainPathsWithProgram(toolchains)
69 | want := []string{path.Join("a", "a")}
70 | if !reflect.DeepEqual(got, want) {
71 | t.Errorf("got toolchains %v, want %v", got, want)
72 | }
73 | }
74 |
75 | func toolchainPathsWithProgram(toolchains []*Info) []string {
76 | paths := make([]string, 0, len(toolchains))
77 | for _, toolchain := range toolchains {
78 | if toolchain.Program != "" {
79 | paths = append(paths, toolchain.Path)
80 | }
81 | }
82 | return paths
83 | }
84 |
--------------------------------------------------------------------------------
/flagutil/marshal.go:
--------------------------------------------------------------------------------
1 | package flagutil
2 |
3 | import (
4 | "fmt"
5 | "reflect"
6 | "strings"
7 |
8 | "sourcegraph.com/sourcegraph/go-flags"
9 | )
10 |
11 | // MarshalArgs takes a struct with go-flags field tags and turns it into an
12 | // equivalent []string for use as command-line args.
13 | func MarshalArgs(v interface{}) ([]string, error) {
14 | parser := flags.NewParser(nil, flags.None)
15 | group, err := parser.AddGroup("", "", v)
16 | if err != nil {
17 | return nil, err
18 | }
19 | return marshalArgsInGroup(group, "")
20 | }
21 |
22 | func marshalArgsInGroup(group *flags.Group, prefix string) ([]string, error) {
23 | var args []string
24 | for _, opt := range group.Options() {
25 | flagStr := opt.String()
26 |
27 | // handle flags with both short and long (just get the long)
28 | if i := strings.Index(flagStr, ", --"); i != -1 {
29 | flagStr = flagStr[i+2:]
30 | }
31 |
32 | switch v := opt.Value().(type) {
33 | case flags.Marshaler:
34 | s, err := v.MarshalFlag()
35 | if err != nil {
36 | return nil, err
37 | }
38 | args = append(args, flagStr, s)
39 | case []string:
40 | for _, s := range v {
41 | args = append(args, flagStr, s)
42 | }
43 | case bool:
44 | if v {
45 | args = append(args, flagStr)
46 | }
47 | default:
48 | if !isDefaultValue(opt, fmt.Sprintf("%v", v)) {
49 | args = append(args, flagStr, fmt.Sprintf("%v", v))
50 | }
51 | }
52 | }
53 | for _, g := range group.Groups() {
54 | // TODO(sqs): assumes that the NamespaceDelimiter is "."
55 | const namespaceDelimiter = "."
56 | groupArgs, err := marshalArgsInGroup(g, g.Namespace+namespaceDelimiter)
57 | if err != nil {
58 | return nil, err
59 | }
60 | args = append(args, groupArgs...)
61 | }
62 |
63 | return args, nil
64 | }
65 |
66 | func isDefaultValue(opt *flags.Option, val string) bool {
67 | var defaultVal string
68 |
69 | switch len(opt.Default) {
70 | case 0:
71 | if k := reflect.TypeOf(opt.Value()).Kind(); k >= reflect.Int && k <= reflect.Uint64 {
72 | defaultVal = "0"
73 | }
74 | case 1:
75 | defaultVal = fmt.Sprintf("%v", opt.Default[0])
76 | default:
77 | return false
78 | }
79 |
80 | return defaultVal == fmt.Sprintf("%v", val)
81 | }
82 |
--------------------------------------------------------------------------------
/ann/annotation.go:
--------------------------------------------------------------------------------
1 | package ann
2 |
3 | import (
4 | "encoding/json"
5 | "fmt"
6 | "net/url"
7 | "strconv"
8 | "strings"
9 | )
10 |
11 | const (
12 | // Link is a type of annotation that refers to an arbitrary URL
13 | // (typically pointing to an external web page).
14 | Link = "link"
15 | )
16 |
17 | // LinkURL parses and returns a's link URL, if a's type is Link and if
18 | // its Data contains a valid URL (encoded as a JSON string).
19 | func (a *Ann) LinkURL() (*url.URL, error) {
20 | if a.Type != Link {
21 | return nil, &ErrType{Expected: Link, Actual: a.Type, Op: "LinkURL"}
22 | }
23 | var urlStr string
24 | if err := json.Unmarshal(a.Data, &urlStr); err != nil {
25 | return nil, fmt.Errorf("could not unmarshal annotation link URL: %s. JSON was %v", err, a.Data)
26 | }
27 | return url.Parse(urlStr)
28 | }
29 |
30 | // SetLinkURL sets a's Type to Link and Data to the JSON
31 | // representation of the URL string. If the URL is invalid, an error
32 | // is returned.
33 | func (a *Ann) SetLinkURL(urlStr string) error {
34 | u, err := url.Parse(urlStr)
35 | if err != nil {
36 | return err
37 | }
38 | b, err := json.Marshal(u.String())
39 | if err != nil {
40 | return err
41 | }
42 | a.Type = Link
43 | a.Data = b
44 | return nil
45 | }
46 |
47 | // ErrType indicates that an operation performed on an annotation
48 | // expected the annotation to be a different type (e.g., calling
49 | // LinkURL on a non-link annotation).
50 | type ErrType struct {
51 | Expected, Actual string // Expected and actual types
52 | Op string // The name of the operation or method that was called
53 | }
54 |
55 | func (e *ErrType) Error() string {
56 | return fmt.Sprintf("%s called on annotation type %q, expected type %q", e.Op, e.Actual, e.Expected)
57 | }
58 |
59 | func (a *Ann) sortKey() string {
60 | return strings.Join([]string{a.Repo, a.CommitID, a.UnitType, a.Unit, a.Type, a.File, strconv.Itoa(int(a.StartLine)), strconv.Itoa(int(a.EndLine))}, ":")
61 | }
62 |
63 | // Sorting
64 |
65 | type Anns []*Ann
66 |
67 | func (vs Anns) Len() int { return len(vs) }
68 | func (vs Anns) Swap(i, j int) { vs[i], vs[j] = vs[j], vs[i] }
69 | func (vs Anns) Less(i, j int) bool { return vs[i].sortKey() < vs[j].sortKey() }
70 |
--------------------------------------------------------------------------------
/docs/theme/images/layers.svg:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
65 |
--------------------------------------------------------------------------------
/config/cached.go:
--------------------------------------------------------------------------------
1 | package config
2 |
3 | import (
4 | "encoding/json"
5 | "fmt"
6 | "os"
7 | "runtime"
8 | "sort"
9 | "strings"
10 |
11 | "github.com/neelance/parallel"
12 |
13 | "github.com/kr/fs"
14 | "golang.org/x/tools/godoc/vfs"
15 | "sourcegraph.com/sourcegraph/rwvfs"
16 | "sourcegraph.com/sourcegraph/srclib/buildstore"
17 | "sourcegraph.com/sourcegraph/srclib/unit"
18 | )
19 |
20 | // ReadCached reads a Tree's configuration from all of its source unit
21 | // definition files (which may either be in a local VFS rooted at a
22 | // .srclib-cache/
30 |
31 | ##Language Toolchains
32 |
33 | To install the language analysis toolchains for
34 | ([Go](toolchains/go.md), [Ruby](toolchains/ruby.md),
35 | [JavaScript](toolchains/javascript.md), and [Python](toolchains/python.md)), run:
36 |
37 | ```
38 | srclib toolchain install go ruby javascript python
39 | ```
40 |
41 | If this command fails, please
42 | [file an issue](https://github.com/sourcegraph/srclib/issues) or skip
43 | one of the languages if you don't need it.
44 |
45 | `srclib toolchain list` helps to verify the currently installed language toolchains like the following example
46 |
47 | ```
48 | $ srclib toolchain list
49 | sourcegraph.com/sourcegraph/srclib-python
50 | sourcegraph.com/sourcegraph/srclib-go
51 | ```
52 |
53 |
54 |
55 | ##Testing srclib
56 |
57 | In order to test srclib we can use it to analyze the already fetched source code for the Go toolchain `srclib-go`.
58 |
59 | First, you need to initialize the git submodules in the root directory of srclib-go
60 | ```
61 | cd src/sourcegraph.com/sourcegraph/srclib-go
62 | git submodule update --init
63 | ```
64 |
65 | Now you can test the srclib-go toolchain with:
66 |
67 | ```
68 | $ srclib do-all
69 | $ srclib store import
70 | ```
71 |
72 | You should have a .srclib-cache directory inside srclib-go that has all of the
73 | build data for the repository. You should also have a .srclib-store directory
74 | corresponding to the analysis information.
75 |
76 | ```
77 | srclib store defs
78 | ```
79 |
80 | should show you the definitions.
81 |
82 |
83 |
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # How to Contribute
2 |
3 | Sourcegraph projects are [MIT licensed](LICENSE) and accept contributions
4 | via GitHub pull requests. This document outlines some of the conventions on
5 | development workflow, commit message formatting, contact points and other
6 | resources to make it easier to get your contribution accepted.
7 |
8 | # Certificate of Origin
9 |
10 | By contributing to this project you agree to the Developer Certificate of Origin
11 | (DCO). This document was created by the Linux Kernel community and is a simple
12 | statement that you, as a contributor, have the legal right to make the
13 | contribution. See the [DCO](DCO) file for details.
14 |
15 | # Email and Chat
16 |
17 | The project currently uses the
18 | [srclib-dev email list](https://groups.google.com/forum/#!forum/srclib-dev).
19 |
20 | Please avoid emailing maintainers found in the MAINTAINERS file directly. They
21 | are very busy and read the mailing lists.
22 |
23 | ## Getting Started
24 |
25 | - Fork the repository on GitHub
26 | - Read the [README](README.md) for build and test instructions
27 | - Play with the project, submit bugs, submit patches!
28 |
29 | ## Contribution Flow
30 |
31 | This is a rough outline of what a contributor's workflow looks like:
32 |
33 | - Create a topic branch from where you want to base your work (usually master).
34 | - Make commits of logical units.
35 | - Make sure your commit messages are in the proper format (see below).
36 | - Push your changes to a topic branch in your fork of the repository.
37 | - Make sure the tests pass, and add any new tests as appropriate.
38 | - Submit a pull request to the original repository.
39 |
40 | Thanks for your contributions!
41 |
42 | ### Format of the Commit Message
43 |
44 | We follow a rough convention for commit messages that is designed to answer two
45 | questions: what changed and why. The subject line should feature the what and
46 | the body of the commit should describe the why.
47 |
48 | ```
49 | scripts: add the test-cluster command
50 |
51 | this uses tmux to setup a test cluster that you can easily kill and
52 | start for debugging.
53 |
54 | Fixes #38
55 | ```
56 |
57 | The format can be described more formally as follows:
58 |
59 | ```
60 |